本地部署

# macOS / Linux
curl -fsSL https://ollama.com/install.sh | sh

# macOS 也可以通过 Homebrew
brew install ollama

# Windows
# 从 https://ollama.com/download 下载安装包

# 启动 Ollama 服务（安装后通常自动启动）
ollama serve

# 运行模型（首次会自动下载）
ollama run llama3.1
ollama run qwen2.5:7b
ollama run deepseek-r1:8b

# 查看已下载的模型
ollama list

# 删除模型
ollama rm llama3.1

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:11434/v1",
    api_key="ollama"  # Ollama 不需要真正的 API key
)

response = client.chat.completions.create(
    model="qwen2.5:7b",
    messages=[
        {"role": "user", "content": "用 Python 写一个快速排序算法"}
    ]
)

print(response.choices[0].message.content)

curl http://localhost:11434/api/generate -d '{
  "model": "qwen2.5:7b",
  "prompt": "什么是 Transformer？",
  "stream": false
}'

# Modelfile
FROM qwen2.5:7b

# 设置系统提示词
SYSTEM "你是一个专业的技术文档助手，使用中文回答问题。"

# 调整参数
PARAMETER temperature 0.7
PARAMETER top_p 0.9
PARAMETER num_ctx 8192

# 创建自定义模型
ollama create my-assistant -f Modelfile

# 运行自定义模型
ollama run my-assistant

# 安装（需要 NVIDIA GPU）
pip install vllm

# 启动 API 服务
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-7B-Instruct \
    --host 0.0.0.0 \
    --port 8000 \
    --max-model-len 8192 \
    --gpu-memory-utilization 0.9

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="token-abc123"  # vLLM 默认不校验
)

response = client.chat.completions.create(
    model="Qwen/Qwen2.5-7B-Instruct",
    messages=[
        {"role": "system", "content": "你是一个有用的助手。"},
        {"role": "user", "content": "解释 PagedAttention 的原理"}
    ],
    temperature=0.7,
    max_tokens=1024,
)

print(response.choices[0].message.content)

# 克隆并编译
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp

# macOS (Metal 加速)
cmake -B build -DGGML_METAL=ON
cmake --build build --config Release

# Linux (CUDA 加速)
cmake -B build -DGGML_CUDA=ON
cmake --build build --config Release

# 纯 CPU
cmake -B build
cmake --build build --config Release

# 交互式对话
./build/bin/llama-cli \
    -m models/qwen2.5-7b-instruct-q4_k_m.gguf \
    -n 512 \
    -t 8 \
    --interactive-first

# 启动 API 服务（兼容 OpenAI 格式）
./build/bin/llama-server \
    -m models/qwen2.5-7b-instruct-q4_k_m.gguf \
    --host 0.0.0.0 \
    --port 8080 \
    -t 8 \
    -ngl 99  # GPU 层数，99 表示尽可能多地放到 GPU

# 安装依赖
pip install -r requirements.txt

# 转换为 GGUF（FP16）
python convert_hf_to_gguf.py /path/to/model --outfile model-f16.gguf

# 量化
./build/bin/llama-quantize model-f16.gguf model-q4_k_m.gguf Q4_K_M

# 使用 huggingface-cli
pip install huggingface_hub
huggingface-cli download Qwen/Qwen2.5-7B-Instruct --local-dir ./qwen2.5-7b

# 下载 GGUF 文件
huggingface-cli download bartowski/Qwen2.5-7B-Instruct-GGUF \
    Qwen2.5-7B-Instruct-Q4_K_M.gguf \
    --local-dir ./models

# 安装
pip install modelscope

# 下载模型
from modelscope import snapshot_download
model_dir = snapshot_download('Qwen/Qwen2.5-7B-Instruct')

# 命令行下载
modelscope download --model Qwen/Qwen2.5-7B-Instruct --local_dir ./qwen2.5-7b

from fastapi import FastAPI
from pydantic import BaseModel
import httpx

app = FastAPI()

class ChatRequest(BaseModel):
    message: str
    model: str = "qwen2.5:7b"
    temperature: float = 0.7

@app.post("/chat")
async def chat(request: ChatRequest):
    async with httpx.AsyncClient() as client:
        response = await client.post(
            "http://localhost:11434/api/chat",
            json={
                "model": request.model,
                "messages": [{"role": "user", "content": request.message}],
                "stream": False,
                "options": {"temperature": request.temperature}
            },
            timeout=120.0
        )
    return response.json()

# docker-compose.yml
version: '3.8'
services:
  ollama:
    image: ollama/ollama
    ports:
      - "11434:11434"
    volumes:
      - ollama_data:/root/.ollama
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

  # 可选：Open WebUI（提供聊天界面）
  open-webui:
    image: ghcr.io/open-webui/open-webui:main
    ports:
      - "3000:8080"
    environment:
      - OLLAMA_BASE_URL=http://ollama:11434
    depends_on:
      - ollama

volumes:
  ollama_data:

docker compose up -d

# 下载模型
docker exec -it ollama ollama pull qwen2.5:7b