【备忘】LLM部署

import os
os.environ['TRANSFORMERS_OFFLINE']="1"  # 让transformers load模型的时候从本地load，不要连接线上避免http timeout
os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'  # 使用hf-mirror下载模型

from typing import Literal
from pydantic import BaseModel, create_model

class Judge(BaseModel):
    rationale: str
    judge: Literal['Yes', 'No']

definitions = dict()
for t in topk_tags:
    definitions[t] = Judge
Results = create_model("Results", **definitions)
LLMOutput = create_model("LLMOutput", description=str, results=Results)


chat_response = self.client.chat.completions.create(
    model="Qwen/Qwen2.5-VL-3B-Instruct",
    messages=[
        {"role": "system", "content": "You are a helpful assistant. Please answer in Chinese."},
        {
            "role": "user",
            "content": [
                content,
                {"type": "text", "text": prompt},
            ],
        }
    ],
    extra_body={
        "guided_json": LLMOutput.model_json_schema(),
        "guided_whitespace_pattern": "",
        "top_k": 1,  # 确定性输出
    },
)