在趋动云上使用xinference部署大模型 原创

# 安装transformers引擎
pip install "xinference[transformers]"

# 安装sentence-transformers
pip install sentence-transformers
1.
2.
3.
4.
5.

git lfs install
git clone https://www.modelscope.cn/Qwen/Qwen2.5-0.5B-Instruct.git
1.
2.

from openai importOpenAI
openai_api_key ="EMPTY"
openai_api_base ="http://direct.virtaicloud.com:40336/v1"

client =OpenAI(
    api_key=openai_api_key,
    base_url=openai_api_base,
)

chat_response = client.chat.completions.create(
    model="Qwen2.5-0.5B-Instruct",
    messages=[
{"role":"system","content":"你是一个很有用的助手。"},
{"role":"user","content":"中华人民共和国的首都是哪里？"},
]
)
print("Chat response:", chat_response)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.

from openai importOpenAI
openai_api_key ="EMPTY"
openai_api_base ="http://direct.virtaicloud.com:40336/v1"

client =OpenAI(
    api_key=openai_api_key,
    base_url=openai_api_base,
)

chat_response = client.chat.completions.create(
    model="qwen2-instruct",
    messages=[
{"role":"system","content":"你是一个很有用的助手。"},
{"role":"user","content":"我最近失眠比较厉害，请问应该如何诊治？"},
]
)
print("Chat response:", chat_response)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.

from openai importOpenAI
import base64

# 配置OpenAI客户端
openai_api_key ="EMPTY"
openai_api_base ="http://direct.virtaicloud.com:40336/v1"# 请根据实际端口映射地址修改

client =OpenAI(
    api_key=openai_api_key,
    base_url=openai_api_base,
)

def encode_image_to_base64(image_path):
"""将图片转换为base64编码"""
withopen(image_path,"rb")as image_file:
return base64.b64encode(image_file.read()).decode('utf-8')

def chat_with_image(image_path, prompt):
"""与多模态模型对话"""
# 将图片转换为base64
    base64_image = encode_image_to_base64(image_path)


    messages=[
    {
     "role":"user",
     "content":[
        {"type":"text","text":prompt},
        {
          "type":"image_url",
          "image_url":{
          "url":f"data:image/png;base64,{base64_image}"
          }
        }
        ]
      }
    ]
# 调用模型
# try:

    response = client.chat.completions.create(
        model="qwen-vl-chat",# 使用部署的多模态模型名称
        messages=messages,
        max_tokens=1024,
        temperature=0.7,
        response_format={"type":"text"}# 指定响应格式为文本
)
return response.choices[0].message.content
# except Exception as e:
#     return f"调用出错: {str(e)}"

if __name__ =="__main__":
# 测试调用
    image_path ="./脑部CT.png"# 替换为实际的图片路径
    prompt ="这张图片中有什么内容？请详细描述。"

    result = chat_with_image(image_path, prompt)
print("模型回复:", result)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.