Megrez-3B-Omni：无问芯穹端侧全模态大模型技术剖析与应用指南

git clone https://github.com/infinigence/Infini-Megrez-Omni.git
pip install -r requirements.txt

sudo apt-get install ffmpeg

# 模型下载 
from modelscope import snapshot_download
model_dir = snapshot_download('InfiniAI/Megrez-3B-Omni', cache_dir='/root/autodl-tmp', revision='master')

import torch
from transformers import AutoModelForCausalLM
 
path = "/mnt/algorithm/user_dir/zhoudong/workspace/models/megrez-o"  # Change this to the path of the model.
 
model = (
    AutoModelForCausalLM.from_pretrained(
        path,
        trust_remote_code=True,
        torch_dtype=torch.bfloat16,
        attn_implementatinotallow="flash_attention_2",
    )
    .eval()
    .cuda()
)
prompt = "hi" * (128 - 1) 
# Chat with text and image
messages = [
    {
        "role": "user",
        "content": {
            "text": prompt,
            "image": "./data/sample_image.jpg",
        },
    },
]
 
# Chat with audio and image
# messages = [
#     {
#         "role": "user",
#         "content": {
#             "image": "./data/sample_image.jpg",
#             "audio": "./data/sample_audio.m4a",
#         },
#     },
# ]
 
MAX_NEW_TOKENS = 100
response = model.chat(
    messages,
    sampling=False,
    max_new_tokens=MAX_NEW_TOKENS,
)
print(response)

pip install vLLM==0.6.3.post1 flash_attn==2.5.8 xformers==0.0.27.post2

from PIL import Image
from vllm import LLM
from vllm import ModelRegistry
from vllm import SamplingParams
 
from megrezo import MegrezOModel
 
ModelRegistry.register_model("MegrezO", MegrezOModel)
 
# Load the model.
# model_path = "{{PATH_TO_HF_PRETRAINED_MODEL}}"  # Change this to the path of the model.
model_path = "/mnt/algorithm/user_dir/zhoudong/workspace/models/megrez-o"  # Change this to the path of the model.
llm = LLM(
    model_path,
    trust_remote_code=True,
    gpu_memory_utilizatinotallow=0.5,
)
 
sampling_params = SamplingParams(
    temperature=0,
    max_tokens=1000,
    repetition_penalty=1.2,
    stop=["<|turn_end|>", "<|eos|>"],
)
 
img = Image.open("../data/sample_image.jpg")
 
conversation = [
    {
        "role": "user",
        "content": {
            "text": "图片的内容是什么？",
            "image": img,
        },
    },
]
 
# Convert the conversation to vLLM acceptable format.
prompt = llm.get_tokenizer().apply_chat_template(
    conversation,
    tokenize=False,
    add_generation_prompt=True,
)
vllm_inputs = [
    {
        "prompt": prompt,
        "multi_modal_data": {
            "image": img,
        },
    }
]
 
# Generate the outputs.
outputs = llm.generate(
    vllm_inputs,
    sampling_params,
)
 
# Print the outputs.
for output in outputs:
    print(output.outputs[0].text)

python gradio_app.py --model_path {model_path} --port {port}