探索阿里通义千问的 Qwen2-VL：新一代视觉语言模型本地体验实战大全

modelscope download --model=qwen/Qwen2-VL-7B-Instruct --local_dir ./Qwen2-VL-7B-Instruct

pip install git+https://github.com/huggingface/transformers


pip install qwen-vl-utils

from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessor
from qwen_vl_utils import process_vision_info
# default: Load the model on the available device(s)
# model = Qwen2VLForConditionalGeneration.from_pretrained(
# "./Qwen2-VL-7B-Instruct", torch_dtype="auto", device_map="auto"
# )
import torch


model = Qwen2VLForConditionalGeneration.from_pretrained(
  "./Qwen2-VL-7B-Instruct",
  torch_dtype=torch.bfloat16,
  attn_implementatinotallow="flash_attention_2",
).to("cuda:0")


# We recommend enabling flash_attention_2 for better acceleration and memory saving, especially in multi-image and video scenarios.
# model = Qwen2VLForConditionalGeneration.from_pretrained(
# "Qwen/Qwen2-VL-7B-Instruct",
# torch_dtype=torch.bfloat16,
# attn_implementatinotallow="flash_attention_2",
# device_map="auto",
# )
# default processer
processor = AutoProcessor.from_pretrained("./Qwen2-VL-7B-Instruct")
# The default range for the number of visual tokens per image in the model is 4-16384.
# You can set min_pixels and max_pixels according to your needs, such as a token range of 256-1280, to balance performance and cost.
# min_pixels = 256*28*28
# max_pixels = 1280*28*28
# processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct", min_pixels=min_pixels, max_pixels=max_pixels)


messages = [
  {
    "role": "user",
    "content": [
      {
        "type": "image",
        "image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg",
      },
      {"type": "text", "text": "Describe this image."},
    ],
  }
]
# Preparation for inference
text = processor.apply_chat_template(
  messages, tokenize=False, add_generation_prompt=True
)
image_inputs, video_inputs = process_vision_info(messages)
inputs = processor(
  text=[text],
  images=image_inputs,
  videos=video_inputs,
  padding=True,
  return_tensors="pt",
)
inputs = inputs.to("cuda")
# Inference: Generation of the output
generated_ids = model.generate(**inputs, max_new_tokens=128)
generated_ids_trimmed = [
  out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
]
output_text = processor.batch_decode(
  generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
)
print(output_text)

git clone https://github.com/modelscope/swift.git
cd swiftpip install -e .[llm]
pip install pyav qwen_vl_utils

# 默认会将lora_target_modules设置为llm的所有linear
CUDA_VISIBLE_DEVICES=0,1,2,3 NPROC_PER_NODE=4 swift sft \
--model_type qwen2-vl-7b-instruct \
--model_id_or_path qwen/Qwen2-VL-7B-Instruct \
--sft_type lora \
--dataset coco-en-mini#20000 \
--deepspeed default-zero2

--dataset train.jsonl \
--val_dataset val.jsonl \

{"query": "<image>55555", "response": "66666", "images": ["image_path"]}
{"query": "eeeee<image>eeeee<image>eeeee", "response": "fffff", "history": [], "images": ["image_path1", "image_path2"]}
{"query": "EEEEE", "response": "FFFFF", "history": [["query1", "response2"

CUDA_VISIBLE_DEVICES=0 swift infer \
--ckpt_dir output/qwen2-vl-7b-instruct/vx-xxx/checkpoint-xxx \
--load_dataset_config true --merge_lora true