我训练了一个医疗多模态大模型帮家里老人看病 原创

pip install datasets

from datasets import load_dataset

# 加载数据集
ds = load_dataset("UCSC-VLAA/MedTrinity-25M", "25M_demo", cache_dir="cache")

# 查看训练集的前1个样本
print(ds['train'][:1])

{
    'image': [<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=512x512 at 0x15DD6D06530>], 
    'id': ['8031efe0-1b5c-11ef-8929-000066532cad'], 
    'caption': ['The image is a non-contrasted computed tomography (CT) scan of the brain, showing the cerebral structures without any medical devices present. The region of interest, located centrally and in the middle of the image, exhibits an area of altered density, which is indicative of a brain hemorrhage. This area is distinct from the surrounding brain tissue, suggesting a possible hematoma or bleeding within the brain parenchyma. The location and characteristics of this abnormality may suggest a relationship with the surrounding brain tissue, potentially causing a mass effect or contributing to increased intracranial pressure.'
    ]
}

# 可视化image内容
from PIL import Image
import matplotlib.pyplot as plt

image = ds['train'][0]['image']  # 获取第一张图像

plt.imshow(image)
plt.axis('off')  # 不显示坐标轴
plt.show()

[
  {
    "messages":[
        {
            "content":"<image>他们是谁？",
            "role":"user"
        },
        {
            "content":"他们是拜仁慕尼黑的凯恩和格雷茨卡。",
            "role":"assistant"
        },
        {
            "content":"他们在做什么？",
            "role":"user"
        },
        {
            "content":"他们在足球场上庆祝。",
            "role":"assistant"
        }
    ],
    "images":[
        "mllm_demo_data/1.jpg"
    ]
  }
]

from datasets import load_dataset
import os
import json
from PIL importImage

defsave_images_and_json(ds, output_dir="mllm_data"):
"""
    将数据集中的图像和对应的 JSON 信息保存到指定目录。

    参数:
    ds: 数据集对象，包含图像和标题。
    output_dir: 输出目录，默认为 "mllm_data"。
    """
# 创建输出目录
ifnot os.path.exists(output_dir):
        os.makedirs(output_dir)

# 创建一个列表来存储所有的消息和图像信息
    all_data =[]

# 遍历数据集中的每个项目
for item in ds:
        img_path =f"{output_dir}/{item['id']}.jpg"# 图像保存路径
        image = item["image"]# 假设这里是一个 PIL 图像对象

# 将图像对象保存为文件
        image.save(img_path)# 使用 PIL 的 save 方法

# 添加消息和图像信息到列表中
        all_data.append(
        {
            "messages":[
            {
                "content":"<image>图片中的诊断结果是怎样?",
                "role":"user",
            },
            {
                "content": item["caption"],# 从数据集中获取的标题
                "role":"assistant",
            },
            ],
            "images":[img_path],# 图像文件路径
        }
)

# 创建 JSON 文件
    json_file_path =f"{output_dir}/mllm_data.json"
withopen(json_file_path,"w", encoding='utf-8')as f:
        json.dump(all_data, f, ensure_ascii=False)# 确保中文字符正常显示

if __name__ =="__main__":
# 加载数据集
    ds = load_dataset("UCSC-VLAA/MedTrinity-25M","25M_demo", cache_dir="cache")

# 保存数据集中的图像和 JSON 信息
    save_images_and_json(ds['train'])

git lfs install
# 下载模型
git clone https://www.modelscope.cn/Qwen/Qwen2-VL-2B-Instruct.git

# 创建python3.10版本虚拟环境
conda create --name train_env pythnotallow=3.10

# 激活环境
conda activate train_env

# 安装依赖包
pip install streamlit torch torchvision

# 安装Qwen2建议的transformers版本
pip install git+https://github.com/huggingface/transformers

# 原始代码
require_version("transformers>=4.41.2,<=4.45.2", "To fix: pip install transformers>=4.41.2,<=4.45.2")
# 修改后代码
require_version("transformers>=4.41.2,<=4.47.0", "To fix: pip install transformers>=4.41.2,<=4.47.0")

llamafactory-cli webui

# 卸载旧的 accelerate
pip uninstall accelerate

# 安装新的 accelerate
pip install accelerate==0.34.0

"mllm_med":{
"file_name":"mllm_data/mllm_data.json",
"formatting":"sharegpt",
"columns":{
"messages":"messages",
"images":"images"
},
"tags":{
"role_tag":"role",
"content_tag":"content",
"user_tag":"user",
"assistant_tag":"assistant"
}
},

llamafactory-cli train \
--do_train True \
--model_name_or_path models/Qwen2-VL-2B-Instruct \
--preprocessing_num_workers 16 \
--finetuning_type lora \
--template qwen2_vl \
--flash_attn auto \
--dataset_dir data \
--dataset mllm_med \
--cutoff_len 1024 \
--learning_rate 5e-05 \
--num_train_epochs 3.0 \
--max_samples 100000 \
--per_device_train_batch_size 2 \
--gradient_accumulation_steps 8 \
--lr_scheduler_type cosine \
--max_grad_norm 1.0 \
--logging_steps 5 \
--save_steps 3000 \
--warmup_steps 0 \
--optim adamw_torch \
--packing False \
--report_to none \
--output_dir saves/Qwen2-VL-2B/full/Qwen2-VL-sft-demo1 \
--bf16 True \
--plot_loss True \
--ddp_timeout 180000000 \
--include_num_input_tokens_seen True \
--lora_rank 8 \
--lora_alpha 16 \
--lora_dropout 0 \
--lora_target all