多模态大模型：算法、应用与微调

# 数据格式 metadata.jsonl + 图片
folder/train/metadata.jsonl #存储caption描述
folder/train/0001.png
folder/train/0002.png
folder/train/0003.png
  # metadata.jsonl中的内容
  {"file_name": "0001.png", "text": "image 1 description"}
  {"file_name": "0002.png", "text": "image 2 description"}
  {"file_name": "0003.png", "text": "image 3 description"}

python
import os
import json
import requests
from bs4 import BeautifulSoup

# 创建文件夹
data_dir = "./train"
if not os.path.exists(data_dir):
    os.makedirs(data_dir)
# 请求数码兽图鉴页面
url = "http://digimons.net/digimon/chn.html"
response = requests.get(url)
soup = BeautifulSoup(response.content, "html.parser")
# 遍历所有的 li 标签
digimon_list = soup.find("ul", id="digimon_list")
for digimon in digimon_list.find_all("li"):
    try:
        # 获取数码兽名称和详情页面链接
        name = digimon.find('a')["href"].split('/')[0]
        detail_url = "http://digimons.net/digimon/" + digimon.find('a')["href"]
        print(f"detail_url: {detail_url}")
        # 进入详情页面，获取数码兽介绍和图片链接
        response = requests.get(detail_url)
        soup = BeautifulSoup(response.content, "html.parser")
        caption = soup.find("div", class_="profile_eng").find('p').text.strip()
        img_url = f"http://digimons.net/digimon/{name}/{name}.jpg"
        # 保存图片
        img_data = requests.get(img_url).content
        file_name = f"{len(os.listdir(data_dir)) + 1:04d}.png"
        with open(os.path.join(data_dir, file_name), "wb") as f:
            f.write(img_data)
        # 将数据整理成指定的格式，并保存到对应的文件中
        metadata = {"file_name": file_name, "text": f"{name}. {caption}"}
        with open(os.path.join(data_dir, "metadata.jsonl"), 'a') as f:
            f.write(json.dumps(metadata) + '\n')
    except Exception as _:
        pass

bash
pip install git+https://github.com/Hugging Face/diffusers

然后我们需要初始化Accelerate分布式训练环境。
bash
> accelerate config                                 
[2023-07-20 18:37:53,537] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect)
NOTE: Redirects are currently not supported in Windows or MacOs.
--------------------------------------------------------------------------------------------------------------------In which compute environment are you running?
This machine
--------------------------------------------------------------------------------------------------------------------Which type of machine are you using?
No distributed training
Do you want to run your training on CPU only (even if a GPU / Apple Silicon device is available)? [yes/NO]: NO        
Do you wish to optimize your script with torch dynamo?[yes/NO]: NO                                                    
Do you want to use DeepSpeed? [yes/NO]: NO                                                                            
What GPU(s) (by id) should be used for training on this machine as a comma-seperated list? [all]: all                  
--------------------------------------------------------------------------------------------------------------------Do you wish to use FP16 or BF16 (mixed precision)?
no
accelerate configuration saved at C:\Users\admin/.cache\Hugging Face\accelerate\default_config.yaml

bash
accelerate launch --mixed_precisinotallow="fp16"  train_text_to_image_lora.py \
   --pretrained_model_name_or_path="runwayml/stable-diffusion-v1-5" \
--train_data_dir="./train_data" \
--dataloader_num_workers=0 \
--resolutinotallow=512  --center_crop  --random_flip \
--train_batch_size=1 \
--gradient_accumulation_steps=4 \
--max_train_steps=15000 \
--learning_rate=1e-04 \
--max_grad_norm=1 \
--lr_scheduler="cosine" --lr_warmup_steps=0 \
--output_dir="./finetune/lora/digimon" \
--checkpointing_steps=500 \
--validation_prompt="Blue Agumon" \
--seed=1024

python
import torch
from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler

model_path = "runwayml/stable-diffusion-v1-5"
LoRA_path = "./finetune/lora/digimon"  # 修改成本地LoRA模型路径
pipe = StableDiffusionPipeline.from_pretrained(model_path, torch_dtype=torch.float16)
pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
pipe.unet.load_attn_procs(LoRA_path)
pipe.to("cuda")
image = pipe("blue skin agumon", num_inference_steps=50).images[0]
image.save("test.png")

python
pipe.unet.load_attn_procs(lora_model_path)
pipe.to("cuda")
image = pipe(
    "A agumon with blue skin.", num_inference_steps=25, guidance_scale=7.5, cross_attention_kwargs={"scale": 0.5}
).images[0]
image.save("blue_pokemon.png")