多模态大模型：算法、应用与微调

# 数据格式 metadata.jsonl + 图片
folder/train/metadata.jsonl #存储caption描述
folder/train/0001.png
folder/train/0002.png
folder/train/0003.png
  # metadata.jsonl中的内容
  {"file_name": "0001.png", "text": "image 1 description"}
  {"file_name": "0002.png", "text": "image 2 description"}
  {"file_name": "0003.png", "text": "image 3 description"}
1.
2.
3.
4.
5.
6.
7.
8.
9.

python
import os
import json
import requests
from bs4 import BeautifulSoup

# 创建文件夹
data_dir = "./train"
if not os.path.exists(data_dir):
    os.makedirs(data_dir)
# 请求数码兽图鉴页面
url = "http://digimons.net/digimon/chn.html"
response = requests.get(url)
soup = BeautifulSoup(response.content, "html.parser")
# 遍历所有的 li 标签
digimon_list = soup.find("ul", id="digimon_list")
for digimon in digimon_list.find_all("li"):
    try:
        # 获取数码兽名称和详情页面链接
        name = digimon.find('a')["href"].split('/')[0]
        detail_url = "http://digimons.net/digimon/" + digimon.find('a')["href"]
        print(f"detail_url: {detail_url}")
        # 进入详情页面，获取数码兽介绍和图片链接
        response = requests.get(detail_url)
        soup = BeautifulSoup(response.content, "html.parser")
        caption = soup.find("div", class_="profile_eng").find('p').text.strip()
        img_url = f"http://digimons.net/digimon/{name}/{name}.jpg"
        # 保存图片
        img_data = requests.get(img_url).content
        file_name = f"{len(os.listdir(data_dir)) + 1:04d}.png"
        with open(os.path.join(data_dir, file_name), "wb") as f:
            f.write(img_data)
        # 将数据整理成指定的格式，并保存到对应的文件中
        metadata = {"file_name": file_name, "text": f"{name}. {caption}"}
        with open(os.path.join(data_dir, "metadata.jsonl"), 'a') as f:
            f.write(json.dumps(metadata) + '\n')
    except Exception as _:
        pass
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.

bash
pip install git+https://github.com/Hugging Face/diffusers

然后我们需要初始化Accelerate分布式训练环境。
bash
> accelerate config                                 
[2023-07-20 18:37:53,537] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect)
NOTE: Redirects are currently not supported in Windows or MacOs.
--------------------------------------------------------------------------------------------------------------------In which compute environment are you running?
This machine
--------------------------------------------------------------------------------------------------------------------Which type of machine are you using?
No distributed training
Do you want to run your training on CPU only (even if a GPU / Apple Silicon device is available)? [yes/NO]: NO        
Do you wish to optimize your script with torch dynamo?[yes/NO]: NO                                                    
Do you want to use DeepSpeed? [yes/NO]: NO                                                                            
What GPU(s) (by id) should be used for training on this machine as a comma-seperated list? [all]: all                  
--------------------------------------------------------------------------------------------------------------------Do you wish to use FP16 or BF16 (mixed precision)?
no
accelerate configuration saved at C:\Users\admin/.cache\Hugging Face\accelerate\default_config.yaml
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.

bash
accelerate launch --mixed_precisinotallow="fp16"  train_text_to_image_lora.py \
   --pretrained_model_name_or_path="runwayml/stable-diffusion-v1-5" \
--train_data_dir="./train_data" \
--dataloader_num_workers=0 \
--resolutinotallow=512  --center_crop  --random_flip \
--train_batch_size=1 \
--gradient_accumulation_steps=4 \
--max_train_steps=15000 \
--learning_rate=1e-04 \
--max_grad_norm=1 \
--lr_scheduler="cosine" --lr_warmup_steps=0 \
--output_dir="./finetune/lora/digimon" \
--checkpointing_steps=500 \
--validation_prompt="Blue Agumon" \
--seed=1024
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.

python
import torch
from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler

model_path = "runwayml/stable-diffusion-v1-5"
LoRA_path = "./finetune/lora/digimon"  # 修改成本地LoRA模型路径
pipe = StableDiffusionPipeline.from_pretrained(model_path, torch_dtype=torch.float16)
pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
pipe.unet.load_attn_procs(LoRA_path)
pipe.to("cuda")
image = pipe("blue skin agumon", num_inference_steps=50).images[0]
image.save("test.png")
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.

python
pipe.unet.load_attn_procs(lora_model_path)
pipe.to("cuda")
image = pipe(
    "A agumon with blue skin.", num_inference_steps=25, guidance_scale=7.5, cross_attention_kwargs={"scale": 0.5}
).images[0]
image.save("blue_pokemon.png")
1.
2.
3.
4.
5.
6.
7.