MLLM QLoRA微调实战：基于最新的袖珍Mini-InternVL模型

quant_config = BitsAndBytesConfig(
 load_in_4bit=True,
 bnb_4bit_quant_type="nf4",
 bnb_4bit_compute_dtype=torch.bfloat16,
)

model = InternVLChatModel.from_pretrained(
 args.path,
 device_map={"": 0},
 quantization_cnotallow=quant_config if args.quant else None,
 torch_dtype=torch.bfloat16,
)

tokenizer = InternLM2Tokenizer.from_pretrained(args.path)
# 在max_num参数中设置图片小块的最大数量

model.eval()

pixel_values = (
 load_image(image_base_path / "X51005255805.jpg", max_num=6)
 .to(torch.bfloat16)
 .cuda()
)

generation_config = dict(
 num_beams=1,
 max_new_tokens=512,
 do_sample=False,
)

# 单轮单图像对话
question = (
 "Extract the company, date, address and total in json format."
 "Respond with a valid JSON only."
)
# print(model)
response = model.chat(tokenizer, pixel_values, question, generation_config)

print(response)
上述代码的输出结果是：
```json
{
 "company": "SAM SAM TRADING CO",
 "date": "Fri, 29-12-2017",
 "address": "67, JLN MENHAW 25/63 TNN SRI HUDA, 40400 SHAH ALAM",
 "total": "RM 14.10"
}
```

_data = load_data(args.data_path, fold="train")

# 量化配置
quant_config = BitsAndBytesConfig(
 load_in_4bit=True,
 bnb_4bit_quant_type="nf4",
 bnb_4bit_compute_dtype=torch.bfloat16,
)

model = InternVLChatModel.from_pretrained(
 path,
 device_map={"": 0},
 quantization_cnotallow=quant_config,
 torch_dtype=torch.bfloat16,
)

tokenizer = InternLM2Tokenizer.from_pretrained(path)

# 在max_num参数中设置图片小块的最大数量
img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
print("img_context_token_id", img_context_token_id)
model.img_context_token_id = img_context_token_id

model.config.llm_config.use_cache = False

model = wrap_lora(model, r=128, lora_alpha=256)

training_data = SFTDataset(
 data=_data, template=model.config.template, tokenizer=tokenizer
)

collator = CustomDataCollator(pad_token=tokenizer.pad_token_id, ignore_index=-100)

img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
print("img_context_token_id", img_context_token_id)
model.img_context_token_id = img_context_token_id
print("model.img_context_token_id", model.img_context_token_id)

train_params = TrainingArguments(
 output_dir=str(BASE_PATH / "results_modified"),
 num_train_epochs=EPOCHS,
 per_device_train_batch_size=1,
 gradient_accumulation_steps=16,
 optim="paged_adamw_32bit",
 save_steps=len(training_data) // 10,
 logging_steps=len(training_data) // 50,
 learning_rate=5e-4,
 lr_scheduler_type="cosine",
 warmup_steps=100,
 weight_decay=0.001,
 max_steps=-1,
 group_by_length=False,
 max_grad_norm=1.0,
)
# 训练器
fine_tuning = SFTTrainer(
 model=model,
 train_dataset=training_data,
 dataset_text_field="###",
 tokenizer=tokenizer,
 args=train_params,
 data_collator=collator,
 max_seq_length=tokenizer.model_max_length,
)

print(fine_tuning.model.print_trainable_parameters())
# 开始训练
fine_tuning.train()
# 保存模型
fine_tuning.model.save_pretrained(refined_model)

Ground Truth: 

{
 "company": "YONG TAT HARDWARE TRADING",
 "date": "13/03/2018",
 "address": "NO 4,JALAN PERJIRANAN 10, TAMAN AIR BIRU, 81700 PASIR GUDANG, JOHOR.",
 "total": "72.00"
}

Prediction Base: KO

```json
{
 "company": "YONG TAT HARDWARE TRADING",
 "date": "13/03/2016",
 "address": "JM092487-D",
 "total": "67.92"
}
```

Prediction QLoRA: OK

{
 "company": "YONG TAT HARDWARE TRADING",
 "date": "13/03/2018",
 "address": "NO 4, JALAN PERUBANAN 10, TAMAN AIR BIRU, 81700 PASIR GUDANG, JOHOR",
 "total": "72.00"
}