Phi-3-Vision-128K大模型，AI助力OCR，文档处理更上一层楼

# 所需包
flash_attn==2.5.8
numpy==1.24.4
Pillow==10.3.0
Requests==2.31.0
torch==2.3.0
torchvision==0.18.0
transformers==4.40.2

from PIL import Image
import requests
from transformers import AutoModelForCausalLM, AutoProcessor

class Phi3VisionModel:
    def __init__(self, model_id="microsoft/Phi-3-vision-128k-instruct", device="cuda"):
        """
        Initialize the Phi3VisionModel with the specified model ID and device.
        
        Args:
            model_id (str): The identifier of the pre-trained model from Hugging Face's model hub.
            device (str): The device to load the model on ("cuda" for GPU or "cpu").
        """
        self.model_id = model_id
        self.device = device
        self.model = self.load_model()  # 在初始化期间加载模型
        self.processor = self.load_processor()  # 在初始化期间加载处理器
    
    def load_model(self):
        """
        Load the pre-trained language model with causal language modeling capabilities.
        
        Returns:
            model (AutoModelForCausalLM): The loaded model.
        """
        print("Loading model...")
        # 使用自动设备映射和数据类型调整加载模型
        return AutoModelForCausalLM.from_pretrained(
            self.model_id, 
            device_map="auto",  # 自动将模型映射到适当的设备
            torch_dtype="auto",  # 根据设备使用适当的 torch 数据类型
            trust_remote_code=True,  # 允许执行自定义代码以加载模型
            _attn_implementation='flash_attention_2'  # 使用优化的注意力实现
        ).to(self.device)  # 将模型移动到指定的设备
    
    def load_processor(self):
        """
        Load the processor associated with the model for processing inputs and outputs.
        
        Returns:
            processor (AutoProcessor): The loaded processor for handling text and images.
        """
        print("Loading processor...")
        # 使用 trust_remote_code=True 加载处理器，以处理任何自定义处理逻辑
        return AutoProcessor.from_pretrained(self.model_id, trust_remote_code=True)
    
    def predict(self, image_url, prompt):
        """
        Perform a prediction using the model given an image and a prompt.
        
        Args:
            image_url (str): The URL of the image to be processed.
            prompt (str): The textual prompt that guides the model's generation.
        
        Returns:
            response (str): The generated response from the model.
        """
        # 从提供的 URL 加载图像
        image = Image.open(requests.get(image_url, stream=True).raw)
        
        # 为模型格式化输入提示模板
        prompt_template = f"<|user|>\n<|image_1|>\n{prompt}<|end|>\n<|assistant|>\n"
        
        # 处理输入，将提示和图像转换为张量格式
        inputs = self.processor(prompt_template, [image], return_tensors="pt").to(self.device)
        
        # 设置模型响应生成的生成参数
        generation_args = {
            "max_new_tokens": 500,  # 生成的最大令牌数
            "temperature": 0.7,     # 生成的采样温度，以增加多样性
            "do_sample": False      # 禁用采样以获得确定性输出
        }
        print("Generating response...")
        # 使用模型生成输出 ID，跳过输入令牌
        output_ids = self.model.generate(**inputs, **generation_args)
        output_ids = output_ids[:, inputs['input_ids'].shape[1]:]  # Ignore the input prompt in the output
        
        # 解码生成的输出令牌以获得响应文本
        response = self.processor.batch_decode(output_ids, skip_special_tokens=True)[0]
        return response

# 初始化模型
phi_model = Phi3VisionModel()

# 示例预测
image_url = "https://example.com/sample_image.png"  # 示例图像的 URL  
prompt = "Extract the data in json format."  # 指导模型的提示  
response = phi_model.predict(image_url, prompt)  # 获取模型的响应 

print("Response:", response)  # 打印生成的响应

{
  "Type/Type": "P",
  "Country code/Code du pays": "UTO",
  "Passport Number/N° de passeport": "L898902C3",
  "Surname/Nom": "ERIKSSON",
  "Given names/Prénoms": "ANNA MARIA",
  "Nationality/Nationalité": "UTOPIAN",
  "Date of Birth/Date de naissance": "12 AUGUST/AOUT 74",
  "Personal No./N° personnel": "Z E 184226 B",
  "Sex/Sexe": "F",
  "Place of birth/Lieu de naissance": "ZENITH",
  "Date of issue/Date de délivrance": "16 APR/AVR 07",
  "Authority/Autorité": "PASSPORT OFFICE",
  "Date of expiry/Date d'expiration": "15 APR/AVR 12",
  "Holder's signature/Signature du titulaire": "anna maria eriksson",
  "Passport/Passeport": "P<UTOERIKSSON<<ANNA<MARIA<<<<<<<<<<<<<<<<<<<<<<<L898902C36UT07408122F1204159ZE184226B<<<<10"
}

{
  "passport": {
    "issuingCountry": "Netherlands",
    "issuingAuthority": "Koninkrijk der Nederlanden",
    "passportNumber": "SPEC12014",
    "issuingDate": "09 MAR 2014",
    "expiryDate": "09 MAR 2024",
    "holder": {
      "gender": "F",
      "nationality": "Netherlands",
      "placeOfBirth": "SPECIMEN",
      "sex": "WF",
      "firstNames": [
        "Willem",
        "Lieselotte"
      ]
    },
    "physicalDescription": {
      "height": "1.75 m",
      "hairColor": "gray",
      "hairLength": "short"
    },
    "issuingOffice": "Burg. van Stad en Dorp",
    "issuingDateAsInt": "14032014",
    "expiryDateAsInt": "14032024",
    "fieldsExtracted": [
      {
        "code": "NL",
        "dateOfBirth": "10 MAR 1965",
        "dateOfIssue": "09 MAR 2014",
        "dateOfExpiry": "09 MAR 2024",
        "firstNames": [
          "Willem",
          "Lieselotte"
        ],
        "nationality": "Netherlands",
        "passportNumber": "SPEC12014",
        "placeOfBirth": "SPECIMEN",
        "sex": "WF"
      }
    ]
  }
}