使用CLIP和LLM构建多模态RAG系统

! pip install -q timm einops wikipedia chromadb open_clip_torch
 !pip install -q transformers==4.36.0
 !pip install -q bitsandbytes==0.41.3 accelerate==0.25.0

import chromadb
 
 from chromadb.utils.embedding_functions import OpenCLIPEmbeddingFunction
 from chromadb.utils.data_loaders import ImageLoader
 from chromadb.config import Settings
 
 
 client = chromadb.PersistentClient(path="DB")
 
 embedding_function = OpenCLIPEmbeddingFunction()
 image_loader = ImageLoader() # must be if you reads from URIs

from chromadb import Documents, EmbeddingFunction, Embeddings
 
 class MyEmbeddingFunction(EmbeddingFunction):
    def __call__(self, input: Documents) -> Embeddings:
        # embed the documents somehow or images
        return embeddings

collection_images = client.create_collection(
    name='multimodal_collection_images', 
    embedding_functinotallow=embedding_function, 
    data_loader=image_loader)
 
 collection_text = client.create_collection(
    name='multimodal_collection_text', 
    embedding_functinotallow=embedding_function, 
    )
 
 # Get the Images
 IMAGE_FOLDER = '/kaggle/working/all_data'
 
 
 image_uris = sorted([os.path.join(IMAGE_FOLDER, image_name) for image_name in os.listdir(IMAGE_FOLDER) if not image_name.endswith('.txt')])
 ids = [str(i) for i in range(len(image_uris))]
 
 collection_images.add(ids=ids, uris=image_uris) #now we have the images collection

from matplotlib import pyplot as plt
 
 retrieved = collection_images.query(query_texts=["tulip"], include=['data'], n_results=3)
 for img in retrieved['data'][0]:
    plt.imshow(img)
    plt.axis("off")
    plt.show()

# now the text DB
 from chromadb.utils import embedding_functions
 default_ef = embedding_functions.DefaultEmbeddingFunction()
 
 text_pth = sorted([os.path.join(IMAGE_FOLDER, image_name) for image_name in os.listdir(IMAGE_FOLDER) if image_name.endswith('.txt')])
 
 list_of_text = []
 for text in text_pth:
    with open(text, 'r') as f:
        text = f.read()
        list_of_text.append(text)
 
 ids_txt_list = ['id'+str(i) for i in range(len(list_of_text))]
 ids_txt_list
 
 collection_text.add(
    documents = list_of_text,
    ids =ids_txt_list
 )

results = collection_text.query(
    query_texts=["What is the bellflower?"],
    n_results=1
 )
 
 results

{'ids': [['id0']],
  'distances': [[0.6072186183744086]],
  'metadatas': [[None]],
  'embeddings': None,
  'documents': [['Campanula () is the type genus of the Campanulaceae family of flowering plants. Campanula are commonly known as bellflowers and take both their common and scientific names from the bell-shaped flowers—campanula is Latin for "little bell".\nThe genus includes over 500 species and several subspecies, distributed across the temperate and subtropical regions of the Northern Hemisphere, with centers of diversity in the Mediterranean region, Balkans, Caucasus and mountains of western Asia. The range also extends into mountains in tropical regions of Asia and Africa.\nThe species include annual, biennial and perennial plants, and vary in habit from dwarf arctic and alpine species under 5 cm high, to large temperate grassland and woodland species growing to 2 metres (6 ft 7 in) tall.']],
  'uris': None,
  'data': None}

query_image = '/kaggle/input/flowers/flowers/rose/00f6e89a2f949f8165d5222955a5a37d.jpg'
 raw_image = Image.open(query_image)
 
 doc = collection_text.query(
    query_embeddings=embedding_function(query_image),
     
    n_results=1,
         
 )['documents'][0][0]

A rose is either a woody perennial flowering plant of the genus Rosa (), in the family Rosaceae (), or the flower it bears. There are over three hundred species and tens of thousands of cultivars. They form a group of plants that can be erect shrubs, climbing, or trailing, with stems that are often armed with sharp prickles. Their flowers vary in size and shape and are usually large and showy, in colours ranging from white through yellows and reds. Most species are native to Asia, with smaller numbers native to Europe, North America, and northwestern Africa. Species, cultivars and hybrids are all widely grown for their beauty and often are fragrant. Roses have acquired cultural significance in many societies. Rose plants range in size from compact, miniature roses, to climbers that can reach seven meters in height. Different species hybridize easily, and this has been used in the development of the wide range of garden roses.

from huggingface_hub import hf_hub_download
 
 hf_hub_download(repo_id="visheratin/LLaVA-3b", filename="configuration_llava.py", local_dir="./", force_download=True)
 hf_hub_download(repo_id="visheratin/LLaVA-3b", filename="configuration_phi.py", local_dir="./", force_download=True)
 hf_hub_download(repo_id="visheratin/LLaVA-3b", filename="modeling_llava.py", local_dir="./", force_download=True)
 hf_hub_download(repo_id="visheratin/LLaVA-3b", filename="modeling_phi.py", local_dir="./", force_download=True)
 hf_hub_download(repo_id="visheratin/LLaVA-3b", filename="processing_llava.py", local_dir="./", force_download=True)

from modeling_llava import LlavaForConditionalGeneration
 import torch
 
 model = LlavaForConditionalGeneration.from_pretrained("visheratin/LLaVA-3b")
 model = model.to("cuda")

from transformers import AutoTokenizer
 
 tokenizer = AutoTokenizer.from_pretrained("visheratin/LLaVA-3b")

from processing_llava import LlavaProcessor, OpenCLIPImageProcessor
 
 image_processor = OpenCLIPImageProcessor(model.config.preprocess_config)
 processor = LlavaProcessor(image_processor, tokenizer)

question = 'Answer with organized answers: What type of rose is in the picture? Mention some of its characteristics and how to take care of it ?'
 
 query_image = '/kaggle/input/flowers/flowers/rose/00f6e89a2f949f8165d5222955a5a37d.jpg'
 raw_image = Image.open(query_image)
 
 doc = collection_text.query(
    query_embeddings=embedding_function(query_image),
     
    n_results=1,
         
 )['documents'][0][0]
 
 plt.imshow(raw_image)
 plt.show()
 imgs = collection_images.query(query_uris=query_image, include=['data'], n_results=3)
 for img in imgs['data'][0][1:]:
    plt.imshow(img)
    plt.axis("off")
    plt.show()

prompt = """<|im_start|>system
 A chat between a curious human and an artificial intelligence assistant.
 The assistant is an exprt in flowers , and gives helpful, detailed, and polite answers to the human's questions.
 The assistant does not hallucinate and pays very close attention to the details.<|im_end|>
 <|im_start|>user
 <image>
 {question} Use the following article as an answer source. Do not write outside its scope unless you find your answer better {article} if you thin your answer is better add it after document.<|im_end|>
 <|im_start|>assistant
 """.format(questinotallow='question', article=doc)