使用大语言模型集成工具 LangChain 创建自己的论文汇总和查询工具

from langchain import PromptTemplate, HuggingFaceHub, LLMChain
 from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.indexes import VectorstoreIndexCreator
 from langchain.callbacks.base import CallbackManager
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 from langchain.chains.qa_with_sources import load_qa_with_sources_chain
 from langchain.document_loaders import DirectoryLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import FAISS
 from huggface_hub import hf_hub_download
 import textwrap
 import glob

HUGGING_FACE_API_KEY = "hf_...."

template = """ You are going to be my assistant.
 Please try to give me the most beneficial answers to my
 question with reasoning for why they are correct.
 
  Question: {input} Answer: """
 prompt = PromptTemplate(template=template, input_variables=["input"])

model = HuggingFaceHub(repo_id="facebook/mbart-large-50",
                        model_kwargs={"temperature": 0, "max_length":200},
                        huggingfacehub_api_token=HUGGING_FACE_API_KEY)
 chain = LLMChain(prompt=prompt, llm=model)

hf_embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2')

my_loader = DirectoryLoader('my_data', glob='**/*.txt')
 docs = my_loader.load()
 text_split = RecursiveCharacterTextSplitter(chunk_size = 700, chunk_overlap = 0)
 text = text_split.split_documents(docs)

vectorstore = FAISS.from_documents(text, hf_embeddings)

my_chain = load_qa_with_sources_chain(model, chain_type="refine")
 query = "Any question that you want to ask the model"
 documents = vectorstore.similarity_search(query)
 result = with_sources_chain({"input_documents": documents, "question": query})

from langchain.chains.summarize import load_summarize_chain
 from langchain.document_loaders import PyPDFLoader
 from langchain import OpenAI, PromptTemplate
 import glob

llm = OpenAI(temperature=0.2)

def summarize_pdfs_from_folder(pdfs_folder):
    summaries = []
    for pdf_file in glob.glob(pdfs_folder + "/*.pdf"):
        loader = PyPDFLoader(pdf_file)
        docs = loader.load_and_split()
        chain = load_summarize_chain(llm, chain_type="map_reduce")
        summary = chain.run(docs)
        print("Summary for: ", pdf_file)
        print(summary)
        print("\n")
        summaries.append(summary)
     
    return summaries

with open("summaries.txt", "w") as f:
    for summary in summaries:
        f.write(summary + "\n"*3)

from langchain.indexes import VectorstoreIndexCreator
 from langchain.document_loaders import PyPDFDirectoryLoader
 loader = PyPDFDirectoryLoader("./pdfs/")
 
 docs = loader.load()
 index = VectorstoreIndexCreator().from_loaders([loader])

query = "What is the core idea behind the CoOP (context optimization) paper?"
 index.query(query)
 
 # Output
 # " The core idea behind the CoOP paper is to model
 # a prompt's context words with learnable vectors
 # while keeping the entire pre-trained parameters fixed,
 # in order to adapt CLIP-like vision-language models for
 # downstream image recognition tasks."

query = "What is the central idea that can allow for scaling transformers to 1 million tokens?"
 
 index.query(query)
 
 # Output
 # ' The central idea is to use the Recurrent Memory Transformer (RMT) architecture to extend the context length of BERT, allowing it to store and process both local and global information across up to 2 million tokens.'