LangChain应用开发指南-TruLens用量化对抗幻觉 精华

pip install trulens_eval

# 导入LangChain和TruLens
from langchain.chains import LLMChain
from langchain.llms import OpenAI
from langchain.prompts import PromptTemplate
from langchain.prompts.chat import ChatPromptTemplate,HumanMessagePromptTemplate
from trulens_eval import TruChain,Feedback, Huggingface, Tru, OpenAI as TruOpenAI
from trulens_eval.feedback.provider.langchain import Langchain

tru = Tru()

# 定义一个问答应用的提示模板
full_prompt = HumanMessagePromptTemplate(
    prompt=PromptTemplate(
        template=
        "Provide a helpful response with relevant background information for the following: {prompt}",
        input_variables=["prompt"],
    )
)

chat_prompt_template = ChatPromptTemplate.from_messages([full_prompt])



# 创建一个LLMChain对象，使用llm和chat_prompt_template作为参数
llm = OpenAI()
chain = LLMChain(llm=llm, prompt=chat_prompt_template, verbose=True)



# Initialize Huggingface-based feedback function collection class:
# Define a language match feedback function using HuggingFace.
hugs = Huggingface()
f_lang_match = Feedback(hugs.language_match).on_input_output()
# Question/answer relevance between overall question and answer.
provider = TruOpenAI()
f_qa_relevance = Feedback(provider.relevance).on_input_output()

# 使用TruChain类来包装chain对象，指定反馈函数和应用ID
tru_recorder = TruChain(
    chain,
    app_id='Chain1_QAApplication',
    feedbacks=[f_lang_match,f_qa_relevance])


# 使用with语句来运行chain对象，并记录反馈数据
with tru_recorder as recording:
    # 输入一个问题，得到一个回答
    chain("What is langchain?")
    # 查看反馈数据
    tru_record = recording.records[0]
    # 打印反馈数据
    print("tru_record:",tru_record)
# 启动tru展示控制台
tru.run_dashboard()

# 导入LangChain和TruLens
from IPython.display import JSON

# Imports main tools:
from trulens_eval import TruChain, Feedback, Huggingface, Tru
from trulens_eval.schema import FeedbackResult
tru = Tru()
tru.reset_database()

# Imports from langchain to build app
import bs4
from langchain import hub
from langchain.chat_models import ChatOpenAI
from langchain.document_loaders import WebBaseLoader
from langchain.embeddings import OpenAIEmbeddings
from langchain.schema import StrOutputParser
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Chroma
from langchain_core.runnables import RunnablePassthrough
from trulens_eval.feedback.provider import OpenAI
import numpy as np
from trulens_eval.app import App
from trulens_eval.feedback import Groundedness


# 加载文件
loader = WebBaseLoader(
    web_paths=("https://lilianweng.github.io/posts/2023-06-23-agent/",),
    bs_kwargs=dict(
        parse_notallow=bs4.SoupStrainer(
            class_=("post-content", "post-title", "post-header")
        )
    ),
)
docs = loader.load()
# 分词
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
splits = text_splitter.split_documents(docs)
# 存入到向量数据库
vectorstore = Chroma.from_documents(documents=splits, embedding=OpenAIEmbeddings(
))
# 定义一个RAG Chain

retriever = vectorstore.as_retriever()

prompt = hub.pull("rlm/rag-prompt")
llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)

def format_docs(docs):
    return "\n\n".join(doc.page_content for doc in docs)

rag_chain = (
    {"context": retriever | format_docs, "question": RunnablePassthrough()}
    | prompt
    | llm
    | StrOutputParser()
)
# 使用TruChain类来包装rag对象，指定反馈函数和应用ID
# Initialize provider class
provider = OpenAI()
# select context to be used in feedback. the location of context is app specific.
context = App.select_context(rag_chain)
grounded = Groundedness(groundedness_provider=provider)
# f_context_relevance, f_groundness, f_answer_relevance 定义反馈函数
# Define a groundedness feedback function
f_groundedness = (
    Feedback(grounded.groundedness_measure_with_cot_reasons)
    .on(context.collect()) # collect context chunks into a list
    .on_output()
    .aggregate(grounded.grounded_statements_aggregator)
)

# Question/answer relevance between overall question and answer.
f_qa_relevance = Feedback(provider.relevance).on_input_output()
# Question/statement relevance between question and each context chunk.
f_context_relevance = (
    Feedback(provider.qs_relevance)
    .on_input()
    .on(context)
    .aggregate(np.mean)
    )
# 使用with语句来运行rag对象，并记录反馈数据
tru_recorder = TruChain(rag_chain,
    app_id='Chain1_ChatApplication',
    feedbacks=[f_qa_relevance, f_context_relevance, f_groundedness])

with tru_recorder as recording:
    # 输入一个问题，得到一个回答，以及检索出的文档
    llm_response = rag_chain.invoke("What is Task Decomposition?")
    # 查看反馈数据
    rec = recording.get() # use .get if only one record
    # 打印反馈数据
    print(rec)
# 启动tru展示控制台
tru.run_dashboard()