大语言模型响应结果的可靠性分析实战

pip install --upgrade cleanlab-studio

from cleanlab_studio import Studio
studio = Studio("<CLEANLAB_API_KEY>") # 从上面获取您的API密钥
tlm = studio.TLM(options={"log": ["explanation"], "model": "gpt-4o"}) # GPT, Claude, etc
#设置提示
out = tlm.prompt("How many vowels are there in the word 'Abracadabra'.?")
#TLM响应包含实际输出的“响应”、可信度评分和解释
print(f"Model's response = {out['response']}")
print(f"Trustworthiness score = {out['trustworthiness_score']}")
print(f"Explanation = {out['log']['explanation']}")

Model's response = The word "Abracadabra" contains 6 vowels. The vowels are: A, a, a, a, a, and a.
Trustworthiness score = 0.6842228802750124
Explanation = This response is untrustworthy due to a lack of consistency in possible responses from the model. Here's one inconsistent alternate response that the model considered (which may not be accurate either):
5.

Model's response = Let me count the vowels in 'Abracadabra':
A-b-r-a-c-a-d-a-b-r-a

The vowels are: A, a, a, a, a

There are 5 vowels in the word 'Abracadabra'.
Trustworthiness score = 0.9378276048845285
Explanation = Did not find a reason to doubt trustworthiness.

python
from cleanlab_studio import Studio
import markdown
from IPython.core.display import display, Markdown

# 使用API密钥初始化Cleanlab Studio
studio = Studio("<CLEANLAB_API_KEY>") #替换为您的实际API密钥

# 要评估的模型列表
models = ["gpt-4o", "claude-3.5-sonnet-v2"]

# 定义提示
prompt_text = "Which one of 9.11 and 9.9 is bigger?"

# 遍历每个模型并进行评估
for model in models:
 tlm = studio.TLM(options={"log": ["explanation"], "model": model})
 out = tlm.prompt(prompt_text)

 md_content = f"""
## 模型: {model}

**响应**: {out['response']}

**可信度评分**: {out['trustworthiness_score']}

**解释**: {out['log']['explanation']}

---
"""
 display(Markdown(md_content))

import streamlit as st
from langchain_groq.chat_models import ChatGroq
import os
os.environ["GROQ_API_KEY"]=st.secrets["GROQ_API_KEY"]
#初始化Groq Llama即时模型
groq_llm = ChatGroq(model="deepseek-r1-distill-llama-70b", temperature=0.5)
prompt = "Which one of 9.11 and 9.9 is bigger?"
# Get the response from the model
response = groq_llm.invoke(prompt)
#初始化Cleanlab的studio
studio = Studio("226eeab91e944b23bd817a46dbe3c8ae") 
cleanlab_tlm = studio.TLM(optinotallow={"log": ["explanation"]}) #供解释
#得到包含可信度得分和解释的输出
output = cleanlab_tlm.get_trustworthiness_score(prompt, respnotallow=response.content.strip())
md_content = f"""
## 模型: {model}
**Response:** {response.content.strip()}
**Trustworthiness Score:** {output['trustworthiness_score']}
**Explanation:** {output['log']['explanation']}
---
"""
display(Markdown(md_content))

pip install llama-parse llama-index-core llama-index-embeddings-huggingface 
llama-index-llms-cleanlab requests beautifulsoup4 pdfkit nest-asyncio

from llama_parse import LlamaParse
from llama_index.core import VectorStoreIndex
import requests
from bs4 import BeautifulSoup
import pdfkit
from llama_index.readers.docling import DoclingReader
from llama_index.core import Settings
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
from llama_index.llms.cleanlab import CleanlabTLM
from typing import Dict, List, ClassVar
from llama_index.core.instrumentation.events import BaseEvent
from llama_index.core.instrumentation.event_handlers import BaseEventHandler
from llama_index.core.instrumentation import get_dispatcher
from llama_index.core.instrumentation.events.llm import LLMCompletionEndEvent
import nest_asyncio
import os

options = {
 "model": "gpt-4o",
 "max_tokens": 512,
 "log": ["explanation"]
}
llm = CleanlabTLM(api_key="<CLEANLAB_API_KEY>", optinotallow=options) # 从https://cleanlab.ai/获取您的免费API
Settings.llm = llm
Settings.embed_model = HuggingFaceEmbedding(
 model_name="BAAI/bge-small-en-v1.5"
)

# 可信度评分事件处理程序
class GetTrustworthinessScore(BaseEventHandler):
 events: ClassVar[List[BaseEvent]] = []
 trustworthiness_score: float = 0.0
 @classmethod
 def class_name(cls) -> str:
 return "GetTrustworthinessScore"
 def handle(self, event: BaseEvent) -> Dict:
 if isinstance(event, LLMCompletionEndEvent):
 self.trustworthiness_score = event.response.additional_kwargs.get("trustworthiness_score", 0.0)
 self.events.append(event)
 return {}

# 显示LLM响应的辅助函数
def display_response(response):
 response_str = response.response
 trustworthiness_score = event_handler.trustworthiness_score
 print(f"Response: {response_str}")
 print(f"Trustworthiness score: {round(trustworthiness_score, 2)}")

##########################################
# 从多个URL生成PDF
##########################################
# 配置wkhtmltopdf路径
wkhtml_path = r'C:\Program Files\wkhtmltopdf\bin\wkhtmltopdf.exe'
config = pdfkit.configuration(wkhtmltopdf=wkhtml_path)
# 定义URL和分配文档名称
urls = {
 "LLMs": "https://en.wikipedia.org/wiki/Large_language_model"
}
# 保存PDF的目录
pdf_directory = "PDFs"
os.makedirs(pdf_directory, exist_ok=True)
pdf_paths = {}
for doc_name, url in urls.items():
 try:
 print(f"Processing {doc_name} from {url} ...")
 response = requests.get(url)
 soup = BeautifulSoup(response.text, "html.parser")
 main_content = soup.find("div", {"id": "mw-content-text"})
 if main_content is None:
 raise ValueError("Main content not found")
 # 将协议相对URL替换为绝对URL
 html_string = str(main_content).replace('src="//', 'src="https://').replace('href="//', 'href="https://')
 pdf_file_path = os.path.join(pdf_directory, f"{doc_name}.pdf")
 pdfkit.from_string(
 html_string,
 pdf_file_path,
 optinotallow={'encoding': 'UTF-8', 'quiet': ''},
 cnotallow=config
 )
 pdf_paths[doc_name] = pdf_file_path
 print(f"Saved PDF for {doc_name} at {pdf_file_path}")
 except Exception as e:
 print(f"Error processing {doc_name}: {e}")

##########################################
# 使用LlamaParse解析PDF并注入元数据
##########################################

# 定义解析指令（如果您的解析器支持）
parsing_instructions = """提取文档的markdown格式内容。
按页将文档拆分为节点（例如）。
确保每个节点具有文档名称和页码的元数据。"""

# 创建LlamaParse实例
parser = LlamaParse(
 api_key="<LLAMACLOUD_API_KEY>", # 替换为您的实际密钥
 parsing_instructinotallow=parsing_instructions,
 result_type="markdown",
 premium_mode=True,
 max_timeout=600
)
# 保存合并的Markdown文件的目录（每个PDF一个）
output_md_dir = os.path.join(pdf_directory, "markdown_docs")
os.makedirs(output_md_dir, exist_ok=True)
# 列表，用于保存所有更新后的节点以供索引
all_nodes = []
for doc_name, pdf_path in pdf_paths.items():
 try:
 print(f"Parsing PDF for {doc_name} from {pdf_path} ...")
 nodes = parser.load_data(pdf_path) # 返回节点列表
 updated_nodes = []
 # 处理每个节点：更新元数据并在文本中注入引用标题。
 for i, node in enumerate(nodes, start=1):
 # 复制现有元数据（如果有），并添加我们自己的键。
 new_metadata = dict(node.metadata) if node.metadata else {}
 new_metadata["document_name"] = doc_name
 if "page_number" not in new_metadata:
 new_metadata["page_number"] = str(i)
 # 构建引用标题。
 citation_header = f"[{new_metadata['document_name']}, page {new_metadata['page_number']}]\n\n"
 # 在节点的文本前添加引用标题。
 updated_text = citation_header + node.text
 new_node = node.__class__(text=updated_text, metadata=new_metadata)
 updated_nodes.append(new_node)
 # 使用更新后的节点文本为文档保存一个合并的Markdown文件。
 combined_texts = [node.text for node in updated_nodes]
 combined_md = "\n\n---\n\n".join(combined_texts)
 md_filename = f"{doc_name}.md"
 md_filepath = os.path.join(output_md_dir, md_filename)
 with open(md_filepath, "w", encoding="utf-8") as f:
 f.write(combined_md)
 print(f"Saved combined markdown for {doc_name} to {md_filepath}")
 # 将更新后的节点添加到全局列表以供索引。
 all_nodes.extend(updated_nodes)
 print(f"Parsed {len(updated_nodes)} nodes from {doc_name}.")
 except Exception as e:
 print(f"Error parsing {doc_name}: {e}")

##########################################

# 创建索引和查询引擎

##########################################

# 从所有节点创建索引。

index = VectorStoreIndex.from_documents(documents=all_nodes)

# 定义一个自定义提示模板，强制包含引用。

prompt_template = """
你是一个具有主题专业知识的AI助手。
仅使用提供的上下文回答问题。
在必要时，以格式良好的Markdown格式回答，包含项目符号和章节。
如果提供的上下文不支持答案，请回复“我不知道。”
上下文：
{context_str}
问题：
{query_str}
答案：
"""
# 使用自定义提示创建查询引擎。
query_engine = index.as_query_engine(similarity_top_k=3, llm=llm, prompt_template=prompt_template)
print("Combined index and query engine created successfully!")

query = "When is mixture of experts approach used?"
response = query_engine.query(query)
display_response(response)

query = "How do you compare Deepseek model with OpenAI's models?"
response = query_engine.query(query)
display_response(response)