没有捷径：RAG入门不推荐直接使用成熟框架

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=200,
    chunk_overlap=20
)
chunks = text_splitter.split_text(text)
1.
2.
3.
4.
5.

results = COLLECTION.query(
    query_embeddings=query_embedding,
    n_results=5,
    include=['documents', 'metadatas']
)
1.
2.
3.
4.
5.

# 检测矛盾
conflict_detected = detect_conflicts(sources_for_conflict_detection)


# 获取可信源
if conflict_detected:
    credible_sources = [s for s in sources_for_conflict_detection 
                      if s['type'] == 'web' and evaluate_source_credibility(s) > 0.7]
1.
2.
3.
4.
5.
6.
7.
8.

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=800,        # 增大到800-1000字符
    chunk_overlap=150,     # 增加重叠以保持连贯性
    separators=["\n\n", "\n", "。", "，", " "] # 优先在自然段落分割
)
1.
2.
3.
4.
5.

# 引入BM25关键词检索
from rank_bm25 import BM25Okapi
tokenized_corpus = [doc.split() for doc in corpus]
bm25 = BM25Okapi(tokenized_corpus)


# 混合检索结果
semantic_results = COLLECTION.query(query_embeddings, n_results=7)
bm25_results = bm25.get_top_n(query.split(), corpus, n=7)
combined_results = hybrid_merge(semantic_results, bm25_results)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.

# 方案A: 使用交叉编码器重排序
from sentence_transformers import CrossEncoder
reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
candidate_pairs = [[query, doc] for doc in retrieved_docs]
scores = reranker.predict(candidate_pairs)
reranked_results = [doc for _, doc in sorted(zip(scores, retrieved_docs), reverse=True)]


# 方案B: 使用LLM进行相关性评分
def llm_rerank(query, docs, llm_client):
    results = []
    for doc in docs:
        prompt = f"问题: {query}\n文档: {doc}\n这个文档与问题的相关性是否高? 0-10分打分:"
        score = float(llm_client.generate(prompt).strip())
        results.append((score, doc))
    return [doc for _, doc in sorted(results, reverse=True)]
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.

def recursive_retrieval(initial_query, max_iteratinotallow=3):
    query = initial_query
    all_contexts = []
    
    for i in range(max_iterations):
        # 当前查询的检索结果
        current_results = retrieve_documents(query)
        all_contexts.extend(current_results)
        
        # 使用LLM分析还需要查询什么
        next_query_prompt = f"""
        基于原始问题: {initial_query}
        以及已检索信息: {summarize(current_results)}
        是否需要进一步查询? 如果需要,请提供新的查询问题:
        """
        
        next_query = llm_client.generate(next_query_prompt)
        if "不需要" in next_query or i == max_iterations-1:
            break
            
        query = next_query
        
    return all_contexts
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.

def compress_context(query, documents, max_tokens=2000):
    compressed_docs = []
    
    for doc in documents:
        # 方案A: 使用map-reduce模式总结
        summary_prompt = f"原文: {doc}\n请提取与问题'{query}'最相关的信息,总结在100词以内:"
        compressed = llm_client.generate(summary_prompt)
        compressed_docs.append(compressed)
        
        # 方案B: 使用抽取式摘要选择关键句子
        # sentences = split_into_sentences(doc)
        # scores = sentence_similarity(query, sentences)
        # top_sentences = [s for s, _ in sorted(zip(sentences, scores), key=lambda x: x[1], reverse=True)[:5]]
        # compressed_docs.append(" ".join(top_sentences))
    
    return compressed_docs
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.

def create_advanced_prompt(query, context, question_type):
    if question_type == "factual":
        template = """
        你是一个精确的事实回答助手。以下是与问题相关的信息:
        {context}
        
        问题: {query}
        请基于以上信息提供精确的事实回答。如信息不足,请明确指出。请在回答末尾标明信息来源。
        """
    elif question_type == "analytical":
        template = """
        你是一个分析型助手。以下是相关参考信息:
        {context}
        
        问题: {query}
        请分析以上信息,提供深入见解。注意分析信息的一致性与可靠性,标明不同来源间的差异。最后给出综合结论并注明信息来源。
        """
    # 更多问题类型...
    
    # 添加思维链指导
    template += """
    思考步骤:
    1. 提取问题关键点
    2. 识别相关文档中最有价值的信息
    3. 对比不同来源信息
    4. 形成清晰、全面的回答
    """
    
    return template.format(cnotallow=context, query=query)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.

# 丰富元数据
metadatas = [{
    "source": file_name,
    "doc_id": doc_id,
    "date_processed": datetime.now().isoformat(),
    "chunk_index": i,
    "total_chunks": len(chunks),
    "document_type": detect_document_type(text),
    "language": detect_language(chunk),
    "entities": extract_entities(chunk)
} for i, chunk in enumerate(chunks)]


# 检索时使用元数据过滤
results = COLLECTION.query(
    query_embeddings=query_embedding,
    n_results=10,
    where={"document_type": {"$in": ["report", "research_paper"]}},
    include=['documents', 'metadatas']
)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.

# 升级为更强大的双语模型
from sentence_transformers import SentenceTransformer
EMBED_MODEL = SentenceTransformer('BAAI/bge-large-zh-v1.5') # 1024维,中文效果更好


# 或考虑OpenAI嵌入模型
import openai
embeddings = openai.Embedding.create(
    input=chunks, 
    model="text-embedding-ada-002"
)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

def evaluate_response(query, response, retrieved_docs):
    # LLM自评估
    evaluation_prompt = f"""
    问题: {query}
    回答: {response}
    请评估这个回答的质量(1-10分),考虑以下标准:
    1. 准确性: 回答是否符合检索文档的事实?
    2. 完整性: 回答是否全面覆盖问题的各个方面?
    3. 相关性: 回答是否切中问题要点?
    4. 一致性: 回答内部是否存在矛盾?
    请详细说明评分理由。
    """
    
    feedback = llm_client.generate(evaluation_prompt)
    
    # 存储评估结果用于系统优化
    save_evaluation_result(query, response, retrieved_docs, feedback)
    
    return feedback
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.

# 问题-检索结果缓存
import hashlib
import pickle
from functools import lru_cache


@lru_cache(maxsize=100)
def cached_retrieval(query):
    # 计算查询哈希
    query_hash = hashlib.md5(query.encode()).hexdigest()
    cache_file = f"cache/{query_hash}.pkl"
    
    # 检查缓存
    try:
        with open(cache_file, 'rb') as f:
            return pickle.load(f)
    except FileNotFoundError:
        # 执行检索
        results = perform_retrieval(query)
        # 保存到缓存
        os.makedirs("cache", exist_ok=True)
        with open(cache_file, 'wb') as f:
            pickle.dump(results, f)
        return results
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.