手搓RAG新增功能：递归检索与迭代查询+重回成熟框架API

# 支持更多文档格式
def extract_text_from_file(file_path):
    """根据文件类型选择适当的提取方法"""
    ext = os.path.splitext(file_path)[1].lower()
   
    if ext == '.pdf':
        return extract_text_from_pdf(file_path)
    elif ext == '.docx':
        return extract_text_from_docx(file_path)
    elif ext == '.pptx':
        return extract_text_from_pptx(file_path)
    elif ext in ['.txt', '.md', '.py', '.java', '.js', '.html', '.css']:
        with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
            return f.read()
    elif ext in ['.csv', '.xlsx', '.xls']:
        return extract_text_from_spreadsheet(file_path)
    else:
        raise ValueError(f"不支持的文件类型: {ext}")
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.

# 多粒度分块示例
def create_multi_granularity_chunks(text):
    """创建多粒度分块"""
    # 大块(1000字符)：适合概述类问题
    large_splitter = RecursiveCharacterTextSplitter(
        chunk_size=1000, chunk_overlap=100
    )
    # 中块(400字符)：平衡粒度
    medium_splitter = RecursiveCharacterTextSplitter(
        chunk_size=400, chunk_overlap=40
    )
    # 小块(150字符)：适合精确问答
    small_splitter = RecursiveCharacterTextSplitter(
        chunk_size=150, chunk_overlap=20
    )
    
    large_chunks = large_splitter.split_text(text)
    medium_chunks = medium_splitter.split_text(text)
    small_chunks = small_splitter.split_text(text)
    
    return {
        "large": large_chunks,
        "medium": medium_chunks,
        "small": small_chunks
    }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.

# 多模型嵌入示例
def generate_multi_model_embeddings(text):
    """使用多个模型生成嵌入并融合"""
    model1 = SentenceTransformer('all-MiniLM-L6-v2')
    model2 = SentenceTransformer('paraphrase-multilingual-mpnet-base-v2')
    
    # 生成嵌入
    emb1 = model1.encode(text)
    emb2 = model2.encode(text)
    
    # 简单融合策略：归一化后拼接
    emb1_norm = emb1 / np.linalg.norm(emb1)
    emb2_norm = emb2 / np.linalg.norm(emb2)
    
    # 返回融合嵌入
    return np.concatenate([emb1_norm, emb2_norm])
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.

# 查询扩展示例
def expand_query(query):
    """使用LLM扩展查询，生成多个角度的查询变体"""
    prompt = f"""
    为以下查询生成3个不同角度的变体，以提高检索召回率：
    
    原始查询: {query}
    
    变体应当包含同义词替换、关键词扩展或问题重构。
    仅返回三个变体，每行一个，不要有额外解释。
    """
    
    response = call_llm_api(prompt)
    variants = [line.strip() for line in response.strip().split('\n')]
    return [query] + variants  # 返回原始查询和变体
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.

# 与现有系统集成示例
from ragflow import RAGPipeline
import your_company_database


# 从公司数据库获取文档
documents = your_company_database.get_latest_documents()


# 使用RAGFlow处理
pipeline = RAGPipeline()
for doc in documents:
    pipeline.add_document(doc)
    
# 将结果同步回公司系统
processed_results = pipeline.get_indexed_status()
your_company_database.update_document_status(processed_results)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.

# 自定义混合检索策略
from ragflow import RetrievalPipeline
from ragflow.retrievers import VectorRetriever, KeywordRetriever


# 创建自定义的检索器组合
def custom_hybrid_retriever(query, top_k=10):
    # 使用向量检索获取语义相关结果
    semantic_results = vector_retriever.retrieve(query, top_k=top_k)
    
    # 使用关键词检索获取精确匹配
    keyword_results = keyword_retriever.retrieve(query, top_k=top_k)
    
    # 自定义融合策略（比如考虑文档及时性等）
    results = custom_merge_function(semantic_results, keyword_results, 
                                    recency_weight=0.2)
    return results
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.

# 创建复杂的数据处理和RAG工作流
from ragflow import DocumentProcessor, Indexer, QueryEngine
import schedule
import time


def daily_knowledge_update():
    # 从多个来源获取新文档
    new_docs = fetch_documents_from_sources()
    
    # 文档预处理（去重、格式转换等）
    processed_docs = DocumentProcessor().process_batch(new_docs)
    
    # 增量更新索引
    indexer = Indexer()
    indexer.update_incrementally(processed_docs)
    
    # 生成日报
    report = generate_daily_summary()
    send_email(report)


# 设置定时任务
schedule.every().day.at("01:00").do(daily_knowledge_update)


while True:
    schedule.run_pending()
    time.sleep(60)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.

# 构建RAG系统评估框架
from ragflow import RAGEvaluator
import matplotlib.pyplot as plt


# 准备测试数据集
test_queries = load_test_queries()
ground_truth = load_ground_truth_answers()


# 测试不同的配置
configurations = [
    {"embeddings": "minilm", "chunk_size": 200, "retriever": "hybrid"},
    {"embeddings": "e5-large", "chunk_size": 500, "retriever": "vector"},
    {"embeddings": "bge-large", "chunk_size": 300, "retriever": "hybrid"}
]


results = {}
for config in configurations:
    # 构建并测试每种配置
    rag_system = build_rag_with_config(config)
    eval_results = RAGEvaluator.evaluate(
        rag_system, test_queries, ground_truth
    )
    results[str(config)] = eval_results


# 可视化不同配置的性能比较
plot_evaluation_results(results)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.

# 医疗领域RAG增强示例
from ragflow import RAGPipeline
from custom_medical_modules import MedicalTermNormalizer, DiseaseEntityExtractor


# 创建领域特定的文档处理器
medical_processor = DocumentProcessor()
medical_processor.add_transformer(MedicalTermNormalizer())
medical_processor.add_transformer(DiseaseEntityExtractor())


# 构建医疗特定的RAG系统
medical_rag = RAGPipeline(
    document_processor=medical_processor,
    retrieval_augmentors=[
        CitationRetriever(),  # 添加医学文献引用
        EvidenceLevelClassifier()  # 分类证据等级
    ],
    response_formatters=[
        MedicalDisclaimerFormatter(),  # 添加医疗免责声明
        CitationFormatter()  # 格式化引用
    ]
)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.