基于Agent的金融问答系统：RAG检索模块初建成 原创

smart-finance-bot \
|- dataset \  # 该目录用于保存PDF以及SQLite数据
|- doc \  # 该目录用于保存文档类文件，例如：需求文档、说明文档、数据文档
|- app \   # 该目录用于服务端代码
|- agent \ # 该目录用于保存agent相关代码
|- rag \   # 该目录用于保存rag相关代码
|-test \   # 该目录用于保存测试类驱动相关代码
|- conf \  # 该目录用于保存配置文件
|-.qwen # 该文件保存QWen的配置文件(请自行添加对应的API KEY)
|-.ernie # 该文件保存百度千帆的配置文件(请自行添加对应的API KEY)
|- chatweb \   # 该目录用于保存前端页面代码
|- scripts \   # 该目录用于保存脚本文件，例如：启动脚本、导入向量数据库脚本等
|- test_result \   # 该目录用于保存测试结果
|- docker \
|- backend \ # 该目录对应后端python服务的Dockerfile
|- frontend \ # 该目录对应前端python服务的Dockerfile
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.

git submodule add https://www.modelscope.cn/datasets/BJQW14B/bs_challenge_financial_14b_dataset.git dataset
1.

from dotenv import load_dotenv
import os

# 获取当前文件的目录
current_dir = os.path.dirname(__file__)

# 构建到 conf/.qwen 的相对路径
conf_file_path_qwen = os.path.join(current_dir,'..','conf','.qwen')

# 加载千问环境变量
load_dotenv(dotenv_path=conf_file_path_qwen)

defget_qwen_models():
"""
    加载千问系列大模型
    """
# llm 大模型
from langchain_community.llms.tongyi importTongyi

    llm =Tongyi(model="qwen-max", temperature=0.1, top_p=0.7, max_tokens=1024)

# chat 大模型
from langchain_community.chat_models importChatTongyi

    chat =ChatTongyi(model="qwen-max", temperature=0.01, top_p=0.2, max_tokens=1024)
# embedding 大模型
from langchain_community.embeddings importDashScopeEmbeddings

    embed =DashScopeEmbeddings(model="text-embedding-v3")

return llm, chat, embed
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.

DASHSCOPE_API_KEY = sk-xxxxxx
1.

import chromadb
from chromadb importSettings
from langchain_chroma importChroma


classChromaDB:
def__init__(self,
                 chroma_server_type="local",  # 服务器类型：http是http方式连接方式，local是本地读取文件方式
                 host="localhost", port=8000,  # 服务器地址，http方式必须指定
                 persist_path="chroma_db",  # 数据库的路径：如果是本地连接，需要指定数据库的路径
                 collection_name="langchain",  # 数据库的集合名称
                 embed=None  # 数据库的向量化函数
                 ):

        self.host = host
        self.port = port
        self.path = persist_path
        self.embed = embed
        self.store =None

# 如果是http协议方式连接数据库
if chroma_server_type =="http":
            client = chromadb.HttpClient(host=host, port=port)

            self.store =Chroma(collection_name=collection_name,
                                embedding_function=embed,
                                client=client)

if chroma_server_type =="local":
            self.store =Chroma(collection_name=collection_name,
                                embedding_function=embed,
                                persist_directory=persist_path)

if self.store isNone:
raiseValueError("Chroma store init failed!")

defadd_with_langchain(self, docs):
"""
        将文档添加到数据库
        """
        self.store.add_documents(documents=docs)

defget_store(self):
"""
        获得向量数据库的对象实例
        """
return self.store
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.

import os
import logging
import time
from tqdm import tqdm
from langchain_community.document_loaders importPyMuPDFLoader
from langchain_text_splitters importRecursiveCharacterTextSplitter
from rag.chroma_conn importChromaDB


classPDFProcessor:
def__init__(self,
                 directory,  # PDF文件所在目录
                 chroma_server_type,  # ChromaDB服务器类型
                 persist_path,  # ChromaDB持久化路径
                 embed):# 向量化函数

        self.directory = directory
        self.file_group_num =80# 每组处理的文件数
        self.batch_num =6# 每次插入的批次数量

        self.chunksize =500# 切分文本的大小
        self.overlap =100# 切分文本的重叠大小

        self.chroma_db =ChromaDB(chroma_server_type=chroma_server_type,
                                  persist_path=persist_path,
                                  embed=embed)
# 配置日志
        logging.basicConfig(
            level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S'
)

defload_pdf_files(self):
"""
        加载目录下的所有PDF文件
        """
        pdf_files =[]
for file in os.listdir(self.directory):
if file.lower().endswith('.pdf'):
                pdf_files.append(os.path.join(self.directory, file))

        logging.info(f"Found {len(pdf_files)} PDF files.")
return pdf_files

defload_pdf_content(self, pdf_path):
"""
        读取PDF文件内容
        """
        pdf_loader =PyMuPDFLoader(file_path=pdf_path)
        docs = pdf_loader.load()
        logging.info(f"Loading content from {pdf_path}.")
return docs

defsplit_text(self, documents):
"""
        将文本切分成小段
        """
# 切分文档
        text_splitter =RecursiveCharacterTextSplitter(
            chunk_size=self.chunksize,
            chunk_overlap=self.overlap,
            length_function=len,
            add_start_index=True,
)

        docs = text_splitter.split_documents(documents)

        logging.info("Split text into smaller chunks with RecursiveCharacterTextSplitter.")
return docs

definsert_docs_chromadb(self, docs, batch_size=6):
"""
        将文档插入到ChromaDB
        """
# 分批入库
        logging.info(f"Inserting {len(docs)} documents into ChromaDB.")

# 记录开始时间
        start_time = time.time()
        total_docs_inserted =0

# 计算总批次
        total_batches =(len(docs)+ batch_size -1)// batch_size

with tqdm(total=total_batches, desc="Inserting batches", unit="batch")as pbar:
for i inrange(0,len(docs), batch_size):
# 获取当前批次的样本
                batch = docs[i:i + batch_size]

# 将样本入库
                self.chroma_db.add_with_langchain(batch)
# self.chroma_db.async_add_with_langchain(batch)

# 更新已插入的文档数量
                total_docs_inserted +=len(batch)

# 计算并显示当前的TPM
                elapsed_time = time.time()- start_time  # 计算已用时间（秒）
if elapsed_time >0:# 防止除以零
                    tpm =(total_docs_inserted / elapsed_time)*60# 转换为每分钟插入的文档数
                    pbar.set_postfix({"TPM":f"{tpm:.2f}"})# 更新进度条的后缀信息

# 更新进度条
                pbar.update(1)

defprocess_pdfs_group(self, pdf_files_group):
# 读取PDF文件内容
        pdf_contents =[]

for pdf_path in pdf_files_group:
# 读取PDF文件内容
            documents = self.load_pdf_content(pdf_path)

# 将documents 逐一添加到pdf_contents
            pdf_contents.extend(documents)

# 将文本切分成小段
        docs = self.split_text(pdf_contents)

# 将文档插入到ChromaDB
        self.insert_docs_chromadb(docs, self.batch_num)

defprocess_pdfs(self):
# 获取目录下所有的PDF文件
        pdf_files = self.load_pdf_files()

        group_num = self.file_group_num

# group_num 个PDF文件为一组，分批处理
for i inrange(0,len(pdf_files), group_num):
            pdf_files_group = pdf_files[i:i + group_num]
            self.process_pdfs_group(pdf_files_group)

print("PDFs processed successfully!")
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.
78.
79.
80.
81.
82.
83.
84.
85.
86.
87.
88.
89.
90.
91.
92.
93.
94.
95.
96.
97.
98.
99.
100.
101.
102.
103.
104.
105.
106.
107.
108.
109.
110.
111.
112.
113.
114.
115.
116.
117.
118.
119.
120.
121.
122.
123.
124.
125.
126.
127.
128.
129.
130.
131.
132.
133.
134.
135.

smart-finance-bot \
    |- app \   # 该目录用于服务端代码
        |- rag \   # 该目录用于保存rag相关代码
            |- pdf_processor.py
            |- chroma_conn.py
        |- test_framework.py
1.
2.
3.
4.
5.
6.

# 测试导入PDF到向量库主流程
deftest_import():
from rag.pdf_processor importPDFProcessor
from utils.util import get_qwen_models

    llm , chat , embed = get_qwen_models()
# embed = get_huggingface_embeddings()

    directory ="./app/dataset/pdf"
    persist_path ="chroma_db"
    server_type ="local"

# 创建 PDFProcessor 实例
    pdf_processor =PDFProcessor(directory=directory,
                                 chroma_server_type=server_type,
                                 persist_path=persist_path,
                                 embed=embed)

# 处理 PDF 文件
    pdf_processor.process_pdfs()

if __name__ =="__main__":
    test_import()
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.

chroma run --path chroma_db --port 8000 --host localhost
1.

import logging
from langchain_core.prompts importChatPromptTemplate
from langchain_core.runnables importRunnablePassthrough
from langchain_core.runnables.base importRunnableLambda
from langchain_core.output_parsers importStrOutputParser
from.chroma_conn importChromaDB

# 配置日志记录
logging.basicConfig(level=logging.INFO,format='%(asctime)s - %(levelname)s - %(message)s')


classRagManager:
def__init__(self,
                 chroma_server_type="http",
                 host="localhost", port=8000,
                 persist_path="chroma_db",
                 llm=None, embed=None):
        self.llm = llm
        self.embed = embed

        chrom_db =ChromaDB(chroma_server_type=chroma_server_type,
                            host=host, port=port,
                            persist_path=persist_path,
                            embed=embed)
        self.store = chrom_db.get_store()

defget_chain(self, retriever):
"""获取RAG查询链"""

# RAG系统经典的 Prompt (A 增强的过程)
        prompt =ChatPromptTemplate.from_messages([
("human","""You are an assistant for question-answering tasks. Use the following pieces 
          of retrieved context to answer the question. 
          If you don't know the answer, just say that you don't know. 
          Use three sentences maximum and keep the answer concise.
          Question: {question} 
          Context: {context} 
          Answer:""")
])
# 将 format_docs 方法包装为 Runnable
        format_docs_runnable =RunnableLambda(self.format_docs)
# RAG 链
        rag_chain =(
{"context": retriever | format_docs_runnable,
"question":RunnablePassthrough()}
| prompt
| self.llm
|StrOutputParser()
)

return rag_chain

defformat_docs(self, docs):
# 返回检索到的资料文件名称
        logging.info(f"检索到资料文件个数：{len(docs)}")
        retrieved_files ="\n".join([doc.metadata["source"]for doc in docs])
        logging.info(f"资料文件分别是:\n{retrieved_files}")

        retrieved_content ="\n\n".join(doc.page_content for doc in docs)
        logging.info(f"检索到的资料为:\n{retrieved_content}")

return retrieved_content

defget_retriever(self, k=4, mutuality=0.3):
        retriever = self.store.as_retriever(search_type="similarity_score_threshold",
                                            search_kwargs={"k": k,"score_threshold": mutuality})

return retriever

defget_result(self, question, k=4, mutuality=0.3):
"""获取RAG查询结果"""

        retriever = self.get_retriever(k, mutuality)

        rag_chain = self.get_chain(retriever)

return rag_chain.invoke(input=question)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.

# 测试RAG主流程
deftest_rag():
from rag.rag importRagManager
from utils.util import get_qwen_models

    llm, chat, embed = get_qwen_models()
    rag =RagManager(host="localhost", port=8000, llm=llm, embed=embed)

    result = rag.get_result("湖南长远锂科股份有限公司变更设立时作为发起人的法人有哪些？")

print(result)


if __name__ =="__main__":
    test_rag()# RAG测试函数
# test_import()         # 批量导入PDF测试函数
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.