Spaces:
Running
Running
蒲源
commited on
Commit
·
b8684d4
1
Parent(s):
d7c8d36
polish(pu): polish readme
Browse files- README.md +19 -27
- README_zh.md +27 -16
- analyze_conversation_history.py +68 -0
- app.py +0 -214
- app_mqa.py +8 -6
- app_mqa_database.py +12 -9
- documents/LightZero_README_zh.md +15 -15
- rag_demo.py +7 -3
README.md
CHANGED
@@ -1,24 +1,11 @@
|
|
1 |
-
|
2 |
-
|
3 |
-
|
4 |
-
colorFrom: yellow
|
5 |
-
colorTo: blue
|
6 |
-
sdk: gradio
|
7 |
-
sdk_version: 4.1.1
|
8 |
-
app_file: app.py
|
9 |
-
pinned: false
|
10 |
-
license: apache-2.0
|
11 |
-
python_version: 3.8
|
12 |
-
---
|
13 |
-
|
14 |
-
# RAG Demo
|
15 |
-
|
16 |
-
English | [简体中文(Simplified Chinese)](https://github.com/puyuan1996/RAG/blob/main/README_zh.md)
|
17 |
|
18 |
## Introduction
|
19 |
|
20 |
-
|
21 |
-
- It utilizes large language models such as GPT-
|
22 |
- The project also includes a web-based interactive application built with Gradio and rag_demo.py.
|
23 |
|
24 |
## rag_demo.py Features
|
@@ -50,7 +37,8 @@ QUESTION_LANG='cn' # The language of the question, currently available option is
|
|
50 |
```
|
51 |
|
52 |
4. Ensure you have available documents as context or use the commented-out code snippet to download the documents you want to reference.
|
53 |
-
5. Run the `python3 -u rag_demo.py` file to
|
|
|
54 |
|
55 |
## Example
|
56 |
|
@@ -60,18 +48,20 @@ if __name__ == "__main__":
|
|
60 |
# Assuming documents are already present locally
|
61 |
file_path = './documents/LightZero_README_zh.md'
|
62 |
# Load and split document
|
63 |
-
chunks = load_and_split_document(file_path)
|
64 |
# Create vector store
|
65 |
-
|
|
|
66 |
# Set up RAG process
|
67 |
-
rag_chain = setup_rag_chain()
|
68 |
|
69 |
# Pose a question and get an answer
|
70 |
query = "Does the AlphaZero algorithm implemented in LightZero support running in the Atari environment? Please explain in detail."
|
71 |
# Use RAG chain to get referenced documents and answer
|
72 |
-
retrieved_documents, result_with_rag = execute_query(retriever, rag_chain, query
|
|
|
73 |
# Get an answer without using RAG chain
|
74 |
-
result_without_rag = execute_query_no_rag(query=query)
|
75 |
|
76 |
# Details of data handling code are omitted here, please refer to the source files in this repository for specifics
|
77 |
|
@@ -91,16 +81,18 @@ if __name__ == "__main__":
|
|
91 |
RAG/
|
92 |
│
|
93 |
├── rag_demo.py # RAG demonstration script with support for outputting retrieved document chunks.
|
94 |
-
├──
|
95 |
-
├──
|
96 |
├── .env # Environment variable configuration file
|
97 |
└── documents/ # Documents folder
|
98 |
└── your_document.txt # Context document
|
|
|
|
|
99 |
```
|
100 |
|
101 |
## Contribution Guide
|
102 |
|
103 |
-
If you would like to contribute code to
|
104 |
|
105 |
1. Fork the project.
|
106 |
2. Create a new branch.
|
|
|
1 |
+
# ZeroPal
|
2 |
+
|
3 |
+
English | [简体中文(Simplified Chinese)](https://github.com/puyuan1996/ZeroPal/blob/main/README_zh.md)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
4 |
|
5 |
## Introduction
|
6 |
|
7 |
+
ZeroPal is a demonstration project for a question-answering system for [LightZero](https://github.com/opendilab/LightZero) based on Retrieval-Augmented Generation (RAG). Zero represents LightZero, and Pal represents a companion.
|
8 |
+
- It utilizes large language models such as Kimi and GPT-4 in conjunction with a document retrieval vector database like Weaviate to respond to user queries by retrieving relevant document contexts and leveraging the generative capabilities of the language model.
|
9 |
- The project also includes a web-based interactive application built with Gradio and rag_demo.py.
|
10 |
|
11 |
## rag_demo.py Features
|
|
|
37 |
```
|
38 |
|
39 |
4. Ensure you have available documents as context or use the commented-out code snippet to download the documents you want to reference.
|
40 |
+
5. Run the `python3 -u rag_demo.py` file to test ZeroPal on the local command line.
|
41 |
+
6. Run the `python3 -u app_mqa_database.py` file to test ZeroPal on a local web page.
|
42 |
|
43 |
## Example
|
44 |
|
|
|
48 |
# Assuming documents are already present locally
|
49 |
file_path = './documents/LightZero_README_zh.md'
|
50 |
# Load and split document
|
51 |
+
chunks = load_and_split_document(file_path, chunk_size=5000, chunk_overlap=500)
|
52 |
# Create vector store
|
53 |
+
vectorstore = create_vector_store(chunks, model=embedding_model)
|
54 |
+
retriever = get_retriever(vectorstore, k=5)
|
55 |
# Set up RAG process
|
56 |
+
rag_chain = setup_rag_chain(model_name=model_name, temperature=temperature)
|
57 |
|
58 |
# Pose a question and get an answer
|
59 |
query = "Does the AlphaZero algorithm implemented in LightZero support running in the Atari environment? Please explain in detail."
|
60 |
# Use RAG chain to get referenced documents and answer
|
61 |
+
retrieved_documents, result_with_rag = execute_query(retriever, rag_chain, query, model_name=model_name,
|
62 |
+
temperature=temperature)
|
63 |
# Get an answer without using RAG chain
|
64 |
+
result_without_rag = execute_query_no_rag(model_name=model_name, query=query, temperature=temperature)
|
65 |
|
66 |
# Details of data handling code are omitted here, please refer to the source files in this repository for specifics
|
67 |
|
|
|
81 |
RAG/
|
82 |
│
|
83 |
├── rag_demo.py # RAG demonstration script with support for outputting retrieved document chunks.
|
84 |
+
├── app_mqa.py # Web-based interactive application built with Gradio and rag_demo.py.
|
85 |
+
├── app_mqa_database.py # Web-based interactive application built with Gradio and rag_demo.py. Supports maintaining the database of conversation history.
|
86 |
├── .env # Environment variable configuration file
|
87 |
└── documents/ # Documents folder
|
88 |
└── your_document.txt # Context document
|
89 |
+
└── database/ # Database folder
|
90 |
+
└── conversation_history.db # Database for conversation history
|
91 |
```
|
92 |
|
93 |
## Contribution Guide
|
94 |
|
95 |
+
If you would like to contribute code to ZeroPal, please follow these steps:
|
96 |
|
97 |
1. Fork the project.
|
98 |
2. Create a new branch.
|
README_zh.md
CHANGED
@@ -1,11 +1,11 @@
|
|
1 |
-
#
|
2 |
|
3 |
-
简体中文 | [English](https://github.com/puyuan1996/
|
4 |
|
5 |
## 简介
|
6 |
|
7 |
-
|
8 |
-
- 它使用大型语言模型(如 GPT-
|
9 |
- 同时提供了一个基于 Gradio 和 rag_demo.py 构建的网页交互式应用。
|
10 |
|
11 |
## rag_demo.py 功能
|
@@ -16,9 +16,9 @@ RAG 是一个基于检索增强生成 (RAG) 的问答系统示例项目。
|
|
16 |
- 支持设置检索增强生成流程,结合文档检索和语言模型生成对用户问题进行回答。
|
17 |
- 支持执行查询并打印结果,可以选择是否通过 RAG 流程。
|
18 |
|
19 |
-
##
|
20 |
|
21 |
-
- 创建一个Gradio应用,用户可以在其中输入问题,应用会使用Retrieval-Augmented Generation (RAG)模型来寻找答案并将结果显示在界面上。
|
22 |
- 其中,检索到的上下文会在Markdown文档中高亮显示,帮助用户理解答案的来源。应用界面分为两部分:顶部是问答区,底部展示了RAG模型参考的上下文。
|
23 |
|
24 |
## 使用方法
|
@@ -37,7 +37,8 @@ QUESTION_LANG='cn' # 问题语言,目前可选值为 'cn'
|
|
37 |
```
|
38 |
|
39 |
4. 确保已经有可用的文档作为上下文,或者使用注释掉的代码段下载你需要参考的文档。
|
40 |
-
5. 执行 `python3 -u rag_demo.py`
|
|
|
41 |
|
42 |
## 示例
|
43 |
|
@@ -46,19 +47,27 @@ QUESTION_LANG='cn' # 问题语言,目前可选值为 'cn'
|
|
46 |
if __name__ == "__main__":
|
47 |
# 假设文档已存在于本地
|
48 |
file_path = './documents/LightZero_README_zh.md'
|
|
|
|
|
|
|
|
|
49 |
# 加载和分割文档
|
50 |
-
chunks = load_and_split_document(file_path)
|
|
|
51 |
# 创建向量存储
|
52 |
-
|
|
|
|
|
53 |
# 设置 RAG 流程
|
54 |
-
rag_chain = setup_rag_chain()
|
55 |
|
56 |
# 提出问题并获取答案
|
57 |
query = "请问 LightZero 里面实现的 AlphaZero 算法支持在 Atari 环境上运行吗?请详细解释原因"
|
58 |
# 使用 RAG 链获取参考的文档与答案
|
59 |
-
retrieved_documents, result_with_rag = execute_query(retriever, rag_chain, query
|
|
|
60 |
# 不使用 RAG 链获取答案
|
61 |
-
result_without_rag = execute_query_no_rag(query=query)
|
62 |
|
63 |
# 此处省略部分数据处理代码,具体细节请参考本仓库中的源文件
|
64 |
|
@@ -75,19 +84,21 @@ if __name__ == "__main__":
|
|
75 |
## 项目结构
|
76 |
|
77 |
```
|
78 |
-
|
79 |
│
|
80 |
├── rag_demo.py # RAG 演示脚本,支持输出检索到的文档块。
|
81 |
-
├──
|
82 |
-
├──
|
83 |
├── .env # 环境变量配置文件
|
84 |
└── documents/ # 文档文件夹
|
85 |
└── your_document.txt # 上下文文档
|
|
|
|
|
86 |
```
|
87 |
|
88 |
## 贡献指南
|
89 |
|
90 |
-
如果您希望为
|
91 |
|
92 |
1. Fork 项目。
|
93 |
2. 创建一个新的分支。
|
|
|
1 |
+
# ZeroPal 使用说明
|
2 |
|
3 |
+
简体中文 | [English](https://github.com/puyuan1996/ZeroPal/blob/main/README.md)
|
4 |
|
5 |
## 简介
|
6 |
|
7 |
+
ZeroPal 是一个基于检索增强生成 (RAG) 技术,关于 [LightZero](https://github.com/opendilab/LightZero) 的问答系统示例项目。其中 Zero 表示 LightZero,Pal 表示伙伴。
|
8 |
+
- 它使用大型语言模型(如 Kimi/GPT-4 等)和文档检索向量数据库(如 Weaviate)来响应用户的问题,通过检索相关的文档上下文以及利用语言模型的生成能力来提供准确的回答。
|
9 |
- 同时提供了一个基于 Gradio 和 rag_demo.py 构建的网页交互式应用。
|
10 |
|
11 |
## rag_demo.py 功能
|
|
|
16 |
- 支持设置检索增强生成流程,结合文档检索和语言模型生成对用户问题进行回答。
|
17 |
- 支持执行查询并打印结果,可以选择是否通过 RAG 流程。
|
18 |
|
19 |
+
## app_mqa.py 功能
|
20 |
|
21 |
+
- 创建一个Gradio应用,用户可以在其中输入问题,应用会使用 Retrieval-Augmented Generation (RAG) 模型来寻找答案并将结果显示在界面上。
|
22 |
- 其中,检索到的上下文会在Markdown文档中高亮显示,帮助用户理解答案的来源。应用界面分为两部分:顶部是问答区,底部展示了RAG模型参考的上下文。
|
23 |
|
24 |
## 使用方法
|
|
|
37 |
```
|
38 |
|
39 |
4. 确保已经有可用的文档作为上下文,或者使用注释掉的代码段下载你需要参考的文档。
|
40 |
+
5. 执行 `python3 -u rag_demo.py` 文件即可在本地命令行测试 ZeroPal。
|
41 |
+
6. 执行 `python3 -u app_mqa_database.py` 文件即可在本地的网页上测试 ZeroPal。
|
42 |
|
43 |
## 示例
|
44 |
|
|
|
47 |
if __name__ == "__main__":
|
48 |
# 假设文档已存在于本地
|
49 |
file_path = './documents/LightZero_README_zh.md'
|
50 |
+
model_name = 'kimi'
|
51 |
+
temperature = 0.01
|
52 |
+
embedding_model = 'OpenAI' # embedding_model=['HuggingFace', 'TensorflowHub', 'OpenAI']
|
53 |
+
|
54 |
# 加载和分割文档
|
55 |
+
chunks = load_and_split_document(file_path, chunk_size=5000, chunk_overlap=500)
|
56 |
+
|
57 |
# 创建向量存储
|
58 |
+
vectorstore = create_vector_store(chunks, model=embedding_model)
|
59 |
+
retriever = get_retriever(vectorstore, k=5)
|
60 |
+
|
61 |
# 设置 RAG 流程
|
62 |
+
rag_chain = setup_rag_chain(model_name=model_name, temperature=temperature)
|
63 |
|
64 |
# 提出问题并获取答案
|
65 |
query = "请问 LightZero 里面实现的 AlphaZero 算法支持在 Atari 环境上运行吗?请详细解释原因"
|
66 |
# 使用 RAG 链获取参考的文档与答案
|
67 |
+
retrieved_documents, result_with_rag = execute_query(retriever, rag_chain, query, model_name=model_name,
|
68 |
+
temperature=temperature)
|
69 |
# 不使用 RAG 链获取答案
|
70 |
+
result_without_rag = execute_query_no_rag(model_name=model_name, query=query, temperature=temperature)
|
71 |
|
72 |
# 此处省略部分数据处理代码,具体细节请参考本仓库中的源文件
|
73 |
|
|
|
84 |
## 项目结构
|
85 |
|
86 |
```
|
87 |
+
ZeroPal/
|
88 |
│
|
89 |
├── rag_demo.py # RAG 演示脚本,支持输出检索到的文档块。
|
90 |
+
├── app_mqa.py # 基于 Gradio 和 rag_demo.py 构建的网页交互式应用。
|
91 |
+
├── app_mqa_database.py # 基于 Gradio 和 rag_demo.py 构建的网页交互式应用。支持使用database保持对话历史。
|
92 |
├── .env # 环境变量配置文件
|
93 |
└── documents/ # 文档文件夹
|
94 |
└── your_document.txt # 上下文文档
|
95 |
+
└── database/ # 数据库文件夹
|
96 |
+
└── conversation_history.db # 对话历史数据库
|
97 |
```
|
98 |
|
99 |
## 贡献指南
|
100 |
|
101 |
+
如果您希望为 ZeroPal 贡献代码,请遵循以下步骤:
|
102 |
|
103 |
1. Fork 项目。
|
104 |
2. 创建一个新的分支。
|
analyze_conversation_history.py
ADDED
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import sqlite3
|
2 |
+
|
3 |
+
|
4 |
+
def analyze_conversation_history():
|
5 |
+
"""
|
6 |
+
分析对话历史数据库中的数据
|
7 |
+
"""
|
8 |
+
# 连接到SQLite数据库
|
9 |
+
conn = sqlite3.connect('database/conversation_history.db')
|
10 |
+
c = conn.cursor()
|
11 |
+
|
12 |
+
# 获取总的对话记录数
|
13 |
+
c.execute("SELECT COUNT(*) FROM history")
|
14 |
+
total_records = c.fetchone()[0]
|
15 |
+
print(f"总对话记录数: {total_records}")
|
16 |
+
|
17 |
+
# 获取不同用户的对话记录数
|
18 |
+
c.execute("SELECT user_id, COUNT(*) as count FROM history GROUP BY user_id")
|
19 |
+
user_records = c.fetchall()
|
20 |
+
print("每个用户的对话记录数:")
|
21 |
+
for user_id, count in user_records:
|
22 |
+
print(f"用户 {user_id}: {count} 条记录")
|
23 |
+
|
24 |
+
# 获取平均对话轮数
|
25 |
+
c.execute("SELECT AVG(cnt) FROM (SELECT user_id, COUNT(*) as cnt FROM history GROUP BY user_id)")
|
26 |
+
avg_turns = c.fetchone()[0]
|
27 |
+
print(f"平均对话轮数: {avg_turns}")
|
28 |
+
|
29 |
+
# 获取最长的用户输入和助手输出
|
30 |
+
c.execute("SELECT MAX(LENGTH(user_input)) FROM history")
|
31 |
+
max_user_input_length = c.fetchone()[0]
|
32 |
+
print(f"最长的用户输入: {max_user_input_length} 个字符")
|
33 |
+
|
34 |
+
c.execute("SELECT MAX(LENGTH(assistant_output)) FROM history")
|
35 |
+
max_assistant_output_length = c.fetchone()[0]
|
36 |
+
print(f"最长的助手输出: {max_assistant_output_length} 个字符")
|
37 |
+
|
38 |
+
# 关闭游标
|
39 |
+
c.close()
|
40 |
+
# 关闭数据库连接
|
41 |
+
conn.close()
|
42 |
+
|
43 |
+
|
44 |
+
def clear_context():
|
45 |
+
"""
|
46 |
+
清除对话历史
|
47 |
+
"""
|
48 |
+
# 连接到SQLite数据库
|
49 |
+
conn = sqlite3.connect('conversation_history.db')
|
50 |
+
c = conn.cursor()
|
51 |
+
c.execute("DELETE FROM history")
|
52 |
+
conn.commit()
|
53 |
+
return "", "", ""
|
54 |
+
|
55 |
+
|
56 |
+
def get_history():
|
57 |
+
"""
|
58 |
+
获取对话历史记录
|
59 |
+
"""
|
60 |
+
# 连接到SQLite数据库
|
61 |
+
conn = sqlite3.connect('conversation_history.db')
|
62 |
+
c = conn.cursor()
|
63 |
+
c.execute("SELECT user_input, assistant_output FROM history")
|
64 |
+
rows = c.fetchall()
|
65 |
+
history = ""
|
66 |
+
for row in rows:
|
67 |
+
history += f"User: {row[0]}\nAssistant: {row[1]}\n\n"
|
68 |
+
return history
|
app.py
DELETED
@@ -1,214 +0,0 @@
|
|
1 |
-
import os
|
2 |
-
import sqlite3
|
3 |
-
import threading
|
4 |
-
|
5 |
-
import gradio as gr
|
6 |
-
from dotenv import load_dotenv
|
7 |
-
from langchain.document_loaders import TextLoader
|
8 |
-
|
9 |
-
from analysis import analyze_conversation_history
|
10 |
-
from rag_demo import load_and_split_document, create_vector_store, setup_rag_chain, execute_query
|
11 |
-
|
12 |
-
# 环境设置
|
13 |
-
load_dotenv() # 加载环境变量
|
14 |
-
QUESTION_LANG = os.getenv("QUESTION_LANG") # 从环境变量获取 QUESTION_LANG
|
15 |
-
assert QUESTION_LANG in ['cn', 'en'], QUESTION_LANG
|
16 |
-
|
17 |
-
if QUESTION_LANG == "cn":
|
18 |
-
title = "ZeroPal"
|
19 |
-
title_markdown = """
|
20 |
-
<div align="center">
|
21 |
-
<img src="https://raw.githubusercontent.com/puyuan1996/RAG/main/assets/banner.svg" width="80%" height="20%" alt="Banner Image">
|
22 |
-
</div>
|
23 |
-
|
24 |
-
📢 **操作说明**:请在下方的“问题”框中输入关于 LightZero 的问题,并点击“提交”按钮。右侧的“回答”框将展示 RAG 模型提供的答案。
|
25 |
-
您可以在问答框下方查看当前“对话历史”,点击“清除上下文”按钮可清空历史记录。在“对话历史”框下方,您将找到相关参考文档,其中相关文段将以黄色高亮显示。
|
26 |
-
如果您喜欢这个项目,请在 GitHub [LightZero RAG Demo](https://github.com/puyuan1996/RAG) 上给我们点赞!✨ 您的支持是我们持续更新的动力。
|
27 |
-
|
28 |
-
<div align="center">
|
29 |
-
<strong>注意:算法模型输出可能包含一定的随机性。结果不代表开发者和相关 AI 服务的态度和意见。本项目开发者不对结果作出任何保证,仅供参考之用。使用该服务即代表同意后文所述的使用条款。</strong>
|
30 |
-
</div>
|
31 |
-
"""
|
32 |
-
tos_markdown = """
|
33 |
-
### 使用条款
|
34 |
-
|
35 |
-
使用本服务的玩家需同意以下条款:
|
36 |
-
|
37 |
-
- 本服务为探索性研究的预览版,仅供非商业用途。
|
38 |
-
- 服务不得用于任何非法、有害、暴力、种族主义或其他令人反感的目的。
|
39 |
-
- 服务提供有限的安全措施,并可能生成令人反感的内容。
|
40 |
-
- 如果您对服务体验不满,请通过 [email protected] 与我们联系!我们承诺修复问题并不断改进项目。
|
41 |
-
- 为了获得最佳体验,请使用台式电脑,因为移动设备可能会影响视觉效果。
|
42 |
-
|
43 |
-
**版权所有 © 2024 OpenDILab。保留所有权利。**
|
44 |
-
"""
|
45 |
-
|
46 |
-
# 路径变量,方便之后的文件使用
|
47 |
-
file_path = './documents/LightZero_README_zh.md'
|
48 |
-
|
49 |
-
# 加载原始Markdown文档
|
50 |
-
loader = TextLoader(file_path)
|
51 |
-
orig_documents = loader.load()
|
52 |
-
|
53 |
-
# 存储对话历史
|
54 |
-
conversation_history = {}
|
55 |
-
|
56 |
-
# 创建线程局部数据对象
|
57 |
-
threadLocal = threading.local()
|
58 |
-
|
59 |
-
|
60 |
-
def get_db_connection():
|
61 |
-
"""
|
62 |
-
返回当前线程的数据库连接
|
63 |
-
"""
|
64 |
-
conn = getattr(threadLocal, 'conn', None)
|
65 |
-
if conn is None:
|
66 |
-
# 连接到SQLite数据库
|
67 |
-
conn = sqlite3.connect('database/conversation_history.db')
|
68 |
-
c = conn.cursor()
|
69 |
-
# Drop the existing 'history' table if it exists
|
70 |
-
# c.execute('DROP TABLE IF EXISTS history')
|
71 |
-
# 创建存储对话历史的表
|
72 |
-
c.execute('''CREATE TABLE IF NOT EXISTS history
|
73 |
-
(id INTEGER PRIMARY KEY AUTOINCREMENT,
|
74 |
-
user_id TEXT NOT NULL,
|
75 |
-
user_input TEXT NOT NULL,
|
76 |
-
assistant_output TEXT NOT NULL,
|
77 |
-
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP)''')
|
78 |
-
threadLocal.conn = conn
|
79 |
-
return conn
|
80 |
-
|
81 |
-
|
82 |
-
def get_db_cursor():
|
83 |
-
"""
|
84 |
-
返回当前线程的数据库游标
|
85 |
-
"""
|
86 |
-
conn = get_db_connection()
|
87 |
-
c = getattr(threadLocal, 'cursor', None)
|
88 |
-
if c is None:
|
89 |
-
c = conn.cursor()
|
90 |
-
threadLocal.cursor = c
|
91 |
-
return c
|
92 |
-
|
93 |
-
|
94 |
-
# 程序结束时清理数据库连接
|
95 |
-
def close_db_connection():
|
96 |
-
conn = getattr(threadLocal, 'conn', None)
|
97 |
-
if conn is not None:
|
98 |
-
conn.close()
|
99 |
-
setattr(threadLocal, 'conn', None)
|
100 |
-
|
101 |
-
c = getattr(threadLocal, 'cursor', None)
|
102 |
-
if c is not None:
|
103 |
-
c.close()
|
104 |
-
setattr(threadLocal, 'cursor', None)
|
105 |
-
|
106 |
-
|
107 |
-
def rag_answer(question, temperature, k, user_id):
|
108 |
-
"""
|
109 |
-
处理用户问题并返回答案和高亮显示的上下文
|
110 |
-
|
111 |
-
:param question: 用户输入的问题
|
112 |
-
:param temperature: 生成答案时使用的温度参数
|
113 |
-
:param k: 检索到的文档块数量
|
114 |
-
:param user_id: 用户ID
|
115 |
-
:return: 模型生成的答案和高亮显示上下文的Markdown文本
|
116 |
-
"""
|
117 |
-
try:
|
118 |
-
chunks = load_and_split_document(file_path, chunk_size=5000, chunk_overlap=500)
|
119 |
-
retriever = create_vector_store(chunks, model='OpenAI', k=k)
|
120 |
-
rag_chain = setup_rag_chain(model_name='kimi', temperature=temperature)
|
121 |
-
|
122 |
-
if user_id not in conversation_history:
|
123 |
-
conversation_history[user_id] = []
|
124 |
-
|
125 |
-
conversation_history[user_id].append((f"User[{user_id}]", question))
|
126 |
-
|
127 |
-
history_str = "\n".join([f"{role}: {text}" for role, text in conversation_history[user_id]])
|
128 |
-
|
129 |
-
retrieved_documents, answer = execute_query(retriever, rag_chain, history_str, model_name='kimi',
|
130 |
-
temperature=temperature)
|
131 |
-
|
132 |
-
############################
|
133 |
-
# 获取当前线程的数据库连接和游标
|
134 |
-
############################
|
135 |
-
conn = get_db_connection()
|
136 |
-
c = get_db_cursor()
|
137 |
-
|
138 |
-
# 分析对话历史
|
139 |
-
# analyze_conversation_history()
|
140 |
-
# 获取总的对话记录数
|
141 |
-
c.execute("SELECT COUNT(*) FROM history")
|
142 |
-
total_records = c.fetchone()[0]
|
143 |
-
print(f"总对话记录数: {total_records}")
|
144 |
-
|
145 |
-
# 将问题和回答存储到数据库
|
146 |
-
c.execute("INSERT INTO history (user_id, user_input, assistant_output) VALUES (?, ?, ?)",
|
147 |
-
(user_id, question, answer))
|
148 |
-
conn.commit()
|
149 |
-
|
150 |
-
# 在文档中高亮显示上下文
|
151 |
-
context = [retrieved_documents[i].page_content for i in range(len(retrieved_documents))]
|
152 |
-
highlighted_document = orig_documents[0].page_content
|
153 |
-
for i in range(len(context)):
|
154 |
-
highlighted_document = highlighted_document.replace(context[i], f"<mark>{context[i]}</mark>")
|
155 |
-
|
156 |
-
conversation_history[user_id].append(("Assistant", answer))
|
157 |
-
|
158 |
-
full_history = "\n".join([f"{role}: {text}" for role, text in conversation_history[user_id]])
|
159 |
-
except Exception as e:
|
160 |
-
print(f"An error occurred: {e}")
|
161 |
-
return "处理您的问题时出现错误,请稍后再试。", "", ""
|
162 |
-
finally:
|
163 |
-
# 不再在这里关闭游标和连接
|
164 |
-
pass
|
165 |
-
|
166 |
-
return answer, highlighted_document, full_history
|
167 |
-
|
168 |
-
|
169 |
-
def clear_context(user_id):
|
170 |
-
"""
|
171 |
-
清除对话历史
|
172 |
-
"""
|
173 |
-
if user_id in conversation_history:
|
174 |
-
conversation_history[user_id] = []
|
175 |
-
return "", "", ""
|
176 |
-
|
177 |
-
|
178 |
-
if __name__ == "__main__":
|
179 |
-
with gr.Blocks(title=title, theme='ParityError/Interstellar') as zero_pal:
|
180 |
-
gr.Markdown(title_markdown)
|
181 |
-
|
182 |
-
with gr.Row():
|
183 |
-
with gr.Column():
|
184 |
-
user_id = gr.Textbox(
|
185 |
-
placeholder="请输入您的真实姓名或昵称作为用户ID",
|
186 |
-
label="用户ID")
|
187 |
-
inputs = gr.Textbox(
|
188 |
-
placeholder="请您在这里输入任何关于 LightZero 的问题。",
|
189 |
-
label="问题")
|
190 |
-
temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.01, step=0.01, label="温度参数")
|
191 |
-
k = gr.Slider(minimum=1, maximum=10, value=5, step=1, label="检索到的文档块数量")
|
192 |
-
with gr.Row():
|
193 |
-
gr_submit = gr.Button('提交')
|
194 |
-
gr_clear = gr.Button('清除上下文')
|
195 |
-
|
196 |
-
outputs_answer = gr.Textbox(placeholder="当你点击提交按钮后,这里会显示 RAG 模型给出的回答。",
|
197 |
-
label="回答")
|
198 |
-
outputs_history = gr.Textbox(label="对话历史")
|
199 |
-
with gr.Row():
|
200 |
-
outputs_context = gr.Markdown(label="参考的文档(检索得到的相关文段用高亮显示)")
|
201 |
-
gr_clear.click(clear_context, inputs=user_id, outputs=[outputs_context, outputs_history])
|
202 |
-
gr_submit.click(
|
203 |
-
rag_answer,
|
204 |
-
inputs=[inputs, temperature, k, user_id],
|
205 |
-
outputs=[outputs_answer, outputs_context, outputs_history],
|
206 |
-
)
|
207 |
-
gr.Markdown(tos_markdown)
|
208 |
-
|
209 |
-
concurrency = int(os.environ.get('CONCURRENCY', os.cpu_count()))
|
210 |
-
favicon_path = os.path.join(os.path.dirname(__file__), 'assets', 'avatar.png')
|
211 |
-
zero_pal.queue().launch(max_threads=concurrency, favicon_path=favicon_path, share=True)
|
212 |
-
|
213 |
-
# 在合适的地方,例如程序退出时,调用close_db_connection函数
|
214 |
-
close_db_connection()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app_mqa.py
CHANGED
@@ -2,7 +2,7 @@ import os
|
|
2 |
import gradio as gr
|
3 |
from dotenv import load_dotenv
|
4 |
from langchain.document_loaders import TextLoader
|
5 |
-
from rag_demo import load_and_split_document, create_vector_store, setup_rag_chain, execute_query
|
6 |
|
7 |
# 环境设置
|
8 |
load_dotenv() # 加载环境变量
|
@@ -13,12 +13,12 @@ if QUESTION_LANG == "cn":
|
|
13 |
title = "ZeroPal"
|
14 |
title_markdown = """
|
15 |
<div align="center">
|
16 |
-
<img src="https://raw.githubusercontent.com/puyuan1996/
|
17 |
</div>
|
18 |
|
19 |
📢 **操作说明**:请在下方的“问题”框中输入关于 LightZero 的问题,并点击“提交”按钮。右侧的“回答”框将展示 RAG 模型提供的答案。
|
20 |
您可以在问答框下方查看当前“对话历史”,点击“清除上下文”按钮可清空历史记录。在“对话历史”框下方,您将找到相关参考文档,其中相关文段将以黄色高亮显示。
|
21 |
-
如果您喜欢这个项目,请在 GitHub [LightZero RAG Demo](https://github.com/puyuan1996/
|
22 |
|
23 |
<div align="center">
|
24 |
<strong>注意:算法模型输出可能包含一定的随机性。结果不代表开发者和相关 AI 服务的态度和意见。本项目开发者不对结果作出任何保证,仅供参考之用。使用该服务即代表同意后文所述的使用条款。</strong>
|
@@ -48,6 +48,9 @@ orig_documents = loader.load()
|
|
48 |
# 存储对话历史
|
49 |
conversation_history = []
|
50 |
|
|
|
|
|
|
|
51 |
|
52 |
def rag_answer(question, temperature, k):
|
53 |
"""
|
@@ -59,8 +62,7 @@ def rag_answer(question, temperature, k):
|
|
59 |
:return: 模型生成的答案和高亮显示上下文的Markdown文本
|
60 |
"""
|
61 |
try:
|
62 |
-
|
63 |
-
retriever = create_vector_store(chunks, model='OpenAI', k=k)
|
64 |
rag_chain = setup_rag_chain(model_name='kimi', temperature=temperature)
|
65 |
|
66 |
# 将问题添加到对话历史中
|
@@ -141,4 +143,4 @@ if __name__ == "__main__":
|
|
141 |
|
142 |
concurrency = int(os.environ.get('CONCURRENCY', os.cpu_count()))
|
143 |
favicon_path = os.path.join(os.path.dirname(__file__), 'assets', 'avatar.png')
|
144 |
-
zero_pal.queue().launch(max_threads=concurrency, favicon_path=favicon_path, share=True)
|
|
|
2 |
import gradio as gr
|
3 |
from dotenv import load_dotenv
|
4 |
from langchain.document_loaders import TextLoader
|
5 |
+
from rag_demo import load_and_split_document, create_vector_store, setup_rag_chain, execute_query, get_retriever
|
6 |
|
7 |
# 环境设置
|
8 |
load_dotenv() # 加载环境变量
|
|
|
13 |
title = "ZeroPal"
|
14 |
title_markdown = """
|
15 |
<div align="center">
|
16 |
+
<img src="https://raw.githubusercontent.com/puyuan1996/ZeroPal/main/assets/banner.svg" width="80%" height="20%" alt="Banner Image">
|
17 |
</div>
|
18 |
|
19 |
📢 **操作说明**:请在下方的“问题”框中输入关于 LightZero 的问题,并点击“提交”按钮。右侧的“回答”框将展示 RAG 模型提供的答案。
|
20 |
您可以在问答框下方查看当前“对话历史”,点击“清除上下文”按钮可清空历史记录。在“对话历史”框下方,您将找到相关参考文档,其中相关文段将以黄色高亮显示。
|
21 |
+
如果您喜欢这个项目,请在 GitHub [LightZero RAG Demo](https://github.com/puyuan1996/ZeroPal) 上给我们点赞!✨ 您的支持是我们持续更新的动力。
|
22 |
|
23 |
<div align="center">
|
24 |
<strong>注意:算法模型输出可能包含一定的随机性。结果不代表开发者和相关 AI 服务的态度和意见。本项目开发者不对结果作出任何保证,仅供参考之用。使用该服务即代表同意后文所述的使用条款。</strong>
|
|
|
48 |
# 存储对话历史
|
49 |
conversation_history = []
|
50 |
|
51 |
+
chunks = load_and_split_document(file_path, chunk_size=5000, chunk_overlap=500)
|
52 |
+
vectorstore = create_vector_store(chunks, model='OpenAI')
|
53 |
+
|
54 |
|
55 |
def rag_answer(question, temperature, k):
|
56 |
"""
|
|
|
62 |
:return: 模型生成的答案和高亮显示上下文的Markdown文本
|
63 |
"""
|
64 |
try:
|
65 |
+
retriever = get_retriever(vectorstore, k)
|
|
|
66 |
rag_chain = setup_rag_chain(model_name='kimi', temperature=temperature)
|
67 |
|
68 |
# 将问题添加到对话历史中
|
|
|
143 |
|
144 |
concurrency = int(os.environ.get('CONCURRENCY', os.cpu_count()))
|
145 |
favicon_path = os.path.join(os.path.dirname(__file__), 'assets', 'avatar.png')
|
146 |
+
zero_pal.queue().launch(max_threads=concurrency, favicon_path=favicon_path, share=True)
|
app_mqa_database.py
CHANGED
@@ -6,8 +6,8 @@ import gradio as gr
|
|
6 |
from dotenv import load_dotenv
|
7 |
from langchain.document_loaders import TextLoader
|
8 |
|
9 |
-
from
|
10 |
-
from rag_demo import load_and_split_document, create_vector_store, setup_rag_chain, execute_query
|
11 |
|
12 |
# 环境设置
|
13 |
load_dotenv() # 加载环境变量
|
@@ -18,12 +18,12 @@ if QUESTION_LANG == "cn":
|
|
18 |
title = "ZeroPal"
|
19 |
title_markdown = """
|
20 |
<div align="center">
|
21 |
-
<img src="https://raw.githubusercontent.com/puyuan1996/
|
22 |
</div>
|
23 |
|
24 |
📢 **操作说明**:请在下方的“问题”框中输入关于 LightZero 的问题,并点击“提交”按钮。右侧的“回答”框将展示 RAG 模型提供的答案。
|
25 |
-
|
26 |
-
如果您喜欢这个项目,请在 GitHub [LightZero RAG Demo](https://github.com/puyuan1996/
|
27 |
|
28 |
<div align="center">
|
29 |
<strong>注意:算法模型输出可能包含一定的随机性。结果不代表开发者和相关 AI 服务的态度和意见。本项目开发者不对结果作出任何保证,仅供参考之用。使用该服务即代表同意后文所述的使用条款。</strong>
|
@@ -104,6 +104,10 @@ def close_db_connection():
|
|
104 |
setattr(threadLocal, 'cursor', None)
|
105 |
|
106 |
|
|
|
|
|
|
|
|
|
107 |
def rag_answer(question, temperature, k, user_id):
|
108 |
"""
|
109 |
处理用户问题并返回答案和高亮显示的上下文
|
@@ -115,8 +119,7 @@ def rag_answer(question, temperature, k, user_id):
|
|
115 |
:return: 模型生成的答案和高亮显示上下文的Markdown文本
|
116 |
"""
|
117 |
try:
|
118 |
-
|
119 |
-
retriever = create_vector_store(chunks, model='OpenAI', k=k)
|
120 |
rag_chain = setup_rag_chain(model_name='kimi', temperature=temperature)
|
121 |
|
122 |
if user_id not in conversation_history:
|
@@ -158,7 +161,7 @@ def rag_answer(question, temperature, k, user_id):
|
|
158 |
full_history = "\n".join([f"{role}: {text}" for role, text in conversation_history[user_id]])
|
159 |
except Exception as e:
|
160 |
print(f"An error occurred: {e}")
|
161 |
-
return "
|
162 |
finally:
|
163 |
# 不再在这里关闭游标和连接
|
164 |
pass
|
@@ -191,7 +194,7 @@ if __name__ == "__main__":
|
|
191 |
k = gr.Slider(minimum=1, maximum=10, value=5, step=1, label="检索到的文档块数量")
|
192 |
with gr.Row():
|
193 |
gr_submit = gr.Button('提交')
|
194 |
-
gr_clear = gr.Button('
|
195 |
|
196 |
outputs_answer = gr.Textbox(placeholder="当你点击提交按钮后,这里会显示 RAG 模型给出的回答。",
|
197 |
label="回答")
|
|
|
6 |
from dotenv import load_dotenv
|
7 |
from langchain.document_loaders import TextLoader
|
8 |
|
9 |
+
from analyze_conversation_history import analyze_conversation_history
|
10 |
+
from rag_demo import load_and_split_document, create_vector_store, setup_rag_chain, execute_query, get_retriever
|
11 |
|
12 |
# 环境设置
|
13 |
load_dotenv() # 加载环境变量
|
|
|
18 |
title = "ZeroPal"
|
19 |
title_markdown = """
|
20 |
<div align="center">
|
21 |
+
<img src="https://raw.githubusercontent.com/puyuan1996/ZeroPal/main/assets/banner.svg" width="80%" height="20%" alt="Banner Image">
|
22 |
</div>
|
23 |
|
24 |
📢 **操作说明**:请在下方的“问题”框中输入关于 LightZero 的问题,并点击“提交”按钮。右侧的“回答”框将展示 RAG 模型提供的答案。
|
25 |
+
您可以在问答框下方查看当前“对话历史”,点击“清除对话历史”按钮可清空历史记录。在“对话历史”框下方,您将找到相关参考文档,其中相关文段将以黄色高亮显示。
|
26 |
+
如果您喜欢这个项目,请在 GitHub [LightZero RAG Demo](https://github.com/puyuan1996/ZeroPal) 上给我们点赞!✨ 您的支持是我们持续更新的动力。
|
27 |
|
28 |
<div align="center">
|
29 |
<strong>注意:算法模型输出可能包含一定的随机性。结果不代表开发者和相关 AI 服务的态度和意见。本项目开发者不对结果作出任何保证,仅供参考之用。使用该服务即代表同意后文所述的使用条款。</strong>
|
|
|
104 |
setattr(threadLocal, 'cursor', None)
|
105 |
|
106 |
|
107 |
+
chunks = load_and_split_document(file_path, chunk_size=5000, chunk_overlap=500)
|
108 |
+
vectorstore = create_vector_store(chunks, model='OpenAI')
|
109 |
+
|
110 |
+
|
111 |
def rag_answer(question, temperature, k, user_id):
|
112 |
"""
|
113 |
处理用户问题并返回答案和高亮显示的上下文
|
|
|
119 |
:return: 模型生成的答案和高亮显示上下文的Markdown文本
|
120 |
"""
|
121 |
try:
|
122 |
+
retriever = get_retriever(vectorstore, k)
|
|
|
123 |
rag_chain = setup_rag_chain(model_name='kimi', temperature=temperature)
|
124 |
|
125 |
if user_id not in conversation_history:
|
|
|
161 |
full_history = "\n".join([f"{role}: {text}" for role, text in conversation_history[user_id]])
|
162 |
except Exception as e:
|
163 |
print(f"An error occurred: {e}")
|
164 |
+
return f"处理您的问题时出现错误,请稍后再试。错误内容为:{e}", "", ""
|
165 |
finally:
|
166 |
# 不再在这里关闭游标和连接
|
167 |
pass
|
|
|
194 |
k = gr.Slider(minimum=1, maximum=10, value=5, step=1, label="检索到的文档块数量")
|
195 |
with gr.Row():
|
196 |
gr_submit = gr.Button('提交')
|
197 |
+
gr_clear = gr.Button('清除对话历史')
|
198 |
|
199 |
outputs_answer = gr.Textbox(placeholder="当你点击提交按钮后,这里会显示 RAG 模型给出的回答。",
|
200 |
label="回答")
|
documents/LightZero_README_zh.md
CHANGED
@@ -109,21 +109,21 @@ LightZero 是基于 [PyTorch](https://pytorch.org/) 实现的 MCTS 算法库,
|
|
109 |
LightZero 目前支持的环境及算法如下表所示:
|
110 |
|
111 |
| Env./Algo. | AlphaZero | MuZero | EfficientZero | Sampled EfficientZero | Gumbel MuZero | Stochastic MuZero |
|
112 |
-
|---------------|
|
113 |
-
| TicTacToe | ✔
|
114 |
-
| Gomoku | ✔
|
115 |
-
| Connect4 | ✔
|
116 |
-
| 2048 |
|
117 |
-
| Chess | 🔒
|
118 |
-
| Go | 🔒
|
119 |
-
| CartPole | ---
|
120 |
-
| Pendulum | ---
|
121 |
-
| LunarLander | ---
|
122 |
-
| BipedalWalker | ---
|
123 |
-
| Atari | ---
|
124 |
-
| MuJoCo | ---
|
125 |
-
| MiniGrid | ---
|
126 |
-
| Bsuite | ---
|
127 |
|
128 |
<sup>(1): "✔" 表示对应的项目已经完成并经过良好的测试。</sup>
|
129 |
|
|
|
109 |
LightZero 目前支持的环境及算法如下表所示:
|
110 |
|
111 |
| Env./Algo. | AlphaZero | MuZero | EfficientZero | Sampled EfficientZero | Gumbel MuZero | Stochastic MuZero |
|
112 |
+
|---------------| -------- | ------ |-------------| ------------------ | ---------- |----------------|
|
113 |
+
| TicTacToe | ✔ | ✔ | 🔒 | 🔒 | ✔ | 🔒 |
|
114 |
+
| Gomoku | ✔ | ✔ | 🔒 | 🔒 | ✔ | 🔒 |
|
115 |
+
| Connect4 | ✔ | ✔ | 🔒 | 🔒 | 🔒 | 🔒 |
|
116 |
+
| 2048 | --- | ✔ | 🔒 | 🔒 | 🔒 | ✔ |
|
117 |
+
| Chess | 🔒 | 🔒 | 🔒 | 🔒 | 🔒 | 🔒 |
|
118 |
+
| Go | 🔒 | 🔒 | 🔒 | 🔒 | 🔒 | 🔒 |
|
119 |
+
| CartPole | --- | ✔ | ✔ | ✔ | ✔ | ✔ |
|
120 |
+
| Pendulum | --- | ✔ | ✔ | ✔ | ✔ | ✔ |
|
121 |
+
| LunarLander | --- | ✔ | ✔ | ✔ | ✔ | ✔ |
|
122 |
+
| BipedalWalker | --- | ✔ | ✔ | ✔ | ✔ | 🔒 |
|
123 |
+
| Atari | --- | ✔ | ✔ | ✔ | ✔ | ✔ |
|
124 |
+
| MuJoCo | --- | ✔ | ✔ | ✔ | 🔒 | 🔒 |
|
125 |
+
| MiniGrid | --- | ✔ | ✔ | ✔ | 🔒 | 🔒 |
|
126 |
+
| Bsuite | --- | ✔ | ✔ | ✔ | 🔒 | 🔒 |
|
127 |
|
128 |
<sup>(1): "✔" 表示对应的项目已经完成并经过良好的测试。</sup>
|
129 |
|
rag_demo.py
CHANGED
@@ -47,7 +47,7 @@ def load_and_split_document(file_path, chunk_size=500, chunk_overlap=50):
|
|
47 |
|
48 |
|
49 |
# 向量存储建立
|
50 |
-
def create_vector_store(chunks, model="OpenAI"
|
51 |
"""将文档块转换为向量并存储到 Weaviate 中"""
|
52 |
client = Client(embedded_options=EmbeddedOptions())
|
53 |
|
@@ -66,6 +66,10 @@ def create_vector_store(chunks, model="OpenAI", k=4):
|
|
66 |
embedding=embedding_model,
|
67 |
by_text=False
|
68 |
)
|
|
|
|
|
|
|
|
|
69 |
return vectorstore.as_retriever(search_kwargs={'k': k})
|
70 |
|
71 |
|
@@ -245,7 +249,8 @@ if __name__ == "__main__":
|
|
245 |
chunks = load_and_split_document(file_path, chunk_size=5000, chunk_overlap=500)
|
246 |
|
247 |
# 创建向量存储
|
248 |
-
|
|
|
249 |
|
250 |
# 设置 RAG 流程
|
251 |
rag_chain = setup_rag_chain(model_name=model_name, temperature=temperature)
|
@@ -266,7 +271,6 @@ if __name__ == "__main__":
|
|
266 |
(11)请问对这个仓库提出详细的改进建议。
|
267 |
"""
|
268 |
|
269 |
-
# query = ("请检索最近关于Transformer+RL的最新论文,并给出详细介绍")
|
270 |
# 使用 RAG 链获取参考的文档与答案
|
271 |
retrieved_documents, result_with_rag = execute_query(retriever, rag_chain, query, model_name=model_name,
|
272 |
temperature=temperature)
|
|
|
47 |
|
48 |
|
49 |
# 向量存储建立
|
50 |
+
def create_vector_store(chunks, model="OpenAI"):
|
51 |
"""将文档块转换为向量并存储到 Weaviate 中"""
|
52 |
client = Client(embedded_options=EmbeddedOptions())
|
53 |
|
|
|
66 |
embedding=embedding_model,
|
67 |
by_text=False
|
68 |
)
|
69 |
+
return vectorstore
|
70 |
+
|
71 |
+
|
72 |
+
def get_retriever(vectorstore, k=4):
|
73 |
return vectorstore.as_retriever(search_kwargs={'k': k})
|
74 |
|
75 |
|
|
|
249 |
chunks = load_and_split_document(file_path, chunk_size=5000, chunk_overlap=500)
|
250 |
|
251 |
# 创建向量存储
|
252 |
+
vectorstore = create_vector_store(chunks, model=embedding_model)
|
253 |
+
retriever = get_retriever(vectorstore, k=4)
|
254 |
|
255 |
# 设置 RAG 流程
|
256 |
rag_chain = setup_rag_chain(model_name=model_name, temperature=temperature)
|
|
|
271 |
(11)请问对这个仓库提出详细的改进建议。
|
272 |
"""
|
273 |
|
|
|
274 |
# 使用 RAG 链获取参考的文档与答案
|
275 |
retrieved_documents, result_with_rag = execute_query(retriever, rag_chain, query, model_name=model_name,
|
276 |
temperature=temperature)
|