huanghai
/
dsProject
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

'commit'

Browse Source
main
HuangHai 1 month ago
parent 111b5a1481
commit 68231fd5f8
4 changed files with 301 additions and 0 deletions
Show Stats Download Patch File Download Diff File

4
dsRag/Config/Config.py
Unescape View File

@ -7,4 +7,8 @@ ES_CONFIG = {
"default_index": "knowledge_base"
}
# 词向量模型路径
WORD2VEC_MODEL_PATH = r"D:\Tencent_AILab_ChineseEmbedding\Tencent_AILab_ChineseEmbedding.txt"
# DeepSeek API KEY
DEEPSEEK_API_KEY = 'sk-44ae895eeb614aa1a9c6460579e322f1'

68
dsRag/T7_Query.py
Unescape View File

@ -0,0 +1,68 @@
from elasticsearch import Elasticsearch
from Util.EmbeddingUtil import text_to_embedding
import random
import Config.Config as config
# 初始化ES连接
es = Elasticsearch(
hosts=config.ES_CONFIG['hosts'],
basic_auth=config.ES_CONFIG['basic_auth'],
verify_certs=config.ES_CONFIG['verify_certs']
)
def vector_search(text):
"""向量相似度搜索"""
vector = text_to_embedding(text)
script_query = {
"script_score": {
"query": {"match_all": {}},
"script": {
"source": "cosineSimilarity(params.query_vector, 'vector') + 1.0",
"params": {"query_vector": vector}
}
}
}
response = es.search(
index='knowledge_base',
body={
"size": 5,
"query": script_query,
"_source": ["text"]
}
)
return [hit['_source']['text'] for hit in response['hits']['hits']]
def text_search(text):
"""文本精确搜索"""
response = es.search(
index='raw_texts',
body={
"query": {
"match_phrase": {
"text": text
}
}
}
)
return [hit['_source']['text'] for hit in response['hits']['hits']]
def test_queries(file_path):
"""从文本文件中随机选取5个句子进行测试"""
with open(file_path, 'r', encoding='utf-8') as f:
sentences = [line.strip() for line in f if line.strip()]
test_samples = random.sample(sentences, min(5, len(sentences)))
for sample in test_samples:
print(f"测试句子: {sample}")
print("向量搜索结果:")
for result in vector_search(sample):
print(f"- {result}")
print("\n文本精确搜索结果:")
for result in text_search(sample):
print(f"- {result}")
print("="*50)
if __name__ == "__main__":
test_queries("人口变化趋势对云南教育的影响.txt")

100
dsRag/T8_RAG.py
Unescape View File

@ -0,0 +1,100 @@
'''
pip install openai
'''
from elasticsearch import Elasticsearch
from Util.EmbeddingUtil import text_to_embedding
import Config.Config as config
from openai import OpenAI
import json
# 初始化ES连接
es = Elasticsearch(
hosts=config.ES_CONFIG['hosts'],
basic_auth=config.ES_CONFIG['basic_auth'],
verify_certs=config.ES_CONFIG['verify_certs']
)
# 初始化DeepSeek客户端
client = OpenAI(api_key=config.DEEPSEEK_API_KEY)
def search_related_data(query):
"""搜索向量数据和原始相关数据"""
# 向量搜索
vector = text_to_embedding(query)
vector_results = es.search(
index='knowledge_base',
body={
"size": 5,
"query": {
"script_score": {
"query": {"match_all": {}},
"script": {
"source": "cosineSimilarity(params.query_vector, 'vector') + 1.0",
"params": {"query_vector": vector}
}
}
},
"_source": ["text"]
}
)
# 文本精确搜索
text_results = es.search(
index='raw_texts',
body={
"query": {
"multi_match": {
"query": query,
"fields": ["text"],
"type": "best_fields"
}
},
"size": 5
}
)
# 合并结果
all_results = [hit['_source']['text'] for hit in vector_results['hits']['hits']]
all_results.extend([hit['_source']['text'] for hit in text_results['hits']['hits']])
return "\n\n".join(all_results)
def generate_report(query, context):
"""使用DeepSeek生成报告"""
prompt = f"""根据以下关于'{query}'的相关信息，整理一份结构化的报告：
要求：
1. 分章节组织内容
2. 包含关键数据和事实
3. 语言简洁专业
相关信息：
{context}"""
response = client.chat.completions.create(
model="deepseek-chat",
messages=[
{"role": "system", "content": "你是一个专业的文档整理助手"},
{"role": "user", "content": prompt}
],
temperature=0.3
)
return response.choices[0].message.content
def process_query(query):
"""处理用户查询并生成报告"""
print(f"正在搜索与'{query}'相关的数据...")
context = search_related_data(query)
print(f"找到{len(context.split('\n\n'))}条相关数据")
print("正在生成报告...")
report = generate_report(query, context)
return report
if __name__ == "__main__":
#user_query = input("请输入您的查询要求：")
user_query = "整理云南省初中在校生情况文档"
report = process_query(user_query)
print("\n=== 生成的报告 ===\n")
print(report)

129
dsRag/运行结果.txt
Unescape View File

@ -0,0 +1,129 @@
D:\anaconda3\envs\rag\python.exe D:\dsWork\dsProject\dsRag\T7_Query.py
D:\anaconda3\envs\rag\lib\site-packages\jieba\_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
import pkg_resources
2025-06-23 20:03:12,025 - INFO - loading projection weights from D:\Tencent_AILab_ChineseEmbedding\Tencent_AILab_ChineseEmbedding.txt
2025-06-23 20:03:13,349 - INFO - KeyedVectors lifecycle event {'msg': 'loaded (10000, 200) matrix of type float32 from D:\\Tencent_AILab_ChineseEmbedding\\Tencent_AILab_ChineseEmbedding.txt', 'binary': False, 'encoding': 'utf8', 'datetime': '2025-06-23T20:03:13.329238', 'gensim': '4.3.3', 'python': '3.10.18 | packaged by conda-forge | (main, Jun 4 2025, 14:42:04) [MSC v.1943 64 bit (AMD64)]', 'platform': 'Windows-10-10.0.19044-SP0', 'event': 'load_word2vec_format'}
2025-06-23 20:03:13,349 - INFO - 模型加载成功，词向量维度: 200
D:\anaconda3\envs\rag\lib\site-packages\elasticsearch\_sync\client\__init__.py:311: SecurityWarning: Connecting to 'https://10.10.14.206:9200' using TLS with verify_certs=False is insecure
_transport = transport_class(
Building prefix dict from the default dictionary ...
2025-06-23 20:03:13,622 - DEBUG - Building prefix dict from the default dictionary ...
Loading model from cache C:\Users\ADMINI~1\AppData\Local\Temp\jieba.cache
2025-06-23 20:03:13,622 - DEBUG - Loading model from cache C:\Users\ADMINI~1\AppData\Local\Temp\jieba.cache
测试句子: （万人） 教职工数需求
向量搜索结果:
Loading model cost 0.655 seconds.
2025-06-23 20:03:14,277 - DEBUG - Loading model cost 0.655 seconds.
Prefix dict has been built successfully.
2025-06-23 20:03:14,278 - DEBUG - Prefix dict has been built successfully.
2025-06-23 20:03:14,278 - INFO - 文本: （万人） 教职工数需求, 分词结果: ['（', '万人', '）', '\t', '教职工', '数', '需求']
2025-06-23 20:03:14,278 - INFO - 有效词向量数量: 5
2025-06-23 20:03:14,278 - INFO - 生成的平均向量: [-0.0246012 -0.1393744 -0.0812634 0.22129479 -0.0203214 ]...
D:\anaconda3\envs\rag\lib\site-packages\urllib3\connectionpool.py:1097: InsecureRequestWarning: Unverified HTTPS request is being made to host '10.10.14.206'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
warnings.warn(
2025-06-23 20:03:14,384 - INFO - POST https://10.10.14.206:9200/knowledge_base/_search [status:200 duration:0.104s]
D:\anaconda3\envs\rag\lib\site-packages\urllib3\connectionpool.py:1097: InsecureRequestWarning: Unverified HTTPS request is being made to host '10.10.14.206'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
warnings.warn(
- （万人） 教职工数需求
- （万人） 教职工数需求
- （万人） 专任教师数需求
- （万人） 教职工数需求
- （万人） 教职工需求
文本精确搜索结果:
2025-06-23 20:03:14,419 - INFO - POST https://10.10.14.206:9200/raw_texts/_search [status:200 duration:0.033s]
2025-06-23 20:03:14,420 - INFO - 文本: 表 6 高等教育规模及资源配置需求预测统计表, 分词结果: ['表', ' ', '6', ' ', '高等教育', '规模', '及', '资源配置', '需求预测', '统计表']
2025-06-23 20:03:14,420 - INFO - 有效词向量数量: 4
2025-06-23 20:03:14,421 - INFO - 生成的平均向量: [ 0.18066275 -0.12073625 -0.12693274 0.219709 0.00525575]...
D:\anaconda3\envs\rag\lib\site-packages\urllib3\connectionpool.py:1097: InsecureRequestWarning: Unverified HTTPS request is being made to host '10.10.14.206'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
warnings.warn(
- （万人） 教职工数需求
- （万人） 教职工数需求
- （万人） 教职工数需求
==================================================
测试句子: 表 6 高等教育规模及资源配置需求预测统计表
向量搜索结果:
- 表 6 高等教育规模及资源配置需求预测统计表
- 表 1 学前教育规模及资源配置需求预测统计表
- 表 5 中等职业教育规模及资源配置需求预测统计表
- 表 4 普通高中教育规模及资源配置需求预测统计表
- 表 3 初中教育规模及资源配置需求预测统计表
文本精确搜索结果:
- 表 6 高等教育规模及资源配置需求预测统计表
==================================================
测试句子: 指标
向量搜索结果:
2025-06-23 20:03:14,431 - INFO - POST https://10.10.14.206:9200/knowledge_base/_search [status:200 duration:0.008s]
D:\anaconda3\envs\rag\lib\site-packages\urllib3\connectionpool.py:1097: InsecureRequestWarning: Unverified HTTPS request is being made to host '10.10.14.206'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
warnings.warn(
2025-06-23 20:03:14,440 - INFO - POST https://10.10.14.206:9200/raw_texts/_search [status:200 duration:0.008s]
2025-06-23 20:03:14,440 - INFO - 文本: 指标, 分词结果: ['指标']
2025-06-23 20:03:14,441 - INFO - 有效词向量数量: 1
2025-06-23 20:03:14,441 - INFO - 生成的平均向量: [-0.24731 -0.30286 -0.223835 0.217864 -0.162194]...
D:\anaconda3\envs\rag\lib\site-packages\urllib3\connectionpool.py:1097: InsecureRequestWarning: Unverified HTTPS request is being made to host '10.10.14.206'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
warnings.warn(
2025-06-23 20:03:14,450 - INFO - POST https://10.10.14.206:9200/knowledge_base/_search [status:200 duration:0.007s]
D:\anaconda3\envs\rag\lib\site-packages\urllib3\connectionpool.py:1097: InsecureRequestWarning: Unverified HTTPS request is being made to host '10.10.14.206'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
warnings.warn(
2025-06-23 20:03:14,456 - INFO - POST https://10.10.14.206:9200/raw_texts/_search [status:200 duration:0.005s]
2025-06-23 20:03:14,457 - INFO - 文本: 二是保障随迁子女受教育权利。健全以居住证为主要依据的农业转移人口随迁子女入学保障政策，以公办学校为主将随迁子女纳入流入地义务教育保障范围，保障享受基本公共教育服务。, 分词结果: ['二是', '保障', '随迁', '子女', '受', '教育', '权利', '。', '健全', '以', '居住证', '为', '主要', '依据', '的', '农业', '转移', '人口', '随迁', '子女', '入学', '保障', '政策', '，', '以', '公办', '学校', '为主', '将', '随迁', '子女', '纳入', '流入地', '义务教育', '保障', '范围', '，', '保障', '享受', '基本', '公共', '教育', '服务', '。']
2025-06-23 20:03:14,457 - INFO - 有效词向量数量: 32
2025-06-23 20:03:14,458 - INFO - 生成的平均向量: [ 0.18174155 -0.13902284 -0.13658895 0.17560197 0.04838478]...
D:\anaconda3\envs\rag\lib\site-packages\urllib3\connectionpool.py:1097: InsecureRequestWarning: Unverified HTTPS request is being made to host '10.10.14.206'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
warnings.warn(
- 指标
- 指标
- 指标
- 指标
- 指标
文本精确搜索结果:
- 指标
- 指标
- 指标
- 指标
- 指标
- 指标
- 2001-2022年云南省人口指标统计表
- 各学段数据及人口指标统计表
- 1.2001-2022年云南省人口指标统计表 33
- 2.各学段数据及人口指标统计表 34
==================================================
测试句子: 二是保障随迁子女受教育权利。健全以居住证为主要依据的农业转移人口随迁子女入学保障政策，以公办学校为主将随迁子女纳入流入地义务教育保障范围，保障享受基本公共教育服务。
向量搜索结果:
- 健全以居住证为主要依据的农业转移人口随迁子女入学保障政策，以公办学校为主将随迁子女纳入流入地义务教育保障范围，保障享受基本公共教育服务
- 在学校布局、入学政策等方面守牢底线，既适应城乡人口变化趋势，又能提供公平的受教育机会，保障不同群体受教育权利
- 健全随迁子女入学保障政策，保障各类特殊群体受教育权利
- 健全随迁子女入学保障政策，保障各类特殊群体受教育权利
- 健全随迁子女入学保障政策，保障各类特殊群体受教育权利
文本精确搜索结果:
2025-06-23 20:03:14,464 - INFO - POST https://10.10.14.206:9200/knowledge_base/_search [status:200 duration:0.005s]
D:\anaconda3\envs\rag\lib\site-packages\urllib3\connectionpool.py:1097: InsecureRequestWarning: Unverified HTTPS request is being made to host '10.10.14.206'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
warnings.warn(
2025-06-23 20:03:14,475 - INFO - POST https://10.10.14.206:9200/raw_texts/_search [status:200 duration:0.010s]
2025-06-23 20:03:14,475 - INFO - 文本: （一）2012—2022年全省人口总体呈现“长增－突降”态势, 分词结果: ['（', '一', '）', '2012', '—', '2022', '年', '全省', '人口', '总体', '呈现', '“', '长增', '－', '突降', '”', '态势']
2025-06-23 20:03:14,476 - INFO - 有效词向量数量: 14
2025-06-23 20:03:14,476 - INFO - 生成的平均向量: [ 0.06827115 -0.212308 -0.07557671 0.21362822 0.12347364]...
D:\anaconda3\envs\rag\lib\site-packages\urllib3\connectionpool.py:1097: InsecureRequestWarning: Unverified HTTPS request is being made to host '10.10.14.206'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
warnings.warn(
2025-06-23 20:03:14,482 - INFO - POST https://10.10.14.206:9200/knowledge_base/_search [status:200 duration:0.005s]
D:\anaconda3\envs\rag\lib\site-packages\urllib3\connectionpool.py:1097: InsecureRequestWarning: Unverified HTTPS request is being made to host '10.10.14.206'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
warnings.warn(
==================================================
测试句子: （一）2012—2022年全省人口总体呈现“长增－突降”态势
向量搜索结果:
- （一）2012—2022年全省人口总体呈现“长增－突降”态势
- （一）2012—2022年全省人口总体呈现“长增－突降”态势 2
- 总的看，2012—2022年全省人口变化呈现先增、突降的态势（图1）
- 结合全国趋势看，预计我省2023—2035年新生人口数持续保持在较低水平（图2），从2024年起，全省人口总数呈缓慢下降趋势，与全国范围人口下降趋势基本同步（图1）
- 按照全省普通高中师生比标准（1：12.5）测算，2023—2035年全省普通高中教师需求量逐年增大，2033年达到峰值（10.31万人），与2022年实际专任教师数7.64万人比，存在2.67万人的缺口（图20）
文本精确搜索结果:
- （一）2012—2022年全省人口总体呈现“长增－突降”态势
- （一）2012—2022年全省人口总体呈现“长增－突降”态势 2
==================================================
2025-06-23 20:03:14,488 - INFO - POST https://10.10.14.206:9200/raw_texts/_search [status:200 duration:0.005s]
进程已结束，退出代码为 0
Write Preview
Loading…
Cancel
Save