From 7e29267f9cd38e53777ec69d7edf65552d35fcb5 Mon Sep 17 00:00:00 2001 From: HuangHai <10402852@qq.com> Date: Tue, 19 Aug 2025 08:08:02 +0800 Subject: [PATCH] 'commit' --- dsSchoolBuddy/ElasticSearch/T2_BgeM3.py | 114 +---------------- dsSchoolBuddy/Util/VectorUtil.py | 115 ++++++++++++++++++ .../__pycache__/VectorUtil.cpython-310.pyc | Bin 0 -> 3516 bytes 3 files changed, 116 insertions(+), 113 deletions(-) create mode 100644 dsSchoolBuddy/Util/VectorUtil.py create mode 100644 dsSchoolBuddy/Util/__pycache__/VectorUtil.cpython-310.pyc diff --git a/dsSchoolBuddy/ElasticSearch/T2_BgeM3.py b/dsSchoolBuddy/ElasticSearch/T2_BgeM3.py index 2e828947..500524b1 100644 --- a/dsSchoolBuddy/ElasticSearch/T2_BgeM3.py +++ b/dsSchoolBuddy/ElasticSearch/T2_BgeM3.py @@ -1,117 +1,5 @@ # pip install pydantic requests -from langchain_core.documents import Document -from langchain_core.vectorstores import InMemoryVectorStore -from langchain_openai import OpenAIEmbeddings -from langchain_text_splitters import RecursiveCharacterTextSplitter -from pydantic import SecretStr -import requests -import json -from Config.Config import ( - EMBED_MODEL_NAME, EMBED_BASE_URL, EMBED_API_KEY, - RERANK_MODEL, RERANK_BASE_URL, RERANK_BINDING_API_KEY -) - - -def text_to_vector_db(text: str, chunk_size: int = 200, chunk_overlap: int = 100) -> tuple: - """ - 将文本存入向量数据库 - - 参数: - text: 要入库的文本 - chunk_size: 文本分割块大小 - chunk_overlap: 文本块重叠大小 - - 返回: - tuple: (向量存储对象, 文档数量, 分割后的文档块数量) - """ - # 创建文档对象 - docs = [Document(page_content=text, metadata={"source": "simulated_document"})] - doc_count = len(docs) - print(f"文档数量：{doc_count} 个") - - # 切割文档 - text_splitter = RecursiveCharacterTextSplitter( - chunk_size=chunk_size, chunk_overlap=chunk_overlap, add_start_index=True - ) - all_splits = text_splitter.split_documents(docs) - split_count = len(all_splits) - print(f"切割后的文档块数量：{split_count}") - - # 嵌入模型 - embeddings = OpenAIEmbeddings( - model=EMBED_MODEL_NAME, - base_url=EMBED_BASE_URL, - api_key=SecretStr(EMBED_API_KEY) # 包装成 SecretStr 类型 - ) - - # 向量存储 - vector_store = InMemoryVectorStore(embeddings) - ids = vector_store.add_documents(documents=all_splits) - - return vector_store, doc_count, split_count - - -def query_vector_db(vector_store: InMemoryVectorStore, query: str, k: int = 4) -> list: - """ - 从向量数据库查询文本 - - 参数: - vector_store: 向量存储对象 - query: 查询字符串 - k: 要返回的结果数量 - - 返回: - list: 重排后的结果列表，每个元素是(文档对象, 可信度分数)的元组 - """ - # 向量查询 - 获取更多结果用于重排 - results = vector_store.similarity_search(query, k=k) - print(f"向量搜索结果数量：{len(results)}") - - # 存储重排后的文档和分数 - reranked_docs_with_scores = [] - - # 调用重排模型 - if len(results) > 1: - # 准备重排请求数据 - rerank_data = { - "model": RERANK_MODEL, - "query": query, - "documents": [doc.page_content for doc in results], - "top_n": len(results) - } - - # 调用SiliconFlow API进行重排 - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {RERANK_BINDING_API_KEY}" - } - - try: - response = requests.post(RERANK_BASE_URL, headers=headers, data=json.dumps(rerank_data)) - response.raise_for_status() # 检查请求是否成功 - rerank_result = response.json() - - # 处理重排结果，提取relevance_score - if "results" in rerank_result: - for item in rerank_result["results"]: - doc_idx = item.get("index") - score = item.get("relevance_score", 0.0) - if 0 <= doc_idx < len(results): - reranked_docs_with_scores.append((results[doc_idx], score)) - else: - print("警告: 无法识别重排API响应格式") - reranked_docs_with_scores = [(doc, 0.0) for doc in results] - - print(f"重排后结果数量：{len(reranked_docs_with_scores)}") - except Exception as e: - print(f"重排模型调用失败: {e}") - print("将使用原始搜索结果") - reranked_docs_with_scores = [(doc, 0.0) for doc in results] - else: - # 只有一个结果，无需重排 - reranked_docs_with_scores = [(doc, 1.0) for doc in results] # 单个结果可信度设为1.0 - - return reranked_docs_with_scores +from Util.VectorUtil import text_to_vector_db, query_vector_db def main(): diff --git a/dsSchoolBuddy/Util/VectorUtil.py b/dsSchoolBuddy/Util/VectorUtil.py new file mode 100644 index 00000000..b695c0c7 --- /dev/null +++ b/dsSchoolBuddy/Util/VectorUtil.py @@ -0,0 +1,115 @@ +# pip install pydantic requests +from langchain_core.documents import Document +from langchain_core.vectorstores import InMemoryVectorStore +from langchain_openai import OpenAIEmbeddings +from langchain_text_splitters import RecursiveCharacterTextSplitter +from pydantic import SecretStr +import requests +import json +from Config.Config import ( + EMBED_MODEL_NAME, EMBED_BASE_URL, EMBED_API_KEY, + RERANK_MODEL, RERANK_BASE_URL, RERANK_BINDING_API_KEY +) + +def text_to_vector_db(text: str, chunk_size: int = 200, chunk_overlap: int = 100) -> tuple: + """ + 将文本存入向量数据库 + + 参数: + text: 要入库的文本 + chunk_size: 文本分割块大小 + chunk_overlap: 文本块重叠大小 + + 返回: + tuple: (向量存储对象, 文档数量, 分割后的文档块数量) + """ + # 创建文档对象 + docs = [Document(page_content=text, metadata={"source": "simulated_document"})] + doc_count = len(docs) + print(f"文档数量：{doc_count} 个") + + # 切割文档 + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=chunk_size, chunk_overlap=chunk_overlap, add_start_index=True + ) + all_splits = text_splitter.split_documents(docs) + split_count = len(all_splits) + print(f"切割后的文档块数量：{split_count}") + + # 嵌入模型 + embeddings = OpenAIEmbeddings( + model=EMBED_MODEL_NAME, + base_url=EMBED_BASE_URL, + api_key=SecretStr(EMBED_API_KEY) # 包装成 SecretStr 类型 + ) + + # 向量存储 + vector_store = InMemoryVectorStore(embeddings) + ids = vector_store.add_documents(documents=all_splits) + + return vector_store, doc_count, split_count + + +def query_vector_db(vector_store: InMemoryVectorStore, query: str, k: int = 4) -> list: + """ + 从向量数据库查询文本 + + 参数: + vector_store: 向量存储对象 + query: 查询字符串 + k: 要返回的结果数量 + + 返回: + list: 重排后的结果列表，每个元素是(文档对象, 可信度分数)的元组 + """ + # 向量查询 - 获取更多结果用于重排 + results = vector_store.similarity_search(query, k=k) + print(f"向量搜索结果数量：{len(results)}") + + # 存储重排后的文档和分数 + reranked_docs_with_scores = [] + + # 调用重排模型 + if len(results) > 1: + # 准备重排请求数据 + rerank_data = { + "model": RERANK_MODEL, + "query": query, + "documents": [doc.page_content for doc in results], + "top_n": len(results) + } + + # 调用SiliconFlow API进行重排 + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {RERANK_BINDING_API_KEY}" + } + + try: + response = requests.post(RERANK_BASE_URL, headers=headers, data=json.dumps(rerank_data)) + response.raise_for_status() # 检查请求是否成功 + rerank_result = response.json() + + # 处理重排结果，提取relevance_score + if "results" in rerank_result: + for item in rerank_result["results"]: + doc_idx = item.get("index") + score = item.get("relevance_score", 0.0) + if 0 <= doc_idx < len(results): + reranked_docs_with_scores.append((results[doc_idx], score)) + else: + print("警告: 无法识别重排API响应格式") + reranked_docs_with_scores = [(doc, 0.0) for doc in results] + + print(f"重排后结果数量：{len(reranked_docs_with_scores)}") + except Exception as e: + print(f"重排模型调用失败: {e}") + print("将使用原始搜索结果") + reranked_docs_with_scores = [(doc, 0.0) for doc in results] + else: + # 只有一个结果，无需重排 + reranked_docs_with_scores = [(doc, 1.0) for doc in results] # 单个结果可信度设为1.0 + + return reranked_docs_with_scores + + diff --git a/dsSchoolBuddy/Util/__pycache__/VectorUtil.cpython-310.pyc b/dsSchoolBuddy/Util/__pycache__/VectorUtil.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0cea75bc7e9bae595fb00eea63ceba517cc11584 GIT binary patch literal 3516 zcmaJ@>vI&x5ue%D?LGBC2oyWcC3ZQ!ijAGu7ZpV!kSZnH$S#CKyoKv(xfw|-?d=}3 zvw&24DaT+SafBe?;4%;EjKN?-gdILOK#F|K->_e!JxQOODt~dMoJY^DbUMqi_iFok zr+a#Odb)o-Za$wu@C^U$%EUk?LjMpS?N1s$eh#m99SA`L+o*~Sj0G>*QdKtOn3rv( zsv2s{E4Efm8L608ZM~W{(lM{u86yMpG&^f#;Xh^Ps(B+{Ef|GrhtW~(G&-wYMpw1l z=mz~>lA)VOmiCYwym`8LOeY1>aZELO$rjQ@Wzr4uO|fhX$a=_TkZmQ}C@O7T0dG+W zhx(vf@vGF~z;7CMcGIfMrhiE*+-0NiQW)-hp+=p7;i2jQN{Ho*docFM9$N94XC0!u zCQN2lIAwe35k6Y8Ee>L;! zD!;Vn`7ob+54<>Bes1XHFt=xD&%nrY$&9dL^{b`byEZU9GB`Z)i}LEUmA}KTRtS(# z;owi0sZ99JRN1p;XqZoU_Yh^aSqnAr&Sy?V7Lizevhc6My9-|LBoK}QI8q{6n8>rz zIIhba2l7!|SGYu!K)Ha3dI`QU;&OlkiD)D>E7#RPomT?6f$JLRYoMP3eLYBBK$oOg zTjd&0ah<1+Vv>d#I?n_;=%s`7WH!)9W){~oK{`-^)R+thmi4KsUzR^=JQ=UsxB`B|o}Mg5$@-^HtvcJd0=^`|QfxZZJlAIxYIJ*6 z)wfMfNtvu=v=ruQ<~S`^T!%x3hkBKAlbGD}JK$)=4%b%t^#0qv0%OI+Pd?aNk{J{_ zVqHLt4wI0w$4$n|mP6n z7;FA{r5uyMS}5Dp3Dp|29L_}c86BReqt-?|CVXQlL3BYhK?LwHk-SD;__%I0%f$B9 z_Yf)|^Wt?O6u!fayhw#|ya*JlkuhyM`SQYSvUH-Cp{<$^bBAJ3fLH)3gtA4v8;E@e z7W&su2K)CDZ=cJi;O|A|PJ$wEqm>EQwRif2Oz(e*TlW40RRp`OHXU}wK5)0ZYL^@+ zfHCiTK)Ulf?!=O;VeP*eS;qz7B`jh1{$i(G+HtWaTNu97n?F;In};p9|z5(}#lW znAE^x%*n!zIt99ru63VbN0@ZOd}V!~`%s{aA_ejM^oJPLa~RF^f!3xVw-yQb>WM9F zZnv}#y$o%qp8Iu9%ul$(_%MQY(~40YP*F4(f6E$PE#5n|0i&AlTwPvxw+(&bT8=j0 zReQqq7rzP?@gxCk@!R1swD5pW?&eiUA>8^}21 z^ebu|2Y3>lkq%0uNln56Js`ayh0?ZXLK*5mF4cq-K@w)Riyc10JElFQdlo&vquHl0>n_cTPr&=on{Sit73A?b)KH|TxTY=lbVdO zVo93G?MhJislC%R=*0s*pKzHq6DyabFf~C5GCPzyu}IhV?{y4`eX2ZCc@Z*{H^e}o;z&` zCmPMSTEgb_Ms)d&<=aP>&Rvh*`)K*j)&63V6t|Br{`G+vJ@roX{u|Aj&{3W zx-fXjEk&AFdN{JyW^3 s7sjnUR0P_@TW$;Wdc5`RaKZ9zx!ktVR5=TKxY10B5a9h5!Hn literal 0 HcmV?d00001