This commit is contained in:
2025-08-19 09:23:36 +08:00
parent 66241b57dd
commit 1fd96fbc4e

View File

@@ -1,18 +1,15 @@
import logging
import warnings
import hashlib # 导入哈希库
import hashlib
import time
from Config.Config import ES_CONFIG
from ElasticSearch.Utils.ElasticsearchConnectionPool import ElasticsearchConnectionPool
from elasticsearch import Elasticsearch
from langchain_core.documents import Document
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_openai import OpenAIEmbeddings
from pydantic import SecretStr
from Config import Config
# 初始化日志
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@@ -111,9 +108,7 @@ class EsSearchUtil:
# 释放连接回连接池
self.es_pool.release_connection(conn)
def split_text_into_chunks(text: str, chunk_size: int = 200, chunk_overlap: int = 0) -> list:
def split_text_into_chunks(self,text: str, chunk_size: int = 200, chunk_overlap: int = 0) -> list:
"""
将文本切割成块
@@ -137,8 +132,7 @@ def split_text_into_chunks(text: str, chunk_size: int = 200, chunk_overlap: int
return [split.page_content for split in all_splits]
def insert_long_text_to_es(long_text: str, tags: list = None) -> bool:
def insert_long_text_to_es(self,long_text: str, tags: list = None) -> bool:
"""
将长文本切割后向量化并插入到Elasticsearch基于文本内容哈希实现去重
@@ -185,7 +179,7 @@ def insert_long_text_to_es(long_text: str, tags: list = None) -> bool:
print(f"索引 '{index_name}' 创建成功")
# 4. 切割文本
text_chunks = split_text_into_chunks(long_text)
text_chunks = self.split_text_into_chunks(long_text)
# 5. 准备标签
if tags is None:
@@ -236,7 +230,6 @@ def insert_long_text_to_es(long_text: str, tags: list = None) -> bool:
search_util.es_pool.release_connection(conn)
# 添加main函数进行测试
if __name__ == "__main__":
try: