'commit'
This commit is contained in:
@@ -1,18 +1,15 @@
|
||||
import logging
|
||||
import warnings
|
||||
import hashlib # 导入哈希库
|
||||
import hashlib
|
||||
import time
|
||||
from Config.Config import ES_CONFIG
|
||||
from ElasticSearch.Utils.ElasticsearchConnectionPool import ElasticsearchConnectionPool
|
||||
from elasticsearch import Elasticsearch
|
||||
from langchain_core.documents import Document
|
||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
from langchain_openai import OpenAIEmbeddings
|
||||
from pydantic import SecretStr
|
||||
|
||||
from Config import Config
|
||||
|
||||
|
||||
# 初始化日志
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
@@ -111,9 +108,7 @@ class EsSearchUtil:
|
||||
# 释放连接回连接池
|
||||
self.es_pool.release_connection(conn)
|
||||
|
||||
|
||||
|
||||
def split_text_into_chunks(text: str, chunk_size: int = 200, chunk_overlap: int = 0) -> list:
|
||||
def split_text_into_chunks(self,text: str, chunk_size: int = 200, chunk_overlap: int = 0) -> list:
|
||||
"""
|
||||
将文本切割成块
|
||||
|
||||
@@ -137,8 +132,7 @@ def split_text_into_chunks(text: str, chunk_size: int = 200, chunk_overlap: int
|
||||
|
||||
return [split.page_content for split in all_splits]
|
||||
|
||||
|
||||
def insert_long_text_to_es(long_text: str, tags: list = None) -> bool:
|
||||
def insert_long_text_to_es(self,long_text: str, tags: list = None) -> bool:
|
||||
"""
|
||||
将长文本切割后向量化并插入到Elasticsearch,基于文本内容哈希实现去重
|
||||
|
||||
@@ -185,7 +179,7 @@ def insert_long_text_to_es(long_text: str, tags: list = None) -> bool:
|
||||
print(f"索引 '{index_name}' 创建成功")
|
||||
|
||||
# 4. 切割文本
|
||||
text_chunks = split_text_into_chunks(long_text)
|
||||
text_chunks = self.split_text_into_chunks(long_text)
|
||||
|
||||
# 5. 准备标签
|
||||
if tags is None:
|
||||
@@ -236,7 +230,6 @@ def insert_long_text_to_es(long_text: str, tags: list = None) -> bool:
|
||||
search_util.es_pool.release_connection(conn)
|
||||
|
||||
|
||||
|
||||
# 添加main函数进行测试
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
|
Reference in New Issue
Block a user