diff --git a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/Test.py b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/Test.py new file mode 100644 index 00000000..4648e701 --- /dev/null +++ b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/Test.py @@ -0,0 +1 @@ +print("Hello, World!") \ No newline at end of file diff --git a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/config/config_loader.py b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/config/config_loader.py index cdb9e2e1..878b125d 100644 --- a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/config/config_loader.py +++ b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/config/config_loader.py @@ -111,6 +111,9 @@ def _make_api_request(api_url, secret, endpoint, json_data=None): def get_config_from_api(config): """从Java API获取配置""" + # 如果配置中指定了manager-api，则从Java API获取配置 + # D:\dsWork\QingLong\XiaoZhi\xiaozhi-esp32-server\main\xiaozhi-server\config_from_api.yaml + # 据说需要从这个config_from_api.yaml拷贝过去才行 api_url = config["manager-api"].get("url", "") secret = config["manager-api"].get("secret", "") @@ -129,7 +132,7 @@ def get_private_config_from_api(config, device_id, client_id): """从Java API获取私有配置""" api_url = config["manager-api"].get("url", "") secret = config["manager-api"].get("secret", "") - + # 调用JAVA接口获取私有配置 return _make_api_request( api_url, secret, diff --git a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py index 5be41341..6dc1af3b 100644 --- a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py +++ b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py @@ -29,12 +29,14 @@ from core.auth import AuthMiddleware, AuthenticationError from core.mcp.manager import MCPManager from config.config_loader import get_private_config_from_api +# 在日志记录中，TAG 通常用于标识日志消息的来源模块 TAG = __name__ -auto_import_modules("plugins_func.functions") +auto_import_modules("plugins_func.functions") # 导入plugins_func.functions包下所有函数 class TTSException(RuntimeError): + # 如果发现TTS异常，那么也不做 pass @@ -198,6 +200,7 @@ class ConnectionHandler: if not read_config_from_api: return """从接口获取差异化的配置进行二次实例化，非全量重新实例化""" + # 配置是两掺的，API中有的，走API；API中没有的，走配置文件 try: private_config = get_private_config_from_api( self.config, @@ -354,6 +357,10 @@ class ConnectionHandler: m.content = prompt def chat(self, query): + ''' + 聊天的主函数 + query:用户说的是什么话 + ''' self.dialogue.put(Message(role="user", content=query)) response_message = [] processed_chars = 0 # 跟踪已处理的字符位置 @@ -361,12 +368,16 @@ class ConnectionHandler: start_time = time.time() # 使用带记忆的对话 future = asyncio.run_coroutine_threadsafe( + # query_memory：似乎是利用memory的内容进行RAG匹配，找到以前讲过的相关性高的内容 + # 这个query_memory函数，是一个接口，有好几个实现，比如memory_llm_local_short.py self.memory.query_memory(query), self.loop ) - memory_str = future.result() + memory_str = future.result() # 从记忆中获取相关性高的内容 self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}") llm_responses = self.llm.response( + # 发送给大模型，参数： + # session_id:会话ID，用于标识会话，这里使用一个随机生成的UUID self.session_id, self.dialogue.get_llm_dialogue_with_memory(memory_str) ) except Exception as e: @@ -375,16 +386,16 @@ class ConnectionHandler: self.llm_finish_task = False text_index = 0 - for content in llm_responses: - response_message.append(content) - if self.client_abort: + for content in llm_responses: # 遍历大模型的返回内容 + response_message.append(content) # 拼接返回内容 + if self.client_abort: # 如果客户端要求中断对话，则中断对话 break - end_time = time.time() + end_time = time.time() # 记录结束时间 self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}") # 合并当前全部文本并处理未分割部分 - full_text = "".join(response_message) + full_text = "".join(response_message) # 拼接全部文本 current_text = full_text[processed_chars:] # 从未处理的位置开始 # 查找最后一个有效标点 @@ -397,21 +408,24 @@ class ConnectionHandler: # 找到分割点则处理 if last_punct_pos != -1: - segment_text_raw = current_text[: last_punct_pos + 1] - segment_text = get_string_no_punctuation_or_emoji(segment_text_raw) - if segment_text: + segment_text_raw = current_text[: last_punct_pos + 1] # 本段的原始文本内容 + segment_text = get_string_no_punctuation_or_emoji(segment_text_raw) # 清理文本内容 + if segment_text: # 如果处理完得到的干净文本内容不为空，则处理 # 强制设置空字符，测试TTS出错返回语音的健壮性 # if text_index % 2 == 0: # segment_text = " " text_index += 1 - self.recode_first_last_text(segment_text, text_index) + self.recode_first_last_text(segment_text, text_index)# 分段记录大模型返回的文本 + # 应该是一般接收大模型返回的每一句话，一边用TTS生成语音，好牛B啊~ future = self.executor.submit( self.speak_and_play, segment_text, text_index ) + # 这里居然有一个 TTS队列，看来是为了解决生成文本太长，最终语音生成时间过长的问题，是一边大模型生成文本，一边进入TTS队列，一边生成返回的意思 self.tts_queue.put(future) processed_chars += len(segment_text_raw) # 更新已处理字符位置 # 处理最后剩余的文本 + # 多么经典的代码！就是每句话都是由下一句话时驱动完成转换并输了的，那么，最后一句话的逻辑就与其它话不同，需要单独处理。 full_text = "".join(response_message) remaining_text = full_text[processed_chars:] if remaining_text: @@ -423,7 +437,7 @@ class ConnectionHandler: self.speak_and_play, segment_text, text_index ) self.tts_queue.put(future) - + # 标识大模型生成任务结束 self.llm_finish_task = True self.dialogue.put(Message(role="assistant", content="".join(response_message))) self.logger.bind(tag=TAG).debug( @@ -439,8 +453,9 @@ class ConnectionHandler: self.dialogue.put(Message(role="user", content=query)) # Define intent functions + # 定义意图感知函数 functions = None - if hasattr(self, "func_handler"): + if hasattr(self, "func_handler"): # 调用对象实体，是不是有属性func_handler? functions = self.func_handler.get_functions() response_message = [] processed_chars = 0 # 跟踪已处理的字符位置 @@ -460,7 +475,7 @@ class ConnectionHandler: llm_responses = self.llm.response_with_functions( self.session_id, self.dialogue.get_llm_dialogue_with_memory(memory_str), - functions=functions, + functions=functions, # 这里调用的是支持function_call的LLM ) except Exception as e: self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}") @@ -505,7 +520,7 @@ class ConnectionHandler: break end_time = time.time() - # self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}") + self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}") # 处理文本分段和TTS逻辑 # 合并当前全部文本并处理未分割部分 diff --git a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/abortHandle.py b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/abortHandle.py index 3183487f..fba22bb1 100644 --- a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/abortHandle.py +++ b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/abortHandle.py @@ -1,11 +1,10 @@ import json -import queue from config.logger import setup_logging TAG = __name__ logger = setup_logging() - +# 打断处理逻辑 async def handleAbortMessage(conn): logger.bind(tag=TAG).info("Abort message received") # 设置成打断状态，会自动打断llm、tts任务 diff --git a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/helloHandle.py b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/helloHandle.py index 4885f9ff..10335149 100644 --- a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/helloHandle.py +++ b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/helloHandle.py @@ -13,7 +13,7 @@ logger = setup_logging() WAKEUP_CONFIG = { "dir": "config/assets/", - "file_name": "wakeup_words", + "file_name": "wakeup_words", # Hello啊，我是台湾女生小智的啦, "create_time": time.time(), "refresh_time": 10, "words": ["你好小智", "你好啊小智", "小智你好", "小智"], diff --git a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/textHandle.py b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/textHandle.py index 43e980e8..d0da2ed1 100644 --- a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/textHandle.py +++ b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/textHandle.py @@ -22,7 +22,7 @@ async def handleTextMessage(conn, message): return if msg_json["type"] == "hello": await handleHelloMessage(conn) - elif msg_json["type"] == "abort": + elif msg_json["type"] == "abort": # 打断 await handleAbortMessage(conn) elif msg_json["type"] == "listen": if "mode" in msg_json: diff --git a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/dialogue.py b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/dialogue.py index d4e66bb6..f0dcc34d 100644 --- a/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/dialogue.py +++ b/XiaoZhi/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/dialogue.py @@ -14,15 +14,16 @@ class Message: class Dialogue: def __init__(self): + # 看这个意思，是维护上下文，就是前几句话说的是什么，那么List[Message]也不能一直装下去吧，是怎么截断的呢？ self.dialogue: List[Message] = [] # 获取当前时间 self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') def put(self, message: Message): - self.dialogue.append(message) + self.dialogue.append(message) # 保存新的内容 def getMessages(self, m, dialogue): - if m.tool_calls is not None: + if m.tool_calls is not None: # 除了记录说了什么内容，还要记录是用哪种角色说的，还要记录tool_calls,tool_call_id等内容 dialogue.append({"role": m.role, "tool_calls": m.tool_calls}) elif m.role == "tool": dialogue.append({"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content}) diff --git a/XiaoZhi/文档/下周小智侧工作重点.md b/XiaoZhi/文档/下周小智侧工作重点.md index 0dc5f923..2e4aa70d 100644 --- a/XiaoZhi/文档/下周小智侧工作重点.md +++ b/XiaoZhi/文档/下周小智侧工作重点.md @@ -1,13 +1,23 @@ -按由易到难的顺序进行整理如下 +**按由易到难的顺序进行整理如下** -**1、小智AI助手与东师理想管理软件融合，无缝升级管理软件系统** -吴缤的实验 +**1.使用dify融合知识库** -**2[、小智AI助手/女友的代码分析 deepseek 豆包 【技术原理，值得一读】](https://blog.csdn.net/zunly/article/details/145564070)** +[AI小智接入扣子知识库](https://www.douyin.com/video/7493494595041381683) -**3、需要花钱** +[图形界面配置dify给你的小智ai上网+查自定义知识库](https://www.bilibili.com/video/BV1wEAreXEBA) + + + +**2.使用n8n开发自定义工作流，比如微信发消息，发通知，申请车等。function call似乎不是最好的解决方案，调研一下n8n。** + +​ 小智AI助手与东师理想管理软件融合，无缝升级管理软件系统 +​ + +**3[、小智AI助手/女友的代码分析 deepseek 豆包 【技术原理，值得一读】](https://blog.csdn.net/zunly/article/details/145564070)** + +**4、需要花钱** (1) 湾湾小何, 就是火山的一个音色,需要购买星火的TTS语音生成模型 https://www.volcengine.com/docs/6561/1257544 @@ -20,20 +30,17 @@ https://www.volcengine.com/docs/6561/1257544 ![](https://dsideal.obs.cn-north-1.myhuaweicloud.com/HuangHai/BlogImages/202504181546253.png) -**4、[小智AI远程控制开关！秒变智能家居，远程开关家电/灯光/设备，手机远程控制所有开关，低成本改造传统家电](https://www.bilibili.com/video/BV1JdoXYvEBt)** - -**5、[小智秒变灯光大师-控制全彩RGB氛围灯](https://www.bilibili.com/video/BV1L5o2YuEdn)** - - -**6、[给小智接入本地知识库，让企业拥有智能客服](https://www.bilibili.com/video/BV1BkfFYnE4o)** +**5、[小智AI远程控制开关！秒变智能家居，远程开关家电/灯光/设备，手机远程控制所有开关，低成本改造传统家电](https://www.bilibili.com/video/BV1JdoXYvEBt)** +**6、[小智秒变灯光大师-控制全彩RGB氛围灯](https://www.bilibili.com/video/BV1L5o2YuEdn)** -**7、[给小智机器人加个移动身体](https://www.bilibili.com/video/BV1xrZ1YREF2)** +**7、[给小智接入本地知识库，让企业拥有智能客服](https://www.bilibili.com/video/BV1BkfFYnE4o)** +**8、[给小智机器人加个移动身体](https://www.bilibili.com/video/BV1xrZ1YREF2)** -**8、[[教程/开源]小智AI接入图像识别+姿态检测+触摸屏教程(基于开源服务器插件)](https://www.bilibili.com/video/BV1RnZwYJEmi)** +**9、[[教程/开源]小智AI接入图像识别+姿态检测+触摸屏教程(基于开源服务器插件)](https://www.bilibili.com/video/BV1RnZwYJEmi)** -**9、[小智AI 增加外挂摄像头能力以及驱动 MIDI 模块 0329 MicroBlocks 与个人计算](https://www.bilibili.com/video/BV14RZHYZEKE)** +**10、[小智AI 增加外挂摄像头能力以及驱动 MIDI 模块 0329 MicroBlocks 与个人计算](https://www.bilibili.com/video/BV14RZHYZEKE)** diff --git a/XiaoZhi/黄海增加的内容/mysql_util.py b/XiaoZhi/黄海增加的内容/mysql_util.py new file mode 100644 index 00000000..c2bb37a4 --- /dev/null +++ b/XiaoZhi/黄海增加的内容/mysql_util.py @@ -0,0 +1,129 @@ +import logging +from aiomysql import create_pool + +# 配置日志 +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger(__name__) + +# 全局 MySQL 连接池 +mysql_pool = None + +# MySQL 配置 +MYSQL_CONFIG = { + "host": "10.10.14.203", + "port": 3306, + "user": "root", + "password": "Password123@mysql", + "db": "xiaozhi_esp32_server", + "minsize": 1, + "maxsize": 20, + "autocommit": True, + "charset": "utf8mb4", +} + + +async def init_mysql_pool(): + """初始化 MySQL 连接池""" + global mysql_pool + try: + mysql_pool = await create_pool(**MYSQL_CONFIG) + logger.info("MySQL连接池创建成功") + except Exception as e: + logger.error(f"创建MySQL连接池失败: {str(e)}") + raise + + +async def close_mysql_pool(): + """关闭 MySQL 连接池""" + global mysql_pool + if mysql_pool: + mysql_pool.close() + await mysql_pool.wait_closed() + logger.info("MySQL连接池已关闭") + + +async def save_chat_history(chat_text, chat_wav, ai_text, person_id, person_name): + """保存聊天记录""" + try: + async with mysql_pool.acquire() as conn: + await conn.ping() + async with conn.cursor() as cur: + sql = """ + INSERT INTO t_chat_history + (chat_text, chat_wav, ai_text, person_id, person_name) + VALUES (%s, %s, %s, %s, %s) + """ + await cur.execute(sql, (chat_text, chat_wav, ai_text, person_id, person_name)) + logger.info(f"成功保存聊天记录: {person_name}") + return cur.lastrowid + except Exception as e: + logger.error(f"保存聊天记录失败: {str(e)}") + raise + + +async def update_chat_history(chat_id, chat_text=None, chat_wav=None, ai_text=None, person_name=None): + """更新聊天记录""" + try: + async with mysql_pool.acquire() as conn: + await conn.ping() + async with conn.cursor() as cur: + updates = [] + params = [] + if chat_text is not None: + updates.append("chat_text = %s") + params.append(chat_text) + if chat_wav is not None: + updates.append("chat_wav = %s") + params.append(chat_wav) + if ai_text is not None: + updates.append("ai_text = %s") + params.append(ai_text) + if person_name is not None: + updates.append("person_name = %s") + params.append(person_name) + + if not updates: + logger.warning("没有提供要更新的字段") + return + + params.append(chat_id) + sql = f"UPDATE t_chat_history SET {', '.join(updates)} WHERE id = %s" + await cur.execute(sql, params) + logger.info(f"成功更新聊天记录 ID: {chat_id}") + except Exception as e: + logger.error(f"更新聊天记录失败: {str(e)}") + raise + + +async def delete_chat_history(chat_id): + """删除聊天记录""" + try: + async with mysql_pool.acquire() as conn: + await conn.ping() + async with conn.cursor() as cur: + sql = "DELETE FROM t_chat_history WHERE id = %s" + await cur.execute(sql, (chat_id,)) + logger.info(f"成功删除聊天记录 ID: {chat_id}") + except Exception as e: + logger.error(f"删除聊天记录失败: {str(e)}") + raise + + +async def get_chat_history(person_id=None, limit=10): + """获取聊天记录""" + try: + async with mysql_pool.acquire() as conn: + await conn.ping() + async with conn.cursor() as cur: + if person_id: + sql = "SELECT * FROM t_chat_history WHERE person_id = %s ORDER BY id DESC LIMIT %s" + await cur.execute(sql, (person_id, limit)) + else: + sql = "SELECT * FROM t_chat_history ORDER BY id DESC LIMIT %s" + await cur.execute(sql, (limit,)) + + result = await cur.fetchall() + return result + except Exception as e: + logger.error(f"获取聊天记录失败: {str(e)}") + raise \ No newline at end of file diff --git a/XiaoZhi/黄海增加的内容/文档.txt b/XiaoZhi/黄海增加的内容/文档.txt new file mode 100644 index 00000000..48a440ef --- /dev/null +++ b/XiaoZhi/黄海增加的内容/文档.txt @@ -0,0 +1,36 @@ +1、app.py +from core.utils.mysql_util import init_mysql_pool, close_mysql_pool + +# 加载配置 + config = load_config() + +# 【黄海】初始化 MySQL 连接池 +await init_mysql_pool() + +... + + finally: + ws_task.cancel() + try: + await ws_task + except asyncio.CancelledError: + pass + # 关闭 MySQL 连接池 + await close_mysql_pool() + print("服务器已关闭，程序退出。") + +2、D:\dsWork\QingLong\XiaoZhi\xiaozhi-esp32-server\main\xiaozhi-server\core\providers\asr\fun_local.py + + text = rich_transcription_postprocess(result[0]["text"]) + logger.bind(tag=TAG).info(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}") + + #【黄海注释】准备在这里进行嵌入代码，把文字+声音的关系记录下来，让用户选择是哪个人进行声纹匹配 + # 保存聊天记录 + await save_chat_history( + chat_text=text, + chat_wav=file_path, + ai_text='',# 暂时没有记录AI的影响 + person_id="FDE43080-122E-4F5D-B860-4E53DEC54BFB", + person_name='黄海' + ) + return text, file_path