main
HuangHai 3 months ago
parent c60f3e2cd8
commit 8023fbe6f4

@ -111,6 +111,9 @@ def _make_api_request(api_url, secret, endpoint, json_data=None):
def get_config_from_api(config):
"""从Java API获取配置"""
# 如果配置中指定了manager-api则从Java API获取配置
# D:\dsWork\QingLong\XiaoZhi\xiaozhi-esp32-server\main\xiaozhi-server\config_from_api.yaml
# 据说需要从这个config_from_api.yaml拷贝过去才行
api_url = config["manager-api"].get("url", "")
secret = config["manager-api"].get("secret", "")
@ -129,7 +132,7 @@ def get_private_config_from_api(config, device_id, client_id):
"""从Java API获取私有配置"""
api_url = config["manager-api"].get("url", "")
secret = config["manager-api"].get("secret", "")
# 调用JAVA接口获取私有配置
return _make_api_request(
api_url,
secret,

@ -29,12 +29,14 @@ from core.auth import AuthMiddleware, AuthenticationError
from core.mcp.manager import MCPManager
from config.config_loader import get_private_config_from_api
# 在日志记录中TAG 通常用于标识日志消息的来源模块
TAG = __name__
auto_import_modules("plugins_func.functions")
auto_import_modules("plugins_func.functions") # 导入plugins_func.functions包下所有函数
class TTSException(RuntimeError):
# 如果发现TTS异常那么也不做
pass
@ -198,6 +200,7 @@ class ConnectionHandler:
if not read_config_from_api:
return
"""从接口获取差异化的配置进行二次实例化,非全量重新实例化"""
# 配置是两掺的API中有的走APIAPI中没有的走配置文件
try:
private_config = get_private_config_from_api(
self.config,
@ -354,6 +357,10 @@ class ConnectionHandler:
m.content = prompt
def chat(self, query):
'''
聊天的主函数
query:用户说的是什么话
'''
self.dialogue.put(Message(role="user", content=query))
response_message = []
processed_chars = 0 # 跟踪已处理的字符位置
@ -361,12 +368,16 @@ class ConnectionHandler:
start_time = time.time()
# 使用带记忆的对话
future = asyncio.run_coroutine_threadsafe(
# query_memory似乎是利用memory的内容进行RAG匹配找到以前讲过的相关性高的内容
# 这个query_memory函数是一个接口有好几个实现比如memory_llm_local_short.py
self.memory.query_memory(query), self.loop
)
memory_str = future.result()
memory_str = future.result() # 从记忆中获取相关性高的内容
self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}")
llm_responses = self.llm.response(
# 发送给大模型,参数:
# session_id:会话ID用于标识会话这里使用一个随机生成的UUID
self.session_id, self.dialogue.get_llm_dialogue_with_memory(memory_str)
)
except Exception as e:
@ -375,16 +386,16 @@ class ConnectionHandler:
self.llm_finish_task = False
text_index = 0
for content in llm_responses:
response_message.append(content)
if self.client_abort:
for content in llm_responses: # 遍历大模型的返回内容
response_message.append(content) # 拼接返回内容
if self.client_abort: # 如果客户端要求中断对话,则中断对话
break
end_time = time.time()
end_time = time.time() # 记录结束时间
self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
# 合并当前全部文本并处理未分割部分
full_text = "".join(response_message)
full_text = "".join(response_message) # 拼接全部文本
current_text = full_text[processed_chars:] # 从未处理的位置开始
# 查找最后一个有效标点
@ -397,21 +408,24 @@ class ConnectionHandler:
# 找到分割点则处理
if last_punct_pos != -1:
segment_text_raw = current_text[: last_punct_pos + 1]
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
if segment_text:
segment_text_raw = current_text[: last_punct_pos + 1] # 本段的原始文本内容
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw) # 清理文本内容
if segment_text: # 如果处理完得到的干净文本内容不为空,则处理
# 强制设置空字符测试TTS出错返回语音的健壮性
# if text_index % 2 == 0:
# segment_text = " "
text_index += 1
self.recode_first_last_text(segment_text, text_index)
self.recode_first_last_text(segment_text, text_index)# 分段记录大模型返回的文本
# 应该是一般接收大模型返回的每一句话一边用TTS生成语音好牛B啊~
future = self.executor.submit(
self.speak_and_play, segment_text, text_index
)
# 这里居然有一个 TTS队列看来是为了解决生成文本太长最终语音生成时间过长的问题是一边大模型生成文本一边进入TTS队列一边生成返回的意思
self.tts_queue.put(future)
processed_chars += len(segment_text_raw) # 更新已处理字符位置
# 处理最后剩余的文本
# 多么经典的代码!就是每句话都是由下一句话时驱动完成转换并输了的,那么,最后一句话的逻辑就与其它话不同,需要单独处理。
full_text = "".join(response_message)
remaining_text = full_text[processed_chars:]
if remaining_text:
@ -423,7 +437,7 @@ class ConnectionHandler:
self.speak_and_play, segment_text, text_index
)
self.tts_queue.put(future)
# 标识大模型生成任务结束
self.llm_finish_task = True
self.dialogue.put(Message(role="assistant", content="".join(response_message)))
self.logger.bind(tag=TAG).debug(
@ -439,8 +453,9 @@ class ConnectionHandler:
self.dialogue.put(Message(role="user", content=query))
# Define intent functions
# 定义意图感知函数
functions = None
if hasattr(self, "func_handler"):
if hasattr(self, "func_handler"): # 调用对象实体是不是有属性func_handler?
functions = self.func_handler.get_functions()
response_message = []
processed_chars = 0 # 跟踪已处理的字符位置
@ -460,7 +475,7 @@ class ConnectionHandler:
llm_responses = self.llm.response_with_functions(
self.session_id,
self.dialogue.get_llm_dialogue_with_memory(memory_str),
functions=functions,
functions=functions, # 这里调用的是支持function_call的LLM
)
except Exception as e:
self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
@ -505,7 +520,7 @@ class ConnectionHandler:
break
end_time = time.time()
# self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
# 处理文本分段和TTS逻辑
# 合并当前全部文本并处理未分割部分

@ -1,11 +1,10 @@
import json
import queue
from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
# 打断处理逻辑
async def handleAbortMessage(conn):
logger.bind(tag=TAG).info("Abort message received")
# 设置成打断状态会自动打断llm、tts任务

@ -13,7 +13,7 @@ logger = setup_logging()
WAKEUP_CONFIG = {
"dir": "config/assets/",
"file_name": "wakeup_words",
"file_name": "wakeup_words", # Hello啊我是台湾女生小智的啦,
"create_time": time.time(),
"refresh_time": 10,
"words": ["你好小智", "你好啊小智", "小智你好", "小智"],

@ -22,7 +22,7 @@ async def handleTextMessage(conn, message):
return
if msg_json["type"] == "hello":
await handleHelloMessage(conn)
elif msg_json["type"] == "abort":
elif msg_json["type"] == "abort": # 打断
await handleAbortMessage(conn)
elif msg_json["type"] == "listen":
if "mode" in msg_json:

@ -14,15 +14,16 @@ class Message:
class Dialogue:
def __init__(self):
# 看这个意思是维护上下文就是前几句话说的是什么那么List[Message]也不能一直装下去吧,是怎么截断的呢?
self.dialogue: List[Message] = []
# 获取当前时间
self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
def put(self, message: Message):
self.dialogue.append(message)
self.dialogue.append(message) # 保存新的内容
def getMessages(self, m, dialogue):
if m.tool_calls is not None:
if m.tool_calls is not None: # 除了记录说了什么内容还要记录是用哪种角色说的还要记录tool_calls,tool_call_id等内容
dialogue.append({"role": m.role, "tool_calls": m.tool_calls})
elif m.role == "tool":
dialogue.append({"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content})

@ -1,13 +1,23 @@
按由易到难的顺序进行整理如下
**按由易到难的顺序进行整理如下**
**1、小智AI助手与东师理想管理软件融合无缝升级管理软件系统**
吴缤的实验
**1.使用dify融合知识库**
**2[、小智AI助手/女友的代码分析 deepseek 豆包 【技术原理,值得一读】](https://blog.csdn.net/zunly/article/details/145564070)**
[AI小智接入扣子知识库](https://www.douyin.com/video/7493494595041381683)
**3、需要花钱**
[图形界面配置dify给你的小智ai上网+查自定义知识库](https://www.bilibili.com/video/BV1wEAreXEBA)
**2.使用n8n开发自定义工作流比如微信发消息发通知申请车等。function call似乎不是最好的解决方案调研一下n8n。**
小智AI助手与东师理想管理软件融合无缝升级管理软件系统
**3[、小智AI助手/女友的代码分析 deepseek 豆包 【技术原理,值得一读】](https://blog.csdn.net/zunly/article/details/145564070)**
**4、需要花钱**
(1) 湾湾小何, 就是火山的一个音色,需要购买星火的TTS语音生成模型
https://www.volcengine.com/docs/6561/1257544
@ -20,20 +30,17 @@ https://www.volcengine.com/docs/6561/1257544
![](https://dsideal.obs.cn-north-1.myhuaweicloud.com/HuangHai/BlogImages/202504181546253.png)
**4、[小智AI远程控制开关秒变智能家居远程开关家电/灯光/设备,手机远程控制所有开关,低成本改造传统家电](https://www.bilibili.com/video/BV1JdoXYvEBt)**
**5、[小智秒变灯光大师-控制全彩RGB氛围灯](https://www.bilibili.com/video/BV1L5o2YuEdn)**
**6、[给小智接入本地知识库,让企业拥有智能客服](https://www.bilibili.com/video/BV1BkfFYnE4o)**
**5、[小智AI远程控制开关秒变智能家居远程开关家电/灯光/设备,手机远程控制所有开关,低成本改造传统家电](https://www.bilibili.com/video/BV1JdoXYvEBt)**
**6、[小智秒变灯光大师-控制全彩RGB氛围灯](https://www.bilibili.com/video/BV1L5o2YuEdn)**
**7、[给小智机器人加个移动身体](https://www.bilibili.com/video/BV1xrZ1YREF2)**
**7、[给小智接入本地知识库,让企业拥有智能客服](https://www.bilibili.com/video/BV1BkfFYnE4o)**
**8、[给小智机器人加个移动身体](https://www.bilibili.com/video/BV1xrZ1YREF2)**
**8、[[教程/开源]小智AI接入图像识别+姿态检测+触摸屏教程(基于开源服务器插件)](https://www.bilibili.com/video/BV1RnZwYJEmi)**
**9、[[教程/开源]小智AI接入图像识别+姿态检测+触摸屏教程(基于开源服务器插件)](https://www.bilibili.com/video/BV1RnZwYJEmi)**
**9、[小智AI 增加外挂摄像头能力以及驱动 MIDI 模块 0329 MicroBlocks 与个人计算](https://www.bilibili.com/video/BV14RZHYZEKE)**
**10、[小智AI 增加外挂摄像头能力以及驱动 MIDI 模块 0329 MicroBlocks 与个人计算](https://www.bilibili.com/video/BV14RZHYZEKE)**

@ -0,0 +1,129 @@
import logging
from aiomysql import create_pool
# 配置日志
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
# 全局 MySQL 连接池
mysql_pool = None
# MySQL 配置
MYSQL_CONFIG = {
"host": "10.10.14.203",
"port": 3306,
"user": "root",
"password": "Password123@mysql",
"db": "xiaozhi_esp32_server",
"minsize": 1,
"maxsize": 20,
"autocommit": True,
"charset": "utf8mb4",
}
async def init_mysql_pool():
"""初始化 MySQL 连接池"""
global mysql_pool
try:
mysql_pool = await create_pool(**MYSQL_CONFIG)
logger.info("MySQL连接池创建成功")
except Exception as e:
logger.error(f"创建MySQL连接池失败: {str(e)}")
raise
async def close_mysql_pool():
"""关闭 MySQL 连接池"""
global mysql_pool
if mysql_pool:
mysql_pool.close()
await mysql_pool.wait_closed()
logger.info("MySQL连接池已关闭")
async def save_chat_history(chat_text, chat_wav, ai_text, person_id, person_name):
"""保存聊天记录"""
try:
async with mysql_pool.acquire() as conn:
await conn.ping()
async with conn.cursor() as cur:
sql = """
INSERT INTO t_chat_history
(chat_text, chat_wav, ai_text, person_id, person_name)
VALUES (%s, %s, %s, %s, %s)
"""
await cur.execute(sql, (chat_text, chat_wav, ai_text, person_id, person_name))
logger.info(f"成功保存聊天记录: {person_name}")
return cur.lastrowid
except Exception as e:
logger.error(f"保存聊天记录失败: {str(e)}")
raise
async def update_chat_history(chat_id, chat_text=None, chat_wav=None, ai_text=None, person_name=None):
"""更新聊天记录"""
try:
async with mysql_pool.acquire() as conn:
await conn.ping()
async with conn.cursor() as cur:
updates = []
params = []
if chat_text is not None:
updates.append("chat_text = %s")
params.append(chat_text)
if chat_wav is not None:
updates.append("chat_wav = %s")
params.append(chat_wav)
if ai_text is not None:
updates.append("ai_text = %s")
params.append(ai_text)
if person_name is not None:
updates.append("person_name = %s")
params.append(person_name)
if not updates:
logger.warning("没有提供要更新的字段")
return
params.append(chat_id)
sql = f"UPDATE t_chat_history SET {', '.join(updates)} WHERE id = %s"
await cur.execute(sql, params)
logger.info(f"成功更新聊天记录 ID: {chat_id}")
except Exception as e:
logger.error(f"更新聊天记录失败: {str(e)}")
raise
async def delete_chat_history(chat_id):
"""删除聊天记录"""
try:
async with mysql_pool.acquire() as conn:
await conn.ping()
async with conn.cursor() as cur:
sql = "DELETE FROM t_chat_history WHERE id = %s"
await cur.execute(sql, (chat_id,))
logger.info(f"成功删除聊天记录 ID: {chat_id}")
except Exception as e:
logger.error(f"删除聊天记录失败: {str(e)}")
raise
async def get_chat_history(person_id=None, limit=10):
"""获取聊天记录"""
try:
async with mysql_pool.acquire() as conn:
await conn.ping()
async with conn.cursor() as cur:
if person_id:
sql = "SELECT * FROM t_chat_history WHERE person_id = %s ORDER BY id DESC LIMIT %s"
await cur.execute(sql, (person_id, limit))
else:
sql = "SELECT * FROM t_chat_history ORDER BY id DESC LIMIT %s"
await cur.execute(sql, (limit,))
result = await cur.fetchall()
return result
except Exception as e:
logger.error(f"获取聊天记录失败: {str(e)}")
raise

@ -0,0 +1,36 @@
1、app.py
from core.utils.mysql_util import init_mysql_pool, close_mysql_pool
# 加载配置
config = load_config()
# 【黄海】初始化 MySQL 连接池
await init_mysql_pool()
...
finally:
ws_task.cancel()
try:
await ws_task
except asyncio.CancelledError:
pass
# 关闭 MySQL 连接池
await close_mysql_pool()
print("服务器已关闭,程序退出。")
2、D:\dsWork\QingLong\XiaoZhi\xiaozhi-esp32-server\main\xiaozhi-server\core\providers\asr\fun_local.py
text = rich_transcription_postprocess(result[0]["text"])
logger.bind(tag=TAG).info(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
#【黄海注释】准备在这里进行嵌入代码,把文字+声音的关系记录下来,让用户选择是哪个人进行声纹匹配
# 保存聊天记录
await save_chat_history(
chat_text=text,
chat_wav=file_path,
ai_text='',# 暂时没有记录AI的影响
person_id="FDE43080-122E-4F5D-B860-4E53DEC54BFB",
person_name='黄海'
)
return text, file_path
Loading…
Cancel
Save