From 67feea9443eb3cf2f8cfa1af556a32f205146039 Mon Sep 17 00:00:00 2001 From: HuangHai <10402852@qq.com> Date: Sun, 31 Aug 2025 12:41:05 +0800 Subject: [PATCH] 'commit' --- .../Config/__pycache__/Config.cpython-310.pyc | Bin 2310 -> 2310 bytes dsLightRag/Routes/XueBanRoute.py | 33 ++++-- .../__pycache__/XueBanRoute.cpython-310.pyc | Bin 8400 -> 8802 bytes dsLightRag/Util/TTS_Pipeline.py | 45 ++++++-- .../__pycache__/TTS_Pipeline.cpython-310.pyc | Bin 6903 -> 7345 bytes dsLightRag/static/YunXiao/xueban.js | 97 ++++++++++++++++-- 6 files changed, 150 insertions(+), 25 deletions(-) diff --git a/dsLightRag/Config/__pycache__/Config.cpython-310.pyc b/dsLightRag/Config/__pycache__/Config.cpython-310.pyc index f982f5691dc6883a9a68d79089ae7476efd39d9d..2c7d682533b14a379d7a334d66470c0f65f370c5 100644 GIT binary patch delta 134 zcmZn@Y7^qk=jG*M00N0~n=`aG@&+<8Ms1E~e9y>zi=!aFxFo$OwfNTNNaieN$6LGs z{=p&cL9W5^9zX^XH^4s#nHw7H8gxq#t~kIkI5^xt$mP~#7Is^p=}zptjBFq~i?}9- Ha%=@_4Z*7iee9y=o#Zi!7T#{atS{$`Gk~xdnF^V_9 zKRCoa$Tc|L1IR$)2KWadb3=n&gQ5iCiUS;jgTwuUT%snku-gJncVg#dWMc(dS;RUy Hlw%_R1KlON diff --git a/dsLightRag/Routes/XueBanRoute.py b/dsLightRag/Routes/XueBanRoute.py index 347631a9..5604e852 100644 --- a/dsLightRag/Routes/XueBanRoute.py +++ b/dsLightRag/Routes/XueBanRoute.py @@ -187,16 +187,37 @@ async def streaming_chat(websocket: WebSocket): logger.error(f"发送音频块失败: {str(e)}") raise - # 获取LLM流式输出并断句 - logger.info("开始LLM处理和TTS合成...") + # 获取学伴响应内容（包含题目信息） + logger.info("获取学伴响应内容...") + llm_chunks = [] + async for chunk in get_xueban_response_async(asr_result['text'], stream=True): + llm_chunks.append(chunk) + full_llm_response = ''.join(llm_chunks) + logger.info(f"学伴响应内容: {full_llm_response}") + + # 定义音频回调函数，将音频块发送给前端 + async def audio_callback(audio_chunk): + logger.info(f"发送音频块，大小: {len(audio_chunk)}") + try: + await websocket.send_bytes(audio_chunk) + logger.info("音频块发送成功") + except Exception as e: + logger.error(f"发送音频块失败: {str(e)}") + raise + + # 实时获取LLM流式输出并处理 + logger.info("开始LLM流式处理和TTS合成...") try: - text_stream = stream_and_split_text(asr_result['text']) + # 直接将LLM流式响应接入TTS + llm_stream = get_xueban_response_async(asr_result['text'], stream=True) + text_stream = stream_and_split_text(llm_stream) # 异步函数调用 # 初始化TTS处理器 - tts = StreamingVolcanoTTS(max_concurrency=2) + tts = StreamingVolcanoTTS(max_concurrency=1) - # 流式处理文本并生成音频 - await tts.synthesize_stream(text_stream, audio_callback) + # 异步迭代文本流 + async for text_chunk in text_stream: + await tts._synthesize_single_with_semaphore(text_chunk, audio_callback) logger.info("TTS合成完成") except Exception as e: logger.error(f"TTS合成失败: {str(e)}") diff --git a/dsLightRag/Routes/__pycache__/XueBanRoute.cpython-310.pyc b/dsLightRag/Routes/__pycache__/XueBanRoute.cpython-310.pyc index 0c80cbb8d0f32ce99a8b7108a723fb1e1bb225c6..e960f473c097c4e9f772e3f92eae6ee010accee9 100644 GIT binary patch delta 1374 zcmZXTe@t6d6vyv*g|&V3cS|WNrG+t|j=}gfHAd48<_0T6HiNC@hnk*HKhQxo2#+Xd~M`QmnG1i#)=Mv+8^PX2=TkOmG-Q#w~K`T7C$SczaG5f7_>sNG;DnXV2u4B_B-7?h6$_uRTH($T^EMY zQg5Bjw45RZQ48r7InYStR(ao=&_|YR{X{0m$s{S%>0+^2uUiKo*Wu;chl$))bnD_e z6l@w41$n=2yG7m=Fn)#1fZQ%eUbPggMyvz9OI?pvFW9xPZeN9hLyO3raw@t81*h)d zOozv zT6yC%5PS@2FG{B#7JjfT>@|da?1J-2=x2AF2VsEy=KKndly1ASU^&i@Br4Ex=Jia% zJvQf=g$?#Iw>Ma~H#8XIVR{5H#IYgJAleDEfu2M=glI;DIX3xwkF9v)aZLQ{t zs^ny8R?X+irj(-57g?x&WI4t|#=hy(Xg!0Odz6bARa2=AJ$n&Ogo`8DBB_fxEv=}T zB}Gy*xl~q>E@ZT{q$=}g&!_WLp%$JObyp|XiNfa)!-!`Q(}-h;1V2tf@pz2ECdF)a zUYbrXTI~lE`apnmOG{mq$ z4`TOzL~Ch^y%cDL1X~F-z(1v%fo}kwXVTvHVTjcP#~N~&pWhY5xD~b0d6o`_P95cc z%3}W9tTL@hbIKxZN5>w%yM?SqA1LS0ZB;1EQ)ymJ8Mm>zu?Ke|&U1XgehKzLyd*YV z5#StqyE%F+#{Cbi*GykQhj9`*x5}=aSxVFSoTlV7W7cK6G3%WeO|VN5FRKr`;VIS; zJ`udkrO~#sIh#+V__5OgEcyh3vbFH{ZDVK}U&{8 delta 1034 zcmZ8fOH30{6rDRD{X&003#HK3QYZ=p3sDyu`G1QDjn$ag05cX!Y4yzzfwX9XQQ{|Q ze3}@I2^fqqaiM99E==6H(1p6~hGq15&!AZnY+GvwfXb=0yY7-MtaZ(P?hzrDvL=h3CxtL6LDkDg1 zL_!iIJS#==^D49LTwVv$Y;iQ=*asb%yN)Y7oThH4ulK(nKt;|oAyM`@kG4j3+Sx>0 z62p^;xGctHGJtgPnYmaj{5wYi)a16%hfWiC=?mweeTiv&o~$aCh(y>nNh_KjK!{U^ o>z!v9L*@6Hz(~39Qy3|?j=hp4s&{*#f%@IGYIDL|t*$GDUzyYN_5c6? diff --git a/dsLightRag/Util/TTS_Pipeline.py b/dsLightRag/Util/TTS_Pipeline.py index e8986bff..bf451fbc 100644 --- a/dsLightRag/Util/TTS_Pipeline.py +++ b/dsLightRag/Util/TTS_Pipeline.py @@ -51,7 +51,37 @@ def stream_and_split_text(query_text): if i+1 < len(sentences): sentence = sentences[i] + sentences[i+1] yield sentence - + + # 保留不完整的部分 + buffer = sentences[-1] + + # 处理最后剩余的部分 + if buffer: + yield buffer + + +# 修改为 +async def stream_and_split_text(llm_stream): + """ + 流式获取LLM输出并按句子分割 + @param llm_stream: LLM流式响应生成器 + @return: 异步生成器，每次产生一个完整句子 + """ + buffer = "" + + # 直接处理LLM流式输出 + async for content in llm_stream: + buffer += content + + # 使用正则表达式检测句子结束 + sentences = re.split(r'([。！？.!?])', buffer) + if len(sentences) > 1: + # 提取完整句子 + for i in range(0, len(sentences)-1, 2): + if i+1 < len(sentences): + sentence = sentences[i] + sentences[i+1] + yield sentence + # 保留不完整的部分 buffer = sentences[-1] @@ -85,15 +115,10 @@ class StreamingVolcanoTTS: text_stream: 文本流生成器 audio_callback: 音频数据回调函数，接收音频片段 """ - # 为每个文本片段创建一个WebSocket连接，但限制并发数 - tasks = [] - for text in text_stream: - if text.strip(): # 忽略空文本 - task = asyncio.create_task(self._synthesize_single_with_semaphore(text, audio_callback)) - tasks.append(task) - - # 等待所有任务完成 - await asyncio.gather(*tasks) + # 实时处理每个文本片段（删除任务列表和gather） + async for text in text_stream: + if text.strip(): + await self._synthesize_single_with_semaphore(text, audio_callback) async def _synthesize_single_with_semaphore(self, text, audio_callback): """使用信号量控制并发数的单个文本合成""" diff --git a/dsLightRag/Util/__pycache__/TTS_Pipeline.cpython-310.pyc b/dsLightRag/Util/__pycache__/TTS_Pipeline.cpython-310.pyc index 254a9d7c95de3b64ee53f8fbd6809e3b17add0eb..63e7b15b6e5c20053d7cea4e470497540e79abae 100644 GIT binary patch delta 1665 zcmZuxTWl0n7(Qoab~~NjZD+UJE{haenravDRulwlQ-cB}*pQfLsA-2~&(IyaH_prk z#o2VRsGvZL2aU!Ds0oNhG+3=j8cp=YC!b6toA}18#1~DBG4VnC|5+`?ILZ0uKmYwU z^T&gix3m-^5sko;TbR$T>N(xAihg~1{h>WfWlcxPsJ?@+W~Ln_Oq-TQqb$V2z>Tpu zi?EiXbhM4NG95T^7G*Jbx0@ZTZBpWf**;AuVf0D=q}nh4JRYFGi{an}+AV$!zBJVX zF*6h+|MLXl-2`WwM2P(s5Jx0NE7T?ZPnkQg*xOMWwJ}>q>AiF zt~^bdGO6%K9hn9Gy{{lFI4wU$#u639RhW_>4ap?oQv~1z8{L zb!FC+gdhh4n}|#KE|(&oNl*=(lv+Lu`DtD7)IU4oEzaCH^|?3m=I-53-1t&>uUz%M zSg4;k?#-O{F3oz!UiOZE;_D8T(mb8-%H{HgJ>_sSo!{C8HVqe^IO$zI^V6BL^<%T% z+ZP)K+;qyku(iuue6fD%{F3mO#S`_9XX+o$UBCJsFt1;!Ucd5z_tA;^8=p3U`qn%Q zNsjOqK;DXg-R3$%luQwlQg|FM?FbzR34m0vruiBA3G!9Q??hOQum+(Ep?e9|B8B4| zf@9-NswzQUR%uuc(-o3V<1|R6swzuW={H5!sB$wS4mZ6^_4!WqaoX1hCLEWL--B?w z=bCC5R>5)%V_dx2yx9A||ERQ1n2y0syIAHK)38`pOlm_z3TWa&eFMivvm?}{uuN^R zOw?Z{*_F?cIVj$RWr9tR=A>a?20I{c5;wK4RH+uQ9d4DxwNQ@&9qSjZ;qIv7n9n)O z6BQ%jj`jiu!lp}Ay;-F(s5$tTsDytQydRjgm|;I(aI&Ut9X1WyDoo_el+61<#0L-_ zM!3Crz6E$M%)cHvCD8%Vtq7X)K)Y+N;! zXjIbTR1K(+aDeyDUx+o)`*1xt>T&c3eM}(k8O9fEr7&U0bqBU zgK~$=_sxgmd!*n_G#(b|j^trzXj69C04c7v_%d}N{Bo2uO(*bHV3tyMc4osMMqKzj(+%V1ix1QkKp%rmw0I9p6#~=w-0svQv87Ei$8;KEAv~S)q#AG gm2>7G1r57Te777%Q6vub=E9394O>o+%xJ|#@6p`m;oJ3DSd z_m5{b7e3G!Wr`6yG zOngOh!>QzTa`8O)1@1*$MpQ;P4OT>5*AIN@y7TH`>W6bzN8UIt4#_pSbOjH)ebL~q z&lc5sdj5_Lkz>%FK?@fEpOdDxZ4Ai!+N!f{4$M7mUlWs(?CAq-&ls2keP7=v+t$F? zGY8f_A=+I_l6B}m(msT=7Br%MOFunfM^-3>-xIHZXq0LQ?n&-SFWgWEnQ14oy#3Fj* z73R^xZ3q-Nw!aE?W94bE-i8)t0aSw&lMbO4Ev79oJ*+!pq<#j2F;2__s-`HT9cTO& zSdk%RcuX5*qlv>J`a#nLuiNu`yzK}4Yw(2>pzqLxZA=)VF?=?@tiSmYjENgK(`fOn zNbkaF_23XAbG_>aqJSQ2Q}~icR!{hW6yq=id-+r?mCF|^gkGo!<`Ca#4i6`I<@y5{P zDUvnPx|tH2!}lh?B&(Y^v4dy;s`^UiaB9m7SeuK(s#VY*BVHmSc_>HK`4OD|ymGc> z8dT51YnO*b#1`D#`da+f8=@bGt&vQubtGtZJU{pqb#y@96!+nQHyOuWw@PkSUy5&! zxCIUo*P?`0d;m-8_mj)rqw~f@M4VT=AZm#^qViwFxzKB!F6*~>jes-6PYc!X-W>kZ ZB(0}OnNTBph1QcrdWt%9EJ@72UjP#?A;JIv diff --git a/dsLightRag/static/YunXiao/xueban.js b/dsLightRag/static/YunXiao/xueban.js index dd1ebd0a..841393b6 100644 --- a/dsLightRag/static/YunXiao/xueban.js +++ b/dsLightRag/static/YunXiao/xueban.js @@ -14,7 +14,10 @@ const AudioState = { playback: { audioElement: null, isPlaying: false, - audioChunks: [] // 存储接收到的音频块 + audioChunks: [], // 存储接收到的音频块 + audioQueue: [], // 音频队列，用于流式播放 + isStreamPlaying: false, // 是否正在流式播放 + currentAudioIndex: 0 // 当前播放的音频索引 }, websocket: { connection: null, @@ -178,12 +181,13 @@ const WebSocketManager = { UIController.toggleElement('thinkingIndicator', false); UIController.setStartRecordButtonEnabled(true); - // 合并所有音频块并播放 - if (AudioState.playback.audioChunks.length > 0) { - console.log('开始合并和播放音频'); - this.combineAndPlayAudio(); - } else { - console.warn('没有收到音频数据，无法播放'); + // 标记流式播放结束 + AudioState.playback.isStreamPlaying = false; + + // 如果有音频数据但尚未开始播放，则开始播放 + if (AudioState.playback.audioQueue.length > 0 && !AudioState.playback.isPlaying) { + console.log('开始播放队列中的音频'); + AudioPlayer.processAudioQueue(); } break; @@ -192,6 +196,9 @@ const WebSocketManager = { console.error('收到错误消息:', data.message); UIController.toggleElement('thinkingIndicator', false); UIController.setStartRecordButtonEnabled(true); + // 重置流式播放状态 + AudioState.playback.isStreamPlaying = false; + AudioState.playback.audioQueue = []; alert('处理失败: ' + data.message); break; @@ -206,8 +213,20 @@ const WebSocketManager = { // 二进制音频数据 console.log('收到音频数据，大小:', event.data.size); console.log('音频数据类型:', event.data.type); + + // 保存到原始音频块数组（保持原有逻辑） AudioState.playback.audioChunks.push(event.data); - console.log('当前音频块数量:', AudioState.playback.audioChunks.length); + + // 添加到音频队列（用于流式播放） + AudioState.playback.audioQueue.push(event.data); + console.log('当前音频队列长度:', AudioState.playback.audioQueue.length); + + // 如果尚未开始流式播放，则开始播放 + if (!AudioState.playback.isStreamPlaying && !AudioState.playback.isPlaying) { + console.log('开始流式播放音频'); + AudioState.playback.isStreamPlaying = true; + AudioPlayer.processAudioQueue(); + } } }, @@ -355,6 +374,64 @@ const AudioPlayer = { const currentTime = AudioState.playback.audioElement.currentTime; const duration = AudioState.playback.audioElement.duration; UIController.updateTimeDisplay(currentTime, duration); + }, + + // 初始化流式播放器 + initStreamPlayer() { + // 创建新的音频元素用于流式播放 + if (!AudioState.playback.streamAudioElement) { + AudioState.playback.streamAudioElement = new Audio(); + + // 监听音频结束事件 + AudioState.playback.streamAudioElement.addEventListener('ended', () => { + // 当前音频播放完毕，处理队列中的下一个音频 + this.processAudioQueue(); + }); + + // 监听错误事件 + AudioState.playback.streamAudioElement.addEventListener('error', (e) => { + console.error('流式播放音频错误:', e); + // 即使出错，也继续处理队列中的下一个音频 + this.processAudioQueue(); + }); + } + }, + + // 处理音频队列 + processAudioQueue() { + // 如果正在播放或队列为空，则返回 + if (AudioState.playback.isStreamPlaying || AudioState.playback.audioQueue.length === 0) { + AudioState.playback.isStreamPlaying = false; + return; + } + + // 设置播放状态 + AudioState.playback.isStreamPlaying = true; + + // 从队列中取出第一个音频块 + const audioBlob = AudioState.playback.audioQueue.shift(); + + // 创建音频URL + const audioUrl = URL.createObjectURL(audioBlob); + + // 设置音频源并播放 + AudioState.playback.streamAudioElement.src = audioUrl; + AudioState.playback.streamAudioElement.play() + .then(() => { + console.log('开始播放音频块'); + }) + .catch(error => { + console.error('播放音频块失败:', error); + // 播放失败，继续处理下一个 + AudioState.playback.isStreamPlaying = false; + this.processAudioQueue(); + }) + .finally(() => { + // 播放完成后释放URL对象 + setTimeout(() => { + URL.revokeObjectURL(audioUrl); + }, 1000); + }); } }; @@ -589,10 +666,12 @@ const RecordingManager = { }; // ==================== 初始化 ==================== -// 页面加载完成后初始化 function initializeApp() { console.log('开始初始化学伴录音功能...'); + // 初始化流式播放器 + AudioPlayer.initStreamPlayer(); + // 检查DOM是否已就绪 if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', () => {