resolve merge conflict

main/xiaozhi-server/core/providers/asr/aliyun_stream.py

@@ -8,8 +8,6 @@ import asyncio
 import requests
 import websockets
 import opuslib_next
-import random
-from typing import Optional, Tuple, List
 from urllib import parse
 from datetime import datetime
 from config.logger import setup_logging
@@ -96,6 +94,8 @@ class ASRProvider(ASRProviderBase):
         self.delete_audio_file = delete_audio_file
         self.expire_time = None
 
+        self.task_id = uuid.uuid4().hex
+
         # Token管理
         if self.access_key_id and self.access_key_secret:
             self._refresh_token()
@@ -137,13 +137,13 @@ class ASRProvider(ASRProviderBase):
         conn.asr_audio.append(audio)
         conn.asr_audio = conn.asr_audio[-10:]
 
-        # 只在有声音且没有连接时建立连接
-        if audio_have_voice and not self.is_processing:
+        # 只在有声音且没有连接时建立连接（排除正在停止的情况）
+        if audio_have_voice and not self.is_processing and not self.asr_ws:
             try:
                 await self._start_recognition(conn)
             except Exception as e:
                 logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}")
-                await self._cleanup(conn)
+                await self._cleanup()
                 return
 
         if self.asr_ws and self.is_processing and self.server_ready:
@@ -169,20 +169,22 @@ class ASRProvider(ASRProviderBase):
             ping_timeout=None,
             close_timeout=5,
         )
-        
+
+        self.task_id = uuid.uuid4().hex
+
+        logger.bind(tag=TAG).debug(f"WebSocket连接建立成功, task_id: {self.task_id}")
+
         self.is_processing = True
         self.server_ready = False  # 重置服务器准备状态
         self.forward_task = asyncio.create_task(self._forward_results(conn))
-        
+
         # 发送开始请求
         start_request = {
             "header": {
                 "namespace": "SpeechTranscriber",
                 "name": "StartTranscription",
-                "status": 20000000,
-                "message_id": ''.join(random.choices('0123456789abcdef', k=32)),
-                "task_id": ''.join(random.choices('0123456789abcdef', k=32)),
-                "status_text": "Gateway:SUCCESS:Success.",
+                "message_id": uuid.uuid4().hex,
+                "task_id": self.task_id,
                 "appkey": self.appkey
             },
             "payload": {
@@ -196,23 +198,26 @@ class ASRProvider(ASRProviderBase):
             }
         }
         await self.asr_ws.send(json.dumps(start_request, ensure_ascii=False))
-        logger.bind(tag=TAG).info("已发送开始请求，等待服务器准备...")
+        logger.bind(tag=TAG).debug("已发送开始请求，等待服务器准备...")
 
     async def _forward_results(self, conn):
         """转发识别结果"""
         try:
-            while self.asr_ws and not conn.stop_event.is_set():
+            while not conn.stop_event.is_set():
                 try:
                     response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
                     result = json.loads(response)
-                    
+
                     header = result.get("header", {})
                     payload = result.get("payload", {})
                     message_name = header.get("name", "")
                     status = header.get("status", 0)
-                    
+
                     if status != 20000000:
-                        if status in [40000004, 40010004]:  # 连接超时或客户端断开
+                        if status == 40010004:
+                            logger.bind(tag=TAG).warning(f"请在服务端响应完成后再关闭链接，状态码: {status}")
+                            break
+                        if status in [40000004, 40010003]:  # 连接超时或客户端断开
                             logger.bind(tag=TAG).warning(f"连接问题，状态码: {status}")
                             break
                         elif status in [40270002, 40270003]:  # 音频问题
@@ -221,12 +226,12 @@ class ASRProvider(ASRProviderBase):
                         else:
                             logger.bind(tag=TAG).error(f"识别错误，状态码: {status}, 消息: {header.get('status_text', '')}")
                             continue
-                    
+
                     # 收到TranscriptionStarted表示服务器准备好接收音频数据
                     if message_name == "TranscriptionStarted":
                         self.server_ready = True
-                        logger.bind(tag=TAG).info("服务器已准备，开始发送缓存音频...")
-                        
+                        logger.bind(tag=TAG).debug("服务器已准备，开始发送缓存音频...")
+
                         # 发送缓存音频
                         if conn.asr_audio:
                             for cached_audio in conn.asr_audio[-10:]:
@@ -237,88 +242,89 @@ class ASRProvider(ASRProviderBase):
                                     logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}")
                                     break
                         continue
-                    
-                    if message_name == "TranscriptionResultChanged":
-                        # 中间结果
-                        text = payload.get("result", "")
-                        if text:
-                            self.text = text
                     elif message_name == "SentenceEnd":
-                        # 最终结果
+                        # 句子结束（每个句子都会触发）
                         text = payload.get("result", "")
                         if text:
-                            self.text = text
-                            conn.reset_vad_states()
-                            # 传递缓存的音频数据
-                            audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
-                            await self.handle_voice_stop(conn, audio_data)
-                            # 清空缓存
-                            conn.asr_audio_for_voiceprint = []
-                            break
-                    elif message_name == "TranscriptionCompleted":
-                        # 识别完成
-                        self.is_processing = False
-                        break
-                        
+                            logger.bind(tag=TAG).info(f"识别到文本: {text}")
+
+                            # 手动模式下累积识别结果
+                            if conn.client_listen_mode == "manual":
+                                if self.text:
+                                    self.text += text
+                                else:
+                                    self.text = text
+
+                                # 手动模式下，只有在收到stop信号后才触发处理（仅处理一次）
+                                if conn.client_voice_stop:
+                                    audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                                    if len(audio_data) > 0:
+                                        logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
+                                        await self.handle_voice_stop(conn, audio_data)
+                                        # 清理音频缓存
+                                        conn.asr_audio.clear()
+                                        conn.reset_vad_states()
+                                    break
+                            else:
+                                # 自动模式下直接覆盖
+                                self.text = text
+                                conn.reset_vad_states()
+                                audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                                await self.handle_voice_stop(conn, audio_data)
+                                break
+
                 except asyncio.TimeoutError:
-                    continue
-                except websockets.exceptions.ConnectionClosed:
+                    logger.bind(tag=TAG).error("接收结果超时")
+                    break
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).info("ASR服务连接已关闭")
+                    self.is_processing = False
                     break
                 except Exception as e:
                     logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}")
                     break
-                    
+
         except Exception as e:
             logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}")
         finally:
-            await self._cleanup(conn)
+            # 清理连接的音频缓存
+            await self._cleanup()
+            if conn:
+                if hasattr(conn, 'asr_audio_for_voiceprint'):
+                    conn.asr_audio_for_voiceprint = []
+                if hasattr(conn, 'asr_audio'):
+                    conn.asr_audio = []
 
-    async def _cleanup(self, conn):
-        """清理资源"""
-        logger.bind(tag=TAG).info(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
-        
-        # 清理连接的音频缓存
-        if conn and hasattr(conn, 'asr_audio_for_voiceprint'):
-            conn.asr_audio_for_voiceprint = []
-        
-        # 判断是否需要发送终止请求
-        should_stop = self.is_processing or self.server_ready
-        
-        # 发送停止识别请求
-        if self.asr_ws and should_stop:
+    async def _send_stop_request(self):
+        """发送停止识别请求（不关闭连接）"""
+        if self.asr_ws:
             try:
+                # 先停止音频发送
+                self.is_processing = False
+
                 stop_msg = {
                     "header": {
                         "namespace": "SpeechTranscriber",
                         "name": "StopTranscription",
-                        "status": 20000000,
-                        "message_id": ''.join(random.choices('0123456789abcdef', k=32)),
-                        "status_text": "Client:Stop",
+                        "message_id": uuid.uuid4().hex,
+                        "task_id": self.task_id,
                         "appkey": self.appkey
                     }
                 }
-                logger.bind(tag=TAG).info("正在发送ASR终止请求")
+                logger.bind(tag=TAG).debug("停止识别请求已发送")
                 await self.asr_ws.send(json.dumps(stop_msg, ensure_ascii=False))
-                await asyncio.sleep(0.1)
-                logger.bind(tag=TAG).info("ASR终止请求已发送")
             except Exception as e:
-                logger.bind(tag=TAG).error(f"ASR终止请求发送失败: {e}")
-        
-        # 状态重置（在终止请求发送后）
+                logger.bind(tag=TAG).error(f"发送停止识别请求失败: {e}")
+
+    async def _cleanup(self):
+        """清理资源（关闭连接）"""
+        logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
+
+        # 状态重置
         self.is_processing = False
         self.server_ready = False
-        logger.bind(tag=TAG).info("ASR状态已重置")
+        logger.bind(tag=TAG).debug("ASR状态已重置")
 
-        # 清理任务
-        if self.forward_task and not self.forward_task.done():
-            self.forward_task.cancel()
-            try:
-                await asyncio.wait_for(self.forward_task, timeout=1.0)
-            except Exception as e:
-                logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}")
-            finally:
-                self.forward_task = None
-        
         # 关闭连接
         if self.asr_ws:
             try:
@@ -329,8 +335,11 @@ class ASRProvider(ASRProviderBase):
                 logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
             finally:
                 self.asr_ws = None
-        
-        logger.bind(tag=TAG).info("ASR会话清理完成")
+
+        # 清理任务引用
+        self.forward_task = None
+
+        logger.bind(tag=TAG).debug("ASR会话清理完成")
 
     async def speech_to_text(self, opus_data, session_id, audio_format):
         """获取识别结果"""
@@ -340,4 +349,11 @@ class ASRProvider(ASRProviderBase):
 
     async def close(self):
         """关闭资源"""
-        await self._cleanup()
+        await self._cleanup(None)
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Aliyun decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Aliyun decoder资源时出错: {e}")

main/xiaozhi-server/core/providers/asr/base.py

@@ -9,7 +9,6 @@ import asyncio
 import traceback
 import threading
 import opuslib_next
-import concurrent.futures
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
@@ -118,121 +117,89 @@ class ASRProviderBase(ABC):
 
     # 接收音频
     async def receive_audio(self, conn, audio, audio_have_voice):
-        if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime":
-            have_voice = audio_have_voice
+        if conn.client_listen_mode == "manual":
+            # 手动模式：缓存音频用于ASR识别
+            conn.asr_audio.append(audio)
         else:
-            have_voice = conn.client_have_voice
-        
-        conn.asr_audio.append(audio)
-        if not have_voice and not conn.client_have_voice:
-            conn.asr_audio = conn.asr_audio[-10:]
-            return
+            # 自动/实时模式：使用VAD检测
+            have_voice = audio_have_voice
 
-        if conn.client_voice_stop:
-            asr_audio_task = conn.asr_audio.copy()
-            conn.asr_audio.clear()
-            conn.reset_vad_states()
+            conn.asr_audio.append(audio)
+            if not have_voice and not conn.client_have_voice:
+                conn.asr_audio = conn.asr_audio[-10:]
+                return
 
-            if len(asr_audio_task) > 15:
-                await self.handle_voice_stop(conn, asr_audio_task)
+            # 自动模式下通过VAD检测到语音停止时触发识别
+            if conn.client_voice_stop:
+                asr_audio_task = conn.asr_audio.copy()
+                conn.asr_audio.clear()
+                conn.reset_vad_states()
+
+                if len(asr_audio_task) > 15:
+                    await self.handle_voice_stop(conn, asr_audio_task)
 
     # 处理语音停止
     async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
         """并行处理ASR和声纹识别"""
         try:
             total_start_time = time.monotonic()
-            
+
             # 准备音频数据
             if conn.audio_format == "pcm":
                 pcm_data = asr_audio_task
             else:
                 pcm_data = self.decode_opus(asr_audio_task)
-            
+
             combined_pcm_data = b"".join(pcm_data)
-            
+
             # 预先准备WAV数据
             wav_data = None
             if conn.voiceprint_provider and combined_pcm_data:
                 wav_data = self._pcm_to_wav(combined_pcm_data)
-            
+
             # 定义ASR任务
-            def run_asr():
-                start_time = time.monotonic()
-                try:
-                    loop = asyncio.new_event_loop()
-                    asyncio.set_event_loop(loop)
-                    try:
-                        result = loop.run_until_complete(
-                            self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
-                        )
-                        end_time = time.monotonic()
-                        logger.bind(tag=TAG).info(f"ASR耗时: {end_time - start_time:.3f}s")
-                        return result
-                    finally:
-                        loop.close()
-                except Exception as e:
-                    end_time = time.monotonic()
-                    logger.bind(tag=TAG).error(f"ASR失败: {e}")
-                    return ("", None)
-            
-            # 定义声纹识别任务
-            def run_voiceprint():
-                if not wav_data:
-                    return None
-                try:
-                    loop = asyncio.new_event_loop()
-                    asyncio.set_event_loop(loop)
-                    try:
-                        # 使用连接的声纹识别提供者
-                        result = loop.run_until_complete(
-                            conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
-                        )
-                        return result
-                    finally:
-                        loop.close()
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"声纹识别失败: {e}")
-                    return None
-            
-            # 使用线程池执行器并行运行
-            with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor:
-                asr_future = thread_executor.submit(run_asr)
-                
-                if conn.voiceprint_provider and wav_data:
-                    voiceprint_future = thread_executor.submit(run_voiceprint)
-                    
-                    # 等待两个线程都完成
-                    asr_result = asr_future.result(timeout=15)
-                    voiceprint_result = voiceprint_future.result(timeout=15)
-                    
-                    results = {"asr": asr_result, "voiceprint": voiceprint_result}
-                else:
-                    asr_result = asr_future.result(timeout=15)
-                    results = {"asr": asr_result, "voiceprint": None}
-            
-            
-            # 处理结果
-            raw_text, _ = results.get("asr", ("", None))
-            speaker_name = results.get("voiceprint", None)
-            
-            # 记录识别结果
+            asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
+
+            if conn.voiceprint_provider and wav_data:
+                voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
+                # 并发等待两个结果
+                asr_result, voiceprint_result = await asyncio.gather(
+                    asr_task, voiceprint_task, return_exceptions=True
+                )
+            else:
+                asr_result = await asr_task
+                voiceprint_result = None
+
+            # 记录识别结果 - 检查是否为异常
+            if isinstance(asr_result, Exception):
+                logger.bind(tag=TAG).error(f"ASR识别失败: {asr_result}")
+                raw_text = ""
+            else:
+                raw_text, _ = asr_result
+
+            if isinstance(voiceprint_result, Exception):
+                logger.bind(tag=TAG).error(f"声纹识别失败: {voiceprint_result}")
+                speaker_name = ""
+            else:
+                speaker_name = voiceprint_result
+
             if raw_text:
                 logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
             if speaker_name:
                 logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
-            
+
             # 性能监控
             total_time = time.monotonic() - total_start_time
-            logger.bind(tag=TAG).info(f"总处理耗时: {total_time:.3f}s")
-            
+            logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s")
+
             # 检查文本长度
             text_len, _ = remove_punctuation_and_length(raw_text)
             self.stop_ws_connection()
-            
+
             if text_len > 0:
                 # 构建包含说话人信息的JSON字符串
                 enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
-                
+
                 # 使用自定义模块进行上报
                 await startToChat(conn, enhanced_text)
                 enqueue_asr_report(conn, enhanced_text, asr_audio_task)
@@ -306,6 +273,7 @@ class ASRProviderBase(ABC):
     @staticmethod
     def decode_opus(opus_data: List[bytes]) -> List[bytes]:
         """将Opus音频数据解码为PCM数据"""
+        decoder = None
         try:
             decoder = opuslib_next.Decoder(16000, 1)
             pcm_data = []
@@ -330,3 +298,9 @@ class ASRProviderBase(ABC):
         except Exception as e:
             logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
             return []
+        finally:
+            if decoder is not None:
+                try:
+                    del decoder
+                except Exception as e:
+                    logger.bind(tag=TAG).debug(f"释放decoder资源时出错: {e}")

main/xiaozhi-server/core/providers/asr/doubao_stream.py

@@ -18,8 +18,6 @@ class ASRProvider(ASRProviderBase):
         self.interface_type = InterfaceType.STREAM
         self.config = config
         self.text = ""
-        self.max_retries = 3
-        self.retry_delay = 2
         self.decoder = opuslib_next.Decoder(16000, 1)
         self.asr_ws = None
         self.forward_task = None
@@ -49,6 +47,8 @@ class ASRProvider(ASRProviderBase):
         self.channel = config.get("channel", 1)
         self.auth_method = config.get("auth_method", "token")
         self.secret = config.get("secret", "access_secret")
+        end_window_size = config.get("end_window_size")
+        self.end_window_size = int(end_window_size) if end_window_size else 200
 
     async def open_audio_channels(self, conn):
         await super().open_audio_channels(conn)
@@ -56,14 +56,13 @@ class ASRProvider(ASRProviderBase):
     async def receive_audio(self, conn, audio, audio_have_voice):
         conn.asr_audio.append(audio)
         conn.asr_audio = conn.asr_audio[-10:]
-        
         # 存储音频数据
         if not hasattr(conn, 'asr_audio_for_voiceprint'):
             conn.asr_audio_for_voiceprint = []
         conn.asr_audio_for_voiceprint.append(audio)
-        
+
         # 当没有音频数据时处理完整语音片段
-        if not audio and len(conn.asr_audio_for_voiceprint) > 0:
+        if conn.client_listen_mode != "manual" and not audio and len(conn.asr_audio_for_voiceprint) > 0:
             await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint)
             conn.asr_audio_for_voiceprint = []
 
@@ -179,6 +178,7 @@ class ASRProvider(ASRProviderBase):
                                 payload.get("audio_info", {}).get("duration", 0) > 2000
                                 and not utterances
                                 and not payload["result"].get("text")
+                                and conn.client_listen_mode != "manual"
                             ):
                                 logger.bind(tag=TAG).error(f"识别文本：空")
                                 self.text = ""
@@ -187,15 +187,44 @@ class ASRProvider(ASRProviderBase):
                                     await self.handle_voice_stop(conn, audio_data)
                                 break
 
+                            # 专门处理没有文本的识别结果（手动模式下可能已经识别完成但是没松按键）
+                            elif not payload["result"].get("text") and not utterances:
+                                if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0:
+                                    logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理")
+                                    await self.handle_voice_stop(conn, audio_data)
+                                    # 清理音频缓存
+                                    conn.asr_audio.clear()
+                                    conn.reset_vad_states()
+                                    break
+
                             for utterance in utterances:
                                 if utterance.get("definite", False):
-                                    self.text = utterance["text"]
+                                    current_text = utterance["text"]
                                     logger.bind(tag=TAG).info(
-                                        f"识别到文本: {self.text}"
+                                        f"识别到文本: {current_text}"
                                     )
-                                    conn.reset_vad_states()
-                                    if len(audio_data) > 15:  # 确保有足够音频数据
-                                        await self.handle_voice_stop(conn, audio_data)
+
+                                    # 手动模式下累积识别结果
+                                    if conn.client_listen_mode == "manual":
+                                        if self.text:
+                                            self.text += current_text
+                                        else:
+                                            self.text = current_text
+
+                                        # 在接收消息中途时收到停止信号
+                                        if conn.client_voice_stop and len(audio_data) > 0:
+                                            logger.bind(tag=TAG).debug("消息中途收到停止信号，触发处理")
+                                            await self.handle_voice_stop(conn, audio_data)
+                                            # 清理音频缓存
+                                            conn.asr_audio.clear()
+                                            conn.reset_vad_states()
+                                        break
+                                    else:
+                                        # 自动模式下直接覆盖
+                                        self.text = current_text
+                                        conn.reset_vad_states()
+                                        if len(audio_data) > 15:  # 确保有足够音频数据
+                                            await self.handle_voice_stop(conn, audio_data)
                                     break
                         elif "error" in payload:
                             error_msg = payload.get("error", "未知错误")
@@ -227,8 +256,6 @@ class ASRProvider(ASRProviderBase):
                     conn.asr_audio_for_voiceprint = []
                 if hasattr(conn, 'asr_audio'):
                     conn.asr_audio = []
-                if hasattr(conn, 'has_valid_voice'):
-                    conn.has_valid_voice = False
 
     def stop_ws_connection(self):
         if self.asr_ws:
@@ -236,6 +263,20 @@ class ASRProvider(ASRProviderBase):
             self.asr_ws = None
         self.is_processing = False
 
+    async def _send_stop_request(self):
+        """发送最后一个音频帧以通知服务器结束"""
+        if self.asr_ws:
+            try:
+                # 发送结束标记的音频帧（gzip压缩的空数据）
+                empty_payload = gzip.compress(b"")
+                last_audio_request = bytearray(self.generate_last_audio_default_header())
+                last_audio_request.extend(len(empty_payload).to_bytes(4, "big"))
+                last_audio_request.extend(empty_payload)
+                await self.asr_ws.send(last_audio_request)
+                logger.bind(tag=TAG).debug("已发送结束音频帧")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"发送结束音频帧时出错: {e}")
+
     def construct_request(self, reqid):
         req = {
             "app": {
@@ -252,7 +293,7 @@ class ASRProvider(ASRProviderBase):
                 "sequence": 1,
                 "boosting_table_name": self.boosting_table_name,
                 "correct_table_name": self.correct_table_name,
-                "end_window_size": 200,
+                "end_window_size": self.end_window_size,
             },
             "audio": {
                 "format": self.format,
@@ -370,6 +411,16 @@ class ASRProvider(ASRProviderBase):
                 pass
             self.forward_task = None
         self.is_processing = False
+        
+        # 显式释放decoder资源
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Doubao decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")
+
         # 清理所有连接的音频缓冲区
         if hasattr(self, '_connections'):
             for conn in self._connections.values():
@@ -377,5 +428,3 @@ class ASRProvider(ASRProviderBase):
                     conn.asr_audio_for_voiceprint = []
                 if hasattr(conn, 'asr_audio'):
                     conn.asr_audio = []
-                if hasattr(conn, 'has_valid_voice'):
-                    conn.has_valid_voice = False

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -1,14 +1,16 @@
-import time
 import os
-import sys
 import io
+import sys
+import time
+import shutil
 import psutil
+import asyncio
+
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
-from core.providers.asr.base import ASRProviderBase
 from funasr import AutoModel
 from funasr.utils.postprocess_utils import rich_transcription_postprocess
-import shutil
+from core.providers.asr.base import ASRProviderBase
 from core.providers.asr.dto.dto import InterfaceType
 
 TAG = __name__
@@ -90,16 +92,17 @@ class ASRProvider(ASRProviderBase):
                 else:
                     file_path = self.save_audio_to_file(pcm_data, session_id)
 
-                # 语音识别
+                # 语音识别 - 使用线程池避免阻塞事件循环
                 start_time = time.time()
-                result = self.model.generate(
+                result = await asyncio.to_thread(
+                    self.model.generate,
                     input=combined_pcm_data,
                     cache={},
                     language="auto",
                     use_itn=True,
                     batch_size_s=60,
                 )
-                text = rich_transcription_postprocess(result[0]["text"])
+                text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"])
                 logger.bind(tag=TAG).debug(
                     f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
                 )

main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py

@@ -0,0 +1,151 @@
+import os
+import tempfile
+from typing import Optional, Tuple, List
+import dashscope
+from config.logger import setup_logging
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+
+tag = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
+        # 音频文件上传类型，流式文本识别输出
+        self.interface_type = InterfaceType.NON_STREAM
+        """Qwen3-ASR-Flash ASR初始化"""
+        
+        # 配置参数
+        self.api_key = config.get("api_key")
+        if not self.api_key:
+            raise ValueError("Qwen3-ASR-Flash 需要配置 api_key")
+            
+        self.model_name = config.get("model_name", "qwen3-asr-flash")
+        self.output_dir = config.get("output_dir", "./audio_output")
+        self.delete_audio_file = delete_audio_file
+        
+        # ASR选项配置
+        self.enable_lid = config.get("enable_lid", True)  # 自动语种检测
+        self.enable_itn = config.get("enable_itn", True)  # 逆文本归一化
+        self.language = config.get("language", None)  # 指定语种，默认自动检测
+        self.context = config.get("context", "")  # 上下文信息，用于提高识别准确率
+        
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def _prepare_audio_file(self, pcm_data: bytes) -> str:
+        """将PCM数据转换为WAV文件并返回文件路径"""
+        try:
+            import wave
+            
+            # 创建临时WAV文件
+            with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file:
+                temp_path = temp_file.name
+                
+            # 写入WAV格式
+            with wave.open(temp_path, 'wb') as wav_file:
+                wav_file.setnchannels(1)      # 单声道
+                wav_file.setsampwidth(2)      # 16位
+                wav_file.setframerate(16000)  # 16kHz采样率
+                wav_file.writeframes(pcm_data)
+                
+            return temp_path
+            
+        except Exception as e:
+            logger.bind(tag=tag).error(f"音频文件准备失败: {e}")
+            return None
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus"
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        temp_file_path = None
+        file_path = None
+        
+        try:
+            # 解码音频数据
+            if audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            
+            combined_pcm_data = b"".join(pcm_data)
+            if len(combined_pcm_data) == 0:
+                logger.bind(tag=tag).warning("音频数据为空")
+                return "", None
+            
+            # 准备音频文件
+            temp_file_path = self._prepare_audio_file(combined_pcm_data)
+            if not temp_file_path:
+                return "", None
+            
+            # 保存音频文件（如果需要）
+            if not self.delete_audio_file:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
+            
+            # 构造请求消息
+            messages = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"audio": temp_file_path}
+                    ]
+                }
+            ]
+            
+            # 如果有上下文信息，添加system消息
+            if self.context:
+                messages.insert(0, {
+                    "role": "system", 
+                    "content": [
+                        {"text": self.context}
+                    ]
+                })
+            
+            # 准备ASR选项
+            asr_options = {
+                "enable_lid": self.enable_lid,
+                "enable_itn": self.enable_itn
+            }
+            
+            # 如果指定了语种，添加到选项中
+            if self.language:
+                asr_options["language"] = self.language
+            
+            # 设置API密钥
+            dashscope.api_key = self.api_key
+            
+            # 发送流式请求
+            response = dashscope.MultiModalConversation.call(
+                model=self.model_name,
+                messages=messages,
+                result_format="message",
+                asr_options=asr_options,
+                stream=True
+            )
+            
+            # 处理流式响应
+            full_text = ""
+            for chunk in response:
+                try:
+                    text = chunk["output"]["choices"][0]["message"].content[0]["text"]
+                    # 更新为最新的完整文本
+                    full_text = text.strip()
+                except:
+                    pass
+            
+            return full_text, file_path
+                
+        except Exception as e:
+            logger.bind(tag=tag).error(f"语音识别失败: {e}")
+            return "", file_path
+            
+        finally:
+            # 清理临时文件
+            if temp_file_path and os.path.exists(temp_file_path):
+                try:
+                    os.unlink(temp_file_path)
+                except Exception as e:
+                    logger.bind(tag=tag).warning(f"清理临时文件失败: {e}")

main/xiaozhi-server/core/providers/asr/vosk.py

@@ -0,0 +1,114 @@
+import os
+import json
+import time
+from typing import Optional, Tuple, List
+from .base import ASRProviderBase
+from config.logger import setup_logging
+from core.providers.asr.dto.dto import InterfaceType
+import vosk
+
+TAG = __name__
+logger = setup_logging()
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool = True):
+        super().__init__()
+        self.interface_type = InterfaceType.LOCAL
+        self.model_path = config.get("model_path")
+        self.output_dir = config.get("output_dir", "tmp/")
+        self.delete_audio_file = delete_audio_file
+        
+        # 初始化VOSK模型
+        self.model = None
+        self.recognizer = None
+        self._load_model()
+        
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def _load_model(self):
+        """加载VOSK模型"""
+        try:
+            if not os.path.exists(self.model_path):
+                raise FileNotFoundError(f"VOSK模型路径不存在: {self.model_path}")
+                
+            logger.bind(tag=TAG).info(f"正在加载VOSK模型: {self.model_path}")
+            self.model = vosk.Model(self.model_path)
+
+            # 初始化VOSK识别器（采样率必须为16kHz）
+            self.recognizer = vosk.KaldiRecognizer(self.model, 16000)
+
+            logger.bind(tag=TAG).info("VOSK模型加载成功")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"加载VOSK模型失败: {e}")
+            raise
+
+    async def speech_to_text(
+        self, audio_data: List[bytes], session_id: str, audio_format: str = "opus"
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        file_path = None
+        try:
+            # 检查模型是否加载成功
+            if not self.model:
+                logger.bind(tag=TAG).error("VOSK模型未加载，无法进行识别")
+                return "", None
+
+            # 解码音频（如果原始格式是Opus）
+            if audio_format == "pcm":
+                pcm_data = audio_data
+            else:
+                pcm_data = self.decode_opus(audio_data)
+                
+            if not pcm_data:
+                logger.bind(tag=TAG).warning("解码后的PCM数据为空，无法进行识别")
+                return "", None
+                
+            # 合并PCM数据
+            combined_pcm_data = b"".join(pcm_data)
+            if len(combined_pcm_data) == 0:
+                logger.bind(tag=TAG).warning("合并后的PCM数据为空")
+                return "", None
+
+            # 判断是否保存为WAV文件
+            if not self.delete_audio_file:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
+
+            start_time = time.time()
+            
+            
+            # 进行识别（VOSK推荐每次送入2000字节的数据）
+            chunk_size = 2000
+            text_result = ""
+            
+            for i in range(0, len(combined_pcm_data), chunk_size):
+                chunk = combined_pcm_data[i:i+chunk_size]
+                if self.recognizer.AcceptWaveform(chunk):
+                    result = json.loads(self.recognizer.Result())
+                    text = result.get('text', '')
+                    if text:
+                        text_result += text + " "
+            
+            # 获取最终结果
+            final_result = json.loads(self.recognizer.FinalResult())
+            final_text = final_result.get('text', '')
+            if final_text:
+                text_result += final_text
+            
+            logger.bind(tag=TAG).debug(
+                f"VOSK语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text_result.strip()}"
+            )
+            
+            return text_result.strip(), file_path
+            
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"VOSK语音识别失败: {e}")
+            return "", None
+        finally:
+            # 文件清理逻辑
+            if self.delete_audio_file and file_path and os.path.exists(file_path):
+                try:
+                    os.remove(file_path)
+                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/asr/xunfei_stream.py

@@ -0,0 +1,372 @@
+import json
+import hmac
+import base64
+import hashlib
+import asyncio
+import websockets
+import opuslib_next
+import gc
+from time import mktime
+from datetime import datetime
+from urllib.parse import urlencode
+from typing import List
+from config.logger import setup_logging
+from wsgiref.handlers import format_date_time
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+# 帧状态常量
+STATUS_FIRST_FRAME = 0  # 第一帧的标识
+STATUS_CONTINUE_FRAME = 1  # 中间帧标识
+STATUS_LAST_FRAME = 2  # 最后一帧的标识
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__()
+        self.interface_type = InterfaceType.STREAM
+        self.config = config
+        self.text = ""
+        self.decoder = opuslib_next.Decoder(16000, 1)
+        self.asr_ws = None
+        self.forward_task = None
+        self.is_processing = False
+        self.server_ready = False
+
+        # 讯飞配置
+        self.app_id = config.get("app_id")
+        self.api_key = config.get("api_key")
+        self.api_secret = config.get("api_secret")
+
+        if not all([self.app_id, self.api_key, self.api_secret]):
+            raise ValueError("必须提供app_id、api_key和api_secret")
+
+        # 识别参数
+        self.iat_params = {
+            "domain": config.get("domain", "slm"),
+            "language": config.get("language", "zh_cn"),
+            "accent": config.get("accent", "mandarin"),
+            "result": {"encoding": "utf8", "compress": "raw", "format": "plain"},
+        }
+
+        self.output_dir = config.get("output_dir", "tmp/")
+        self.delete_audio_file = delete_audio_file
+
+    def create_url(self) -> str:
+        """生成认证URL"""
+        url = "ws://iat.cn-huabei-1.xf-yun.com/v1"
+        # 生成RFC1123格式的时间戳
+        now = datetime.now()
+        date = format_date_time(mktime(now.timetuple()))
+
+        # 拼接字符串
+        signature_origin = "host: " + "iat.cn-huabei-1.xf-yun.com" + "\n"
+        signature_origin += "date: " + date + "\n"
+        signature_origin += "GET " + "/v1 " + "HTTP/1.1"
+
+        # 进行hmac-sha256进行加密
+        signature_sha = hmac.new(
+            self.api_secret.encode("utf-8"),
+            signature_origin.encode("utf-8"),
+            digestmod=hashlib.sha256,
+        ).digest()
+        signature_sha = base64.b64encode(signature_sha).decode(encoding="utf-8")
+
+        authorization_origin = (
+            'api_key="%s", algorithm="%s", headers="%s", signature="%s"'
+            % (self.api_key, "hmac-sha256", "host date request-line", signature_sha)
+        )
+        authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(
+            encoding="utf-8"
+        )
+
+        # 将请求的鉴权参数组合为字典
+        v = {
+            "authorization": authorization,
+            "date": date,
+            "host": "iat.cn-huabei-1.xf-yun.com",
+        }
+
+        # 拼接鉴权参数，生成url
+        url = url + "?" + urlencode(v)
+        return url
+
+    async def open_audio_channels(self, conn):
+        await super().open_audio_channels(conn)
+
+    async def receive_audio(self, conn, audio, audio_have_voice):
+        # 先调用父类方法处理基础逻辑
+        await super().receive_audio(conn, audio, audio_have_voice)
+
+        # 存储音频数据用于声纹识别
+        if not hasattr(conn, "asr_audio_for_voiceprint"):
+            conn.asr_audio_for_voiceprint = []
+        conn.asr_audio_for_voiceprint.append(audio)
+
+        # 如果本次有声音，且之前没有建立连接
+        if audio_have_voice and self.asr_ws is None and not self.is_processing:
+            try:
+                await self._start_recognition(conn)
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
+                await self._cleanup()
+                return
+
+        # 发送当前音频数据
+        if self.asr_ws and self.is_processing and self.server_ready:
+            try:
+                pcm_frame = self.decoder.decode(audio, 960)
+                await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"发送音频数据时发生错误: {e}")
+                await self._cleanup()
+
+    async def _start_recognition(self, conn):
+        """开始识别会话"""
+        try:
+            self.is_processing = True
+            # 建立WebSocket连接
+            ws_url = self.create_url()
+            logger.bind(tag=TAG).info(f"正在连接ASR服务: {ws_url[:50]}...")
+
+            # 如果为手动模式,设置超时时长为一分钟
+            if conn.client_listen_mode == "manual":
+                self.iat_params["eos"] = 60000
+
+            self.asr_ws = await websockets.connect(
+                ws_url,
+                max_size=1000000000,
+                ping_interval=None,
+                ping_timeout=None,
+                close_timeout=10,
+            )
+
+            logger.bind(tag=TAG).info("ASR WebSocket连接已建立")
+            self.server_ready = False
+            self.forward_task = asyncio.create_task(self._forward_results(conn))
+
+            # 发送首帧音频
+            if conn.asr_audio and len(conn.asr_audio) > 0:
+                first_audio = conn.asr_audio[-1] if conn.asr_audio else b""
+                pcm_frame = (
+                    self.decoder.decode(first_audio, 960) if first_audio else b""
+                )
+                await self._send_audio_frame(pcm_frame, STATUS_FIRST_FRAME)
+                self.server_ready = True
+                logger.bind(tag=TAG).info("已发送首帧，开始识别")
+
+                # 发送缓存的音频数据
+                for cached_audio in conn.asr_audio[-10:]:
+                    try:
+                        pcm_frame = self.decoder.decode(cached_audio, 960)
+                        await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
+                    except Exception as e:
+                        logger.bind(tag=TAG).info(f"发送缓存音频数据时发生错误: {e}")
+                        break
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
+            if hasattr(e, "__cause__") and e.__cause__:
+                logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+            if self.asr_ws:
+                await self.asr_ws.close()
+                self.asr_ws = None
+            self.is_processing = False
+            raise
+
+    async def _send_audio_frame(self, audio_data: bytes, status: int):
+        """发送音频帧"""
+        if not self.asr_ws:
+            return
+
+        audio_b64 = base64.b64encode(audio_data).decode("utf-8")
+
+        frame_data = {
+            "header": {"status": status, "app_id": self.app_id},
+            "parameter": {"iat": self.iat_params},
+            "payload": {
+                "audio": {"audio": audio_b64, "sample_rate": 16000, "encoding": "raw"}
+            },
+        }
+
+        await self.asr_ws.send(json.dumps(frame_data, ensure_ascii=False))
+
+    async def _forward_results(self, conn):
+        """转发识别结果"""
+        try:
+            while not conn.stop_event.is_set():
+                try:
+                    response = await asyncio.wait_for(self.asr_ws.recv(), timeout=60)
+                    result = json.loads(response)
+                    logger.bind(tag=TAG).debug(f"收到ASR结果: {result}")
+
+                    header = result.get("header", {})
+                    payload = result.get("payload", {})
+                    code = header.get("code", 0)
+                    status = header.get("status", 0)
+
+                    if code != 0:
+                        logger.bind(tag=TAG).error(
+                            f"识别错误，错误码: {code}, 消息: {header.get('message', '')}"
+                        )
+                        if code in [10114, 10160]:  # 连接问题
+                            break
+                        continue
+
+                    # 处理识别结果
+                    if payload and "result" in payload:
+                        text_data = payload["result"]["text"]
+                        if text_data:
+                            # 解码base64文本
+                            decoded_text = base64.b64decode(text_data).decode("utf-8")
+                            text_json = json.loads(decoded_text)
+                            # 提取文本内容
+                            text_ws = text_json.get("ws", [])
+                            for i in text_ws:
+                                for j in i.get("cw", []):
+                                    w = j.get("w", "")
+                                    self.text += w
+
+                    if status == 2:
+                        if conn.client_listen_mode == "manual":
+                            audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                            if len(audio_data) > 0:
+                                logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
+                                await self.handle_voice_stop(conn, audio_data)
+                                # 清理音频缓存
+                                conn.asr_audio.clear()
+                        conn.reset_vad_states()
+                        break
+
+                except asyncio.TimeoutError:
+                    logger.bind(tag=TAG).error("接收结果超时")
+                    break
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).info("ASR服务连接已关闭")
+                    self.is_processing = False
+                    break
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"处理ASR结果时发生错误: {str(e)}")
+                    if hasattr(e, "__cause__") and e.__cause__:
+                        logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+                    self.is_processing = False
+                    break
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR结果转发任务发生错误: {str(e)}")
+            if hasattr(e, "__cause__") and e.__cause__:
+                logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+        finally:
+            # 清理连接资源
+            await self._cleanup()
+
+            # 清理连接的音频缓存
+            if conn:
+                if hasattr(conn, "asr_audio_for_voiceprint"):
+                    conn.asr_audio_for_voiceprint = []
+                if hasattr(conn, "asr_audio"):
+                    conn.asr_audio = []
+
+    async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
+        """处理语音停止，发送最后一帧并处理识别结果"""
+        try:
+            # 先发送最后一帧表示音频结束
+            if self.asr_ws and self.is_processing:
+                try:
+                    await self._send_audio_frame(b"", STATUS_LAST_FRAME)
+                    logger.bind(tag=TAG).debug(f"已发送停止请求")
+
+                    await asyncio.sleep(0.25)
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
+
+            await super().handle_voice_stop(conn, asr_audio_task)
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
+            import traceback
+
+            logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
+
+    def stop_ws_connection(self):
+        if self.asr_ws:
+            asyncio.create_task(self.asr_ws.close())
+            self.asr_ws = None
+        self.is_processing = False
+
+    async def _send_stop_request(self):
+        """发送停止识别请求（不关闭连接）"""
+        if self.asr_ws:
+            try:
+                # 先停止音频发送
+                self.is_processing = False
+                await self._send_audio_frame(b"", STATUS_LAST_FRAME)
+                logger.bind(tag=TAG).debug("已发送停止请求")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
+
+    async def _cleanup(self):
+        """清理资源（关闭连接）"""
+        logger.bind(tag=TAG).debug(
+            f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}"
+        )
+
+        # 状态重置
+        self.is_processing = False
+        self.server_ready = False
+        logger.bind(tag=TAG).debug("ASR状态已重置")
+
+        # 关闭连接
+        if self.asr_ws:
+            try:
+                logger.bind(tag=TAG).debug("正在关闭WebSocket连接")
+                await asyncio.wait_for(self.asr_ws.close(), timeout=2.0)
+                logger.bind(tag=TAG).debug("WebSocket连接已关闭")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
+            finally:
+                self.asr_ws = None
+
+        # 清理任务引用
+        self.forward_task = None
+
+        logger.bind(tag=TAG).debug("ASR会话清理完成")
+
+    async def speech_to_text(self, opus_data, session_id, audio_format):
+        """获取识别结果"""
+        result = self.text
+        self.text = ""
+        return result, None
+
+    async def close(self):
+        """资源清理方法"""
+        if self.asr_ws:
+            await self.asr_ws.close()
+            self.asr_ws = None
+        if self.forward_task:
+            self.forward_task.cancel()
+            try:
+                await self.forward_task
+            except asyncio.CancelledError:
+                pass
+            self.forward_task = None
+        self.is_processing = False
+
+        # 显式释放decoder资源
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Xunfei decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
+
+        # 清理所有连接的音频缓冲区
+        if hasattr(self, "_connections"):
+            for conn in self._connections.values():
+                if hasattr(conn, "asr_audio_for_voiceprint"):
+                    conn.asr_audio_for_voiceprint = []
+                if hasattr(conn, "asr_audio"):
+                    conn.asr_audio = []

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -53,24 +53,32 @@ class IntentProvider(IntentProviderBase):
             functions_desc += "---\n"
 
         prompt = (
+            "【严格格式要求】你必须只能返回JSON格式，绝对不能返回任何自然语言！\n\n"
             "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
+            "【重要规则】以下类型的查询请直接返回result_for_context，无需调用函数：\n"
+            "- 询问当前时间（如：现在几点、当前时间、查询时间等）\n"
+            "- 询问今天日期（如：今天几号、今天星期几、今天是什么日期等）\n"
+            "- 询问今天农历（如：今天农历几号、今天什么节气等）\n"
+            "- 询问所在城市（如：我现在在哪里、你知道我在哪个城市吗等）"
+            "系统会根据上下文信息直接构建回答。\n\n"
             "- 如果用户使用疑问词（如'怎么'、'为什么'、'如何'）询问退出相关的问题（例如'怎么退出了？'），注意这不是让你退出，请返回 {'function_call': {'name': 'continue_chat'}\n"
             "- 仅当用户明确使用'退出系统'、'结束对话'、'我不想和你说话了'等指令时，才触发 handle_exit_intent\n\n"
             f"{functions_desc}\n"
             "处理步骤:\n"
             "1. 分析用户输入，确定用户意图\n"
-            "2. 从可用函数列表中选择最匹配的函数\n"
-            "3. 如果找到匹配的函数，生成对应的function_call 格式\n"
-            '4. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
+            "2. 检查是否为上述基础信息查询（时间、日期等），如是则返回result_for_context\n"
+            "3. 从可用函数列表中选择最匹配的函数\n"
+            "4. 如果找到匹配的函数，生成对应的function_call 格式\n"
+            '5. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
             "返回格式要求：\n"
-            "1. 必须返回纯JSON格式\n"
+            "1. 必须返回纯JSON格式，不要包含任何其他文字\n"
             "2. 必须包含function_call字段\n"
             "3. function_call必须包含name字段\n"
             "4. 如果函数需要参数，必须包含arguments字段\n\n"
             "示例：\n"
             "```\n"
             "用户: 现在几点了？\n"
-            '返回: {"function_call": {"name": "get_time"}}\n'
+            '返回: {"function_call": {"name": "result_for_context"}}\n'
             "```\n"
             "```\n"
             "用户: 当前电池电量是多少？\n"
@@ -94,12 +102,15 @@ class IntentProvider(IntentProviderBase):
             "```\n\n"
             "注意：\n"
             "1. 只返回JSON格式，不要包含任何其他文字\n"
-            '2. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
-            "3. 确保返回的JSON格式正确，包含所有必要的字段\n"
+            '2. 优先检查用户查询是否为基础信息（时间、日期等），如是则返回{"function_call": {"name": "result_for_context"}}，不需要arguments参数\n'
+            '3. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
+            "4. 确保返回的JSON格式正确，包含所有必要的字段\n"
+            "5. result_for_context不需要任何参数，系统会自动从上下文获取信息\n"
             "特殊说明：\n"
             "- 当用户单次输入包含多个指令时（如'打开灯并且调高音量'）\n"
             "- 请返回多个function_call组成的JSON数组\n"
-            "- 示例：{'function_calls': [{name:'light_on'}, {name:'volume_up'}]}"
+            "- 示例：{'function_calls': [{name:'light_on'}, {name:'volume_up'}]}\n\n"
+            "【最终警告】绝对禁止输出任何自然语言、表情符号或解释文字！只能输出有效JSON格式！违反此规则将导致系统错误！"
         )
         return prompt
 
@@ -190,7 +201,7 @@ class IntentProvider(IntentProviderBase):
         # 记录LLM调用完成时间
         llm_time = time.time() - llm_start_time
         logger.bind(tag=TAG).debug(
-            f"LLM意图识别完成, 模型: {model_info}, 调用耗时: {llm_time:.4f}秒"
+            f"外挂的大模型意图识别完成, 模型: {model_info}, 调用耗时: {llm_time:.4f}秒"
         )
 
         # 记录后处理开始时间
@@ -223,8 +234,15 @@ class IntentProvider(IntentProviderBase):
                     f"llm 识别到意图: {function_name}, 参数: {function_args}"
                 )
 
-                # 如果是继续聊天，清理工具调用相关的历史消息
-                if function_name == "continue_chat":
+                # 处理不同类型的意图
+                if function_name == "result_for_context":
+                    # 处理基础信息查询，直接从context构建结果
+                    logger.bind(tag=TAG).info(
+                        "检测到result_for_context意图，将使用上下文信息直接回答"
+                    )
+
+                elif function_name == "continue_chat":
+                    # 处理普通对话
                     # 保留非工具相关的消息
                     clean_history = [
                         msg
@@ -233,25 +251,15 @@ class IntentProvider(IntentProviderBase):
                     ]
                     conn.dialogue.dialogue = clean_history
 
-                # 添加到缓存
-                self.cache_manager.set(self.CacheType.INTENT, cache_key, intent)
+                else:
+                    # 处理函数调用
+                    logger.bind(tag=TAG).info(f"检测到函数调用意图: {function_name}")
 
-                # 后处理时间
-                postprocess_time = time.time() - postprocess_start_time
-                logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
-
-                # 确保返回完全序列化的JSON字符串
-                return intent
-            else:
-                # 添加到缓存
-                self.cache_manager.set(self.CacheType.INTENT, cache_key, intent)
-
-                # 后处理时间
-                postprocess_time = time.time() - postprocess_start_time
-                logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
-
-                # 返回普通意图
-                return intent
+            # 统一缓存处理和返回
+            self.cache_manager.set(self.CacheType.INTENT, cache_key, intent)
+            postprocess_time = time.time() - postprocess_start_time
+            logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
+            return intent
         except json.JSONDecodeError:
             # 后处理时间
             postprocess_time = time.time() - postprocess_start_time
@@ -259,4 +267,4 @@ class IntentProvider(IntentProviderBase):
                 f"无法解析意图JSON: {intent}, 后处理耗时: {postprocess_time:.4f}秒"
             )
             # 如果解析失败，默认返回继续聊天意图
-            return '{"intent": "继续聊天"}'
+            return '{"function_call": {"name": "continue_chat"}}'

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -17,27 +17,26 @@ class LLMProvider(LLMProviderBase):
             self.base_url = config.get("base_url")
         else:
             self.base_url = config.get("url")
-        # 增加timeout的配置项，单位为秒
         timeout = config.get("timeout", 300)
         self.timeout = int(timeout) if timeout else 300
 
         param_defaults = {
-            "max_tokens": (500, int),
-            "temperature": (0.7, lambda x: round(float(x), 1)),
-            "top_p": (1.0, lambda x: round(float(x), 1)),
-            "frequency_penalty": (0, lambda x: round(float(x), 1)),
+            "max_tokens": int,
+            "temperature": lambda x: round(float(x), 1),
+            "top_p": lambda x: round(float(x), 1),
+            "frequency_penalty": lambda x: round(float(x), 1),
         }
 
-        for param, (default, converter) in param_defaults.items():
+        for param, converter in param_defaults.items():
             value = config.get(param)
             try:
                 setattr(
                     self,
                     param,
-                    converter(value) if value not in (None, "") else default,
+                    converter(value) if value not in (None, "") else None,
                 )
             except (ValueError, TypeError):
-                setattr(self, param, default)
+                setattr(self, param, None)
 
         logger.debug(
             f"意图识别参数初始化: {self.temperature}, {self.max_tokens}, {self.top_p}, {self.frequency_penalty}"
@@ -48,34 +47,46 @@ class LLMProvider(LLMProviderBase):
             logger.bind(tag=TAG).error(model_key_msg)
         self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url, timeout=httpx.Timeout(self.timeout))
 
+    @staticmethod
+    def normalize_dialogue(dialogue):
+        """自动修复 dialogue 中缺失 content 的消息"""
+        for msg in dialogue:
+            if "role" in msg and "content" not in msg:
+                msg["content"] = ""
+        return dialogue
+
     def response(self, session_id, dialogue, **kwargs):
         try:
-            responses = self.client.chat.completions.create(
-                model=self.model_name,
-                messages=dialogue,
-                stream=True,
-                max_tokens=kwargs.get("max_tokens", self.max_tokens),
-                temperature=kwargs.get("temperature", self.temperature),
-                top_p=kwargs.get("top_p", self.top_p),
-                frequency_penalty=kwargs.get(
-                    "frequency_penalty", self.frequency_penalty
-                ),
-            )
+            dialogue = self.normalize_dialogue(dialogue)
+
+            request_params = {
+                "model": self.model_name,
+                "messages": dialogue,
+                "stream": True,
+            }
+
+            # 添加可选参数,只有当参数不为None时才添加
+            optional_params = {
+                "max_tokens": kwargs.get("max_tokens", self.max_tokens),
+                "temperature": kwargs.get("temperature", self.temperature),
+                "top_p": kwargs.get("top_p", self.top_p),
+                "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
+            }
+
+            for key, value in optional_params.items():
+                if value is not None:
+                    request_params[key] = value
+
+            responses = self.client.chat.completions.create(**request_params)
 
             is_active = True
             for chunk in responses:
                 try:
-                    # 检查是否存在有效的choice且content不为空
-                    delta = (
-                        chunk.choices[0].delta
-                        if getattr(chunk, "choices", None)
-                        else None
-                    )
-                    content = delta.content if hasattr(delta, "content") else ""
+                    delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
+                    content = getattr(delta, "content", "") if delta else ""
                 except IndexError:
                     content = ""
                 if content:
-                    # 处理标签跨多个chunk的情况
                     if "<think>" in content:
                         is_active = False
                         content = content.split("<think>")[0]
@@ -88,19 +99,36 @@ class LLMProvider(LLMProviderBase):
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")
 
-    def response_with_functions(self, session_id, dialogue, functions=None):
+    def response_with_functions(self, session_id, dialogue, functions=None, **kwargs):
         try:
-            stream = self.client.chat.completions.create(
-                model=self.model_name, messages=dialogue, stream=True, tools=functions
-            )
+            dialogue = self.normalize_dialogue(dialogue)
+
+            request_params = {
+                "model": self.model_name,
+                "messages": dialogue,
+                "stream": True,
+                "tools": functions,
+            }
+
+            optional_params = {
+                "max_tokens": kwargs.get("max_tokens", self.max_tokens),
+                "temperature": kwargs.get("temperature", self.temperature),
+                "top_p": kwargs.get("top_p", self.top_p),
+                "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
+            }
+
+            for key, value in optional_params.items():
+                if value is not None:
+                    request_params[key] = value
+
+            stream = self.client.chat.completions.create(**request_params)
 
             for chunk in stream:
-                # 检查是否存在有效的choice且content不为空
                 if getattr(chunk, "choices", None):
-                    yield chunk.choices[0].delta.content, chunk.choices[
-                        0
-                    ].delta.tool_calls
-                # 存在 CompletionUsage 消息时，生成 Token 消耗 log
+                    delta = chunk.choices[0].delta
+                    content = getattr(delta, "content", "")
+                    tool_calls = getattr(delta, "tool_calls", None)
+                    yield content, tool_calls
                 elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
                     usage_info = getattr(chunk, "usage", None)
                     logger.bind(tag=TAG).info(

main/xiaozhi-server/core/providers/memory/base.py

@@ -14,7 +14,7 @@ class MemoryProviderBase(ABC):
         self.llm = llm
 
     @abstractmethod
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         """Save a new memory for specific role and return memory ID"""
         print("this is base func", msgs)
 

main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py

@@ -28,7 +28,7 @@ class MemoryProvider(MemoryProviderBase):
             logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
             self.use_mem0 = False
 
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         if not self.use_mem0:
             return None
         if len(msgs) < 2:
@@ -41,9 +41,7 @@ class MemoryProvider(MemoryProviderBase):
                 for message in msgs
                 if message.role != "system"
             ]
-            result = self.client.add(
-                messages, user_id=self.role_id, output_format=self.api_version
-            )
+            result = self.client.add(messages, user_id=self.role_id)
             logger.bind(tag=TAG).debug(f"Save memory result: {result}")
         except Exception as e:
             logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
@@ -53,9 +51,12 @@ class MemoryProvider(MemoryProviderBase):
         if not self.use_mem0:
             return ""
         try:
-            results = self.client.search(
-                query, user_id=self.role_id, output_format=self.api_version
-            )
+            if not getattr(self, "role_id", None):
+                return ""
+
+            filters = {"user_id": self.role_id}
+
+            results = self.client.search(query, filters=filters)
             if not results or "results" not in results:
                 return ""
 

main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -4,7 +4,8 @@ import json
 import os
 import yaml
 from config.config_loader import get_project_dir
-from config.manage_api_client import save_mem_local_short
+from config.manage_api_client import generate_and_save_chat_summary
+import asyncio
 from core.utils.util import check_model_key
 
 
@@ -74,18 +75,6 @@ short_term_memory_prompt = """
 ```
 """
 
-short_term_memory_prompt_only_content = """
-你是一个经验丰富的记忆总结者，擅长将对话内容进行总结摘要，遵循以下规则：
-1、总结user的重要信息，以便在未来的对话中提供更个性化的服务
-2、不要重复总结，不要遗忘之前记忆，除非原来的记忆超过了1800字内，否则不要遗忘、不要压缩用户的历史记忆
-3、用户操控的设备音量、播放音乐、天气、退出、不想对话等和用户本身无关的内容，这些信息不需要加入到总结中
-4、聊天内容中的今天的日期时间、今天的天气情况与用户事件无关的数据，这些信息如果当成记忆存储会影响后序对话，这些信息不需要加入到总结中
-5、不要把设备操控的成果结果和失败结果加入到总结中，也不要把用户的一些废话加入到总结中
-6、不要为了总结而总结，如果用户的聊天没有意义，请返回原来的历史记录也是可以的
-7、只需要返回总结摘要，严格控制在1800字内
-8、不要包含代码、xml，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
-"""
-
 
 def extract_json_data(json_code):
     start = json_code.find("```json")
@@ -143,7 +132,7 @@ class MemoryProvider(MemoryProviderBase):
         with open(self.memory_path, "w", encoding="utf-8") as f:
             yaml.dump(all_memory, f, allow_unicode=True)
 
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         # 打印使用的模型信息
         model_info = getattr(self.llm, "model_name", str(self.llm.__class__.__name__))
         logger.bind(tag=TAG).debug(f"使用记忆保存模型: {model_info}")
@@ -187,14 +176,12 @@ class MemoryProvider(MemoryProviderBase):
             except Exception as e:
                 print("Error:", e)
         else:
-            result = self.llm.response_no_stream(
-                short_term_memory_prompt_only_content,
-                msgStr,
-                max_tokens=2000,
-                temperature=0.2,
-            )
-            save_mem_local_short(self.role_id, result)
-        logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
+            # 当save_to_file为False时，调用Java端的聊天记录总结接口
+            summary_id = session_id if session_id else self.role_id
+            await generate_and_save_chat_summary(summary_id)
+        logger.bind(tag=TAG).info(
+            f"Save memory successful - Role: {self.role_id}, Session: {session_id}"
+        )
 
         return self.short_memory
 

main/xiaozhi-server/core/providers/memory/nomem/nomem.py

@@ -11,7 +11,7 @@ class MemoryProvider(MemoryProviderBase):
     def __init__(self, config, summary_memory=None):
         super().__init__(config)
 
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
         return None
 

main/xiaozhi-server/core/providers/tools/device_mcp/mcp_handler.py

@@ -116,14 +116,14 @@ async def send_mcp_message(conn, payload: dict, transport=None):
             await conn.transport.send(message)
         else:
             raise AttributeError("无法找到可用的传输层接口")
-        logger.bind(tag=TAG).info(f"成功发送MCP消息: {message}")
+        logger.bind(tag=TAG).debug(f"成功发送MCP消息: {message}")
     except Exception as e:
         logger.bind(tag=TAG).error(f"发送MCP消息失败: {e}")
 
 
 async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict, transport=None):
     """处理MCP消息,包括初始化、工具列表和工具调用响应等"""
-    logger.bind(tag=TAG).info(f"处理MCP消息: {str(payload)[:100]}")
+    logger.bind(tag=TAG).debug(f"处理MCP消息: {str(payload)[:100]}")
 
     if not isinstance(payload, dict):
         logger.bind(tag=TAG).error("MCP消息缺少payload字段或格式错误")
@@ -148,7 +148,7 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict, transpo
             if isinstance(server_info, dict):
                 name = server_info.get("name")
                 version = server_info.get("version")
-                logger.bind(tag=TAG).info(
+                logger.bind(tag=TAG).debug(
                     f"客户端MCP服务器信息: name={name}, version={version}"
                 )
             return
@@ -205,11 +205,11 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict, transpo
 
                 next_cursor = result.get("nextCursor", "")
                 if next_cursor:
-                    logger.bind(tag=TAG).info(f"有更多工具，nextCursor: {next_cursor}")
+                    logger.bind(tag=TAG).debug(f"有更多工具，nextCursor: {next_cursor}")
                     await send_mcp_tools_list_continue_request(conn, next_cursor, transport)
                 else:
                     await mcp_client.set_ready(True)
-                    logger.bind(tag=TAG).info("所有工具已获取，MCP客户端准备就绪")
+                    logger.bind(tag=TAG).debug("所有工具已获取，MCP客户端准备就绪")
 
                     # 刷新工具缓存，确保MCP工具被包含在函数列表中
                     if hasattr(conn, "func_handler") and conn.func_handler:
@@ -265,7 +265,7 @@ async def send_mcp_initialize_message(conn, transport=None):
             },
         },
     }
-    logger.bind(tag=TAG).info("发送MCP初始化消息")
+    logger.bind(tag=TAG).debug("发送MCP初始化消息")
     await send_mcp_message(conn, payload, transport)
 
 

main/xiaozhi-server/core/providers/tools/mcp_endpoint/mcp_endpoint_handler.py

@@ -358,7 +358,9 @@ async def call_mcp_endpoint_tool(
     }
 
     message = json.dumps(payload)
-    logger.bind(tag=TAG).info(f"发送MCP接入点工具调用请求: {actual_name}，参数: {args}")
+    logger.bind(tag=TAG).info(
+        f"发送MCP接入点工具调用请求: {actual_name}，参数: {json.dumps(arguments, ensure_ascii=False)}"
+    )
     await mcp_client.send_message(message)
 
     try:

main/xiaozhi-server/core/providers/tools/server_mcp/mcp_client.py

@@ -10,9 +10,13 @@ import concurrent.futures
 from contextlib import AsyncExitStack
 from typing import Optional, List, Dict, Any
 
-from mcp import ClientSession, StdioServerParameters
+from mcp import ClientSession, StdioServerParameters, Implementation
+from mcp.client.session import SamplingFnT, ElicitationFnT, ListRootsFnT, LoggingFnT, MessageHandlerFnT
 from mcp.client.stdio import stdio_client
 from mcp.client.sse import sse_client
+from mcp.client.streamable_http import streamablehttp_client
+from mcp.shared.session import ProgressFnT
+
 from config.logger import setup_logging
 from core.utils.util import sanitize_tool_name
 
@@ -40,13 +44,25 @@ class ServerMCPClient:
         self.tools_dict: Dict[str, Any] = {}
         self.name_mapping: Dict[str, str] = {}
 
-    async def initialize(self):
+    async def initialize(self, read_timeout_seconds: timedelta | None = None,
+             sampling_callback: SamplingFnT | None = None,
+             elicitation_callback: ElicitationFnT | None = None,
+             list_roots_callback: ListRootsFnT | None = None,
+             logging_callback: LoggingFnT | None = None,
+             message_handler: MessageHandlerFnT | None = None,
+             client_info: Implementation | None = None):
         """初始化MCP客户端连接"""
         if self._worker_task:
             return
 
         self._worker_task = asyncio.create_task(
-            self._worker(), name="ServerMCPClientWorker"
+            self._worker(read_timeout_seconds=read_timeout_seconds,
+                        sampling_callback=sampling_callback,
+                        elicitation_callback=elicitation_callback,
+                        list_roots_callback=list_roots_callback,
+                        logging_callback=logging_callback,
+                        message_handler=message_handler,
+                        client_info=client_info), name="ServerMCPClientWorker"
         )
         await self._ready_evt.wait()
 
@@ -96,12 +112,15 @@ class ServerMCPClient:
             for name, tool in self.tools_dict.items()
         ]
 
-    async def call_tool(self, name: str, args: dict) -> Any:
+    async def call_tool(self, name: str, arguments: dict, read_timeout_seconds: timedelta | None = None, progress_callback: ProgressFnT | None = None, *, meta: dict[str, Any] | None = None) -> Any:
         """调用指定工具
 
         Args:
             name: 工具名称
-            args: 工具参数
+            arguments: 工具参数
+            read_timeout_seconds:
+            progress_callback: 进度回调函数
+            meta:
 
         Returns:
             Any: 工具执行结果
@@ -114,7 +133,7 @@ class ServerMCPClient:
 
         real_name = self.name_mapping.get(name, name)
         loop = self._worker_task.get_loop()
-        coro = self.session.call_tool(real_name, args)
+        coro = self.session.call_tool(real_name, arguments=arguments, read_timeout_seconds=read_timeout_seconds, progress_callback=progress_callback, meta=meta)
 
         if loop is asyncio.get_running_loop():
             return await coro
@@ -143,7 +162,13 @@ class ServerMCPClient:
         # 所有检查都通过，连接正常
         return True
 
-    async def _worker(self):
+    async def _worker(self, read_timeout_seconds: timedelta | None = None,
+             sampling_callback: SamplingFnT | None = None,
+             elicitation_callback: ElicitationFnT | None = None,
+             list_roots_callback: ListRootsFnT | None = None,
+             logging_callback: LoggingFnT | None = None,
+             message_handler: MessageHandlerFnT | None = None,
+             client_info: Implementation | None = None):
         """MCP客户端工作协程"""
         async with AsyncExitStack() as stack:
             try:
@@ -167,16 +192,38 @@ class ServerMCPClient:
 
                 # 建立SSEClient
                 elif "url" in self.config:
+                    headers = dict(self.config.get("headers", {}))
+                    # TODO 兼容旧版本
                     if "API_ACCESS_TOKEN" in self.config:
-                        headers = {
-                            "Authorization": f"Bearer {self.config['API_ACCESS_TOKEN']}"
-                        }
+                        headers["Authorization"] = f"Bearer {self.config['API_ACCESS_TOKEN']}"
+                        self.logger.bind(tag=TAG).warning(f"你正在使用旧过时的配置 API_ACCESS_TOKEN ，请在.mcp_server_settings.json中将API_ACCESS_TOKEN直接设置在headers中，例如 'Authorization': 'Bearer API_ACCESS_TOKEN'")
+                   
+                    # 根据transport类型选择不同的客户端，默认为SSE
+                    transport_type = self.config.get("transport", "sse")
+
+                    if transport_type == "streamable-http" or transport_type == "http":
+                        # 使用 Streamable HTTP 传输
+                        http_r, http_w, get_session_id = await stack.enter_async_context(
+                            streamablehttp_client(
+                                url=self.config["url"],
+                                headers=headers,
+                                timeout=self.config.get("timeout", 30),
+                                sse_read_timeout=self.config.get("sse_read_timeout", 60 * 5),
+                                terminate_on_close=self.config.get("terminate_on_close", True)
+                            )
+                        )
+                        read_stream, write_stream = http_r, http_w
                     else:
-                        headers = {}
-                    sse_r, sse_w = await stack.enter_async_context(
-                        sse_client(self.config["url"], headers=headers)
-                    )
-                    read_stream, write_stream = sse_r, sse_w
+                        # 使用传统的 SSE 传输
+                        sse_r, sse_w = await stack.enter_async_context(
+                            sse_client(
+                                url=self.config["url"],
+                                headers=headers,
+                                timeout=self.config.get("timeout", 5),
+                                sse_read_timeout=self.config.get("sse_read_timeout", 60 * 5)
+                            )
+                        )
+                        read_stream, write_stream = sse_r, sse_w
 
                 else:
                     raise ValueError("MCP客户端配置必须包含'command'或'url'")
@@ -185,7 +232,13 @@ class ServerMCPClient:
                     ClientSession(
                         read_stream=read_stream,
                         write_stream=write_stream,
-                        read_timeout_seconds=timedelta(seconds=15),
+                        read_timeout_seconds=read_timeout_seconds,
+                        sampling_callback=sampling_callback,
+                        elicitation_callback=elicitation_callback,
+                        list_roots_callback=list_roots_callback,
+                        logging_callback=logging_callback,
+                        message_handler=message_handler,
+                        client_info=client_info
                     )
                 )
                 await self.session.initialize()

main/xiaozhi-server/core/providers/tools/server_mcp/mcp_manager.py

@@ -4,6 +4,9 @@ import asyncio
 import os
 import json
 from typing import Dict, Any, List
+
+from mcp.types import LoggingMessageNotificationParams
+
 from config.config_loader import get_project_dir
 from config.logger import setup_logging
 from .mcp_client import ServerMCPClient
@@ -26,6 +29,7 @@ class ServerMCPManager:
             )
         self.clients: Dict[str, ServerMCPClient] = {}
         self.tools = []
+        self._init_lock = asyncio.Lock()
 
     def load_config(self) -> Dict[str, Any]:
         """加载MCP服务配置"""
@@ -42,29 +46,50 @@ class ServerMCPManager:
             )
             return {}
 
+    async def _init_server(self, name: str, srv_config: Dict[str, Any]):
+        """初始化单个MCP服务"""
+        client = None
+        try:
+            # 初始化服务端MCP客户端
+            logger.bind(tag=TAG).info(f"初始化服务端MCP客户端: {name}")
+            client = ServerMCPClient(srv_config)
+            # 设置超时时间10秒
+            await asyncio.wait_for(client.initialize(logging_callback=self.logging_callback), timeout=10)
+
+            # 使用锁保护共享状态的修改
+            async with self._init_lock:
+                self.clients[name] = client
+                client_tools = client.get_available_tools()
+                self.tools.extend(client_tools)
+
+        except asyncio.TimeoutError:
+            logger.bind(tag=TAG).error(
+                f"Failed to initialize MCP server {name}: Timeout"
+            )
+            if client:
+                await client.cleanup()
+        except Exception as e:
+            logger.bind(tag=TAG).error(
+                f"Failed to initialize MCP server {name}: {e}"
+            )
+            if client:
+                await client.cleanup()
+
     async def initialize_servers(self) -> None:
         """初始化所有MCP服务"""
         config = self.load_config()
+        tasks = []
         for name, srv_config in config.items():
             if not srv_config.get("command") and not srv_config.get("url"):
                 logger.bind(tag=TAG).warning(
                     f"Skipping server {name}: neither command nor url specified"
                 )
                 continue
-
-            try:
-                # 初始化服务端MCP客户端
-                logger.bind(tag=TAG).info(f"初始化服务端MCP客户端: {name}")
-                client = ServerMCPClient(srv_config)
-                await client.initialize()
-                self.clients[name] = client
-                client_tools = client.get_available_tools()
-                self.tools.extend(client_tools)
-
-            except Exception as e:
-                logger.bind(tag=TAG).error(
-                    f"Failed to initialize MCP server {name}: {e}"
-                )
+            
+            tasks.append(self._init_server(name, srv_config))
+        
+        if tasks:
+            await asyncio.gather(*tasks)
 
         # 输出当前支持的服务端MCP工具列表
         if hasattr(self.conn, "func_handler") and self.conn.func_handler:
@@ -109,7 +134,7 @@ class ServerMCPManager:
         # 带重试机制的工具调用
         for attempt in range(max_retries):
             try:
-                return await target_client.call_tool(tool_name, arguments)
+                return await target_client.call_tool(tool_name, arguments, progress_callback=self.progress_callback)
             except Exception as e:
                 # 最后一次尝试失败时直接抛出异常
                 if attempt == max_retries - 1:
@@ -131,7 +156,7 @@ class ServerMCPManager:
                     config = self.load_config()
                     if client_name in config:
                         client = ServerMCPClient(config[client_name])
-                        await client.initialize()
+                        await client.initialize(logging_callback=self.logging_callback)
                         self.clients[client_name] = client
                         target_client = client
                         logger.bind(tag=TAG).info(
@@ -159,3 +184,11 @@ class ServerMCPManager:
             except (asyncio.TimeoutError, Exception) as e:
                 logger.bind(tag=TAG).error(f"关闭服务端MCP客户端 {name} 时出错: {e}")
         self.clients.clear()
+
+    # 可选回调方法
+
+    async def logging_callback(self, params: LoggingMessageNotificationParams):
+        logger.bind(tag=TAG).info(f"[Server Log - {params.level.upper()}] {params.data}")
+
+    async def progress_callback(self, progress: float, total: float | None, message: str | None) -> None:
+        logger.bind(tag=TAG).info(f"[Progress {progress}/{total}]: {message}")

main/xiaozhi-server/core/providers/tools/server_plugins/plugin_executor.py

@@ -71,6 +71,19 @@ class ServerPluginExecutor(ToolExecutor):
         for func_name in all_required_functions:
             func_item = all_function_registry.get(func_name)
             if func_item:
+                # 从函数注册中获取描述
+                fun_description = (
+                    self.config.get("plugins", {})
+                    .get(func_name, {})
+                    .get("description", "")
+                )
+                if fun_description is not None and len(fun_description) > 0:
+                    if "function" in func_item.description and isinstance(
+                        func_item.description["function"], dict
+                    ):
+                        func_item.description["function"][
+                            "description"
+                        ] = fun_description
                 tools[func_name] = ToolDefinition(
                     name=func_name,
                     description=func_item.description,

main/xiaozhi-server/core/providers/tools/unified_tool_handler.py

@@ -69,7 +69,7 @@ class UnifiedToolHandler:
             self._initialize_home_assistant()
 
             self.finish_init = True
-            self.logger.info("统一工具处理器初始化完成")
+            self.logger.debug("统一工具处理器初始化完成")
 
             # 输出当前支持的所有工具列表
             self.current_support_functions()

main/xiaozhi-server/core/providers/tools/unified_tool_manager.py

@@ -20,7 +20,7 @@ class ToolManager:
         """注册工具执行器"""
         self.executors[tool_type] = executor
         self._invalidate_cache()
-        self.logger.info(f"注册工具执行器: {tool_type.value}")
+        self.logger.debug(f"注册工具执行器: {tool_type.value}")
 
     def _invalidate_cache(self):
         """使缓存失效"""
@@ -109,7 +109,7 @@ class ToolManager:
     def refresh_tools(self):
         """刷新工具缓存"""
         self._invalidate_cache()
-        self.logger.info("工具缓存已刷新")
+        self.logger.debug("工具缓存已刷新")
 
     def get_tool_statistics(self) -> Dict[str, int]:
         """获取工具统计信息"""

main/xiaozhi-server/core/providers/tts/alibl_stream.py

@@ -0,0 +1,522 @@
+import os
+import uuid
+import json
+import time
+import queue
+import asyncio
+import traceback
+import websockets
+from asyncio import Task
+from config.logger import setup_logging
+from core.utils import opus_encoder_utils
+from core.utils.tts import MarkdownCleaner
+from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+
+        self.interface_type = InterfaceType.DUAL_STREAM
+        # 基础配置
+        self.api_key = config.get("api_key")
+        if not self.api_key:
+            raise ValueError("api_key is required for CosyVoice TTS")
+
+        # WebSocket配置
+        self.ws_url = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/"
+        self.ws = None
+        self._monitor_task = None
+        self.last_active_time = None
+
+        # 模型和音色配置
+        self.model = config.get("model", "cosyvoice-v2")
+        self.voice = config.get("voice", "longxiaochun_v2")  # 默认音色
+        if config.get("private_voice"):
+            self.voice = config.get("private_voice")
+
+        # 音频参数配置
+        self.format = config.get("format", "pcm")
+        sample_rate = config.get("sample_rate", "24000")
+        self.sample_rate = int(sample_rate) if sample_rate else 24000
+
+        volume = config.get("volume", "50")
+        self.volume = int(volume) if volume else 50
+
+        rate = config.get("rate", "1.0")
+        self.rate = float(rate) if rate else 1.0
+
+        pitch = config.get("pitch", "1.0")
+        self.pitch = float(pitch) if pitch else 1.0
+
+        self.header = {
+            "Authorization": f"Bearer {self.api_key}",
+            # "user-agent": "your_platform_info", // 可选
+            # "X-DashScope-WorkSpace": workspace, // 可选，阿里云百炼业务空间ID
+            "X-DashScope-DataInspection": "enable",
+        }
+
+        # 创建Opus编码器
+        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
+            sample_rate=self.sample_rate, channels=1, frame_size_ms=60
+        )
+
+    async def _ensure_connection(self):
+        """确保WebSocket连接可用，支持60秒内连接复用"""
+        try:
+            current_time = time.time()
+            if self.ws and current_time - self.last_active_time < 60:
+                # 一分钟内才可以复用链接进行连续对话
+                logger.bind(tag=TAG).info(f"使用已有链接...")
+                return self.ws
+            logger.bind(tag=TAG).info("开始建立新连接...")
+
+            self.ws = await websockets.connect(
+                self.ws_url,
+                additional_headers=self.header,
+                ping_interval=30,
+                ping_timeout=10,
+                close_timeout=10,
+            )
+
+            logger.bind(tag=TAG).info("WebSocket连接建立成功")
+            self.last_active_time = current_time
+            return self.ws
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}")
+            self.ws = None
+            self.last_active_time = None
+            raise
+
+    def tts_text_priority_thread(self):
+        """流式TTS文本处理线程"""
+        while not self.conn.stop_event.is_set():
+            try:
+                message = self.tts_text_queue.get(timeout=1)
+                logger.bind(tag=TAG).debug(
+                    f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}"
+                )
+
+                if message.sentence_type == SentenceType.FIRST:
+                    self.conn.client_abort = False
+
+                if self.conn.client_abort:
+                    try:
+                        logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
+                        continue
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"取消TTS会话失败: {str(e)}")
+                        continue
+
+                if message.sentence_type == SentenceType.FIRST:
+                    # 初始化会话
+                    try:
+                        if not getattr(self.conn, "sentence_id", None): 
+                            self.conn.sentence_id = uuid.uuid4().hex
+                            logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}")
+
+                        logger.bind(tag=TAG).info("开始启动TTS会话...")
+                        future = asyncio.run_coroutine_threadsafe(
+                            self.start_session(self.conn.sentence_id),
+                            loop=self.conn.loop,
+                        )
+                        future.result()
+                        self.before_stop_play_files.clear()
+                        logger.bind(tag=TAG).info("TTS会话启动成功")
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
+                        continue
+
+                elif ContentType.TEXT == message.content_type:
+                    if message.content_detail:
+                        try:
+                            logger.bind(tag=TAG).debug(
+                                f"开始发送TTS文本: {message.content_detail}"
+                            )
+                            future = asyncio.run_coroutine_threadsafe(
+                                self.text_to_speak(message.content_detail, None),
+                                loop=self.conn.loop,
+                            )
+                            future.result()
+                            logger.bind(tag=TAG).debug("TTS文本发送成功")
+                        except Exception as e:
+                            logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
+                            continue
+
+                elif ContentType.FILE == message.content_type:
+                    logger.bind(tag=TAG).info(
+                        f"添加音频文件到待播放列表: {message.content_file}"
+                    )
+                    if message.content_file and os.path.exists(message.content_file):
+                        # 先处理文件音频数据
+                        self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
+
+                if message.sentence_type == SentenceType.LAST:
+                    try:
+                        logger.bind(tag=TAG).info("开始结束TTS会话...")
+                        future = asyncio.run_coroutine_threadsafe(
+                            self.finish_session(self.conn.sentence_id),
+                            loop=self.conn.loop,
+                        )
+                        future.result()
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"结束TTS会话失败: {str(e)}")
+                        continue
+
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
+                )
+                continue
+
+    async def text_to_speak(self, text, _):
+        """发送文本到TTS服务进行合成"""
+        try:
+            if self.ws is None:
+                logger.bind(tag=TAG).warning("WebSocket连接不存在，终止发送文本")
+                return
+
+            # 过滤Markdown
+            filtered_text = MarkdownCleaner.clean_markdown(text)
+
+            # 发送continue-task消息
+            continue_task_message = {
+                "header": {
+                    "action": "continue-task",
+                    "task_id": self.conn.sentence_id,
+                    "streaming": "duplex",
+                },
+                "payload": {"input": {"text": filtered_text}},
+            }
+
+            await self.ws.send(json.dumps(continue_task_message))
+            self.last_active_time = time.time()
+            logger.bind(tag=TAG).debug(f"已发送文本: {filtered_text}")
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
+            if self.ws:
+                try:
+                    await self.ws.close()
+                except:
+                    pass
+                self.ws = None
+            raise
+
+    async def start_session(self, session_id):
+        """启动TTS会话"""
+        logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
+        try:
+            # 检查并清理上一个会话的监听任务
+            if (
+                self._monitor_task is not None
+                and isinstance(self._monitor_task, Task)
+                and not self._monitor_task.done()
+            ):
+                logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务...")
+                await self.close()
+
+            # 确保连接可用
+            await self._ensure_connection()
+
+            # 启动监听任务
+            self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
+
+            # 发送run-task消息启动会话
+            run_task_message = {
+                "header": {
+                    "action": "run-task",
+                    "task_id": session_id,
+                    "streaming": "duplex",
+                },
+                "payload": {
+                    "task_group": "audio",
+                    "task": "tts",
+                    "function": "SpeechSynthesizer",
+                    "model": self.model,
+                    "parameters": {
+                        "text_type": "PlainText",
+                        "voice": self.voice,
+                        "format": self.format,
+                        "sample_rate": self.sample_rate,
+                        "volume": self.volume,
+                        "rate": self.rate,
+                        "pitch": self.pitch,
+                    },
+                    "input": {}
+                },
+            }
+
+            await self.ws.send(json.dumps(run_task_message))
+            self.last_active_time = time.time()
+            logger.bind(tag=TAG).info("会话启动请求已发送")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
+            await self.close()
+            raise
+
+    async def finish_session(self, session_id):
+        """结束TTS会话"""
+        logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
+        try:
+            if self.ws and session_id:
+                # 发送finish-task消息
+                finish_task_message = {
+                    "header": {
+                        "action": "finish-task",
+                        "task_id": session_id,
+                        "streaming": "duplex",
+                    },
+                    "payload": {
+                        "input": {}
+                    }
+                }
+
+                await self.ws.send(json.dumps(finish_task_message))
+                self.last_active_time = time.time()
+                logger.bind(tag=TAG).info("会话结束请求已发送")
+                # 等待监听任务完成
+                if self._monitor_task:
+                    try:
+                        await self._monitor_task
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(
+                            f"等待监听任务完成时发生错误: {str(e)}"
+                        )
+                    finally:
+                        self._monitor_task = None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
+            await self.close()
+            raise
+
+    async def close(self):
+        """清理资源"""
+        # 取消监听任务
+        if self._monitor_task:
+            try:
+                self._monitor_task.cancel()
+                await self._monitor_task
+            except asyncio.CancelledError:
+                pass
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}")
+            self._monitor_task = None
+
+        # 关闭WebSocket连接
+        if self.ws:
+            try:
+                await self.ws.close()
+            except:
+                pass
+            self.ws = None
+            self.last_active_time = None
+
+    async def _start_monitor_tts_response(self):
+        """监听TTS响应"""
+        try:
+            session_finished = False
+            while not self.conn.stop_event.is_set():
+                try:
+                    msg = await self.ws.recv()
+                    self.last_active_time = time.time()
+
+                    # 检查客户端是否中止
+                    if self.conn.client_abort:
+                        logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应")
+                        break
+
+                    if isinstance(msg, str):  # JSON控制消息
+                        try:
+                            data = json.loads(msg)
+                            event = data["header"].get("event")
+
+                            if event == "task-started":
+                                logger.bind(tag=TAG).debug("TTS任务启动成功~")
+                                self.tts_audio_queue.put((SentenceType.FIRST, [], None))
+                            elif event == "result-generated":
+                                # 发送缓存的数据
+                                if self.conn.tts_MessageText:
+                                    logger.bind(tag=TAG).info(
+                                        f"句子语音生成成功： {self.conn.tts_MessageText}"
+                                    )
+                                    self.tts_audio_queue.put(
+                                        (SentenceType.FIRST, [], self.conn.tts_MessageText)
+                                    )
+                                    self.conn.tts_MessageText = None
+                            elif event == "task-finished":
+                                logger.bind(tag=TAG).debug("TTS任务完成~")
+                                self._process_before_stop_play_files()
+                                session_finished = True
+                                break
+                            elif event == "task-failed":
+                                error_code = data["header"].get("error_code", "unknown")
+                                error_message = data["header"].get("error_message", "未知错误")
+                                logger.bind(tag=TAG).error(
+                                    f"TTS任务失败: {error_code} - {error_message}"
+                                )
+                                break
+                        except json.JSONDecodeError:
+                            logger.bind(tag=TAG).warning("收到无效的JSON消息")
+                    elif isinstance(msg, (bytes, bytearray)):
+                        self.opus_encoder.encode_pcm_to_opus_stream(
+                            msg, False, callback=self.handle_opus
+                        )
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).warning("WebSocket连接已关闭")
+                    break
+                except Exception as e:
+                    logger.bind(tag=TAG).error(
+                        f"处理TTS响应时出错: {e}\n{traceback.format_exc()}"
+                    )
+                    break
+
+            # 仅在连接异常且非正常结束时才关闭连接
+            if not session_finished and self.ws:
+                try:
+                    await self.ws.close()
+                except:
+                    pass
+                self.ws = None
+        # 监听任务退出时清理引用
+        finally:
+            self._monitor_task = None
+
+    def to_tts(self, text: str) -> list:
+        """非流式生成音频数据，用于生成音频及测试场景"""
+        try:
+            # 创建事件循环
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+            # 生成会话ID
+            session_id = uuid.uuid4().hex
+            # 存储音频数据
+            audio_data = []
+
+            async def _generate_audio():
+                ws = await websockets.connect(
+                    self.ws_url,
+                    additional_headers=self.header,
+                    ping_interval=30,
+                    ping_timeout=10,
+                    close_timeout=10,
+                    max_size=10 * 1024 * 1024,
+                )
+
+                try:
+                    # 发送run-task消息启动会话
+                    run_task_message = {
+                        "header": {
+                            "action": "run-task",
+                            "task_id": session_id,
+                            "streaming": "duplex",
+                        },
+                        "payload": {
+                            "task_group": "audio",
+                            "task": "tts",
+                            "function": "SpeechSynthesizer",
+                            "model": self.model,
+                            "parameters": {
+                                "text_type": "PlainText",
+                                "voice": self.voice,
+                                "format": self.format,
+                                "sample_rate": self.sample_rate,
+                                "volume": self.volume,
+                                "rate": self.rate,
+                                "pitch": self.pitch,
+                            },
+                            "input": {}
+                        },
+                    }
+                    await ws.send(json.dumps(run_task_message))
+
+                    # 等待任务启动
+                    task_started = False
+                    while not task_started:
+                        msg = await ws.recv()
+                        if isinstance(msg, str):
+                            data = json.loads(msg)
+                            header = data.get("header", {})
+                            if header.get("event") == "task-started":
+                                task_started = True
+                                logger.bind(tag=TAG).debug("TTS任务已启动")
+                            elif header.get("event") == "task-failed":
+                                error_code = header.get("error_code", "unknown")
+                                error_message = header.get("error_message", "未知错误")
+                                raise Exception(
+                                    f"启动任务失败: {error_code} - {error_message}"
+                                )
+
+                    # 发送文本
+                    filtered_text = MarkdownCleaner.clean_markdown(text)
+                    # 发送continue-task消息
+                    continue_task_message = {
+                        "header": {
+                            "action": "continue-task",
+                            "task_id": session_id,
+                            "streaming": "duplex",
+                        },
+                        "payload": {"input": {"text": filtered_text}},
+                    }
+                    await ws.send(json.dumps(continue_task_message))
+
+                    # 发送finish-task消息
+                    finish_task_message = {
+                        "header": {
+                            "action": "finish-task",
+                            "task_id": session_id,
+                            "streaming": "duplex",
+                        },
+                        "payload": {
+                            "input": {}
+                        }
+                    }
+                    await ws.send(json.dumps(finish_task_message))
+
+                    # 接收音频数据
+                    task_finished = False
+                    while not task_finished:
+                        msg = await ws.recv()
+                        if isinstance(msg, (bytes, bytearray)):
+                            self.opus_encoder.encode_pcm_to_opus_stream(
+                                msg,
+                                end_of_stream=False,
+                                callback=lambda opus: audio_data.append(opus)
+                            )
+                        elif isinstance(msg, str):
+                            data = json.loads(msg)
+                            header = data.get("header", {})
+                            if header.get("event") == "task-finished":
+                                task_finished = True
+                                logger.bind(tag=TAG).debug("TTS任务完成")
+                            elif header.get("event") == "task-failed":
+                                error_code = header.get("error_code", "unknown")
+                                error_message = header.get("error_message", "未知错误")
+                                raise Exception(
+                                    f"合成失败: {error_code} - {error_message}"
+                                )
+
+                finally:
+                    # 清理资源
+                    try:
+                        await ws.close()
+                    except:
+                        pass
+
+            # 运行异步任务
+            loop.run_until_complete(_generate_audio())
+            loop.close()
+
+            return audio_data
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}")
+            return []

main/xiaozhi-server/core/providers/tts/aliyun_stream.py

@@ -1,3 +1,4 @@
+import random
 import uuid
 import json
 import hmac
@@ -131,7 +132,7 @@ class TTSProvider(TTSProviderBase):
         self.last_active_time = None
 
         # 专属tts设置
-        self.message_id = ""
+        self.task_id = uuid.uuid4().hex
 
         # 创建Opus编码器
         self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
@@ -185,9 +186,10 @@ class TTSProvider(TTSProviderBase):
             current_time = time.time()
             if self.ws and current_time - self.last_active_time < 10:
                 # 10秒内才可以复用链接进行连续对话
-                logger.bind(tag=TAG).info(f"使用已有链接...")
+                self.task_id = uuid.uuid4().hex
+                logger.bind(tag=TAG).info(f"使用已有链接..., task_id: {self.task_id}")
                 return self.ws
-            logger.bind(tag=TAG).info("开始建立新连接...")
+            logger.bind(tag=TAG).debug("开始建立新连接...")
 
             self.ws = await websockets.connect(
                 self.ws_url,
@@ -196,7 +198,8 @@ class TTSProvider(TTSProviderBase):
                 ping_timeout=10,
                 close_timeout=10,
             )
-            logger.bind(tag=TAG).info("WebSocket连接建立成功")
+            self.task_id = uuid.uuid4().hex
+            logger.bind(tag=TAG).debug(f"WebSocket连接建立成功, task_id: {self.task_id}")
             self.last_active_time = time.time()
             return self.ws
         except Exception as e:
@@ -224,23 +227,14 @@ class TTSProvider(TTSProviderBase):
                 if message.sentence_type == SentenceType.FIRST:
                     # 初始化参数
                     try:
-                        if not getattr(self.conn, "sentence_id", None):
-                            self.conn.sentence_id = uuid.uuid4().hex
-                            logger.bind(tag=TAG).info(
-                                f"自动生成新的 会话ID: {self.conn.sentence_id}"
-                            )
-
-                        # aliyunStream独有的参数生成
-                        self.message_id = str(uuid.uuid4().hex)
-
-                        logger.bind(tag=TAG).info("开始启动TTS会话...")
+                        logger.bind(tag=TAG).debug("开始启动TTS会话...")
                         future = asyncio.run_coroutine_threadsafe(
-                            self.start_session(self.conn.sentence_id),
+                            self.start_session(self.task_id),
                             loop=self.conn.loop,
                         )
                         future.result()
                         self.before_stop_play_files.clear()
-                        logger.bind(tag=TAG).info("TTS会话启动成功")
+                        logger.bind(tag=TAG).debug("TTS会话启动成功")
 
                     except Exception as e:
                         logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
@@ -271,9 +265,9 @@ class TTSProvider(TTSProviderBase):
                         self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
                 if message.sentence_type == SentenceType.LAST:
                     try:
-                        logger.bind(tag=TAG).info("开始结束TTS会话...")
+                        logger.bind(tag=TAG).debug("开始结束TTS会话...")
                         future = asyncio.run_coroutine_threadsafe(
-                            self.finish_session(self.conn.sentence_id),
+                            self.finish_session(self.task_id),
                             loop=self.conn.loop,
                         )
                         future.result()
@@ -296,8 +290,8 @@ class TTSProvider(TTSProviderBase):
             filtered_text = MarkdownCleaner.clean_markdown(text)
             run_request = {
                 "header": {
-                    "message_id": self.message_id,
-                    "task_id": self.conn.sentence_id,
+                    "message_id": uuid.uuid4().hex,
+                    "task_id": self.task_id,
                     "namespace": "FlowingSpeechSynthesizer",
                     "name": "RunSynthesis",
                     "appkey": self.appkey,
@@ -318,8 +312,8 @@ class TTSProvider(TTSProviderBase):
                 self.ws = None
             raise
 
-    async def start_session(self, session_id):
-        logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
+    async def start_session(self, task_id):
+        logger.bind(tag=TAG).debug("开始会话～～")
         try:
             # 会话开始时检测上个会话的监听状态
             if (
@@ -340,8 +334,8 @@ class TTSProvider(TTSProviderBase):
 
             start_request = {
                 "header": {
-                    "message_id": self.message_id,
-                    "task_id": self.conn.sentence_id,
+                    "message_id": uuid.uuid4().hex,
+                    "task_id": self.task_id,
                     "namespace": "FlowingSpeechSynthesizer",
                     "name": "StartSynthesis",
                     "appkey": self.appkey,
@@ -358,28 +352,28 @@ class TTSProvider(TTSProviderBase):
             }
             await self.ws.send(json.dumps(start_request))
             self.last_active_time = time.time()
-            logger.bind(tag=TAG).info("会话启动请求已发送")
+            logger.bind(tag=TAG).debug("会话启动请求已发送")
         except Exception as e:
             logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
             # 确保清理资源
             await self.close()
             raise
 
-    async def finish_session(self, session_id):
-        logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
+    async def finish_session(self, task_id):
+        logger.bind(tag=TAG).debug(f"关闭会话～～{task_id}")
         try:
             if self.ws:
                 stop_request = {
                     "header": {
-                        "message_id": self.message_id,
-                        "task_id": self.conn.sentence_id,
+                        "message_id": uuid.uuid4().hex,
+                        "task_id": self.task_id,
                         "namespace": "FlowingSpeechSynthesizer",
                         "name": "StopSynthesis",
                         "appkey": self.appkey,
                     }
                 }
                 await self.ws.send(json.dumps(stop_request))
-                logger.bind(tag=TAG).info("会话结束请求已发送")
+                logger.bind(tag=TAG).debug("会话结束请求已发送")
                 self.last_active_time = time.time()
                 if self._monitor_task:
                     try:
@@ -457,7 +451,6 @@ class TTSProvider(TTSProviderBase):
                             logger.bind(tag=TAG).warning("收到无效的JSON消息")
                     # 二进制消息（音频数据）
                     elif isinstance(msg, (bytes, bytearray)):
-                        logger.bind(tag=TAG).debug(f"推送数据到队列里面～～")
                         self.opus_encoder.encode_pcm_to_opus_stream(msg, False, self.handle_opus)
                 except websockets.ConnectionClosed:
                     logger.bind(tag=TAG).warning("WebSocket连接已关闭")
@@ -485,8 +478,6 @@ class TTSProvider(TTSProviderBase):
             loop = asyncio.new_event_loop()
             asyncio.set_event_loop(loop)
 
-            # 生成会话ID
-            session_id = uuid.uuid4().hex
             # 存储音频数据
             audio_data = []
 
@@ -505,11 +496,10 @@ class TTSProvider(TTSProviderBase):
                 )
                 try:
                     # 发送StartSynthesis请求
-                    start_message_id = str(uuid.uuid4().hex)
                     start_request = {
                         "header": {
-                            "message_id": start_message_id,
-                            "task_id": session_id,
+                            "message_id": uuid.uuid4().hex,
+                            "task_id": self.task_id,
                             "namespace": "FlowingSpeechSynthesizer",
                             "name": "StartSynthesis",
                             "appkey": self.appkey,
@@ -550,11 +540,10 @@ class TTSProvider(TTSProviderBase):
 
                     # 发送文本合成请求
                     filtered_text = MarkdownCleaner.clean_markdown(text)
-                    run_message_id = str(uuid.uuid4().hex)
                     run_request = {
                         "header": {
-                            "message_id": run_message_id,
-                            "task_id": session_id,
+                            "message_id": uuid.uuid4().hex,
+                            "task_id": self.task_id,
                             "namespace": "FlowingSpeechSynthesizer",
                             "name": "RunSynthesis",
                             "appkey": self.appkey,
@@ -564,11 +553,10 @@ class TTSProvider(TTSProviderBase):
                     await ws.send(json.dumps(run_request))
 
                     # 发送停止合成请求
-                    stop_message_id = str(uuid.uuid4().hex)
                     stop_request = {
                         "header": {
-                            "message_id": stop_message_id,
-                            "task_id": session_id,
+                            "message_id": uuid.uuid4().hex,
+                            "task_id": self.task_id,
                             "namespace": "FlowingSpeechSynthesizer",
                             "name": "StopSynthesis",
                             "appkey": self.appkey,

main/xiaozhi-server/core/providers/tts/base.py

@@ -395,14 +395,6 @@ class TTSProviderBase(ABC):
 
                 # 收到下一个文本开始或会话结束时进行上报
                 if sentence_type is not SentenceType.MIDDLE:
-                    # 重置音频流控状态（新句子开始或者结束）
-                    if hasattr(self.conn, 'audio_flow_control'):
-                        self.conn.audio_flow_control = {
-                            'last_send_time': 0,
-                            'packet_count': 0,
-                            'start_time': time.perf_counter()
-                        }
-                    
                     # 上报TTS数据
                     if enqueue_text is not None and enqueue_audio is not None:
                         enqueue_tts_report(self.conn, enqueue_text, enqueue_audio)

main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py

@@ -149,6 +149,7 @@ class TTSProvider(TTSProviderBase):
         self.access_token = config.get("access_token")
         self.cluster = config.get("cluster")
         self.resource_id = config.get("resource_id")
+        self.activate_session = False
         if config.get("private_voice"):
             self.voice = config.get("private_voice")
         else:
@@ -162,7 +163,8 @@ class TTSProvider(TTSProviderBase):
         self.ws_url = config.get("ws_url")
         self.authorization = config.get("authorization")
         self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
-        self.enable_two_way = True
+        enable_ws_reuse_value = config.get("enable_ws_reuse", True)
+        self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() in ('false', 'False') else True
         self.tts_text = ""
         self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
             sample_rate=16000, channels=1, frame_size_ms=60
@@ -180,12 +182,18 @@ class TTSProvider(TTSProviderBase):
             raise
 
     async def _ensure_connection(self):
-        """建立新的WebSocket连接"""
+        """建立新的WebSocket连接，并启动监听任务（仅第一次）"""
         try:
             if self.ws:
-                logger.bind(tag=TAG).info(f"使用已有链接...")
-                return self.ws
-            logger.bind(tag=TAG).info("开始建立新连接...")
+                if self.enable_ws_reuse:
+                    logger.bind(tag=TAG).info(f"使用已有链接...")
+                    return self.ws
+                else:
+                    try:
+                        await self.finish_connection()
+                    except:
+                        pass
+            logger.bind(tag=TAG).debug("开始建立新连接...")
             ws_header = {
                 "X-Api-App-Key": self.appId,
                 "X-Api-Access-Key": self.access_token,
@@ -195,12 +203,34 @@ class TTSProvider(TTSProviderBase):
             self.ws = await websockets.connect(
                 self.ws_url, additional_headers=ws_header, max_size=1000000000
             )
-            logger.bind(tag=TAG).info("WebSocket连接建立成功")
+            logger.bind(tag=TAG).debug("WebSocket连接建立成功")
+            
+            # 连接建立成功后，启动监听任务
+            if self._monitor_task is None or self._monitor_task.done():
+                logger.bind(tag=TAG).debug("启动监听任务...")
+                self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
+            
             return self.ws
         except Exception as e:
             logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}")
             self.ws = None
             raise
+    
+    async def finish_connection(self):
+        """发送 FinishConnection 事件，等待服务端返回 EVENT_ConnectionFinished"""
+        try:
+            if self.ws:
+                logger.bind(tag=TAG).debug("开始关闭连接...")
+                header = Header(
+                    message_type=FULL_CLIENT_REQUEST,
+                    message_type_specific_flags=MsgTypeFlagWithEvent,
+                    serial_method=JSON,
+                ).as_bytes()
+                optional = Optional(event=EVENT_FinishConnection).as_bytes()
+                payload = str.encode("{}")
+                await self.send_event(self.ws, header, optional, payload)
+        except:
+            pass
 
     def tts_text_priority_thread(self):
         """火山引擎双流式TTS的文本处理线程"""
@@ -217,10 +247,16 @@ class TTSProvider(TTSProviderBase):
                 if self.conn.client_abort:
                     try:
                         logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
-                        asyncio.run_coroutine_threadsafe(
-                            self.cancel_session(self.conn.sentence_id),
-                            loop=self.conn.loop,
-                        )
+                        if self.enable_ws_reuse:
+                            asyncio.run_coroutine_threadsafe(
+                                self.cancel_session(self.conn.sentence_id),
+                                loop=self.conn.loop,
+                            )
+                        else:
+                            asyncio.run_coroutine_threadsafe(
+                                self.finish_connection(),
+                                loop=self.conn.loop,
+                            )
                         continue
                     except Exception as e:
                         logger.bind(tag=TAG).error(f"取消TTS会话失败: {str(e)}")
@@ -231,16 +267,16 @@ class TTSProvider(TTSProviderBase):
                     try:
                         if not getattr(self.conn, "sentence_id", None): 
                             self.conn.sentence_id = uuid.uuid4().hex
-                            logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}")
+                            logger.bind(tag=TAG).debug(f"自动生成新的 会话ID: {self.conn.sentence_id}")
 
-                        logger.bind(tag=TAG).info("开始启动TTS会话...")
+                        logger.bind(tag=TAG).debug("开始启动TTS会话...")
                         future = asyncio.run_coroutine_threadsafe(
                             self.start_session(self.conn.sentence_id),
                             loop=self.conn.loop,
                         )
                         future.result()
                         self.before_stop_play_files.clear()
-                        logger.bind(tag=TAG).info("TTS会话启动成功")
+                        logger.bind(tag=TAG).debug("TTS会话启动成功")
                     except Exception as e:
                         logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
                         continue
@@ -270,7 +306,7 @@ class TTSProvider(TTSProviderBase):
                         self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
                 if message.sentence_type == SentenceType.LAST:
                     try:
-                        logger.bind(tag=TAG).info("开始结束TTS会话...")
+                        logger.bind(tag=TAG).debug("开始结束TTS会话...")
                         future = asyncio.run_coroutine_threadsafe(
                             self.finish_session(self.conn.sentence_id),
                             loop=self.conn.loop,
@@ -313,23 +349,25 @@ class TTSProvider(TTSProviderBase):
             raise
 
     async def start_session(self, session_id):
-        logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
-        try:
-            # 会话开始时检测上个会话的监听状态
-            if (
-                self._monitor_task is not None
-                and isinstance(self._monitor_task, Task)
-                and not self._monitor_task.done()
-            ):
-                logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务和连接...")
+        logger.bind(tag=TAG).debug(f"开始会话～～{session_id}")
+        try:       
+            # 等待上一个会话结束，最多等待3次
+            for _ in range(3):
+                if not self.activate_session:
+                    break
+                logger.bind(tag=TAG).debug(f"等待上一个会话结束...")
+                await asyncio.sleep(0.1)
+            else:
+                # 等待超时，强制清除连接状态
+                logger.bind(tag=TAG).debug("等待上一个会话超时，清除连接状态...")
                 await self.close()
-
-            # 建立新连接
+            
+            # 设置会话激活标志
+            self.activate_session = True
+            
+            # 确保连接建立
             await self._ensure_connection()
 
-            # 启动监听任务
-            self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
-
             header = Header(
                 message_type=FULL_CLIENT_REQUEST,
                 message_type_specific_flags=MsgTypeFlagWithEvent,
@@ -342,7 +380,7 @@ class TTSProvider(TTSProviderBase):
                 event=EVENT_StartSession, speaker=self.voice
             )
             await self.send_event(self.ws, header, optional, payload)
-            logger.bind(tag=TAG).info("会话启动请求已发送")
+            logger.bind(tag=TAG).debug("会话启动请求已发送")
         except Exception as e:
             logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
             # 确保清理资源
@@ -350,7 +388,7 @@ class TTSProvider(TTSProviderBase):
             raise
 
     async def finish_session(self, session_id):
-        logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
+        logger.bind(tag=TAG).debug(f"关闭会话～～{session_id}")
         try:
             if self.ws:
                 header = Header(
@@ -363,18 +401,7 @@ class TTSProvider(TTSProviderBase):
                 ).as_bytes()
                 payload = str.encode("{}")
                 await self.send_event(self.ws, header, optional, payload)
-                logger.bind(tag=TAG).info("会话结束请求已发送")
-
-                # 等待监听任务完成
-                if self._monitor_task:
-                    try:
-                        await self._monitor_task
-                    except Exception as e:
-                        logger.bind(tag=TAG).error(
-                            f"等待监听任务完成时发生错误: {str(e)}"
-                        )
-                    finally:
-                        self._monitor_task = None
+                logger.bind(tag=TAG).debug("会话结束请求已发送")
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
@@ -383,7 +410,7 @@ class TTSProvider(TTSProviderBase):
             raise
 
     async def cancel_session(self,session_id):
-        logger.bind(tag=TAG).info(f"取消会话，释放服务端资源～～{session_id}")
+        logger.bind(tag=TAG).debug(f"取消会话，释放服务端资源～～{session_id}")
         try:
             if self.ws:
                 header = Header(
@@ -396,7 +423,7 @@ class TTSProvider(TTSProviderBase):
                 ).as_bytes()
                 payload = str.encode("{}")
                 await self.send_event(self.ws, header, optional, payload)
-                logger.bind(tag=TAG).info("会话取消请求已发送")
+                logger.bind(tag=TAG).debug("会话取消请求已发送")
         except Exception as e:
             logger.bind(tag=TAG).error(f"取消会话失败: {str(e)}")
             # 确保清理资源
@@ -405,6 +432,7 @@ class TTSProvider(TTSProviderBase):
 
     async def close(self):
         """资源清理方法"""
+        self.activate_session = False
         # 取消监听任务
         if self._monitor_task:
             try:
@@ -424,9 +452,8 @@ class TTSProvider(TTSProviderBase):
             self.ws = None
 
     async def _start_monitor_tts_response(self):
-        """监听TTS响应"""
+        """监听TTS响应 - 长期运行"""
         try:
-            session_finished = False  # 标记会话是否正常结束
             while not self.conn.stop_event.is_set():
                 try:
                     # 确保 `recv()` 运行在同一个 event loop
@@ -434,10 +461,22 @@ class TTSProvider(TTSProviderBase):
                     res = self.parser_response(msg)
                     self.print_response(res, "send_text res:")
 
+                    # 优先处理连接级别事件
+                    if res.optional.event == EVENT_ConnectionFinished:
+                        logger.bind(tag=TAG).debug(f"链接关闭成功～～")
+                        break
+
+                    # 只处理当前活跃会话的响应
+                    if res.optional.sessionId and self.conn.sentence_id != res.optional.sessionId:
+                        # 如果是会话结束相关事件，即使会话ID不匹配也要重置状态
+                        if res.optional.event in [EVENT_SessionCanceled, EVENT_SessionFailed, EVENT_SessionFinished]:
+                            logger.bind(tag=TAG).debug(f"收到残余下行结束响应重置会话状态～～")
+                            self.activate_session = False
+                        continue
+
                     if res.optional.event == EVENT_SessionCanceled:
                         logger.bind(tag=TAG).debug(f"释放服务端资源成功～～")
-                        session_finished = True
-                        break
+                        self.activate_session = False
                     elif res.optional.event == EVENT_TTSSentenceStart:
                         json_data = json.loads(res.payload.decode("utf-8"))
                         self.tts_text = json_data.get("text", "")
@@ -449,15 +488,16 @@ class TTSProvider(TTSProviderBase):
                         res.optional.event == EVENT_TTSResponse
                         and res.header.message_type == AUDIO_ONLY_RESPONSE
                     ):
-                        logger.bind(tag=TAG).debug(f"推送数据到队列里面～～")
                         self.wav_to_opus_data_audio_raw_stream(res.payload, callback=self.handle_opus)
                     elif res.optional.event == EVENT_TTSSentenceEnd:
                         logger.bind(tag=TAG).info(f"句子语音生成成功：{self.tts_text}")
                     elif res.optional.event == EVENT_SessionFinished:
                         logger.bind(tag=TAG).debug(f"会话结束～～")
+                        self.activate_session = False
                         self._process_before_stop_play_files()
-                        session_finished = True
-                        break
+                        # 非复用模式下，会话结束后发送 FinishConnection
+                        if not self.enable_ws_reuse:
+                            await self.finish_connection()
                 except websockets.ConnectionClosed:
                     logger.bind(tag=TAG).warning("WebSocket连接已关闭")
                     break
@@ -467,8 +507,8 @@ class TTSProvider(TTSProviderBase):
                     )
                     traceback.print_exc()
                     break
-            # 仅在连接异常时才关闭
-            if not session_finished and self.ws:
+            # 连接异常时关闭WebSocket
+            if self.ws:
                 try:
                     await self.ws.close()
                 except:
@@ -476,6 +516,7 @@ class TTSProvider(TTSProviderBase):
                 self.ws = None
         # 监听任务退出时清理引用
         finally:
+            self.activate_session = False
             self._monitor_task = None
 
     async def send_event(
@@ -514,7 +555,7 @@ class TTSProvider(TTSProviderBase):
     def read_res_content(self, res: bytes, offset: int):
         content_size = int.from_bytes(res[offset : offset + 4], "big", signed=True)
         offset += 4
-        content = str(res[offset : offset + content_size])
+        content = res[offset : offset + content_size].decode('utf-8')
         offset += content_size
         return content, offset
 
@@ -616,12 +657,13 @@ class TTSProvider(TTSProviderBase):
                             "speech_rate": self.speech_rate,
                             "loudness_rate": self.loudness_rate
                         },
+                        "additions": json.dumps({
+                            "post_process": {
+                                "pitch": self.pitch
+                            }
+                        })
                     },
-                    "additions": {
-                        "post_process": {
-                            "pitch": self.pitch
-                        }
-                    }
+
                 }
             )
         )

main/xiaozhi-server/core/providers/tts/index_stream.py

@@ -190,7 +190,7 @@ class TTSProvider(TTSProviderBase):
         start_time = time.time()
         text = MarkdownCleaner.clean_markdown(text)
 
-        payload = {"text": text, "character": self.character}
+        payload = {"text": text, "character": self.voice}
 
         try:
             with requests.post(self.api_url, json=payload, timeout=5) as response:

main/xiaozhi-server/core/providers/tts/minimax.py

@@ -1,95 +0,0 @@
-import os
-import uuid
-import json
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-from core.utils.util import parse_string_to_list
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.group_id = config.get("group_id")
-        self.api_key = config.get("api_key")
-        self.model = config.get("model")
-        if config.get("private_voice"):
-            self.voice = config.get("private_voice")
-        else:
-            self.voice = config.get("voice_id")
-
-        default_voice_setting = {
-            "voice_id": "female-shaonv",
-            "speed": 1,
-            "vol": 1,
-            "pitch": 0,
-            "emotion": "happy",
-        }
-        default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}
-        defult_audio_setting = {
-            "sample_rate": 32000,
-            "bitrate": 128000,
-            "format": "mp3",
-            "channel": 1,
-        }
-        self.voice_setting = {
-            **default_voice_setting,
-            **config.get("voice_setting", {}),
-        }
-        self.pronunciation_dict = {
-            **default_pronunciation_dict,
-            **config.get("pronunciation_dict", {}),
-        }
-        self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
-        self.timber_weights = parse_string_to_list(config.get("timber_weights"))
-
-        if self.voice:
-            self.voice_setting["voice_id"] = self.voice
-
-        self.host = "api.minimax.chat"
-        self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
-        self.header = {
-            "Content-Type": "application/json",
-            "Authorization": f"Bearer {self.api_key}",
-        }
-        self.audio_file_type = defult_audio_setting.get("format", "mp3")
-
-    def generate_filename(self, extension=".mp3"):
-        return os.path.join(
-            self.output_file,
-            f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
-        )
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "model": self.model,
-            "text": text,
-            "stream": False,
-            "voice_setting": self.voice_setting,
-            "pronunciation_dict": self.pronunciation_dict,
-            "audio_setting": self.audio_setting,
-        }
-
-        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
-            request_json["timber_weights"] = self.timber_weights
-            request_json["voice_setting"]["voice_id"] = ""
-
-        try:
-            resp = requests.post(
-                self.api_url, json.dumps(request_json), headers=self.header
-            )
-            # 检查返回请求数据的status_code是否为0
-            if resp.json()["base_resp"]["status_code"] == 0:
-                data = resp.json()["data"]["audio"]
-                audio_bytes = bytes.fromhex(data)
-                if output_file:
-                    with open(output_file, "wb") as file_to_save:
-                        file_to_save.write(audio_bytes)
-                else:
-                    return audio_bytes
-            else:
-                raise Exception(
-                    f"{__name__} status_code: {resp.status_code} response: {resp.content}"
-                )
-        except Exception as e:
-            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/minimax_httpstream.py

@@ -1,11 +1,20 @@
 import os
-import uuid
 import json
+import time
+import queue
+import asyncio
+import aiohttp
 import requests
-from datetime import datetime
-from typing import Iterator, Optional, Union
-from core.providers.tts.base import TTSProviderBase
+import traceback
+from config.logger import setup_logging
+from core.utils.tts import MarkdownCleaner
 from core.utils.util import parse_string_to_list
+from core.providers.tts.base import TTSProviderBase
+from core.utils import opus_encoder_utils, textUtils
+from core.providers.tts.dto.dto import SentenceType, ContentType
+
+TAG = __name__
+logger = setup_logging()
 
 
 class TTSProvider(TTSProviderBase):
@@ -28,9 +37,9 @@ class TTSProvider(TTSProviderBase):
         }
         default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}
         defult_audio_setting = {
-            "sample_rate": 32000,
+            "sample_rate": 24000,
             "bitrate": 128000,
-            "format": "mp3",
+            "format": "pcm",
             "channel": 1,
         }
         self.voice_setting = {
@@ -47,66 +56,101 @@ class TTSProvider(TTSProviderBase):
         if self.voice:
             self.voice_setting["voice_id"] = self.voice
 
-        self.host = "api.minimax.chat"
+        self.host = "api.minimaxi.com"  # 备用地址：api-bj.minimaxi.com
         self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
         self.header = {
             "Content-Type": "application/json",
             "Authorization": f"Bearer {self.api_key}",
         }
-        self.audio_file_type = defult_audio_setting.get("format", "mp3")
+        self.audio_file_type = defult_audio_setting.get("format", "pcm")
 
-    def generate_filename(self, extension=".mp3"):
-        return os.path.join(
-            self.output_file,
-            f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
+        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
+            sample_rate=24000, channels=1, frame_size_ms=60
         )
 
-    async def text_to_speak(self, text, output_file):
-        """非流式语音合成（保留原有实现）"""
-        request_json = {
-            "model": self.model,
-            "text": text,
-            "stream": False,
-            "voice_setting": self.voice_setting,
-            "pronunciation_dict": self.pronunciation_dict,
-            "audio_setting": self.audio_setting,
-        }
+        # PCM缓冲区
+        self.pcm_buffer = bytearray()
 
-        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
-            request_json["timber_weights"] = self.timber_weights
-            request_json["voice_setting"]["voice_id"] = ""
+    def tts_text_priority_thread(self):
+        """流式文本处理线程"""
+        while not self.conn.stop_event.is_set():
+            try:
+                message = self.tts_text_queue.get(timeout=1)
+                if message.sentence_type == SentenceType.FIRST:
+                    # 初始化参数
+                    self.tts_stop_request = False
+                    self.processed_chars = 0
+                    self.tts_text_buff = []
+                    self.before_stop_play_files.clear()
+                elif ContentType.TEXT == message.content_type:
+                    self.tts_text_buff.append(message.content_detail)
+                    segment_text = self._get_segment_text()
+                    if segment_text:
+                        self.to_tts_single_stream(segment_text)
 
-        try:
-            resp = requests.post(
-                self.api_url, json.dumps(request_json), headers=self.header
-            )
-            if resp.json()["base_resp"]["status_code"] == 0:
-                data = resp.json()["data"]["audio"]
-                audio_bytes = bytes.fromhex(data)
-                if output_file:
-                    with open(output_file, "wb") as file_to_save:
-                        file_to_save.write(audio_bytes)
-                else:
-                    return audio_bytes
+                elif ContentType.FILE == message.content_type:
+                    logger.bind(tag=TAG).info(
+                        f"添加音频文件到待播放列表: {message.content_file}"
+                    )
+                    if message.content_file and os.path.exists(message.content_file):
+                        # 先处理文件音频数据
+                        self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
+                if message.sentence_type == SentenceType.LAST:
+                    # 处理剩余的文本
+                    self._process_remaining_text_stream(True)
+
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
+                )
+
+    def _process_remaining_text_stream(self, is_last=False):
+        """处理剩余的文本并生成语音
+        Returns:
+            bool: 是否成功处理了文本
+        """
+        full_text = "".join(self.tts_text_buff)
+        remaining_text = full_text[self.processed_chars :]
+        if remaining_text:
+            segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text)
+            if segment_text:
+                self.to_tts_single_stream(segment_text, is_last)
+                self.processed_chars += len(full_text)
             else:
-                raise Exception(
-                    f"{__name__} status_code: {resp.status_code} response: {resp.content}"
+                self._process_before_stop_play_files()
+        else:
+            self._process_before_stop_play_files()
+
+    def to_tts_single_stream(self, text, is_last=False):
+        try:
+            max_repeat_time = 5
+            text = MarkdownCleaner.clean_markdown(text)
+            try:
+                asyncio.run(self.text_to_speak(text, is_last))
+            except Exception as e:
+                logger.bind(tag=TAG).warning(
+                    f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}"
+                )
+                max_repeat_time -= 1
+
+            if max_repeat_time > 0:
+                logger.bind(tag=TAG).info(
+                    f"语音生成成功: {text}，重试{5 - max_repeat_time}次"
+                )
+            else:
+                logger.bind(tag=TAG).error(
+                    f"语音生成失败: {text}，请检查网络或服务是否正常"
                 )
         except Exception as e:
-            raise Exception(f"{__name__} error: {e}")
+            logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
+        finally:
+            return None
 
-    def text_to_speak_stream(
-        self, 
-        text: str, 
-        chunk_callback: Optional[callable] = None
-    ) -> Iterator[bytes]:
-        """
-        流式语音合成方法
-        :param text: 要合成的文本
-        :param chunk_callback: 可选的回调函数，用于处理每个音频块
-        :return: 生成器，每次产生一个音频数据块(bytes)
-        """
-        request_json = {
+    async def text_to_speak(self, text, is_last):
+        """流式处理TTS音频，每句只推送一次音频列表"""
+        payload = {
             "model": self.model,
             "text": text,
             "stream": True,
@@ -115,116 +159,183 @@ class TTSProvider(TTSProviderBase):
             "audio_setting": self.audio_setting,
         }
 
-        if isinstance(self.timber_weights, list) and len(self.timber_weights) > 0:
-            request_json["timber_weights"] = self.timber_weights
-            request_json["voice_setting"]["voice_id"] = ""
+        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
+            payload["timber_weights"] = self.timber_weights
+            payload["voice_setting"]["voice_id"] = ""
+
+        frame_bytes = int(
+            self.opus_encoder.sample_rate
+            * self.opus_encoder.channels  # 1
+            * self.opus_encoder.frame_size_ms
+            / 1000
+            * 2
+        )  # 16-bit = 2 bytes
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(
+                    self.api_url,
+                    headers=self.header,
+                    data=json.dumps(payload),
+                    timeout=10,
+                ) as resp:
+
+                    if resp.status != 200:
+                        logger.bind(tag=TAG).error(
+                            f"TTS请求失败: {resp.status}, {await resp.text()}"
+                        )
+                        self.tts_audio_queue.put((SentenceType.LAST, [], None))
+                        return
+
+                    self.pcm_buffer.clear()
+                    self.tts_audio_queue.put((SentenceType.FIRST, [], text))
+
+                    # 处理音频流数据
+                    buffer = b""
+                    async for chunk in resp.content.iter_any():
+                        if not chunk:
+                            continue
+
+                        buffer += chunk
+                        while True:
+                            # 查找数据块分隔符
+                            header_pos = buffer.find(b"data: ")
+                            if header_pos == -1:
+                                break
+
+                            end_pos = buffer.find(b"\n\n", header_pos)
+                            if end_pos == -1:
+                                break
+
+                            # 提取单个完整JSON块
+                            json_str = buffer[header_pos + 6 : end_pos].decode("utf-8")
+                            buffer = buffer[end_pos + 2 :]
+
+                            try:
+                                data = json.loads(json_str)
+                                status = data.get("data", {}).get("status", 1)
+                                audio_hex = data.get("data", {}).get("audio")
+
+                                # 仅处理status=1的有效音频块 忽略status=2的结束汇总块
+                                if status == 1 and audio_hex:
+                                    pcm_data = bytes.fromhex(audio_hex)
+                                    self.pcm_buffer.extend(pcm_data)
+
+                            except json.JSONDecodeError as e:
+                                logger.bind(tag=TAG).error(f"JSON解析失败: {e}")
+                                continue
+
+                        while len(self.pcm_buffer) >= frame_bytes:
+                            frame = bytes(self.pcm_buffer[:frame_bytes])
+                            del self.pcm_buffer[:frame_bytes]
+
+                            self.opus_encoder.encode_pcm_to_opus_stream(
+                                frame, end_of_stream=False, callback=self.handle_opus
+                            )
+
+                    # flush 剩余不足一帧的数据
+                    if self.pcm_buffer:
+                        self.opus_encoder.encode_pcm_to_opus_stream(
+                            bytes(self.pcm_buffer),
+                            end_of_stream=True,
+                            callback=self.handle_opus,
+                        )
+                        self.pcm_buffer.clear()
+
+                    # 如果是最后一段，输出音频获取完毕
+                    if is_last:
+                        self._process_before_stop_play_files()
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
+            self.tts_audio_queue.put((SentenceType.LAST, [], None))
+
+    async def close(self):
+        """资源清理"""
+        await super().close()
+        if hasattr(self, "opus_encoder"):
+            self.opus_encoder.close()
+
+    def to_tts(self, text: str) -> list:
+        """非流式TTS处理，用于测试及保存音频文件的场景
+        Args:
+            text: 要转换的文本
+        Returns:
+            list: 返回opus编码后的音频数据列表
+        """
+        start_time = time.time()
+        text = MarkdownCleaner.clean_markdown(text)
+
+        payload = {
+            "model": self.model,
+            "text": text,
+            "stream": True,
+            "voice_setting": self.voice_setting,
+            "pronunciation_dict": self.pronunciation_dict,
+            "audio_setting": self.audio_setting,
+        }
+
+        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
+            payload["timber_weights"] = self.timber_weights
+            payload["voice_setting"]["voice_id"] = ""
+
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}",
+        }
 
         try:
             with requests.post(
-                self.api_url,
-                data=json.dumps(request_json),
-                headers=self.header,
-                stream=True
+                self.api_url, data=json.dumps(payload), headers=headers, timeout=5
             ) as response:
-                
-                # 检查HTTP状态码
                 if response.status_code != 200:
-                    raise Exception(
-                        f"HTTP error: {response.status_code}, response: {response.text}"
+                    logger.bind(tag=TAG).error(
+                        f"TTS请求失败: {response.status_code}, {response.text}"
                     )
-                
-                # 处理流式响应
-                for line in response.iter_lines():
-                    if line:  # 过滤空行
-                        # 检查是否为数据行 (SSE格式)
-                        if line.startswith(b'data:'):
-                            try:
-                                data = json.loads(line[5:].strip())  # 去掉"data:"前缀
-                                
-                                # 检查API状态码
-                                if data.get("base_resp", {}).get("status_code", -1) != 0:
-                                    raise Exception(
-                                        f"API error: {data.get('base_resp', {}).get('status_msg')}"
-                                    )
-                                
-                                # 跳过非音频数据块
-                                if "extra_info" in data:
-                                    continue
-                                    
-                                # 提取音频数据
-                                audio_hex = data.get("data", {}).get("audio")
-                                if audio_hex:
-                                    audio_chunk = bytes.fromhex(audio_hex)
-                                    if chunk_callback:
-                                        chunk_callback(audio_chunk)
-                                    yield audio_chunk
-                                    
-                            except json.JSONDecodeError:
-                                # 忽略JSON解析错误（可能是心跳包等）
-                                continue
-                            except Exception as e:
-                                raise e
-                                
-        except Exception as e:
-            raise Exception(f"{__name__} stream error: {e}")
+                    return []
 
-    def save_stream_to_file(
-        self, 
-        text: str, 
-        output_file: Optional[str] = None,
-        progress_callback: Optional[callable] = None
-    ) -> str:
-        """
-        流式合成并保存到文件
-        :param text: 要合成的文本
-        :param output_file: 输出文件路径，如果为None则自动生成
-        :param progress_callback: 可选的回调函数，接收已写入的字节数
-        :return: 保存的文件路径
-        """
-        if not output_file:
-            output_file = self.generate_filename(extension=f".{self.audio_file_type}")
-        
-        os.makedirs(os.path.dirname(output_file), exist_ok=True)
-        
-        total_bytes = 0
-        try:
-            with open(output_file, "wb") as audio_file:
-                for audio_chunk in self.text_to_speak_stream(text):
-                    audio_file.write(audio_chunk)
-                    audio_file.flush()
-                    total_bytes += len(audio_chunk)
-                    if progress_callback:
-                        progress_callback(total_bytes)
-            return output_file
-        except Exception as e:
-            # 清理可能创建的不完整文件
-            if os.path.exists(output_file):
-                os.remove(output_file)
-            raise e
+                logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒")
+
+                # 使用opus编码器处理PCM数据
+                opus_datas = []
+                full_content = response.content.decode('utf-8')
+                pcm_data = bytearray()
+                for data_block in full_content.split('\n\n'):
+                    if not data_block.startswith('data: '):
+                        continue
+
+                    try:
+                        json_str = data_block[6:]  # 去除'data: '前缀
+                        data = json.loads(json_str)
+                        if data.get('data', {}).get('status') == 1:
+                            audio_hex = data['data']['audio']
+                            pcm_data.extend(bytes.fromhex(audio_hex))
+                    except (json.JSONDecodeError, KeyError) as e:
+                        logger.bind(tag=TAG).warning(f"无效数据块: {e}")
+                        continue
+
+                # 计算每帧的字节数
+                frame_bytes = int(
+                    self.opus_encoder.sample_rate
+                    * self.opus_encoder.channels
+                    * self.opus_encoder.frame_size_ms
+                    / 1000
+                    * 2
+                )
+
+                # 分帧处理合并后的PCM数据
+                for i in range(0, len(pcm_data), frame_bytes):
+                    frame = bytes(pcm_data[i:i+frame_bytes])
+                    if len(frame) < frame_bytes:
+                        frame += b"\x00" * (frame_bytes - len(frame))
+ 
+                    self.opus_encoder.encode_pcm_to_opus_stream(
+                        frame,
+                        end_of_stream=(i + frame_bytes >= len(pcm_data)),
+                        callback=lambda opus: opus_datas.append(opus)
+                    )
+
+                return opus_datas
 
-    def stream_to_audio_player(self, text: str, player_command: list = None):
-        """
-        流式合成并直接播放音频
-        :param text: 要合成的文本
-        :param player_command: 音频播放器命令，默认使用mpv
-        """
-        if player_command is None:
-            player_command = ["mpv", "--no-cache", "--no-terminal", "--", "fd://0"]
-        
-        try:
-            import subprocess
-            player_process = subprocess.Popen(
-                player_command,
-                stdin=subprocess.PIPE,
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.DEVNULL,
-            )
-            
-            for audio_chunk in self.text_to_speak_stream(text):
-                player_process.stdin.write(audio_chunk)
-                player_process.stdin.flush()
-                
-            player_process.stdin.close()
-            player_process.wait()
         except Exception as e:
-            raise Exception(f"Audio player error: {e}")
+            logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
+            return []

main/xiaozhi-server/core/providers/tts/minimax_webSocket.py

@@ -1,180 +0,0 @@
-import os
-import uuid
-import json
-import asyncio
-import websockets
-import ssl
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-from core.utils.util import parse_string_to_list
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.group_id = config.get("group_id")
-        self.api_key = config.get("api_key")
-        self.model = config.get("model")
-        
-        # 初始化语音设置
-        default_voice_setting = {
-            "voice_id": "female-shaonv",
-            "speed": 1,
-            "vol": 1,
-            "pitch": 0,
-            "emotion": "happy",
-        }
-        default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}
-        default_audio_setting = {
-            "sample_rate": 32000,
-            "bitrate": 128000,
-            "format": "mp3",
-            "channel": 1,
-        }
-        
-        # 合并配置
-        self.voice_setting = {
-            **default_voice_setting,
-            **config.get("voice_setting", {}),
-        }
-        self.pronunciation_dict = {
-            **default_pronunciation_dict,
-            **config.get("pronunciation_dict", {}),
-        }
-        self.audio_setting = {
-            **default_audio_setting, 
-            **config.get("audio_setting", {})
-        }
-        self.timber_weights = parse_string_to_list(config.get("timber_weights"))
-        
-        # 设置语音ID
-        if config.get("private_voice"):
-            self.voice_setting["voice_id"] = config.get("private_voice")
-        elif config.get("voice_id"):
-            self.voice_setting["voice_id"] = config.get("voice_id")
-        
-        # WebSocket配置
-        self.ws_url = "wss://api.minimaxi.com/ws/v1/t2a_v2"
-        self.headers = {
-            "Authorization": f"Bearer {self.api_key}",
-            "GroupId": self.group_id
-        }
-        self.audio_file_type = self.audio_setting.get("format", "mp3")
-
-    def generate_filename(self, extension=".mp3"):
-        """生成唯一的音频文件名"""
-        return os.path.join(
-            self.output_file,
-            f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
-        )
-
-    async def _establish_connection(self):
-        """建立WebSocket连接"""
-        ssl_context = ssl.create_default_context()
-        ssl_context.check_hostname = False
-        ssl_context.verify_mode = ssl.CERT_NONE
-
-        try:
-            ws = await websockets.connect(
-                self.ws_url,
-                additional_headers=self.headers,
-                ssl=ssl_context
-            )
-            connected = json.loads(await ws.recv())
-            if connected.get("event") == "connected_success":
-                print("连接成功")
-                return ws
-            return None
-        except Exception as e:
-            print(f"连接失败: {e}")
-            return None
-
-    async def _start_task(self, websocket):
-        """发送任务开始请求"""
-        start_msg = {
-            "event": "task_start",
-            "model": self.model,
-            "voice_setting": self.voice_setting,
-            "pronunciation_dict": self.pronunciation_dict,
-            "audio_setting": self.audio_setting
-        }
-        
-        if self.timber_weights and len(self.timber_weights) > 0:
-            start_msg["timber_weights"] = self.timber_weights
-            start_msg["voice_setting"]["voice_id"] = ""
-
-        await websocket.send(json.dumps(start_msg))
-        response = json.loads(await websocket.recv())
-        return response.get("event") == "task_started"
-
-    async def _continue_task(self, websocket, text):
-        """发送继续请求并收集音频数据"""
-        await websocket.send(json.dumps({
-            "event": "task_continue",
-            "text": text
-        }))
-
-        audio_chunks = []
-        while True:
-            response = json.loads(await websocket.recv())
-            if "data" in response and "audio" in response["data"]:
-                audio_chunks.append(response["data"]["audio"])
-            if response.get("is_final"):
-                break
-        return "".join(audio_chunks)
-
-    async def _close_connection(self, websocket):
-        """关闭连接"""
-        if websocket:
-            await websocket.send(json.dumps({"event": "task_finish"}))
-            await websocket.close()
-            print("连接已关闭")
-
-    async def text_to_speak(self, text, output_file=None):
-        """主方法：文本转语音"""
-        ws = await self._establish_connection()
-        if not ws:
-            raise Exception("无法建立WebSocket连接")
-
-        try:
-            if not await self._start_task(ws):
-                raise Exception("任务启动失败")
-
-            hex_audio = await self._continue_task(ws, text)
-            audio_bytes = bytes.fromhex(hex_audio)
-
-            # 保存到文件或返回二进制数据
-            if output_file:
-                with open(output_file, "wb") as f:
-                    f.write(audio_bytes)
-                print(f"音频已保存为{output_file}")
-                return output_file
-            else:
-                # 返回音频二进制数据（不播放）
-                return audio_bytes
-
-        finally:
-            await self._close_connection(ws)
-
-
-async def main():
-    """测试用主函数"""
-    # 示例配置
-    config = {
-        "group_id": "YOUR_GROUP_ID",  # 替换为实际的group_id
-        "api_key": "YOUR_API_KEY",    # 替换为实际的api_key
-        "model": "your-model",        # 替换为实际的模型名称
-        "voice_id": "male-qn-qingse",
-        "voice_setting": {
-            "speed": 1.2,
-            "emotion": "happy"
-        }
-    }
-    
-    tts = TTSProvider(config, delete_audio_file=True)
-    output_file = tts.generate_filename()
-    await tts.text_to_speak("这是一个测试文本，用于验证流式语音合成功能", output_file)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

main/xiaozhi-server/core/providers/tts/xunfei_stream.py

@@ -0,0 +1,527 @@
+import os
+import time
+import uuid
+import json
+import hmac
+import queue
+import base64
+import hashlib
+import asyncio
+import traceback
+import websockets
+from asyncio import Task
+from config.logger import setup_logging
+from core.utils import opus_encoder_utils
+from core.utils.tts import MarkdownCleaner
+from urllib.parse import urlencode, urlparse
+from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+
+class XunfeiWSAuth:
+    @staticmethod
+    def create_auth_url(api_key, api_secret, api_url):
+        """生成讯飞WebSocket认证URL"""
+        parsed_url = urlparse(api_url)
+        host = parsed_url.netloc
+        path = parsed_url.path
+
+        # 获取UTC时间，讯飞要求使用RFC1123格式
+        now = time.gmtime()
+        date = time.strftime('%a, %d %b %Y %H:%M:%S GMT', now)
+
+        # 构造签名字符串
+        signature_origin = f"host: {host}\ndate: {date}\nGET {path} HTTP/1.1"
+
+        # 计算签名
+        signature_sha = hmac.new(
+            api_secret.encode('utf-8'),
+            signature_origin.encode('utf-8'),
+            digestmod=hashlib.sha256
+        ).digest()
+        signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding='utf-8')
+
+        # 构造authorization
+        authorization_origin = f'api_key="{api_key}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha_base64}"'
+        authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8')
+
+        # 构造最终的WebSocket URL
+        v = {
+            "authorization": authorization,
+            "date": date,
+            "host": host
+        }
+        url = api_url + '?' + urlencode(v)
+        return url
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+
+        # 设置为流式接口类型
+        self.interface_type = InterfaceType.DUAL_STREAM
+
+        # 基础配置
+        self.app_id = config.get("app_id")
+        self.api_key = config.get("api_key")
+        self.api_secret = config.get("api_secret")
+
+        # 接口地址
+        self.api_url = config.get("api_url", "wss://cbm01.cn-huabei-1.xf-yun.com/v1/private/mcd9m97e6")
+
+        # 音色配置
+        self.voice = config.get("voice", "x5_lingxiaoxuan_flow")
+        if config.get("private_voice"):
+            self.voice = config.get("private_voice")
+
+        # 音频参数配置
+        speed = config.get("speed", "50")
+        self.speed = int(speed) if speed else 50
+
+        volume = config.get("volume", "50")
+        self.volume = int(volume) if volume else 50
+
+        pitch = config.get("pitch", "50")
+        self.pitch = int(pitch) if pitch else 50
+
+        # 音频编码配置
+        self.format = config.get("format", "raw")
+
+        sample_rate = config.get("sample_rate", "24000")
+        self.sample_rate = int(sample_rate) if sample_rate else 24000
+
+        # 口语化配置
+        self.oral_level = config.get("oral_level", "mid")
+
+        spark_assist = config.get("spark_assist", "1")
+        self.spark_assist = int(spark_assist) if spark_assist else 1
+
+        stop_split = config.get("stop_split", "0")
+        self.stop_split = int(stop_split) if stop_split else 0
+    
+        remain = config.get("remain", "0")
+        self.remain = int(remain) if remain else 0
+
+        # WebSocket配置
+        self.ws = None
+        self._monitor_task = None
+
+        # 序列号管理
+        self.text_seq = 0
+
+        # 创建Opus编码器
+        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
+            sample_rate=self.sample_rate, channels=1, frame_size_ms=60
+        )
+
+        # 验证必需参数
+        if not all([self.app_id, self.api_key, self.api_secret]):
+            raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
+
+    async def _ensure_connection(self):
+        """确保WebSocket连接可用"""
+        try:
+            logger.bind(tag=TAG).info("开始建立新连接...")
+
+            # 生成认证URL
+            auth_url = XunfeiWSAuth.create_auth_url(
+                self.api_key, self.api_secret, self.api_url
+            )
+
+            self.ws = await websockets.connect(
+                auth_url,
+                ping_interval=30,
+                ping_timeout=10,
+                close_timeout=10,
+            )
+            logger.bind(tag=TAG).info("WebSocket连接建立成功")
+            return self.ws
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}")
+            self.ws = None
+            raise
+
+    def tts_text_priority_thread(self):
+        """流式文本处理线程"""
+        while not self.conn.stop_event.is_set():
+            try:
+                message = self.tts_text_queue.get(timeout=1)
+                logger.bind(tag=TAG).debug(
+                    f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}"
+                )
+
+                if message.sentence_type == SentenceType.FIRST:
+                    # 重置序列号
+                    self.text_seq = 0
+                    self.conn.client_abort = False
+                # 增加序列号
+                self.text_seq += 1
+                if self.conn.client_abort:
+                    logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
+                    continue
+
+                if message.sentence_type == SentenceType.FIRST:
+                    # 初始化参数
+                    try:
+                        if not getattr(self.conn, "sentence_id", None):
+                            self.conn.sentence_id = uuid.uuid4().hex
+                            logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}")
+
+                        logger.bind(tag=TAG).info("开始启动TTS会话...")
+                        future = asyncio.run_coroutine_threadsafe(
+                            self.start_session(self.conn.sentence_id),
+                            loop=self.conn.loop,
+                        )
+                        future.result()
+                        self.before_stop_play_files.clear()
+                        logger.bind(tag=TAG).info("TTS会话启动成功")
+
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
+                        continue
+
+                # 处理文本内容
+                if ContentType.TEXT == message.content_type:
+                    if message.content_detail:
+                        try:
+                            logger.bind(tag=TAG).debug(
+                                f"开始发送TTS文本: {message.content_detail}"
+                            )
+                            future = asyncio.run_coroutine_threadsafe(
+                                self.text_to_speak(message.content_detail, None),
+                                loop=self.conn.loop,
+                            )
+                            future.result()
+                            logger.bind(tag=TAG).debug("TTS文本发送成功")
+                        except Exception as e:
+                            logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
+                            # 不使用continue，确保后续处理不被中断
+
+                # 处理文件内容
+                if ContentType.FILE == message.content_type:
+                    logger.bind(tag=TAG).info(
+                        f"添加音频文件到待播放列表: {message.content_file}"
+                    )
+                    if message.content_file and os.path.exists(message.content_file):
+                        # 先处理文件音频数据
+                        self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
+
+                # 处理会话结束
+                if message.sentence_type == SentenceType.LAST:
+                    try:
+                        logger.bind(tag=TAG).info("开始结束TTS会话...")
+                        asyncio.run_coroutine_threadsafe(
+                            self.finish_session(self.conn.sentence_id),
+                            loop=self.conn.loop,
+                        )
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"结束TTS会话失败: {str(e)}")
+                        continue
+
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
+                )
+
+    async def text_to_speak(self, text, _):
+        """发送文本到TTS服务进行合成"""
+        try:
+            if self.ws is None:
+                logger.bind(tag=TAG).warning(f"WebSocket连接不存在，终止发送文本")
+                return
+
+            filtered_text = MarkdownCleaner.clean_markdown(text)
+
+            # 发送文本合成请求
+            run_request = self._build_base_request(status=1,text=filtered_text)
+            await self.ws.send(json.dumps(run_request))
+            return
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
+            if self.ws:
+                try:
+                    await self.ws.close()
+                except:
+                    pass
+                self.ws = None
+            raise
+
+    async def start_session(self, session_id):
+        logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
+        try:
+            # 会话开始时检测上个会话的监听状态
+            if (
+                self._monitor_task is not None
+                and isinstance(self._monitor_task, Task)
+                and not self._monitor_task.done()
+            ):
+                logger.bind(tag=TAG).info(
+                    "检测到未完成的上个会话，关闭监听任务和连接..."
+                )
+                await self.close()
+
+            # 建立新连接
+            await self._ensure_connection()
+
+            # 启动监听任务
+            self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
+
+            # 发送会话启动请求
+            start_request = self._build_base_request(status=0)
+
+            await self.ws.send(json.dumps(start_request))
+            logger.bind(tag=TAG).info("会话启动请求已发送")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
+            # 确保清理资源
+            await self.close()
+            raise
+
+    async def finish_session(self, session_id):
+        logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
+        try:
+            if self.ws:
+                # 发送会话结束请求
+                stop_request = self._build_base_request(status=2)
+                await self.ws.send(json.dumps(stop_request))
+                logger.bind(tag=TAG).info("会话结束请求已发送")
+
+                if self._monitor_task:
+                    try:
+                        await self._monitor_task
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"等待监听任务完成时发生错误: {str(e)}")
+                    finally:
+                        self._monitor_task = None
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
+            await self.close()
+            raise
+
+    async def close(self):
+        """资源清理"""
+        if self._monitor_task:
+            try:
+                self._monitor_task.cancel()
+                await self._monitor_task
+            except asyncio.CancelledError:
+                pass
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}")
+            self._monitor_task = None
+
+        if self.ws:
+            try:
+                await self.ws.close()
+            except:
+                pass
+            self.ws = None
+
+    async def _start_monitor_tts_response(self):
+        """监听TTS响应"""
+        try:
+            while not self.conn.stop_event.is_set():
+                try:
+                    msg = await self.ws.recv()
+
+                    # 检查客户端是否中止
+                    if self.conn.client_abort:
+                        logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应")
+                        break
+
+                    try:
+                        data = json.loads(msg)
+                        header = data.get("header", {})
+                        code = header.get("code")
+
+                        if code == 0:
+                            payload = data.get("payload", {})
+                            audio_payload = payload.get("audio", {})
+
+                            if audio_payload:
+                                status = audio_payload.get("status", 0)
+                                audio_data = audio_payload.get("audio", "")
+                                if status == 0:
+                                    logger.bind(tag=TAG).debug("TTS合成已启动")
+                                    self.tts_audio_queue.put(
+                                        (SentenceType.FIRST, [], None)
+                                    )
+                                elif status == 2:
+                                    logger.bind(tag=TAG).debug("收到结束状态的音频数据，TTS合成完成")
+                                    self._process_before_stop_play_files()
+                                    break
+                                else:
+                                    if self.conn.tts_MessageText:
+                                        logger.bind(tag=TAG).info(
+                                            f"句子语音生成成功： {self.conn.tts_MessageText}"
+                                        )
+                                        self.tts_audio_queue.put(
+                                            (SentenceType.FIRST, [], self.conn.tts_MessageText)
+                                        )
+                                        self.conn.tts_MessageText = None
+                                    try:
+                                        audio_bytes = base64.b64decode(audio_data)
+                                        self.opus_encoder.encode_pcm_to_opus_stream(
+                                            audio_bytes, False, self.handle_opus
+                                        )
+
+                                    except Exception as e:
+                                        logger.bind(tag=TAG).error(f"处理音频数据失败: {e}")
+
+                        else:
+                            message = header.get("message", "未知错误")
+                            logger.bind(tag=TAG).error(f"TTS合成错误: {code} - {message}")
+                            break
+
+                    except json.JSONDecodeError:
+                        logger.bind(tag=TAG).warning("收到无效的JSON消息")
+
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).warning("WebSocket连接已关闭")
+                    break
+
+                except Exception as e:
+                    logger.bind(tag=TAG).error(
+                        f"处理TTS响应时出错: {e}\n{traceback.format_exc()}"
+                    )
+                    break
+
+            # 链接不可复用
+            if self.ws:
+                try:
+                    await self.ws.close()
+                except:
+                    pass
+                self.ws = None
+        # 监听任务退出时清理引用
+        finally:
+            self._monitor_task = None
+
+    def to_tts(self, text: str) -> list:
+        """非流式TTS处理，用于测试及保存音频文件的场景"""
+        try:
+            # 创建新的事件循环
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+            # 存储音频数据
+            audio_data = []
+
+            async def _generate_audio():
+                # 生成认证URL
+                auth_url = XunfeiWSAuth.create_auth_url(
+                    self.api_key, self.api_secret, self.api_url
+                )
+
+                # 建立WebSocket连接
+                ws = await websockets.connect(
+                    auth_url,
+                    ping_interval=30,
+                    ping_timeout=10,
+                    close_timeout=10,
+                )
+
+                try:
+                    filtered_text = MarkdownCleaner.clean_markdown(text)
+
+                    text_request = self._build_base_request(status=2,text=filtered_text)
+
+                    await ws.send(json.dumps(text_request))
+
+                    task_finished = False
+                    while not task_finished:
+                        msg = await ws.recv()
+
+                        data = json.loads(msg)
+                        header = data.get("header", {})
+                        code = header.get("code")
+
+                        if code == 0:
+                            payload = data.get("payload", {})
+                            audio_payload = payload.get("audio", {})
+                            if audio_payload:
+                                status = audio_payload.get("status", 0)
+                                audio_base64 = audio_payload.get("audio", "")
+                                if status == 1:
+                                    try:
+                                        audio_bytes = base64.b64decode(audio_base64)
+                                        self.opus_encoder.encode_pcm_to_opus_stream(
+                                            audio_bytes,
+                                            end_of_stream=False,
+                                            callback=lambda opus: audio_data.append(opus)
+                                        )
+                                    except Exception as e:
+                                        logger.bind(tag=TAG).error(f"处理音频数据失败: {e}")
+                                elif status == 2:
+                                    task_finished = True
+                                    logger.bind(tag=TAG).debug("TTS任务完成")
+
+                        else:
+                            message = header.get("message", "未知错误")
+                            raise Exception(f"合成失败: {code} - {message}")
+
+                finally:
+                    # 清理资源
+                    try:
+                        await ws.close()
+                    except:
+                        pass
+
+            loop.run_until_complete(_generate_audio())
+            loop.close()
+
+            return audio_data
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}")
+            return []        
+    
+    def _build_base_request(self, status,text=" "):
+        """构建基础请求结构"""
+        return {
+            "header": {
+                "app_id": self.app_id,
+                "status": status,
+            },
+            "parameter": {
+                "oral": {
+                    "oral_level": self.oral_level,
+                    "spark_assist": self.spark_assist,
+                    "stop_split": self.stop_split,
+                    "remain": self.remain
+                },
+                "tts": {
+                    "vcn": self.voice,
+                    "speed": self.speed,
+                    "volume": self.volume,
+                    "pitch": self.pitch,
+                    "bgs": 0,
+                    "reg": 0,
+                    "rdn": 0,
+                    "rhy": 0,
+                    "audio": {
+                        "encoding": self.format,
+                        "sample_rate": self.sample_rate,
+                        "channels": 1,
+                        "bit_depth": 16,
+                        "frame_size": 0
+                    }
+                }
+            },
+            "payload": {
+                "text": {
+                    "encoding": "utf8",
+                    "compress": "raw",
+                    "format": "plain",
+                    "status": status,
+                    "seq": self.text_seq,
+                    "text": base64.b64encode(text.encode('utf-8')).decode('utf-8')
+                }
+            }
+        }

main/xiaozhi-server/core/providers/vad/silero.py

@@ -36,7 +36,18 @@ class VADProvider(VADProviderBase):
         # 至少要多少帧才算有语音
         self.frame_window_threshold = 3
 
+    def __del__(self):
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+            except Exception:
+                pass
+
     def is_vad(self, conn, opus_packet):
+        # 手动模式：直接返回True，不进行实时VAD检测，所有音频都缓存
+        if conn.client_listen_mode == "manual":
+            return True
+            
         try:
             pcm_frame = self.decoder.decode(opus_packet, 960)
             conn.client_audio_buffer.extend(pcm_frame)  # 将新数据加入缓冲区
@@ -70,7 +81,9 @@ class VADProvider(VADProviderBase):
 
                 # 更新滑动窗口
                 conn.client_voice_window.append(is_voice)
-                client_have_voice = (conn.client_voice_window.count(True) >= self.frame_window_threshold)
+                client_have_voice = (
+                    conn.client_voice_window.count(True) >= self.frame_window_threshold
+                )
 
                 # 如果之前有声音，但本次没有声音，且与上次有声音的时间差已经超过了静默阈值，则认为已经说完一句话
                 if conn.client_have_voice and not client_have_voice: