Merge branch 'main' into py_test_tts

main/xiaozhi-server/core/connection.py

@@ -17,6 +17,7 @@ from core.utils.util import (
     filter_sensitive_info,
 )
 from typing import Dict, Any
+from collections import deque
 from core.utils.modules_initialize import (
     initialize_modules,
     initialize_tts,
@@ -110,10 +111,10 @@ class ConnectionHandler:
         # vad相关变量
         self.client_audio_buffer = bytearray()
         self.client_have_voice = False
-        self.client_have_voice_last_time = 0.0
-        self.client_no_voice_last_time = 0.0
+        self.last_activity_time = 0.0  # 统一的活动时间戳（毫秒）
         self.client_voice_stop = False
-        self.client_voice_frame_count = 0
+        self.client_voice_window = deque(maxlen=5)
+        self.last_is_voice = False
 
         # asr相关变量
         # 因为实际部署时可能会用到公共的本地ASR，不能把变量暴露给公共ASR
@@ -143,10 +144,10 @@ class ConnectionHandler:
         self.load_function_plugin = False
         self.intent_type = "nointent"
 
-        self.timeout_task = None
         self.timeout_seconds = (
             int(self.config.get("close_connection_no_voice_time", 120)) + 60
         )  # 在原来第一道关闭的基础上加60秒，进行二道关闭
+        self.timeout_task = None
 
         # {"mcp":true} 表示启用MCP功能
         self.features = None
@@ -187,12 +188,14 @@ class ConnectionHandler:
             self.websocket = ws
             self.device_id = self.headers.get("device-id", None)
 
+            # 初始化活动时间戳
+            self.last_activity_time = time.time() * 1000
+
             # 启动超时检查任务
             self.timeout_task = asyncio.create_task(self._check_timeout())
 
             self.welcome_msg = self.config["xiaozhi"]
             self.welcome_msg["session_id"] = self.session_id
-            await self.websocket.send(json.dumps(self.welcome_msg))
 
             # 获取差异化配置
             self._initialize_private_config()
@@ -213,7 +216,17 @@ class ConnectionHandler:
             self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}")
             return
         finally:
-            await self._save_and_close(ws)
+            try:
+                await self._save_and_close(ws)
+            except Exception as final_error:
+                self.logger.bind(tag=TAG).error(f"最终清理时出错: {final_error}")
+                # 确保即使保存记忆失败，也要关闭连接
+                try:
+                    await self.close(ws)
+                except Exception as close_error:
+                    self.logger.bind(tag=TAG).error(
+                        f"强制关闭连接时出错: {close_error}"
+                    )
 
     async def _save_and_close(self, ws):
         """保存记忆并关闭连接"""
@@ -231,7 +244,10 @@ class ConnectionHandler:
                     except Exception as e:
                         self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
                     finally:
-                        loop.close()
+                        try:
+                            loop.close()
+                        except Exception:
+                            pass
 
                 # 启动线程保存记忆，不等待完成
                 threading.Thread(target=save_memory_task, daemon=True).start()
@@ -239,16 +255,17 @@ class ConnectionHandler:
             self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
         finally:
             # 立即关闭连接，不等待记忆保存完成
-            await self.close(ws)
+            try:
+                await self.close(ws)
+            except Exception as close_error:
+                self.logger.bind(tag=TAG).error(
+                    f"保存记忆后关闭连接失败: {close_error}"
+                )
 
     async def _route_message(self, message):
         """消息路由"""
-        # 重置超时计时器
-        if self.timeout_task:
-            self.timeout_task.cancel()
-            self.timeout_task = asyncio.create_task(self._check_timeout())
-
         if isinstance(message, str):
+            self.last_activity_time = time.time() * 1000
             await handleTextMessage(self, message)
         elif isinstance(message, bytes):
             if self.vad is None:
@@ -594,13 +611,24 @@ class ConnectionHandler:
         # 更新系统prompt至上下文
         self.dialogue.update_system_message(self.prompt)
 
-    def chat(self, query, tool_call=False):
+    def chat(self, query, tool_call=False, depth=0):
         self.logger.bind(tag=TAG).info(f"大模型收到用户消息: {query}")
         self.llm_finish_task = False
 
         if not tool_call:
             self.dialogue.put(Message(role="user", content=query))
 
+        # 为最顶层时新建会话ID和发送FIRST请求
+        if depth == 0:
+            self.sentence_id = str(uuid.uuid4().hex)
+            self.tts.tts_text_queue.put(
+                TTSMessageDTO(
+                    sentence_id=self.sentence_id,
+                    sentence_type=SentenceType.FIRST,
+                    content_type=ContentType.ACTION,
+                )
+            )
+
         # Define intent functions
         functions = None
         if self.intent_type == "function_call" and hasattr(self, "func_handler"):
@@ -616,8 +644,6 @@ class ConnectionHandler:
                 )
                 memory_str = future.result()
 
-            self.sentence_id = str(uuid.uuid4().hex)
-
             if self.intent_type == "function_call" and functions is not None:
                 # 使用支持functions的streaming接口
                 llm_responses = self.llm.response_with_functions(
@@ -640,7 +666,6 @@ class ConnectionHandler:
         function_id = None
         function_arguments = ""
         content_arguments = ""
-        text_index = 0
         self.client_abort = False
         for response in llm_responses:
             if self.client_abort:
@@ -670,14 +695,6 @@ class ConnectionHandler:
             if content is not None and len(content) > 0:
                 if not tool_call_flag:
                     response_message.append(content)
-                    if text_index == 0:
-                        self.tts.tts_text_queue.put(
-                            TTSMessageDTO(
-                                sentence_id=self.sentence_id,
-                                sentence_type=SentenceType.FIRST,
-                                content_type=ContentType.ACTION,
-                            )
-                        )
                     self.tts.tts_text_queue.put(
                         TTSMessageDTO(
                             sentence_id=self.sentence_id,
@@ -686,7 +703,6 @@ class ConnectionHandler:
                             content_detail=content,
                         )
                     )
-                    text_index += 1
         # 处理function call
         if tool_call_flag:
             bHasError = False
@@ -711,6 +727,11 @@ class ConnectionHandler:
                         f"function call error: {content_arguments}"
                     )
             if not bHasError:
+                # 如需要大模型先处理一轮，添加相关处理后的日志情况
+                if len(response_message) > 0:
+                    self.dialogue.put(
+                        Message(role="assistant", content="".join(response_message))
+                    )
                 response_message.clear()
                 self.logger.bind(tag=TAG).debug(
                     f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}"
@@ -728,14 +749,14 @@ class ConnectionHandler:
                     ),
                     self.loop,
                 ).result()
-                self._handle_function_result(result, function_call_data)
+                self._handle_function_result(result, function_call_data, depth=depth)
 
         # 存储对话内容
         if len(response_message) > 0:
             self.dialogue.put(
                 Message(role="assistant", content="".join(response_message))
             )
-        if text_index > 0:
+        if depth == 0:
             self.tts.tts_text_queue.put(
                 TTSMessageDTO(
                     sentence_id=self.sentence_id,
@@ -750,7 +771,7 @@ class ConnectionHandler:
 
         return True
 
-    def _handle_function_result(self, result, function_call_data):
+    def _handle_function_result(self, result, function_call_data, depth):
         if result.action == Action.RESPONSE:  # 直接回复前端
             text = result.response
             self.tts.tts_one_sentence(self, ContentType.TEXT, content_detail=text)
@@ -787,7 +808,7 @@ class ConnectionHandler:
                         content=text,
                     )
                 )
-                self.chat(text, tool_call=True)
+                self.chat(text, tool_call=True, depth=depth + 1)
         elif result.action == Action.NOTFOUND or result.action == Action.ERROR:
             text = result.response if result.response else result.result
             self.tts.tts_one_sentence(self, ContentType.TEXT, content_detail=text)
@@ -840,13 +861,22 @@ class ConnectionHandler:
         """资源清理方法"""
         try:
             # 取消超时任务
-            if self.timeout_task:
+            if self.timeout_task and not self.timeout_task.done():
                 self.timeout_task.cancel()
+                try:
+                    await self.timeout_task
+                except asyncio.CancelledError:
+                    pass
                 self.timeout_task = None
 
             # 清理工具处理器资源
             if hasattr(self, "func_handler") and self.func_handler:
-                await self.func_handler.cleanup()
+                try:
+                    await self.func_handler.cleanup()
+                except Exception as cleanup_error:
+                    self.logger.bind(tag=TAG).error(
+                        f"清理工具处理器时出错: {cleanup_error}"
+                    )
 
             # 触发停止事件
             if self.stop_event:
@@ -856,19 +886,61 @@ class ConnectionHandler:
             self.clear_queues()
 
             # 关闭WebSocket连接
-            if ws:
-                await ws.close()
-            elif self.websocket:
-                await self.websocket.close()
+            try:
+                if ws:
+                    # 安全地检查WebSocket状态并关闭
+                    try:
+                        if hasattr(ws, "closed") and not ws.closed:
+                            await ws.close()
+                        elif hasattr(ws, "state") and ws.state.name != "CLOSED":
+                            await ws.close()
+                        else:
+                            # 如果没有closed属性，直接尝试关闭
+                            await ws.close()
+                    except Exception:
+                        # 如果关闭失败，忽略错误
+                        pass
+                elif self.websocket:
+                    try:
+                        if (
+                            hasattr(self.websocket, "closed")
+                            and not self.websocket.closed
+                        ):
+                            await self.websocket.close()
+                        elif (
+                            hasattr(self.websocket, "state")
+                            and self.websocket.state.name != "CLOSED"
+                        ):
+                            await self.websocket.close()
+                        else:
+                            # 如果没有closed属性，直接尝试关闭
+                            await self.websocket.close()
+                    except Exception:
+                        # 如果关闭失败，忽略错误
+                        pass
+            except Exception as ws_error:
+                self.logger.bind(tag=TAG).error(f"关闭WebSocket连接时出错: {ws_error}")
+
+            if self.tts:
+                await self.tts.close()
 
             # 最后关闭线程池（避免阻塞）
             if self.executor:
-                self.executor.shutdown(wait=False)
+                try:
+                    self.executor.shutdown(wait=False)
+                except Exception as executor_error:
+                    self.logger.bind(tag=TAG).error(
+                        f"关闭线程池时出错: {executor_error}"
+                    )
                 self.executor = None
 
             self.logger.bind(tag=TAG).info("连接资源已释放")
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"关闭连接时出错: {e}")
+        finally:
+            # 确保停止事件被设置
+            if self.stop_event:
+                self.stop_event.set()
 
     def clear_queues(self):
         """清空所有任务队列"""
@@ -898,7 +970,6 @@ class ConnectionHandler:
     def reset_vad_states(self):
         self.client_audio_buffer = bytearray()
         self.client_have_voice = False
-        self.client_have_voice_last_time = 0
         self.client_voice_stop = False
         self.logger.bind(tag=TAG).debug("VAD states reset.")
 
@@ -917,10 +988,28 @@ class ConnectionHandler:
         """检查连接超时"""
         try:
             while not self.stop_event.is_set():
-                await asyncio.sleep(self.timeout_seconds)
-                if not self.stop_event.is_set():
-                    self.logger.bind(tag=TAG).info("连接超时，准备关闭")
-                    await self.close(self.websocket)
-                    break
+                # 检查是否超时（只有在时间戳已初始化的情况下）
+                if self.last_activity_time > 0.0:
+                    current_time = time.time() * 1000
+                    if (
+                        current_time - self.last_activity_time
+                        > self.timeout_seconds * 1000
+                    ):
+                        if not self.stop_event.is_set():
+                            self.logger.bind(tag=TAG).info("连接超时，准备关闭")
+                            # 设置停止事件，防止重复处理
+                            self.stop_event.set()
+                            # 使用 try-except 包装关闭操作，确保不会因为异常而阻塞
+                            try:
+                                await self.close(self.websocket)
+                            except Exception as close_error:
+                                self.logger.bind(tag=TAG).error(
+                                    f"超时关闭连接时出错: {close_error}"
+                                )
+                        break
+                # 每10秒检查一次，避免过于频繁
+                await asyncio.sleep(10)
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"超时检查任务出错: {e}")
+        finally:
+            self.logger.bind(tag=TAG).info("超时检查任务已退出")

main/xiaozhi-server/core/handle/intentHandler.py

@@ -6,9 +6,8 @@ from core.handle.helloHandle import checkWakeupWords
 from core.utils.util import remove_punctuation_and_length
 from core.providers.tts.dto.dto import ContentType
 from core.utils.dialogue import Message
-from core.providers.tools.device_mcp import call_mcp_tool
 from plugins_func.register import Action, ActionResponse
-from loguru import logger
+from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType
 
 TAG = __name__
 
@@ -29,6 +28,8 @@ async def handle_user_intent(conn, text):
     intent_result = await analyze_intent_with_llm(conn, text)
     if not intent_result:
         return False
+    # 会话开始时生成sentence_id
+    conn.sentence_id = str(uuid.uuid4().hex)
     # 处理各种意图
     return await process_intent_result(conn, intent_result, text)
 
@@ -79,11 +80,6 @@ async def process_intent_result(conn, intent_result, original_text):
             if function_name == "continue_chat":
                 return False
 
-            if function_name == "play_music":
-                funcItem = conn.func_handler.get_function(function_name)
-                if not funcItem:
-                    conn.func_handler.function_registry.register_function("play_music")
-
             function_args = {}
             if "arguments" in intent_data["function_call"]:
                 function_args = intent_data["function_call"]["arguments"]
@@ -158,5 +154,19 @@ async def process_intent_result(conn, intent_result, original_text):
 
 
 def speak_txt(conn, text):
+    conn.tts.tts_text_queue.put(
+        TTSMessageDTO(
+            sentence_id=conn.sentence_id,
+            sentence_type=SentenceType.FIRST,
+            content_type=ContentType.ACTION,
+        )
+    )
     conn.tts.tts_one_sentence(conn, ContentType.TEXT, content_detail=text)
+    conn.tts.tts_text_queue.put(
+        TTSMessageDTO(
+            sentence_id=conn.sentence_id,
+            sentence_type=SentenceType.LAST,
+            content_type=ContentType.ACTION,
+        )
+    )
     conn.dialogue.put(Message(role="assistant", content=text))

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -66,12 +66,11 @@ async def startToChat(conn, text):
 
 async def no_voice_close_connect(conn, have_voice):
     if have_voice:
-        conn.client_no_voice_last_time = 0.0
+        conn.last_activity_time = time.time() * 1000
         return
-    if conn.client_no_voice_last_time == 0.0:
-        conn.client_no_voice_last_time = time.time() * 1000
-    else:
-        no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
+    # 只有在已经初始化过时间戳的情况下才进行超时检查
+    if conn.last_activity_time > 0.0:
+        no_voice_time = time.time() * 1000 - conn.last_activity_time
         close_connection_no_voice_time = int(
             conn.config.get("close_connection_no_voice_time", 120)
         )

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -93,7 +93,7 @@ async def sendAudio(conn, audios, pre_buffer=True):
             break
 
         # 重置没有声音的状态
-        conn.client_no_voice_last_time = 0.0
+        conn.last_activity_time = time.time() * 1000
 
         # 计算预期发送时间
         expected_time = start_time + (play_position / 1000)

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -1,3 +1,4 @@
+import httpx
 import openai
 from openai.types import CompletionUsage
 from config.logger import setup_logging
@@ -16,6 +17,9 @@ class LLMProvider(LLMProviderBase):
             self.base_url = config.get("base_url")
         else:
             self.base_url = config.get("url")
+        # 增加timeout的配置项，单位为秒
+        timeout = config.get("timeout", 300)
+        self.timeout = int(timeout) if timeout else 300
 
         param_defaults = {
             "max_tokens": (500, int),
@@ -42,7 +46,7 @@ class LLMProvider(LLMProviderBase):
         model_key_msg = check_model_key("LLM", self.api_key)
         if model_key_msg:
             logger.bind(tag=TAG).error(model_key_msg)
-        self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
+        self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url, timeout=httpx.Timeout(self.timeout))
 
     def response(self, session_id, dialogue, **kwargs):
         try:

main/xiaozhi-server/core/providers/tts/base.py

@@ -161,13 +161,6 @@ class TTSProviderBase(ABC):
             else:
                 sentence_id = str(uuid.uuid4()).replace("-", "")
                 conn.sentence_id = sentence_id
-        self.tts_text_queue.put(
-            TTSMessageDTO(
-                sentence_id=sentence_id,
-                sentence_type=SentenceType.FIRST,
-                content_type=ContentType.ACTION,
-            )
-        )
         # 对于单句的文本，进行分段处理
         segments = re.split(r"([。！？!?；;\n])", content_detail)
         for seg in segments:
@@ -180,13 +173,6 @@ class TTSProviderBase(ABC):
                     content_file=content_file,
                 )
             )
-        self.tts_text_queue.put(
-            TTSMessageDTO(
-                sentence_id=sentence_id,
-                sentence_type=SentenceType.LAST,
-                content_type=ContentType.ACTION,
-            )
-        )
 
     async def open_audio_channels(self, conn):
         self.conn = conn
@@ -209,6 +195,8 @@ class TTSProviderBase(ABC):
         while not self.conn.stop_event.is_set():
             try:
                 message = self.tts_text_queue.get(timeout=1)
+                if message.sentence_type == SentenceType.FIRST:
+                    self.conn.client_abort = False
                 if self.conn.client_abort:
                     logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
                     continue
@@ -362,10 +350,8 @@ class TTSProviderBase(ABC):
         return audio_datas
 
     def _process_before_stop_play_files(self):
-        for tts_file, text in self.before_stop_play_files:
-            if tts_file and os.path.exists(tts_file):
-                audio_datas = self._process_audio_file(tts_file)
-                self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text))
+        for audio_datas, text in self.before_stop_play_files:
+            self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text))
         self.before_stop_play_files.clear()
         self.tts_audio_queue.put((SentenceType.LAST, [], None))
 

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -85,8 +85,12 @@ class TTSProvider(TTSProviderBase):
         self.reference_id = (
             None if not config.get("reference_id") else config.get("reference_id")
         )
-        self.reference_audio = parse_string_to_list(config.get("reference_audio"))
-        self.reference_text = parse_string_to_list(config.get("reference_text"))
+        self.reference_audio = parse_string_to_list(
+             config.get('ref_audio')if config.get('ref_audio') else config.get("reference_audio")
+        )
+        self.reference_text = parse_string_to_list(
+             config.get('ref_text')if config.get('ref_text') else config.get("reference_text")
+        )
         self.format = config.get("response_format", "wav")
         self.audio_file_type = config.get("response_format", "wav")
         self.api_key = config.get("api_key", "YOUR_API_KEY")

main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py

@@ -12,8 +12,8 @@ class TTSProvider(TTSProviderBase):
         super().__init__(config, delete_audio_file)
         self.url = config.get("url")
         self.text_lang = config.get("text_lang", "zh")
-        self.ref_audio_path = config.get("ref_audio_path")
-        self.prompt_text = config.get("prompt_text")
+        self.ref_audio_path = config.get('ref_audio') if config.get('ref_audio') else config.get("ref_audio_path")
+        self.prompt_text = config.get('ref_text') if config.get('ref_text') else config.get("prompt_text")
         self.prompt_lang = config.get("prompt_lang", "zh")
 
         # 处理空字符串的情况

main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py

@@ -11,8 +11,8 @@ class TTSProvider(TTSProviderBase):
     def __init__(self, config, delete_audio_file):
         super().__init__(config, delete_audio_file)
         self.url = config.get("url")
-        self.refer_wav_path = config.get("refer_wav_path")
-        self.prompt_text = config.get("prompt_text")
+        self.refer_wav_path = config.get('ref_audio')if config.get('ref_audio') else config.get("refer_wav_path")
+        self.prompt_text = config.get('ref_text')if config.get('ref_text') else config.get("prompt_text")
         self.prompt_language = config.get("prompt_language")
         self.text_language = config.get("text_language", "audo")
 

main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py

@@ -10,7 +10,6 @@ from config.logger import setup_logging
 from core.utils import opus_encoder_utils
 from core.utils.util import check_model_key
 from core.providers.tts.base import TTSProviderBase
-from core.handle.abortHandle import handleAbortMessage
 from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
 from asyncio import Task
 
@@ -175,6 +174,9 @@ class TTSProvider(TTSProviderBase):
     async def _ensure_connection(self):
         """建立新的WebSocket连接"""
         try:
+            if self.ws:
+                logger.bind(tag=TAG).info(f"使用已有链接...")
+                return self.ws
             logger.bind(tag=TAG).info("开始建立新连接...")
             ws_header = {
                 "X-Api-App-Key": self.appId,
@@ -200,6 +202,10 @@ class TTSProvider(TTSProviderBase):
                 logger.bind(tag=TAG).debug(
                     f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}"
                 )
+
+                if message.sentence_type == SentenceType.FIRST:
+                    self.conn.client_abort = False
+
                 if self.conn.client_abort:
                     logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
                     continue
@@ -244,9 +250,12 @@ class TTSProvider(TTSProviderBase):
                     logger.bind(tag=TAG).info(
                         f"添加音频文件到待播放列表: {message.content_file}"
                     )
-                    self.before_stop_play_files.append(
-                        (message.content_file, message.content_detail)
-                    )
+                    if message.content_file and os.path.exists(message.content_file):
+                        # 先处理文件音频数据
+                        file_audio = self._process_audio_file(message.content_file)
+                        self.before_stop_play_files.append(
+                            (file_audio, message.content_detail)
+                        )
 
                 if message.sentence_type == SentenceType.LAST:
                     try:
@@ -295,25 +304,15 @@ class TTSProvider(TTSProviderBase):
     async def start_session(self, session_id):
         logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
         try:
-            task = self._monitor_task
+            # 会话开始时检测上个会话的监听状态
             if (
-                task is not None
-                and isinstance(task, Task)
-                and not task.done()
+                self._monitor_task is not None
+                and isinstance(self._monitor_task, Task)
+                and not self._monitor_task.done()
             ):
-                logger.bind(tag=TAG).info("等待上一个监听任务结束...")
-                if self.ws is not None:
-                    logger.bind(tag=TAG).info("强制关闭上一个WebSocket连接以唤醒监听任务...")
-                    try:
-                        await self.ws.close()
-                    except Exception as e:
-                        logger.bind(tag=TAG).warning(f"关闭上一个ws异常: {e}")
-                    self.ws = None
-                try:
-                    await asyncio.wait_for(task, timeout=8)
-                except Exception as e:
-                    logger.bind(tag=TAG).warning(f"等待监听任务异常: {e}")
-                self._monitor_task = None
+                logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务和连接...")
+                await self.close()
+
             # 建立新连接
             await self._ensure_connection()
 
@@ -336,19 +335,7 @@ class TTSProvider(TTSProviderBase):
         except Exception as e:
             logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
             # 确保清理资源
-            if hasattr(self, "_monitor_task"):
-                try:
-                    self._monitor_task.cancel()
-                    await self._monitor_task
-                except:
-                    pass
-                self._monitor_task = None
-            if self.ws:
-                try:
-                    await self.ws.close()
-                except:
-                    pass
-                self.ws = None
+            await self.close()
             raise
 
     async def finish_session(self, session_id):
@@ -368,7 +355,7 @@ class TTSProvider(TTSProviderBase):
                 logger.bind(tag=TAG).info("会话结束请求已发送")
 
                 # 等待监听任务完成
-                if hasattr(self, "_monitor_task"):
+                if self._monitor_task:
                     try:
                         await self._monitor_task
                     except Exception as e:
@@ -378,28 +365,25 @@ class TTSProvider(TTSProviderBase):
                     finally:
                         self._monitor_task = None
 
-                # 关闭连接
-                await self.close()
         except Exception as e:
             logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
             # 确保清理资源
-            if hasattr(self, "_monitor_task"):
-                try:
-                    self._monitor_task.cancel()
-                    await self._monitor_task
-                except:
-                    pass
-                self._monitor_task = None
-            if self.ws:
-                try:
-                    await self.ws.close()
-                except:
-                    pass
-                self.ws = None
+            await self.close()
             raise
 
     async def close(self):
         """资源清理方法"""
+        # 取消监听任务
+        if self._monitor_task:
+            try:
+                self._monitor_task.cancel()
+                await self._monitor_task
+            except asyncio.CancelledError:
+                pass
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}")
+            self._monitor_task = None
+
         if self.ws:
             try:
                 await self.ws.close()
@@ -413,6 +397,7 @@ class TTSProvider(TTSProviderBase):
         is_first_sentence = True
         first_sentence_segment_count = 0  # 添加计数器
         try:
+            session_finished = False  # 标记会话是否正常结束
             while not self.conn.stop_event.is_set():
                 try:
                     # 确保 `recv()` 运行在同一个 event loop
@@ -466,6 +451,7 @@ class TTSProvider(TTSProviderBase):
                     elif res.optional.event == EVENT_SessionFinished:
                         logger.bind(tag=TAG).debug(f"会话结束～～")
                         self._process_before_stop_play_files()
+                        session_finished = True
                         break
                 except websockets.ConnectionClosed:
                     logger.bind(tag=TAG).warning("WebSocket连接已关闭")
@@ -476,15 +462,15 @@ class TTSProvider(TTSProviderBase):
                     )
                     traceback.print_exc()
                     break
-        finally:
-            # 确保清理资源
-            if self.ws:
+            # 仅在连接异常时才关闭
+            if not session_finished and self.ws:
                 try:
                     await self.ws.close()
                 except:
                     pass
                 self.ws = None
             # 监听任务退出时清理引用
+        finally:
             self._monitor_task = None
 
     async def send_event(

main/xiaozhi-server/core/providers/tts/linkerai.py

@@ -1,3 +1,4 @@
+import os
 import queue
 import asyncio
 import traceback
@@ -63,9 +64,12 @@ class TTSProvider(TTSProviderBase):
                     logger.bind(tag=TAG).info(
                         f"添加音频文件到待播放列表: {message.content_file}"
                     )
-                    self.before_stop_play_files.append(
-                        (message.content_file, message.content_detail)
-                    )
+                    if message.content_file and os.path.exists(message.content_file):
+                        # 先处理文件音频数据
+                        file_audio = self._process_audio_file(message.content_file)
+                        self.before_stop_play_files.append(
+                            (file_audio, message.content_detail)
+                        )
 
                 if message.sentence_type == SentenceType.LAST:
                     # 处理剩余的文本

main/xiaozhi-server/core/providers/vad/silero.py

@@ -23,22 +23,24 @@ class VADProvider(VADProviderBase):
 
         # 处理空字符串的情况
         threshold = config.get("threshold", "0.5")
+        threshold_low = config.get("threshold_low", "0.2")
         min_silence_duration_ms = config.get("min_silence_duration_ms", "1000")
 
         self.vad_threshold = float(threshold) if threshold else 0.5
+        self.vad_threshold_low = float(threshold_low) if threshold_low else 0.2
+
         self.silence_threshold_ms = (
             int(min_silence_duration_ms) if min_silence_duration_ms else 1000
         )
 
+        # 至少要多少帧才算有语音
+        self.frame_window_threshold = 3
+
     def is_vad(self, conn, opus_packet):
         try:
             pcm_frame = self.decoder.decode(opus_packet, 960)
             conn.client_audio_buffer.extend(pcm_frame)  # 将新数据加入缓冲区
 
-            # 初始化帧计数器
-            if not hasattr(conn, "client_voice_frame_count"):
-                conn.client_voice_frame_count = 0
-
             # 处理缓冲区中的完整帧（每次处理512采样点）
             client_have_voice = False
             while len(conn.client_audio_buffer) >= 512 * 2:
@@ -54,26 +56,30 @@ class VADProvider(VADProviderBase):
                 # 检测语音活动
                 with torch.no_grad():
                     speech_prob = self.model(audio_tensor, 16000).item()
-                is_voice = speech_prob >= self.vad_threshold
 
-                if is_voice:
-                    conn.client_voice_frame_count += 1
+                # 双阈值判断
+                if speech_prob >= self.vad_threshold:
+                    is_voice = True
+                elif speech_prob <= self.vad_threshold_low:
+                    is_voice = False
                 else:
-                    conn.client_voice_frame_count = 0
+                    is_voice = conn.last_is_voice
 
-                # 只有连续4帧检测到语音才认为有语音
-                client_have_voice = conn.client_voice_frame_count >= 4
+                # 声音没低于最低值则延续前一个状态，判断为有声音
+                conn.last_is_voice = is_voice
+
+                # 更新滑动窗口
+                conn.client_voice_window.append(is_voice)
+                client_have_voice = (conn.client_voice_window.count(True) >= self.frame_window_threshold)
 
                 # 如果之前有声音，但本次没有声音，且与上次有声音的时间差已经超过了静默阈值，则认为已经说完一句话
                 if conn.client_have_voice and not client_have_voice:
-                    stop_duration = (
-                        time.time() * 1000 - conn.client_have_voice_last_time
-                    )
+                    stop_duration = time.time() * 1000 - conn.last_activity_time
                     if stop_duration >= self.silence_threshold_ms:
                         conn.client_voice_stop = True
                 if client_have_voice:
                     conn.client_have_voice = True
-                    conn.client_have_voice_last_time = time.time() * 1000
+                    conn.last_activity_time = time.time() * 1000
 
             return client_have_voice
         except opuslib_next.OpusError as e:

main/xiaozhi-server/core/providers/vllm/openai.py

@@ -40,6 +40,7 @@ class VLLMProvider(VLLMProviderBase):
         self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
 
     def response(self, question, base64_image):
+        question = question + "(请使用中文回复)"
         try:
             messages = [
                 {

main/xiaozhi-server/core/utils/util.py

@@ -980,4 +980,30 @@ def is_valid_image_file(file_data: bytes) -> bool:
 
 def sanitize_tool_name(name: str) -> str:
     """Sanitize tool names for OpenAI compatibility."""
-    return re.sub(r"[^a-zA-Z0-9_-]", "_", name)
+    # 支持中文、英文字母、数字、下划线和连字符
+    return re.sub(r"[^a-zA-Z0-9_\-\u4e00-\u9fff]", "_", name)
+
+
+def validate_mcp_endpoint(mcp_endpoint: str) -> bool:
+    """
+    校验MCP接入点格式
+
+    Args:
+        mcp_endpoint: MCP接入点字符串
+
+    Returns:
+        bool: 是否有效
+    """
+    # 1. 检查是否以ws开头
+    if not mcp_endpoint.startswith("ws"):
+        return False
+
+    # 2. 检查是否包含key、call字样
+    if "key" in mcp_endpoint.lower() or "call" in mcp_endpoint.lower():
+        return False
+
+    # 3. 检查是否包含/mcp/字样
+    if "/mcp/" not in mcp_endpoint:
+        return False
+
+    return True

main/xiaozhi-server/core/websocket_server.py

@@ -57,8 +57,25 @@ class WebSocketServer:
         self.active_connections.add(handler)
         try:
             await handler.handle_connection(websocket)
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"处理连接时出错: {e}")
         finally:
+            # 确保从活动连接集合中移除
             self.active_connections.discard(handler)
+            # 强制关闭连接（如果还没有关闭的话）
+            try:
+                # 安全地检查WebSocket状态并关闭
+                if hasattr(websocket, "closed") and not websocket.closed:
+                    await websocket.close()
+                elif hasattr(websocket, "state") and websocket.state.name != "CLOSED":
+                    await websocket.close()
+                else:
+                    # 如果没有closed属性，直接尝试关闭
+                    await websocket.close()
+            except Exception as close_error:
+                self.logger.bind(tag=TAG).error(
+                    f"服务器端强制关闭连接时出错: {close_error}"
+                )
 
     async def _http_response(self, websocket, request_headers):
         # 检查是否为 WebSocket 升级请求