Merge pull request #1642 from xinnan-tech/py_tts_listen

update:更新版本号

main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java

@@ -227,7 +227,7 @@ public interface Constant {
     /**
      * 版本号
      */
-    public static final String VERSION = "0.5.7";
+    public static final String VERSION = "0.5.8";
 
     /**
      * 无效固件URL

main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentTemplateServiceImpl.java

@@ -56,6 +56,9 @@ public class AgentTemplateServiceImpl extends ServiceImpl<AgentTemplateDao, Agen
                 wrapper.set("tts_model_id", modelId);
                 wrapper.set("tts_voice_id", null);
                 break;
+            case "VLLM":
+                wrapper.set("vllm_model_id", modelId);
+                break;
             case "MEMORY":
                 wrapper.set("mem_model_id", modelId);
                 break;

main/manager-api/src/main/resources/db/changelog/202506191643.sql

@@ -0,0 +1,19 @@
+-- LLM意图识别配置说明
+UPDATE `ai_model_config` SET 
+`doc_link` = NULL,
+`remark` = 'LLM意图识别配置说明：
+1. 使用独立的LLM进行意图识别
+2. 默认使用selected_module.LLM的模型
+3. 可以配置使用独立的LLM（如免费的ChatGLMLLM）
+4. 通用性强，但会增加处理时间
+配置说明：
+1. 在llm字段中指定使用的LLM模型
+2. 如果不指定，则使用selected_module.LLM的模型' WHERE `id` = 'Intent_intent_llm';
+
+-- 函数调用意图识别配置说明
+UPDATE `ai_model_config` SET 
+`doc_link` = NULL,
+`remark` = '函数调用意图识别配置说明：
+1. 使用LLM的function_call功能进行意图识别
+2. 需要所选择的LLM支持function_call
+3. 按需调用工具，处理速度快' WHERE `id` = 'Intent_function_call';

main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -211,4 +211,11 @@ databaseChangeLog:
       changes:
         - sqlFile:
             encoding: utf8
-            path: classpath:db/changelog/202506161101.sql
+            path: classpath:db/changelog/202506161101.sql
+  - changeSet:
+      id: 202506191643
+      author: hrz
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202506191643.sql

main/xiaozhi-server/config/logger.py

@@ -5,7 +5,7 @@ from config.config_loader import load_config
 from config.settings import check_config_file
 from datetime import datetime
 
-SERVER_VERSION = "0.5.7"
+SERVER_VERSION = "0.5.8"
 _logger_initialized = False
 
 

main/xiaozhi-server/core/api/vision_handler.py

@@ -8,6 +8,7 @@ from config.config_loader import get_private_config_from_api
 from core.utils.auth import AuthToken
 import base64
 from typing import Tuple, Optional
+from plugins_func.register import Action
 
 TAG = __name__
 
@@ -122,7 +123,8 @@ class VisionHandler:
 
             return_json = {
                 "success": True,
-                "result": result,
+                "action": Action.RESPONSE.name,
+                "response": result,
             }
 
             response = web.Response(

main/xiaozhi-server/core/connection.py

@@ -115,6 +115,7 @@ class ConnectionHandler:
         self.client_have_voice_last_time = 0.0
         self.client_no_voice_last_time = 0.0
         self.client_voice_stop = False
+        self.client_voice_frame_count = 0
 
         # asr相关变量
         # 因为实际部署时可能会用到公共的本地ASR，不能把变量暴露给公共ASR
@@ -627,8 +628,8 @@ class ConnectionHandler:
                 )
                 memory_str = future.result()
 
-            uuid_str = str(uuid.uuid4()).replace("-", "")
-            self.sentence_id = uuid_str
+            self.sentence_id = str(uuid.uuid4().hex)
+
 
             if self.intent_type == "function_call" and functions is not None:
                 # 使用支持functions的streaming接口
@@ -751,9 +752,31 @@ class ConnectionHandler:
                             self.loop,
                         ).result()
                         self.logger.bind(tag=TAG).debug(f"MCP工具调用结果: {result}")
-                        result = ActionResponse(
-                            action=Action.REQLLM, result=result, response=""
-                        )
+
+                        resultJson = None
+                        if isinstance(result, str):
+                            try:
+                                resultJson = json.loads(result)
+                            except Exception as e:
+                                self.logger.bind(tag=TAG).error(
+                                    f"解析MCP工具返回结果失败: {e}"
+                                )
+
+                        # 视觉大模型不经过二次LLM处理
+                        if (
+                            resultJson is not None
+                            and isinstance(resultJson, dict)
+                            and "action" in resultJson
+                        ):
+                            result = ActionResponse(
+                                action=Action[resultJson["action"]],
+                                result=None,
+                                response=resultJson.get("response", ""),
+                            )
+                        else:
+                            result = ActionResponse(
+                                action=Action.REQLLM, result=result, response=""
+                            )
                     except Exception as e:
                         self.logger.bind(tag=TAG).error(f"MCP工具调用失败: {e}")
                         result = ActionResponse(

main/xiaozhi-server/core/mcp/MCPClient.py

@@ -80,6 +80,27 @@ class MCPClient:
         fut: concurrent.futures.Future = asyncio.run_coroutine_threadsafe(coro, loop)
         return await asyncio.wrap_future(fut)
 
+    def is_connected(self) -> bool:
+        """检查MCP客户端是否连接正常
+
+        Returns:
+            bool: 如果客户端已连接并正常工作，返回True，否则返回False
+        """
+        # 检查工作任务是否存在
+        if self._worker_task is None:
+            return False
+            
+        # 检查工作任务是否已经完成或取消
+        if self._worker_task.done():
+            return False
+            
+        # 检查会话是否存在
+        if self.session is None:
+            return False
+            
+        # 所有检查都通过，连接正常
+        return True
+
     async def _worker(self):
         async with AsyncExitStack() as stack:
             try:

main/xiaozhi-server/core/mcp/manager.py

@@ -100,7 +100,7 @@ class MCPManager:
         return False
 
     async def execute_tool(self, tool_name: str, arguments: Dict[str, Any]) -> Any:
-        """执行工具调用
+        """执行工具调用，失败时会尝试重新连接
         Args:
             tool_name: 工具名称
             arguments: 工具参数
@@ -112,11 +112,64 @@ class MCPManager:
         self.conn.logger.bind(tag=TAG).info(
             f"Executing tool {tool_name} with arguments: {arguments}"
         )
-        for client in self.client.values():
+        
+        max_retries = 3  # 最大重试次数
+        retry_interval = 2  # 重试间隔(秒)
+        
+        # 找到对应的客户端
+        client_name = None
+        target_client = None
+        for name, client in self.client.items():
             if client.has_tool(tool_name):
-                return await client.call_tool(tool_name, arguments)
-
-        raise ValueError(f"Tool {tool_name} not found in any MCP server")
+                client_name = name
+                target_client = client
+                break
+        
+        if not target_client:
+            raise ValueError(f"Tool {tool_name} not found in any MCP server")
+        
+        # 带重试机制的工具调用
+        for attempt in range(max_retries):
+            try:
+                return await target_client.call_tool(tool_name, arguments)
+            except Exception as e:
+                # 最后一次尝试失败时直接抛出异常
+                if attempt == max_retries - 1:
+                    raise
+                
+                self.conn.logger.bind(tag=TAG).warning(
+                    f"执行工具 {tool_name} 失败 (尝试 {attempt+1}/{max_retries}): {e}"
+                )
+                
+                # 尝试重新连接
+                self.conn.logger.bind(tag=TAG).info(
+                    f"重试前尝试重新连接 MCP 客户端 {client_name}"
+                )
+                try:
+                    # 关闭旧的连接
+                    await target_client.cleanup()
+                    
+                    # 重新初始化客户端
+                    config = self.load_config()
+                    if client_name in config:
+                        client = MCPClient(config[client_name])
+                        await client.initialize()
+                        self.client[client_name] = client
+                        target_client = client                        
+                        self.conn.logger.bind(tag=TAG).info(
+                            f"成功重新连接 MCP 客户端: {client_name}"
+                        )
+                    else:
+                        self.conn.logger.bind(tag=TAG).error(
+                            f"Cannot reconnect MCP client {client_name}: config not found"
+                        )
+                except Exception as reconnect_error:
+                    self.conn.logger.bind(tag=TAG).error(
+                        f"Failed to reconnect MCP client {client_name}: {reconnect_error}"
+                    )
+                
+                # 等待一段时间再重试
+                await asyncio.sleep(retry_interval)
 
     async def cleanup_all(self) -> None:
         """依次关闭所有 MCPClient，不让异常阻断整体流程。"""

main/xiaozhi-server/core/providers/asr/doubao_stream.py

@@ -93,9 +93,7 @@ class ASRProvider(ASRProviderBase):
 
                     # 检查初始化响应
                     if "code" in result and result["code"] != 1000:
-                        error_msg = f"ASR服务初始化失败: {result.get('payload_msg', {}).get('message', '未知错误')}"
-                        if "payload_msg" in result:
-                            error_msg += f"\n详细错误信息: {json.dumps(result['payload_msg'], ensure_ascii=False)}"
+                        error_msg = f"ASR服务初始化失败: {result.get('payload_msg', {}).get('error', '未知错误')}"
                         logger.bind(tag=TAG).error(error_msg)
                         raise Exception(error_msg)
 
@@ -256,7 +254,6 @@ class ASRProvider(ASRProviderBase):
             "X-Api-Access-Key": self.access_token,
             "X-Api-Resource-Id": "volc.bigasr.sauc.duration",
             "X-Api-Connect-Id": str(uuid.uuid4()),
-            "Host": "openspeech.bytedance.com",
         }
 
     def generate_header(
@@ -309,9 +306,14 @@ class ASRProvider(ASRProviderBase):
 
             # 如果是错误响应
             if message_type == 0x0F:  # SERVER_ERROR_RESPONSE
-                code = int.from_bytes(header[4:8], "big", signed=False)
-                error_msg = res[8:].decode("utf-8")
-                return {"code": code, "error": error_msg}
+                code = int.from_bytes(res[4:8], "big", signed=False)
+                msg_length = int.from_bytes(res[8:12], "big", signed=False)
+                error_msg = json.loads(res[12:].decode("utf-8"))
+                return {
+                    "code": code,
+                    "msg_length": msg_length,
+                    "payload_msg": error_msg,
+                }
 
             # 获取JSON数据（跳过12字节头部）
             try:

main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -5,6 +5,7 @@ import os
 import yaml
 from config.config_loader import get_project_dir
 from config.manage_api_client import save_mem_local_short
+from core.utils.util import check_model_key
 
 
 short_term_memory_prompt = """
@@ -145,6 +146,10 @@ class MemoryProvider(MemoryProviderBase):
         # 打印使用的模型信息
         model_info = getattr(self.llm, "model_name", str(self.llm.__class__.__name__))
         logger.bind(tag=TAG).debug(f"使用记忆保存模型: {model_info}")
+        api_key = getattr(self.llm, "api_key", None)
+        memory_key_msg = check_model_key("记忆总结专用LLM", api_key)
+        if memory_key_msg:
+            logger.bind(tag=TAG).error(memory_key_msg)
         if self.llm is None:
             logger.bind(tag=TAG).error("LLM is not set for memory provider")
             return None

main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py

@@ -12,6 +12,8 @@ from core.utils.util import check_model_key
 from core.providers.tts.base import TTSProviderBase
 from core.handle.abortHandle import handleAbortMessage
 from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
+from asyncio import Task
+
 
 TAG = __name__
 logger = setup_logging()
@@ -141,6 +143,7 @@ class TTSProvider(TTSProviderBase):
         super().__init__(config, delete_audio_file)
         self.ws = None
         self.interface_type = InterfaceType.DUAL_STREAM
+        self._monitor_task = None  # 监听任务引用
         self.appId = config.get("appid")
         self.access_token = config.get("access_token")
         self.cluster = config.get("cluster")
@@ -270,8 +273,7 @@ class TTSProvider(TTSProviderBase):
         try:
             # 建立新连接
             if self.ws is None:
-                await handleAbortMessage(self.conn)
-                logger.bind(tag=TAG).error(f"WebSocket连接不存在，终止发送文本")
+                logger.bind(tag=TAG).warning(f"WebSocket连接不存在，终止发送文本")
                 return
 
             #  过滤Markdown
@@ -293,6 +295,25 @@ class TTSProvider(TTSProviderBase):
     async def start_session(self, session_id):
         logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
         try:
+            task = self._monitor_task
+            if (
+                task is not None
+                and isinstance(task, Task)
+                and not task.done()
+            ):
+                logger.bind(tag=TAG).info("等待上一个监听任务结束...")
+                if self.ws is not None:
+                    logger.bind(tag=TAG).info("强制关闭上一个WebSocket连接以唤醒监听任务...")
+                    try:
+                        await self.ws.close()
+                    except Exception as e:
+                        logger.bind(tag=TAG).warning(f"关闭上一个ws异常: {e}")
+                    self.ws = None
+                try:
+                    await asyncio.wait_for(task, timeout=8)
+                except Exception as e:
+                    logger.bind(tag=TAG).warning(f"等待监听任务异常: {e}")
+                self._monitor_task = None
             # 建立新连接
             await self._ensure_connection()
 
@@ -463,6 +484,8 @@ class TTSProvider(TTSProviderBase):
                 except:
                     pass
                 self.ws = None
+            # 监听任务退出时清理引用
+            self._monitor_task = None
 
     async def send_event(
         self,

main/xiaozhi-server/core/providers/vad/silero.py

@@ -35,6 +35,10 @@ class VADProvider(VADProviderBase):
             pcm_frame = self.decoder.decode(opus_packet, 960)
             conn.client_audio_buffer.extend(pcm_frame)  # 将新数据加入缓冲区
 
+            # 初始化帧计数器
+            if not hasattr(conn, "client_voice_frame_count"):
+                conn.client_voice_frame_count = 0
+
             # 处理缓冲区中的完整帧（每次处理512采样点）
             client_have_voice = False
             while len(conn.client_audio_buffer) >= 512 * 2:
@@ -50,7 +54,15 @@ class VADProvider(VADProviderBase):
                 # 检测语音活动
                 with torch.no_grad():
                     speech_prob = self.model(audio_tensor, 16000).item()
-                client_have_voice = speech_prob >= self.vad_threshold
+                is_voice = speech_prob >= self.vad_threshold
+
+                if is_voice:
+                    conn.client_voice_frame_count += 1
+                else:
+                    conn.client_voice_frame_count = 0
+
+                # 只有连续4帧检测到语音才认为有语音
+                client_have_voice = conn.client_voice_frame_count >= 4
 
                 # 如果之前有声音，但本次没有声音，且与上次有声音的时间差已经超过了静默阈值，则认为已经说完一句话
                 if conn.client_have_voice and not client_have_voice:

main/xiaozhi-server/plugins_func/functions/hass_set_state.py

@@ -31,7 +31,8 @@ hass_set_state_function_desc = {
                             "description": "只有在设置静音操作时才需要,设置静音的时候该值为true,取消静音时该值为false",
                         },
                         "rgb_color": {
-                            "type": "list",
+                            "type": "array",
+                            "items": {"type": "integer"},
                             "description": "只有在设置颜色时需要,这里填目标颜色的rgb值",
                         },
                     },