Merge branch 'custom_tts_api' into custom_tts_api

main/xiaozhi-server/core/connection.py

@@ -22,6 +22,7 @@ from core.utils.util import (
     initialize_modules,
     check_vad_update,
     check_asr_update,
+    filter_sensitive_info,
 )
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 from core.handle.sendAudioHandle import sendAudioMessage
@@ -273,9 +274,10 @@ class ConnectionHandler:
             await self.websocket.send(
                 json.dumps(
                     {
-                        "type": "server_response",
+                        "type": "server",
                         "status": "success",
                         "message": "服务器重启中...",
+                        "content": {"action": "restart"},
                     }
                 )
             )
@@ -302,9 +304,10 @@ class ConnectionHandler:
             await self.websocket.send(
                 json.dumps(
                     {
-                        "type": "server_response",
+                        "type": "server",
                         "status": "error",
                         "message": f"Restart failed: {str(e)}",
+                        "content": {"action": "restart"},
                     }
                 )
             )
@@ -1087,37 +1090,3 @@ class ConnectionHandler:
                     break
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"超时检查任务出错: {e}")
-
-
-def filter_sensitive_info(config: dict) -> dict:
-    """
-    过滤配置中的敏感信息
-    Args:
-        config: 原始配置字典
-    Returns:
-        过滤后的配置字典
-    """
-    sensitive_keys = [
-        "api_key",
-        "personal_access_token",
-        "access_token",
-        "token",
-        "secret",
-        "access_key_secret",
-        "secret_key",
-    ]
-
-    def _filter_dict(d: dict) -> dict:
-        filtered = {}
-        for k, v in d.items():
-            if any(sensitive in k.lower() for sensitive in sensitive_keys):
-                filtered[k] = "***"
-            elif isinstance(v, dict):
-                filtered[k] = _filter_dict(v)
-            elif isinstance(v, list):
-                filtered[k] = [_filter_dict(i) if isinstance(i, dict) else i for i in v]
-            else:
-                filtered[k] = v
-        return filtered
-
-    return _filter_dict(copy.deepcopy(config))

main/xiaozhi-server/core/handle/functionHandler.py

@@ -1,6 +1,12 @@
 from config.logger import setup_logging
 import json
-from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType
+from plugins_func.register import (
+    FunctionRegistry,
+    ActionResponse,
+    Action,
+    ToolType,
+    DeviceTypeRegistry,
+)
 from plugins_func.functions.hass_init import append_devices_to_prompt
 
 TAG = __name__
@@ -10,6 +16,7 @@ class FunctionHandler:
     def __init__(self, conn):
         self.conn = conn
         self.config = conn.config
+        self.device_type_registry = DeviceTypeRegistry()
         self.function_registry = FunctionRegistry()
         self.register_nessary_functions()
         self.register_config_functions()
@@ -54,7 +61,7 @@ class FunctionHandler:
         self.function_registry.register_function("plugin_loader")
         self.function_registry.register_function("get_time")
         self.function_registry.register_function("get_lunar")
-        self.function_registry.register_function("handle_device")
+        self.function_registry.register_function("handle_speaker_volume_or_screen_brightness")
 
     def register_config_functions(self):
         """注册配置中的函数,可以不同客户端使用不同的配置"""

main/xiaozhi-server/core/handle/helloHandle.py

@@ -40,8 +40,8 @@ async def checkWakeupWords(conn, text):
     if not enable_wakeup_words_response_cache:
         return False
     """检查是否是唤醒词"""
-    _, text = remove_punctuation_and_length(text)
-    if text in conn.config.get("wakeup_words"):
+    _, filtered_text = remove_punctuation_and_length(text)
+    if filtered_text in conn.config.get("wakeup_words"):
         await send_stt_message(conn, text)
         conn.tts_first_text_index = 0
         conn.tts_last_text_index = 0

main/xiaozhi-server/core/handle/intentHandler.py

@@ -13,10 +13,11 @@ TAG = __name__
 
 async def handle_user_intent(conn, text):
     # 检查是否有明确的退出命令
-    if await check_direct_exit(conn, text):
+    filtered_text = remove_punctuation_and_length(text)[1]
+    if await check_direct_exit(conn, filtered_text):
         return True
     # 检查是否是唤醒词
-    if await checkWakeupWords(conn, text):
+    if await checkWakeupWords(conn, filtered_text):
         return True
 
     if conn.intent_type == "function_call":

main/xiaozhi-server/core/handle/iotHandle.py

@@ -1,9 +1,8 @@
 import json
 import asyncio
-from config.logger import setup_logging
 from plugins_func.register import (
-    device_type_registry,
-    register_function,
+    FunctionItem,
+    register_device_function,
     ActionResponse,
     Action,
     ToolType,
@@ -177,7 +176,7 @@ class IotDescriptor:
                 self.methods.append(method)
 
 
-def register_device_type(descriptor):
+def register_device_type(descriptor, device_type_registry):
     """注册设备类型及其功能"""
     device_name = descriptor["name"]
     type_id = device_type_registry.generate_device_type_id(descriptor)
@@ -213,10 +212,12 @@ def register_device_type(descriptor):
             },
         }
         query_func = create_iot_query_function(device_name, prop_name, prop_info)
-        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
-            query_func
+        decorated_func = register_device_function(
+            func_name, func_desc, ToolType.IOT_CTL
+        )(query_func)
+        functions[func_name] = FunctionItem(
+            func_name, func_desc, decorated_func, ToolType.IOT_CTL
         )
-        functions[func_name] = decorated_func
 
     # 为每个方法创建控制函数
     for method_name, method_info in descriptor["methods"].items():
@@ -267,10 +268,12 @@ def register_device_type(descriptor):
             },
         }
         control_func = create_iot_function(device_name, method_name, method_info)
-        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
-            control_func
+        decorated_func = register_device_function(
+            func_name, func_desc, ToolType.IOT_CTL
+        )(control_func)
+        functions[func_name] = FunctionItem(
+            func_name, func_desc, decorated_func, ToolType.IOT_CTL
         )
-        functions[func_name] = decorated_func
 
     device_type_registry.register_device_type(type_id, functions)
     return type_id
@@ -289,7 +292,6 @@ async def handleIotDescriptors(conn, descriptors):
     functions_changed = False
 
     for descriptor in descriptors:
-
         # 如果descriptor没有properties和methods，则直接跳过
         if "properties" not in descriptor and "methods" not in descriptor:
             continue
@@ -319,13 +321,16 @@ async def handleIotDescriptors(conn, descriptors):
 
         if conn.load_function_plugin:
             # 注册或获取设备类型
-            type_id = register_device_type(descriptor)
+            device_type_registry = conn.func_handler.device_type_registry
+            type_id = register_device_type(descriptor, device_type_registry)
             device_functions = device_type_registry.get_device_functions(type_id)
 
             # 在连接级注册设备函数
             if hasattr(conn, "func_handler"):
-                for func_name in device_functions:
-                    conn.func_handler.function_registry.register_function(func_name)
+                for func_name, func_item in device_functions.items():
+                    conn.func_handler.function_registry.register_function(
+                        func_name, func_item
+                    )
                     conn.logger.bind(tag=TAG).info(
                         f"注册IOT函数到function handler: {func_name}"
                     )

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -39,14 +39,14 @@ async def handleAudioMessage(conn, audio):
         if len(conn.asr_audio) < 15:
             conn.asr_server_receive = True
         else:
-            text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
-            conn.logger.bind(tag=TAG).info(f"识别文本: {text}")
-            text_len, _ = remove_punctuation_and_length(text)
+            raw_text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)  # 确保ASR模块返回原始文本
+            conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
+            text_len, _ = remove_punctuation_and_length(raw_text)
             if text_len > 0:
                 # 使用自定义模块进行上报
-                enqueue_asr_report(conn, text, copy.deepcopy(conn.asr_audio))
+                enqueue_asr_report(conn, raw_text, copy.deepcopy(conn.asr_audio))
 
-                await startToChat(conn, text)
+                await startToChat(conn, raw_text)
             else:
                 conn.asr_server_receive = True
         conn.asr_audio.clear()

main/xiaozhi-server/core/handle/textHandle.py

@@ -1,7 +1,7 @@
 import json
 from core.handle.abortHandle import handleAbortMessage
 from core.handle.helloHandle import handleHelloMessage
-from core.utils.util import remove_punctuation_and_length
+from core.utils.util import remove_punctuation_and_length, filter_sensitive_info
 from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
 from core.handle.sendAudioHandle import send_stt_message, send_tts_message
 from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
@@ -13,17 +13,20 @@ TAG = __name__
 
 async def handleTextMessage(conn, message):
     """处理文本消息"""
-    conn.logger.bind(tag=TAG).info(f"收到文本消息：{message}")
     try:
         msg_json = json.loads(message)
         if isinstance(msg_json, int):
+            conn.logger.bind(tag=TAG).info(f"收到文本消息：{message}")
             await conn.websocket.send(message)
             return
         if msg_json["type"] == "hello":
+            conn.logger.bind(tag=TAG).info(f"收到hello消息：{message}")
             await handleHelloMessage(conn, msg_json)
         elif msg_json["type"] == "abort":
+            conn.logger.bind(tag=TAG).info(f"收到abort消息：{message}")
             await handleAbortMessage(conn)
         elif msg_json["type"] == "listen":
+            conn.logger.bind(tag=TAG).info(f"收到listen消息：{message}")
             if "mode" in msg_json:
                 conn.client_listen_mode = msg_json["mode"]
                 conn.logger.bind(tag=TAG).debug(
@@ -42,17 +45,17 @@ async def handleTextMessage(conn, message):
                 conn.client_have_voice = False
                 conn.asr_audio.clear()
                 if "text" in msg_json:
-                    text = msg_json["text"]
-                    _, text = remove_punctuation_and_length(text)
+                    original_text = msg_json["text"]  # 保留原始文本
+                    filtered_len, filtered_text = remove_punctuation_and_length(original_text)
 
                     # 识别是否是唤醒词
-                    is_wakeup_words = text in conn.config.get("wakeup_words")
+                    is_wakeup_words = filtered_text in conn.config.get("wakeup_words")
                     # 是否开启唤醒词回复
                     enable_greeting = conn.config.get("enable_greeting", True)
 
                     if is_wakeup_words and not enable_greeting:
                         # 如果是唤醒词，且关闭了唤醒词回复，就不用回答
-                        await send_stt_message(conn, text)
+                        await send_stt_message(conn, original_text)
                         await send_tts_message(conn, "stop", None)
                     elif is_wakeup_words:
                         # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
@@ -60,15 +63,20 @@ async def handleTextMessage(conn, message):
                         await startToChat(conn, "嘿，你好呀")
                     else:
                         # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
-                        enqueue_asr_report(conn, text, [])
+                        enqueue_asr_report(conn, original_text, [])
                         # 否则需要LLM对文字内容进行答复
-                        await startToChat(conn, text)
+                        await startToChat(conn, original_text)
         elif msg_json["type"] == "iot":
+            conn.logger.bind(tag=TAG).info(f"收到iot消息：{message}")
             if "descriptors" in msg_json:
                 asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
             if "states" in msg_json:
                 asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
         elif msg_json["type"] == "server":
+            # 记录日志时过滤敏感信息
+            conn.logger.bind(tag=TAG).info(
+                f"收到服务器消息：{filter_sensitive_info(msg_json)}"
+            )
             # 如果配置是从API读取的，则需要验证secret
             if not conn.read_config_from_api:
                 return
@@ -95,9 +103,10 @@ async def handleTextMessage(conn, message):
                         await conn.websocket.send(
                             json.dumps(
                                 {
-                                    "type": "config_update_response",
+                                    "type": "server",
                                     "status": "error",
                                     "message": "无法获取服务器实例",
+                                    "content": {"action": "update_config"},
                                 }
                             )
                         )
@@ -107,9 +116,10 @@ async def handleTextMessage(conn, message):
                         await conn.websocket.send(
                             json.dumps(
                                 {
-                                    "type": "config_update_response",
+                                    "type": "server",
                                     "status": "error",
                                     "message": "更新服务器配置失败",
+                                    "content": {"action": "update_config"},
                                 }
                             )
                         )
@@ -119,9 +129,10 @@ async def handleTextMessage(conn, message):
                     await conn.websocket.send(
                         json.dumps(
                             {
-                                "type": "config_update_response",
+                                "type": "server",
                                 "status": "success",
                                 "message": "配置更新成功",
+                                "content": {"action": "update_config"},
                             }
                         )
                     )
@@ -130,9 +141,10 @@ async def handleTextMessage(conn, message):
                     await conn.websocket.send(
                         json.dumps(
                             {
-                                "type": "config_update_response",
+                                "type": "server",
                                 "status": "error",
                                 "message": f"更新配置失败: {str(e)}",
+                                "content": {"action": "update_config"},
                             }
                         )
                     )

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -53,6 +53,8 @@ class IntentProvider(IntentProviderBase):
 
         prompt = (
             "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
+            "- 如果用户使用疑问词（如'怎么'、'为什么'、'如何'）询问退出相关的问题（例如'怎么退出了？'），注意这不是让你退出，请返回 {'function_call': {'name': 'continue_chat'}\n"
+            "- 仅当用户明确使用'退出系统'、'结束对话'、'我不想和你说话了'等指令时，才触发 handle_exit_intent\n\n"
             f"{functions_desc}\n"
             "处理步骤:\n"
             "1. 分析用户输入，确定用户意图\n"
@@ -218,6 +220,15 @@ class IntentProvider(IntentProviderBase):
                     f"llm 识别到意图: {function_name}, 参数: {function_args}"
                 )
 
+                # 如果是继续聊天，清理工具调用相关的历史消息
+                if function_name == "continue_chat":
+                    # 保留非工具相关的消息
+                    clean_history = [
+                        msg for msg in conn.dialogue.dialogue
+                        if msg.role not in ["tool", "function"]
+                    ]
+                    conn.dialogue.dialogue = clean_history
+
                 # 添加到缓存
                 self.intent_cache[cache_key] = {
                     "intent": intent,

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -130,7 +130,7 @@ class TTSProvider(TTSProviderBase):
             "yes",
         )
         self.use_memory_cache = config.get("use_memory_cache", "on")
-        self.seed = config.get("seed") or None
+        self.seed = int(config.get("seed")) if config.get("seed") else None
         self.api_url = config.get("api_url", "http://127.0.0.1:8080/v1/tts")
 
     def generate_filename(self, extension=".wav"):

main/xiaozhi-server/core/utils/util.py

@@ -9,6 +9,7 @@ import opuslib_next
 from pydub import AudioSegment
 from typing import Dict, Any
 from core.utils import tts, llm, intent, memory, vad, asr
+import copy
 
 TAG = __name__
 emoji_map = {
@@ -319,7 +320,7 @@ def initialize_modules(
         modules["memory"] = memory.create_instance(
             memory_type,
             config["Memory"][select_memory_module],
-            config.get('summaryMemory', None),
+            config.get("summaryMemory", None),
         )
         logger.bind(tag=TAG).info(f"初始化组件: memory成功 {select_memory_module}")
 
@@ -956,3 +957,37 @@ def check_asr_update(before_config, new_config):
     )
     update_asr = current_asr_type != new_asr_type
     return update_asr
+
+
+def filter_sensitive_info(config: dict) -> dict:
+    """
+    过滤配置中的敏感信息
+    Args:
+        config: 原始配置字典
+    Returns:
+        过滤后的配置字典
+    """
+    sensitive_keys = [
+        "api_key",
+        "personal_access_token",
+        "access_token",
+        "token",
+        "secret",
+        "access_key_secret",
+        "secret_key",
+    ]
+
+    def _filter_dict(d: dict) -> dict:
+        filtered = {}
+        for k, v in d.items():
+            if any(sensitive in k.lower() for sensitive in sensitive_keys):
+                filtered[k] = "***"
+            elif isinstance(v, dict):
+                filtered[k] = _filter_dict(v)
+            elif isinstance(v, list):
+                filtered[k] = [_filter_dict(i) if isinstance(i, dict) else i for i in v]
+            else:
+                filtered[k] = v
+        return filtered
+
+    return _filter_dict(copy.deepcopy(config))

main/xiaozhi-server/core/websocket_server.py

@@ -82,11 +82,13 @@ class WebSocketServer:
                 if new_config is None:
                     self.logger.bind(tag=TAG).error("获取新配置失败")
                     return False
-
+                self.logger.bind(tag=TAG).info(f"获取新配置成功")
                 # 检查 VAD 和 ASR 类型是否需要更新
                 update_vad = check_vad_update(self.config, new_config)
                 update_asr = check_asr_update(self.config, new_config)
-
+                self.logger.bind(tag=TAG).info(
+                    f"检查VAD和ASR类型是否需要更新: {update_vad} {update_asr}"
+                )
                 # 更新配置
                 self.config = new_config
                 # 重新初始化组件
@@ -114,7 +116,7 @@ class WebSocketServer:
                     self._intent = modules["intent"]
                 if "memory" in modules:
                     self._memory = modules["memory"]
-
+                self.logger.bind(tag=TAG).info(f"更新配置任务执行完毕")
                 return True
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"更新服务器配置失败: {str(e)}")