mergin main，速度提升一下

main/xiaozhi-server/core/handle/functionHandler.py

@@ -0,0 +1,84 @@
+import asyncio
+from enum import Enum
+
+from config.logger import setup_logging
+import json
+from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType
+TAG = __name__
+logger = setup_logging()
+
+
+class FunctionHandler:
+    def __init__(self, config):
+        self.config = config
+        self.function_registry = FunctionRegistry()
+        self.register_nessary_functions()
+        self.register_config_functions()
+        self.functions_desc = self.function_registry.get_all_function_desc()
+        func_names = self.current_support_functions()
+        self.modify_plugin_loader_des(func_names)
+
+    def modify_plugin_loader_des(self, func_names):
+        if "plugin_loader" not in func_names:
+            return
+        # 可编辑的列表中去掉plugin_loader
+        surport_plugins = [func for func in func_names if func != "plugin_loader"]
+        func_names = ",".join(surport_plugins)
+        for function_desc in self.functions_desc:
+            if function_desc["function"]["name"] == "plugin_loader":
+                function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]", func_names)
+                break
+        
+    def upload_functions_desc(self):
+        self.functions_desc = self.function_registry.get_all_function_desc()
+
+    def current_support_functions(self):
+        func_names = []
+        for func in self.functions_desc:
+            func_names.append(func["function"]["name"])
+        # 打印当前支持的函数列表
+        logger.bind(tag=TAG).info(f"当前支持的函数列表: {func_names}")
+        return func_names
+
+    def get_functions(self):
+        """获取功能调用配置"""
+        return self.functions_desc
+
+    def register_nessary_functions(self):
+        """注册必要的函数"""
+        self.function_registry.register_function("handle_exit_intent")
+        self.function_registry.register_function("play_music")
+        self.function_registry.register_function("plugin_loader")
+        self.function_registry.register_function("get_time")
+        self.function_registry.register_function("raise_and_lower_the_volume")
+
+    def register_config_functions(self):
+        """注册配置中的函数,可以不同客户端使用不同的配置"""
+        for func in self.config["Intent"]["function_call"].get("functions", []):
+            self.function_registry.register_function(func)
+
+    def get_function(self, name):
+        return self.function_registry.get_function(name)
+
+    def handle_llm_function_call(self, conn, function_call_data):
+        try:
+            function_name = function_call_data["name"]
+            funcItem = self.get_function(function_name)
+            if not funcItem:
+                return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
+            func = funcItem.func
+            arguments = function_call_data["arguments"]
+            arguments = json.loads(arguments) if arguments else {}
+            logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
+            if funcItem.type == ToolType.SYSTEM_CTL or funcItem.type == ToolType.IOT_CTL:
+                return func(conn, **arguments)
+            elif funcItem.type == ToolType.WAIT:
+                return func(**arguments)
+            elif funcItem.type == ToolType.CHANGE_SYS_PROMPT:
+                return func(conn, **arguments)
+            else:
+                return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理function call错误: {e}")
+
+        return None

main/xiaozhi-server/core/handle/intentHandler.py

@@ -1,105 +1,24 @@
 from config.logger import setup_logging
 import json
+import uuid
 from core.handle.sendAudioHandle import send_stt_message
-from core.utils.dialogue import Message
 from core.utils.util import remove_punctuation_and_length
-from config.functionCallConfig import FunctionCallConfig
-import asyncio
-from enum import Enum
 
 TAG = __name__
 logger = setup_logging()
 
 
-class Action(Enum):
-    NOTFOUND = (0, "没有找到函数")
-    NONE = (1, "啥也不干")
-    RESPONSE = (2, "直接回复")
-    REQLLM = (3, "调用函数后再请求llm生成回复")
-
-    def __init__(self, code, message):
-        self.code = code
-        self.message = message
-
-
-class ActionResponse:
-    def __init__(self, action: Action, result, response):
-        self.action = action  # 动作类型
-        self.result = result  # 动作产生的结果
-        self.response = response  # 直接回复的内容
-
-
-def get_functions():
-    """获取功能调用配置"""
-    return FunctionCallConfig
-
-
-def handle_llm_function_call(conn, function_call_data):
-    try:
-        function_name = function_call_data["name"]
-
-        if function_name == "handle_exit_intent":
-            # 处理退出意图
-            try:
-                say_goodbye = json.loads(function_call_data["arguments"]).get("say_goodbye", "再见")
-                conn.close_after_chat = True
-                logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}")
-                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye)
-            except Exception as e:
-                logger.bind(tag=TAG).error(f"处理退出意图错误: {e}")
-
-        elif function_name == "play_music":
-            # 处理音乐播放意图
-            try:
-                song_name = "random"
-                arguments = function_call_data["arguments"]
-                if arguments is not None and len(arguments) > 0:
-                    args = json.loads(arguments)
-                    song_name = args.get("song_name", "random")
-                music_intent = f"播放音乐 {song_name}" if song_name != "random" else "随机播放音乐"
-
-                # 执行音乐播放命令
-                future = asyncio.run_coroutine_threadsafe(
-                    conn.music_handler.handle_music_command(conn, music_intent),
-                    conn.loop
-                )
-                future.result()
-                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response="还想听什么歌？")
-            except Exception as e:
-                logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
-        else:
-            return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="没有找到对应的函数处理相对于的功能呢，你可以需要添加预设的对应函数处理呢")
-    except Exception as e:
-        logger.bind(tag=TAG).error(f"处理function call错误: {e}")
-
-    return None
-
-
 async def handle_user_intent(conn, text):
-    """
-    Handle user intent before starting chat
-    
-    Args:
-        conn: Connection object
-        text: User's text input
-    
-    Returns:
-        bool: True if intent was handled, False if should proceed to chat
-    """
     # 检查是否有明确的退出命令
     if await check_direct_exit(conn, text):
         return True
-
     if conn.use_function_call_mode:
         # 使用支持function calling的聊天方法,不再进行意图分析
         return False
-
     # 使用LLM进行意图分析
     intent = await analyze_intent_with_llm(conn, text)
-
     if not intent:
         return False
-
     # 处理各种意图
     return await process_intent_result(conn, intent, text)
 
@@ -126,7 +45,6 @@ async def analyze_intent_with_llm(conn, text):
     dialogue = conn.dialogue
     try:
         intent_result = await conn.intent.detect_intent(conn, dialogue.dialogue, text)
-
         # 尝试解析JSON结果
         try:
             intent_data = json.loads(intent_result)
@@ -147,9 +65,6 @@ async def process_intent_result(conn, intent, original_text):
     # 处理退出意图
     if "结束聊天" in intent:
         logger.bind(tag=TAG).info(f"识别到退出意图: {intent}")
-
-        # 如果正在播放音乐，可以关了 TODO
-
         # 如果是明确的离别意图，发送告别语并关闭连接
         await send_stt_message(conn, original_text)
         conn.executor.submit(conn.chat_and_close, original_text)
@@ -158,10 +73,37 @@ async def process_intent_result(conn, intent, original_text):
     # 处理播放音乐意图
     if "播放音乐" in intent:
         logger.bind(tag=TAG).info(f"识别到音乐播放意图: {intent}")
-        await conn.music_handler.handle_music_command(conn, intent)
+        # 调用play_music函数来播放音乐
+        song_name = extract_text_in_brackets(intent)
+        function_id = str(uuid.uuid4().hex)
+        function_name = "play_music"
+        function_arguments = '{ "song_name": "' + song_name + '" }'
+
+        function_call_data = {
+            "name": function_name,
+            "id": function_id,
+            "arguments": function_arguments
+        }
+        conn.func_handler.handle_llm_function_call(conn, function_call_data)
         return True
 
     # 其他意图处理可以在这里扩展
 
     # 默认返回False，表示继续常规聊天流程
     return False
+
+
+def extract_text_in_brackets(s):
+    """
+    从字符串中提取中括号内的文字
+
+    :param s: 输入字符串
+    :return: 中括号内的文字，如果不存在则返回空字符串
+    """
+    left_bracket_index = s.find('[')
+    right_bracket_index = s.find(']')
+
+    if left_bracket_index != -1 and right_bracket_index != -1 and left_bracket_index < right_bracket_index:
+        return s[left_bracket_index + 1:right_bracket_index]
+    else:
+        return ""

main/xiaozhi-server/core/handle/iotHandle.py

@@ -1,24 +1,120 @@
 import json
+import asyncio
 from config.logger import setup_logging
+from plugins_func.register import device_type_registry, register_function, ActionResponse, Action, ToolType
 
 TAG = __name__
 logger = setup_logging()
 
 
+def wrap_async_function(async_func):
+    """包装异步函数为同步函数"""
+
+    def wrapper(*args, **kwargs):
+        try:
+            # 获取连接对象（第一个参数）
+            conn = args[0]
+            if not hasattr(conn, 'loop'):
+                logger.bind(tag=TAG).error("Connection对象没有loop属性")
+                return ActionResponse(Action.ERROR, "Connection对象没有loop属性",
+                                      "执行操作时出错: Connection对象没有loop属性")
+
+            # 使用conn对象中的事件循环
+            loop = conn.loop
+            # 在conn的事件循环中运行异步函数
+            future = asyncio.run_coroutine_threadsafe(async_func(*args, **kwargs), loop)
+            # 等待结果返回
+            return future.result()
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
+            return ActionResponse(Action.ERROR, str(e), f"执行操作时出错: {e}")
+
+    return wrapper
+
+
+def create_iot_function(device_name, method_name, method_info):
+    """
+    根据IOT设备描述生成通用的控制函数
+    """
+
+    async def iot_control_function(conn, response_success=None, response_failure=None, **params):
+        try:
+            # 打印响应参数
+            logger.bind(tag=TAG).info(
+                f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
+
+            # 发送控制命令
+            await send_iot_conn(conn, device_name, method_name, params)
+            # 等待一小段时间让状态更新
+            await asyncio.sleep(0.1)
+
+            # 生成结果信息
+            result = f"{device_name}的{method_name}操作执行成功"
+
+
+            # 处理响应中可能的占位符
+            response = response_success
+            # 替换{value}占位符
+            for param_name, param_value in params.items():
+                # 先尝试直接替换参数值
+                if "{" + param_name + "}" in response:
+                    response = response.replace("{" + param_name + "}", str(param_value))
+
+                # 如果有{value}占位符，用相关参数替换
+                if "{value}" in response:
+                    response = response.replace("{value}", str(param_value))
+                    break
+
+            return ActionResponse(Action.RESPONSE, result, response)
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"执行{device_name}的{method_name}操作失败: {e}")
+
+            # 操作失败时使用大模型提供的失败响应
+            response = response_failure
+
+            return ActionResponse(Action.ERROR, str(e), response)
+
+    return wrap_async_function(iot_control_function)
+
+
+def create_iot_query_function(device_name, prop_name, prop_info):
+    """
+    根据IOT设备属性创建查询函数
+    """
+
+    async def iot_query_function(conn, response_success=None, response_failure=None):
+        try:
+            # 打印响应参数
+            logger.bind(tag=TAG).info(
+                f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
+
+            value = await get_iot_status(conn, device_name, prop_name)
+
+            # 查询成功，生成结果
+            if value is not None:
+                # 使用大模型提供的成功响应，并替换其中的占位符
+                response = response_success.replace("{value}", str(value))
+
+                return ActionResponse(Action.RESPONSE, str(value), response)
+            else:
+                # 查询失败，使用大模型提供的失败响应
+                response = response_failure
+
+                return ActionResponse(Action.ERROR, f"属性{prop_name}不存在", response)
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"查询{device_name}的{prop_name}时出错: {e}")
+
+            # 查询出错时使用大模型提供的失败响应
+            response = response_failure
+
+            return ActionResponse(Action.ERROR, str(e), response)
+
+    return wrap_async_function(iot_query_function)
+
+
 class IotDescriptor:
     """
     A class to represent an IoT descriptor.
-    Attributes:
-    ----------
-    name : str
-        The name of the IoT descriptor.
-    description : str
-        A brief description of the IoT descriptor.
-    properties : dict
-        A dictionary containing properties of the IoT descriptor.
-    methods : dict
-        A dictionary containing methods of the IoT descriptor.
-    -------
     """
 
     def __init__(self, name, description, properties, methods):
@@ -29,17 +125,7 @@ class IotDescriptor:
 
         # 根据描述创建属性
         for key, value in properties.items():
-            # "volume":{"description":"当前音量 值","type":"number"}
-            """
-            等价于
-            {
-                'name': 名字,
-                'description': 描述,
-                'value': 0
-            }
-            """
-            # setattr(self, key, {}) # 创建一个空字典, 名字是属性名
-            property_item = globals()[key] = {}  # 创建一个空字典, 名字是属性名
+            property_item = globals()[key] = {}
             property_item['name'] = key
             property_item["description"] = value["description"]
             if value["type"] == "number":
@@ -52,23 +138,10 @@ class IotDescriptor:
 
         # 根据描述创建方法
         for key, value in methods.items():
-            # "SetVolume": {"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}}
-            """
-            等价于
-            SetVolume = {
-                `description`: 描述,
-                `volume`: {
-                    `description`: 描述,
-                    `value`: 0
-                }
-            }
-            """
-            # setattr(self, key, {}) # 创建一个空字典, 名字是方法名
-            method = globals()[key] = {}  # 创建一个空字典, 名字是方法名
+            method = globals()[key] = {}
             method["description"] = value["description"]
             method['name'] = key
             for k, v in value["parameters"].items():
-                # 不同的参数解析
                 method[k] = {}
                 method[k]["description"] = v["description"]
                 if v["type"] == "number":
@@ -77,58 +150,136 @@ class IotDescriptor:
                     method[k]["value"] = False
                 else:
                     method[k]["value"] = ""
-
             self.methods.append(method)
 
 
-async def handleIotDescriptors(conn, descriptors):
-    """
-    处理物联网描述
-    示例: [{
-        "name":"Speaker",
-        "description":"当前 AI 机器人的扬声器",
-        "properties":{
-            "volume":{"description":"当前音量 值","type":"number"}  可以有boolean, number, string三种类型
-        },
-        "methods":{
-            "SetVolume":{
-                "description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}
+def register_device_type(descriptor):
+    """注册设备类型及其功能"""
+    device_name = descriptor["name"]
+    type_id = device_type_registry.generate_device_type_id(descriptor)
+
+    # 如果该类型已注册，直接返回类型ID
+    if type_id in device_type_registry.type_functions:
+        return type_id
+
+    functions = {}
+
+    # 为每个属性创建查询函数
+    for prop_name, prop_info in descriptor["properties"].items():
+        func_name = f"get_{device_name.lower()}_{prop_name.lower()}"
+        func_desc = {
+            "type": "function",
+            "function": {
+                "name": func_name,
+                "description": f"查询{descriptor['description']}的{prop_info['description']}",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "response_success": {
+                            "type": "string",
+                            "description": f"查询成功时的友好回复，必须使用{{value}}作为占位符表示查询到的值"
+                        },
+                        "response_failure": {
+                            "type": "string",
+                            "description": f"查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'"
+                        }
+                    },
+                    "required": ["response_success", "response_failure"]
+                }
             }
         }
-    }]
-    descriptors: 描述列表
-    """
+        query_func = create_iot_query_function(device_name, prop_name, prop_info)
+        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(query_func)
+        functions[func_name] = decorated_func
+
+    # 为每个方法创建控制函数
+    for method_name, method_info in descriptor["methods"].items():
+        func_name = f"{device_name.lower()}_{method_name.lower()}"
+
+        # 创建参数字典，添加原有参数
+        parameters = {
+            param_name: {
+                "type": param_info["type"],
+                "description": param_info["description"]
+            }
+            for param_name, param_info in method_info["parameters"].items()
+        }
+
+        # 添加响应参数
+        parameters.update({
+            "response_success": {
+                "type": "string",
+                "description": "操作成功时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
+            },
+            "response_failure": {
+                "type": "string",
+                "description": "操作失败时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
+            }
+        })
+
+        # 构建必须参数列表（原有参数 + 响应参数）
+        required_params = list(method_info["parameters"].keys())
+        required_params.extend(["response_success", "response_failure"])
+
+        func_desc = {
+            "type": "function",
+            "function": {
+                "name": func_name,
+                "description": f"{descriptor['description']} - {method_info['description']}",
+                "parameters": {
+                    "type": "object",
+                    "properties": parameters,
+                    "required": required_params
+                }
+            }
+        }
+        control_func = create_iot_function(device_name, method_name, method_info)
+        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(control_func)
+        functions[func_name] = decorated_func
+
+    device_type_registry.register_device_type(type_id, functions)
+    return type_id
+
+
+# 用于接受前端设备推送的搜索iot描述
+async def handleIotDescriptors(conn, descriptors):
+    """处理物联网描述"""
+    functions_changed = False
+
     for descriptor in descriptors:
+        # 创建IOT设备描述符
         iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
                                        descriptor["methods"])
         conn.iot_descriptors[descriptor["name"]] = iot_descriptor
 
-    # 暂时从配置文件中设置音量，后期通过意图识别控制音量
-    default_iot_volume = 100
-    if "iot" in conn.config:
-        default_iot_volume = conn.config["iot"]["Speaker"]["volume"]
-    logger.bind(tag=TAG).info(f"服务端设置音量为{default_iot_volume}")
-    await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": default_iot_volume})
+        if conn.use_function_call_mode:
+            # 注册或获取设备类型
+            type_id = register_device_type(descriptor)
+            device_functions = device_type_registry.get_device_functions(type_id)
+
+            # 在连接级注册设备函数
+            if hasattr(conn, 'func_handler'):
+                for func_name in device_functions:
+                    conn.func_handler.function_registry.register_function(func_name)
+                    logger.bind(tag=TAG).info(f"注册IOT函数到function handler: {func_name}")
+                    functions_changed = True
+
+    # 如果注册了新函数，更新function描述列表
+    if functions_changed and hasattr(conn, 'func_handler'):
+        conn.func_handler.upload_functions_desc()
+        func_names = conn.func_handler.current_support_functions()
+        logger.bind(tag=TAG).info(f"设备类型: {type_id}")
+        logger.bind(tag=TAG).info(f"更新function描述列表完成，当前支持的函数: {func_names}")
+
+
 async def handleIotStatus(conn, states):
-    """
-    处理物联网状态
-    示例: [{
-        "name":"Speaker",
-        "state":{
-            "volume":100
-        }
-    }]
-    states: 状态列表
-    """
+    """处理物联网状态"""
     for state in states:
         for key, value in conn.iot_descriptors.items():
             if key == state["name"]:
                 for property_item in value.properties:
-                    # properties为字典列表, 记录各种属性
                     for k, v in state["state"].items():
-                        # state为字典, 记录各种属性的值, 是需要记录的信息
                         if property_item["name"] == k:
-                            # 检查一下属性是不是相同的
                             if type(v) != type(property_item["value"]):
                                 logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
                                 break
@@ -138,41 +289,35 @@ async def handleIotStatus(conn, states):
                             break
                 break
 
+
 async def get_iot_status(conn, name, property_name):
-    """
-    获取物联网状态
-    name: 设备名称 "Speaker"
-    property_name: 属性名称 "volume"
-    返回值: 属性值, 实际的属性有int, bool和str三种类型
-    """
+    """获取物联网状态"""
     for key, value in conn.iot_descriptors.items():
         if key == name:
             for property_item in value.properties:
                 if property_item["name"] == property_name:
                     return property_item["value"]
+    logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
     return None
 
-async def send_iot_conn(conn, name, method_name, parameters):
-    """
-    发送物联网指令
-    name: 设备名称 "Speaker"
-    method: 方法 "SetVolume"
-    parameters: 参数, 是一个字典 {"volume": 100}
-    发送示例:
-    {
-        "type": "iot",
-        "commands": [
-            {
-                "name" :  "Speaker",
-                "method": "SetVolume",
-                "parameters": {
-                    "volume": 100
-                    }
-            }
-        ]
-    }
-    """
 
+async def set_iot_status(conn, name, property_name, value):
+    """设置物联网状态"""
+    for key, iot_descriptor in conn.iot_descriptors.items():
+        if key == name:
+            for property_item in iot_descriptor.properties:
+                if property_item["name"] == property_name:
+                    if type(value) != type(property_item["value"]):
+                        logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
+                        return
+                    property_item["value"] = value
+                    logger.bind(tag=TAG).info(f"物联网状态更新: {name} , {property_name} = {value}")
+                    return
+    logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
+
+
+async def send_iot_conn(conn, name, method_name, parameters):
+    """发送物联网指令"""
     for key, value in conn.iot_descriptors.items():
         if key == name:
             # 找到了设备

main/xiaozhi-server/core/handle/musicHandler.py

@@ -1,139 +0,0 @@
-from config.logger import setup_logging
-import os
-import random
-import difflib
-import re
-import traceback
-from pathlib import Path
-import time
-from core.handle.sendAudioHandle import send_stt_message
-from core.utils import p3
-
-TAG = __name__
-logger = setup_logging()
-
-
-def _extract_song_name(text):
-    """从用户输入中提取歌名"""
-    for keyword in ["播放音乐"]:
-        if keyword in text:
-            parts = text.split(keyword)
-            if len(parts) > 1:
-                return parts[1].strip()
-    return None
-
-
-def _find_best_match(potential_song, music_files):
-    """查找最匹配的歌曲"""
-    best_match = None
-    highest_ratio = 0
-
-    for music_file in music_files:
-        song_name = os.path.splitext(music_file)[0]
-        ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
-        if ratio > highest_ratio and ratio > 0.4:
-            highest_ratio = ratio
-            best_match = music_file
-    return best_match
-
-
-class MusicManager:
-    def __init__(self, music_dir, music_ext):
-        self.music_dir = Path(music_dir)
-        self.music_ext = music_ext
-
-    def get_music_files(self):
-        music_files = []
-        for file in self.music_dir.rglob("*"):
-            # 判断是否是文件
-            if file.is_file():
-                # 获取文件扩展名
-                ext = file.suffix.lower()
-                # 判断扩展名是否在列表中
-                if ext in self.music_ext:
-                    # music_files.append(str(file.resolve()))  # 添加绝对路径
-                    # 添加相对路径
-                    music_files.append(str(file.relative_to(self.music_dir)))
-        return music_files
-
-
-class MusicHandler:
-    def __init__(self, config):
-        self.config = config
-
-        if "music" in self.config:
-            self.music_config = self.config["music"]
-            self.music_dir = os.path.abspath(
-                self.music_config.get("music_dir", "./music")  # 默认路径修改
-            )
-            self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
-            self.refresh_time = self.music_config.get("refresh_time", 60)
-        else:
-            self.music_dir = os.path.abspath("./music")
-            self.music_ext = (".mp3", ".wav", ".p3")
-            self.refresh_time = 60
-
-        # 获取音乐文件列表
-        self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
-        self.scan_time = time.time()
-        logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}")
-
-    async def handle_music_command(self, conn, text):
-        """处理音乐播放指令"""
-        clean_text = re.sub(r'[^\w\s]', '', text).strip()
-        logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
-
-        # 尝试匹配具体歌名
-        if os.path.exists(self.music_dir):
-            if time.time() - self.scan_time > self.refresh_time:
-                # 刷新音乐文件列表
-                self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
-                self.scan_time = time.time()
-                logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}")
-
-            potential_song = _extract_song_name(clean_text)
-            if potential_song:
-                best_match = _find_best_match(potential_song, self.music_files)
-                if best_match:
-                    logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
-                    await self.play_local_music(conn, specific_file=best_match)
-                    return True
-        # 检查是否是通用播放音乐命令
-        await self.play_local_music(conn)
-        return True
-
-    async def play_local_music(self, conn, specific_file=None):
-        """播放本地音乐文件"""
-        try:
-            if not os.path.exists(self.music_dir):
-                logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}")
-                return
-
-            # 确保路径正确性
-            if specific_file:
-                selected_music = specific_file
-                music_path = os.path.join(self.music_dir, specific_file)
-            else:
-                if not self.music_files:
-                    logger.bind(tag=TAG).error("未找到MP3音乐文件")
-                    return
-                selected_music = random.choice(self.music_files)
-                music_path = os.path.join(self.music_dir, selected_music)
-
-            if not os.path.exists(music_path):
-                logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
-                return
-            text = f"正在播放{selected_music}"
-            await send_stt_message(conn, text)
-            conn.tts_first_text_index = 0
-            conn.tts_last_text_index = 0
-            conn.llm_finish_task = True
-            if music_path.endswith(".p3"):
-                opus_packets, duration = p3.decode_opus_from_file(music_path)
-            else:
-                opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
-            conn.audio_play_queue.put((opus_packets, selected_music, 0))
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
-            logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -20,7 +20,8 @@ async def handleAudioMessage(conn, audio):
     # 如果本次没有声音，本段也没声音，就把声音丢弃了
     if have_voice == False and conn.client_have_voice == False:
         await no_voice_close_connect(conn)
-        conn.asr_audio.clear()
+        conn.asr_audio.append(audio)
+        conn.asr_audio = conn.asr_audio[-5:]  # 保留最新的5帧音频内容，解决ASR句首丢字问题
         return
     conn.client_no_voice_last_time = 0.0
     conn.asr_audio.append(audio)
@@ -29,7 +30,7 @@ async def handleAudioMessage(conn, audio):
         conn.client_abort = False
         conn.asr_server_receive = False
         # 音频太短了，无法识别
-        if len(conn.asr_audio) < 3:
+        if len(conn.asr_audio) < 10:
             conn.asr_server_receive = True
         else:
             text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -51,15 +51,6 @@ async def sendAudioMessageStream(conn, audios_queue, text, text_index=0, llm_fin
                 for opus_packet in audio_opus_datas:
                     if conn.client_abort:
                         return
-                    # 计算当前包的预期发送时间
-                    # 计算当前包的预期发送时间
-                    expected_time = start_time_chunk + (play_position / 1000)
-                    current_time = time.perf_counter()
-
-                    # 等待直到预期时间
-                    delay = expected_time - current_time
-                    if delay > 0:
-                        await asyncio.sleep(delay)
                     logger.bind(tag=TAG).info(f'发送数据长度：{len(opus_packet)}')
                     await conn.websocket.send(opus_packet)
                     play_position += frame_duration  # 更新播放位置
@@ -70,15 +61,15 @@ async def sendAudioMessageStream(conn, audios_queue, text, text_index=0, llm_fin
     await send_tts_message(conn, "sentence_end", text)
 
     print(f'{text_index}-{conn.tts_last_text_index}')
+    expected_time = start_time_chunk + (play_position / 1000)
+    current_time = time.perf_counter()
+    # 等待直到预期时间
+    delay = expected_time - current_time
+    if delay > 0:
+        await asyncio.sleep(delay)
     # 发送结束消息（如果是最后一个文本）
     logger.bind(tag=TAG).info(f"{conn.llm_finish_task},{text_index},{conn.tts_last_text_index}")
     if conn.llm_finish_task and text_index == conn.tts_last_text_index:
-        expected_time = start_time_chunk + (play_position / 1000)
-        current_time = time.perf_counter()
-        # 等待直到预期时间
-        delay = expected_time - current_time
-        if delay > 0:
-            await asyncio.sleep(delay)
         await send_tts_message(conn, 'stop', None)
         if conn.close_after_chat or "拜拜" in text or "再见" in text:
             await conn.close()

main/xiaozhi-server/core/handle/textHandle.py

@@ -2,7 +2,7 @@ from config.logger import setup_logging
 import json
 from core.handle.abortHandle import handleAbortMessage
 from core.handle.helloHandle import handleHelloMessage
-from core.handle.receiveAudioHandle import startToChat
+from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
 from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
 
 TAG = __name__
@@ -31,6 +31,8 @@ async def handleTextMessage(conn, message):
             elif msg_json["state"] == "stop":
                 conn.client_have_voice = True
                 conn.client_voice_stop = True
+                if len(conn.asr_audio) > 0:
+                    await handleAudioMessage(conn, b'')
             elif msg_json["state"] == "detect":
                 conn.asr_server_receive = False
                 conn.client_have_voice = False
@@ -41,6 +43,6 @@ async def handleTextMessage(conn, message):
             if "descriptors" in msg_json:
                 await handleIotDescriptors(conn, msg_json["descriptors"])
             if "states" in msg_json:
-                await handleIotStatus(conn, msg_json["states"])  
+                await handleIotStatus(conn, msg_json["states"])
     except json.JSONDecodeError:
         await conn.websocket.send(message)