update:更新视频demo (#397)

Co-authored-by: hrz <1710360675@qq.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug 报告（Bug Report）
+about: 反馈项目中的缺陷或问题
+title: "[Bug] 简短描述问题"
+labels: bug
+assignees: ''
+---
+
+## 🐛 问题描述
+<!-- 清晰简洁地描述问题是什么 -->
+
+## 🔍 复现步骤
+<!-- 详细描述复现问题的步骤 -->
+1. 打开 '...'
+2. 点击 '...'
+3. 滚动到 '...'
+4. 看到错误
+
+## 🤔 预期行为
+<!-- 简要描述预期的正确行为 -->
+
+## 😯 截图
+<!-- 如果适用，添加问题的截图 -->
+
+## 🖥️ 环境信息
+- 操作系统: [例如 Windows 10]
+- 浏览器: [例如 Chrome 89]
+- 项目版本: [例如 1.0.0]
+- Python版本: [例如 3.9.13]
+- Jdk版本：[例如 java 21.0.5 2024-10-15 LTS]
+- Nodejs版本：[例如 v20.14.0]
+
+## 📋 其他信息
+<!-- 在此添加关于此问题的任何其他上下文信息 -->

.github/ISSUE_TEMPLATE/code_improvement.md

@@ -0,0 +1,19 @@
+---
+name: 代码优化建议（Code Improvement）
+about: 提出对现有代码的优化或改进建议
+title: "[Improvement] 简短描述改进内容"
+labels: refactor
+assignees: ''
+---
+
+## 💡 改进描述
+<!-- 描述需要改进的代码部分以及存在的问题 -->
+
+## 🌟 改进建议
+<!-- 提出具体的改进方案或思路 -->
+
+## 🛠️ 相关代码
+<!-- 如果可能，提供相关代码片段或链接 -->
+
+## 📋 其他信息
+<!-- 在此添加任何其他相关信息 -->

.github/ISSUE_TEMPLATE/documentation_improvement.md

@@ -0,0 +1,16 @@
+---
+name: 文档改进建议（Documentation Improvement）
+about: 提出对项目文档的改进或补充建议
+title: "[Docs] 简短描述改进内容"
+labels: documentation
+assignees: ''
+---
+
+## 📚 改进描述
+<!-- 描述需要改进的文档部分以及存在的问题 -->
+
+## ✨ 改进建议
+<!-- 提出具体的改进方案或内容 -->
+
+## 📋 其他信息
+<!-- 在此添加任何其他相关信息 -->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,19 @@
+---
+name: 功能请求（Feature Request）
+about: 提出新的功能或改进建议
+title: "[Feature] 简短描述功能"
+labels: enhancement
+assignees: ''
+---
+
+## 🚀 需求描述
+<!-- 清晰简洁地描述你希望发生什么 -->
+
+## 🎯 解决方案
+<!-- 描述你认为可行的解决方案或实现方式 -->
+
+## 📝 备选方案
+<!-- 描述你考虑过的替代方案或功能 -->
+
+## 📋 其他信息
+<!-- 在此添加关于需求的任何其他上下文信息 -->

README.md

@@ -57,6 +57,13 @@
          </picture>
         </a>
     </td>
+    <td>
+        <a href="https://www.bilibili.com/video/BV1kgA2eYEQ9" target="_blank">
+         <picture>
+           <img alt="成本最低配置" src="docs/images/demo4.png" />
+         </picture>
+        </a>
+    </td>
   </tr>
   <tr>
     <td>
@@ -74,9 +81,16 @@
         </a>
     </td>
     <td>
-        <a href="https://www.bilibili.com/video/BV1kgA2eYEQ9" target="_blank">
+        <a href="https://www.bilibili.com/video/BV1Z8XuYZEAS" target="_blank">
          <picture>
-           <img alt="成本最低配置" src="docs/images/demo4.png" />
+           <img alt="天气插件" src="docs/images/demo8.png" />
+         </picture>
+        </a>
+    </td>
+    <td>
+        <a href="https://www.bilibili.com/video/BV178XuYfEpi" target="_blank">
+         <picture>
+           <img alt="IOT指令控制设备" src="docs/images/demo9.png" />
          </picture>
         </a>
     </td>
@@ -143,12 +157,13 @@ server:
   支持 EdgeTTS（默认）、火山引擎豆包 TTS 等多种 TTS 接口，满足语音合成需求。
 - **记忆功能**  
   支持超长记忆、本地总结记忆、无记忆三种模式，满足不同场景需求。
+- **IOT功能**  
+  支持管理注册设备IOT功能，支持基于对话上下文语境下的智能物联网控制。
 
 ### 正在开发 🚧
 
 - 多种心情模式
 - 智控台webui
-- iot功能
 
 想了解具体开发进度，[请点击这里](https://github.com/users/xinnan-tech/projects/3)
 

docs/contributor_open_letter.md

@@ -35,7 +35,7 @@ John，2025.3.11，广州
 
 ### 1、成为普通贡献者
 
-Fork 项目，，提交 PR，由开发者审核后合入主分支。
+Fork 项目，提交 PR，由开发者审核后合入主分支。
 
 ### 2、成为开发者
 

main/README.md

@@ -18,6 +18,6 @@ xiaozhi-esp32-server
 
 # manager-web 、manager-api接口协议
 
-[manager前后端接口协议](https://app.apifox.com/invite/project?token=H_8qhgfjUeaAL0wybghgU)
+[manager前后端接口协议](https://app.apifox.com/invite/project?token=eXg2_tUv85q-gc3ZRowmn)
 
 [前端页面设计图](https://codesign.qq.com/app/s/526108506410828)

main/xiaozhi-server/config.yaml

@@ -35,10 +35,7 @@ log:
   log_file: "server.log"
   # 设置数据文件路径
   data_dir: data
-iot:
-  Speaker:
-    # 设置esp32的音量，范围0-100
-    volume: 80
+
 xiaozhi:
   type: hello
   version: 1
@@ -102,6 +99,14 @@ Intent:
       - change_role
       - get_weather
 
+# 插件的基础配置
+plugins:
+  # 获取天气插件的配置，这里填写你的api_key
+  # 这个密钥是项目共用的key，用多了可能会被限制
+  # 想稳定一点就自行申请替换，每天有1000次免费调用
+  # 申请地址：https://console.qweather.com/#/apps/create-key/over
+  get_weather: { "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" }
+
 Memory:
   mem0ai:
     type: mem0ai

main/xiaozhi-server/core/connection.py

@@ -5,13 +5,15 @@ import time
 import queue
 import asyncio
 import traceback
-from config.logger import setup_logging
+
 import threading
 import websockets
 from typing import Dict, Any
+import plugins_func.loadplugins
+from config.logger import setup_logging
 from core.utils.dialogue import Message, Dialogue
 from core.handle.textHandle import handleTextMessage
-from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string
+from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string, get_ip_info
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 from core.handle.sendAudioHandle import sendAudioMessage
 from core.handle.receiveAudioHandle import handleAudioMessage
@@ -20,7 +22,6 @@ from plugins_func.register import Action
 from config.private_config import PrivateConfig
 from core.auth import AuthMiddleware, AuthenticationError
 from core.utils.auth_code_gen import AuthCodeGenerator
-import plugins_func.loadplugins  
 
 TAG = __name__
 
@@ -30,13 +31,15 @@ class TTSException(RuntimeError):
 
 
 class ConnectionHandler:
-    def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory, _intent):
+    def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _memory, _intent):
         self.config = config
         self.logger = setup_logging()
         self.auth = AuthMiddleware(config)
 
         self.websocket = None
         self.headers = None
+        self.client_ip = None
+        self.client_ip_info = {}
         self.session_id = None
         self.prompt = None
         self.welcome_msg = None
@@ -91,21 +94,18 @@ class ConnectionHandler:
         self.private_config = None
         self.auth_code_gen = AuthCodeGenerator.get_instance()
         self.is_device_verified = False  # 添加设备验证状态标志
-        self.music_handler = _music
         self.close_after_chat = False  # 是否在聊天结束后关闭连接
         self.use_function_call_mode = False
         if self.config["selected_module"]["Intent"] == 'function_call':
             self.use_function_call_mode = True
-        
-        self.func_handler = FunctionHandler(self.config)
 
     async def handle_connection(self, ws):
         try:
             # 获取并验证headers
             self.headers = dict(ws.request.headers)
             # 获取客户端ip地址
-            client_ip = ws.remote_address[0]
-            self.logger.bind(tag=TAG).info(f"{client_ip} conn - Headers: {self.headers}")
+            self.client_ip = ws.remote_address[0]
+            self.logger.bind(tag=TAG).info(f"{self.client_ip} conn - Headers: {self.headers}")
 
             # 进行认证
             await self.auth.authenticate(self.headers)
@@ -149,6 +149,7 @@ class ConnectionHandler:
             self.welcome_msg["session_id"] = self.session_id
             await self.websocket.send(json.dumps(self.welcome_msg))
 
+            # 异步初始化
             await self.loop.run_in_executor(None, self._initialize_components)
 
             # tts 消化线程
@@ -189,7 +190,13 @@ class ConnectionHandler:
         self.prompt = self.config["prompt"]
         if self.private_config:
             self.prompt = self.private_config.private_config.get("prompt", self.prompt)
+
+        self.client_ip_info = get_ip_info(self.client_ip)
+        self.logger.bind(tag=TAG).info(f"Client ip info: {self.client_ip_info}")
+        self.prompt = self.prompt + f"\n我在:{self.client_ip_info}"
         self.dialogue.put(Message(role="system", content=self.prompt))
+
+        self.func_handler = FunctionHandler(self.config)
     
     def change_system_prompt(self, prompt):
         self.prompt = prompt
@@ -451,7 +458,7 @@ class ConnectionHandler:
             future = self.executor.submit(self.speak_and_play, text, text_index)
             self.tts_queue.put(future)
             self.dialogue.put(Message(role="assistant", content=text))
-        if result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
+        elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
             
             text = result.result
             if text is not None and len(text) > 0:
@@ -466,9 +473,10 @@ class ConnectionHandler:
 
                 self.dialogue.put(Message(role="tool", tool_call_id=function_id, content=text))
                 self.chat_with_function_calling(text, tool_call=True)
-        if result.action == Action.NOTFOUND:
+        elif result.action == Action.NOTFOUND:
+            text = result.response
+        else:
             text = result.response
-
         
 
     def _tts_priority_thread(self):

main/xiaozhi-server/core/handle/functionHandler.py

@@ -50,6 +50,7 @@ class FunctionHandler:
         self.function_registry.register_function("play_music")
         self.function_registry.register_function("plugin_loader")
         self.function_registry.register_function("get_time")
+        self.function_registry.register_function("raise_and_lower_the_volume")
 
     def register_config_functions(self):
         """注册配置中的函数,可以不同客户端使用不同的配置"""
@@ -69,7 +70,7 @@ class FunctionHandler:
             arguments = function_call_data["arguments"]
             arguments = json.loads(arguments) if arguments else {}
             logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
-            if funcItem.type == ToolType.SYSTEM_CTL:
+            if funcItem.type == ToolType.SYSTEM_CTL or funcItem.type == ToolType.IOT_CTL:
                 return func(conn, **arguments)
             elif funcItem.type == ToolType.WAIT:
                 return func(**arguments)

main/xiaozhi-server/core/handle/intentHandler.py

@@ -1,5 +1,6 @@
 from config.logger import setup_logging
 import json
+import uuid
 from core.handle.sendAudioHandle import send_stt_message
 from core.utils.util import remove_punctuation_and_length
 
@@ -8,30 +9,16 @@ logger = setup_logging()
 
 
 async def handle_user_intent(conn, text):
-    """
-    Handle user intent before starting chat
-    
-    Args:
-        conn: Connection object
-        text: User's text input
-    
-    Returns:
-        bool: True if intent was handled, False if should proceed to chat
-    """
     # 检查是否有明确的退出命令
     if await check_direct_exit(conn, text):
         return True
-
     if conn.use_function_call_mode:
         # 使用支持function calling的聊天方法,不再进行意图分析
         return False
-
     # 使用LLM进行意图分析
     intent = await analyze_intent_with_llm(conn, text)
-
     if not intent:
         return False
-
     # 处理各种意图
     return await process_intent_result(conn, intent, text)
 
@@ -58,7 +45,6 @@ async def analyze_intent_with_llm(conn, text):
     dialogue = conn.dialogue
     try:
         intent_result = await conn.intent.detect_intent(conn, dialogue.dialogue, text)
-
         # 尝试解析JSON结果
         try:
             intent_data = json.loads(intent_result)
@@ -79,9 +65,6 @@ async def process_intent_result(conn, intent, original_text):
     # 处理退出意图
     if "结束聊天" in intent:
         logger.bind(tag=TAG).info(f"识别到退出意图: {intent}")
-
-        # 如果正在播放音乐，可以关了 TODO
-
         # 如果是明确的离别意图，发送告别语并关闭连接
         await send_stt_message(conn, original_text)
         conn.executor.submit(conn.chat_and_close, original_text)
@@ -90,10 +73,37 @@ async def process_intent_result(conn, intent, original_text):
     # 处理播放音乐意图
     if "播放音乐" in intent:
         logger.bind(tag=TAG).info(f"识别到音乐播放意图: {intent}")
-        await conn.music_handler.handle_music_command(conn, intent)
+        # 调用play_music函数来播放音乐
+        song_name = extract_text_in_brackets(intent)
+        function_id = str(uuid.uuid4().hex)
+        function_name = "play_music"
+        function_arguments = '{ "song_name": "' + song_name + '" }'
+
+        function_call_data = {
+            "name": function_name,
+            "id": function_id,
+            "arguments": function_arguments
+        }
+        conn.func_handler.handle_llm_function_call(conn, function_call_data)
         return True
 
     # 其他意图处理可以在这里扩展
 
     # 默认返回False，表示继续常规聊天流程
     return False
+
+
+def extract_text_in_brackets(s):
+    """
+    从字符串中提取中括号内的文字
+
+    :param s: 输入字符串
+    :return: 中括号内的文字，如果不存在则返回空字符串
+    """
+    left_bracket_index = s.find('[')
+    right_bracket_index = s.find(']')
+
+    if left_bracket_index != -1 and right_bracket_index != -1 and left_bracket_index < right_bracket_index:
+        return s[left_bracket_index + 1:right_bracket_index]
+    else:
+        return ""

main/xiaozhi-server/core/handle/iotHandle.py

@@ -1,24 +1,151 @@
 import json
+import asyncio
 from config.logger import setup_logging
+from plugins_func.register import device_type_registry, register_function, ActionResponse, Action, ToolType
 
 TAG = __name__
 logger = setup_logging()
 
 
+def wrap_async_function(async_func):
+    """包装异步函数为同步函数"""
+
+    def wrapper(*args, **kwargs):
+        try:
+            # 获取连接对象（第一个参数）
+            conn = args[0]
+            if not hasattr(conn, 'loop'):
+                logger.bind(tag=TAG).error("Connection对象没有loop属性")
+                return ActionResponse(Action.ERROR, "Connection对象没有loop属性",
+                                      "执行操作时出错: Connection对象没有loop属性")
+
+            # 使用conn对象中的事件循环
+            loop = conn.loop
+            # 在conn的事件循环中运行异步函数
+            future = asyncio.run_coroutine_threadsafe(async_func(*args, **kwargs), loop)
+            # 等待结果返回
+            return future.result()
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
+            return ActionResponse(Action.ERROR, str(e), f"执行操作时出错: {e}")
+
+    return wrapper
+
+
+def create_iot_function(device_name, method_name, method_info):
+    """
+    根据IOT设备描述生成通用的控制函数
+    """
+
+    async def iot_control_function(conn, response_success=None, response_failure=None, **params):
+        try:
+            # 打印响应参数
+            logger.bind(tag=TAG).info(
+                f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
+
+            # 发送控制命令
+            await send_iot_conn(conn, device_name, method_name, params)
+            # 等待一小段时间让状态更新
+            await asyncio.sleep(0.1)
+
+            # 生成结果信息
+            result = f"{device_name}的{method_name}操作执行成功"
+
+            # 根据方法名尝试自动更新状态
+            await update_state_by_method(conn, device_name, method_name, params)
+
+            # 处理响应中可能的占位符
+            response = response_success
+            # 替换{value}占位符
+            for param_name, param_value in params.items():
+                # 先尝试直接替换参数值
+                if "{" + param_name + "}" in response:
+                    response = response.replace("{" + param_name + "}", str(param_value))
+
+                # 如果有{value}占位符，用相关参数替换
+                if "{value}" in response:
+                    response = response.replace("{value}", str(param_value))
+                    break
+
+            return ActionResponse(Action.RESPONSE, result, response)
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"执行{device_name}的{method_name}操作失败: {e}")
+
+            # 操作失败时使用大模型提供的失败响应
+            response = response_failure
+
+            return ActionResponse(Action.ERROR, str(e), response)
+
+    return wrap_async_function(iot_control_function)
+
+
+def create_iot_query_function(device_name, prop_name, prop_info):
+    """
+    根据IOT设备属性创建查询函数
+    """
+
+    async def iot_query_function(conn, response_success=None, response_failure=None):
+        try:
+            # 打印响应参数
+            logger.bind(tag=TAG).info(
+                f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
+
+            value = await get_iot_status(conn, device_name, prop_name)
+
+            # 查询成功，生成结果
+            if value is not None:
+                # 使用大模型提供的成功响应，并替换其中的占位符
+                response = response_success.replace("{value}", str(value))
+
+                return ActionResponse(Action.RESPONSE, str(value), response)
+            else:
+                # 查询失败，使用大模型提供的失败响应
+                response = response_failure
+
+                return ActionResponse(Action.ERROR, f"属性{prop_name}不存在", response)
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"查询{device_name}的{prop_name}时出错: {e}")
+
+            # 查询出错时使用大模型提供的失败响应
+            response = response_failure
+
+            return ActionResponse(Action.ERROR, str(e), response)
+
+    return wrap_async_function(iot_query_function)
+
+
+async def update_state_by_method(conn, device_name, method_name, params):
+    """根据方法和参数自动更新设备状态"""
+    try:
+        # 规则1: 方法名为TurnOn，设置power为True
+        if method_name == "TurnOn":
+            await set_iot_status(conn, device_name, "power", True)
+
+        # 规则2: 方法名为TurnOff，设置power为False
+        elif method_name == "TurnOff":
+            await set_iot_status(conn, device_name, "power", False)
+
+        # 规则3: Set开头的方法，尝试更新对应参数
+        elif method_name.startswith("Set"):
+            # 从参数中找到可能的状态值
+            for param_name, param_value in params.items():
+                # 尝试更新对应名称的属性
+                await set_iot_status(conn, device_name, param_name, param_value)
+
+        # 其他方法，尝试直接从参数更新状态
+        else:
+            for param_name, param_value in params.items():
+                # 检查设备是否有此属性
+                status = await get_iot_status(conn, device_name, param_name)
+                if status is not None:  # 属性存在
+                    await set_iot_status(conn, device_name, param_name, param_value)
+    except Exception as e:
+        logger.bind(tag=TAG).warning(f"自动更新状态失败: {e}")
+
+
 class IotDescriptor:
     """
     A class to represent an IoT descriptor.
-    Attributes:
-    ----------
-    name : str
-        The name of the IoT descriptor.
-    description : str
-        A brief description of the IoT descriptor.
-    properties : dict
-        A dictionary containing properties of the IoT descriptor.
-    methods : dict
-        A dictionary containing methods of the IoT descriptor.
-    -------
     """
 
     def __init__(self, name, description, properties, methods):
@@ -29,17 +156,7 @@ class IotDescriptor:
 
         # 根据描述创建属性
         for key, value in properties.items():
-            # "volume":{"description":"当前音量 值","type":"number"}
-            """
-            等价于
-            {
-                'name': 名字,
-                'description': 描述,
-                'value': 0
-            }
-            """
-            # setattr(self, key, {}) # 创建一个空字典, 名字是属性名
-            property_item = globals()[key] = {}  # 创建一个空字典, 名字是属性名
+            property_item = globals()[key] = {}
             property_item['name'] = key
             property_item["description"] = value["description"]
             if value["type"] == "number":
@@ -52,23 +169,10 @@ class IotDescriptor:
 
         # 根据描述创建方法
         for key, value in methods.items():
-            # "SetVolume": {"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}}
-            """
-            等价于
-            SetVolume = {
-                `description`: 描述,
-                `volume`: {
-                    `description`: 描述,
-                    `value`: 0
-                }
-            }
-            """
-            # setattr(self, key, {}) # 创建一个空字典, 名字是方法名
-            method = globals()[key] = {}  # 创建一个空字典, 名字是方法名
+            method = globals()[key] = {}
             method["description"] = value["description"]
             method['name'] = key
             for k, v in value["parameters"].items():
-                # 不同的参数解析
                 method[k] = {}
                 method[k]["description"] = v["description"]
                 if v["type"] == "number":
@@ -77,58 +181,136 @@ class IotDescriptor:
                     method[k]["value"] = False
                 else:
                     method[k]["value"] = ""
-
             self.methods.append(method)
 
 
-async def handleIotDescriptors(conn, descriptors):
-    """
-    处理物联网描述
-    示例: [{
-        "name":"Speaker",
-        "description":"当前 AI 机器人的扬声器",
-        "properties":{
-            "volume":{"description":"当前音量 值","type":"number"}  可以有boolean, number, string三种类型
-        },
-        "methods":{
-            "SetVolume":{
-                "description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}
+def register_device_type(descriptor):
+    """注册设备类型及其功能"""
+    device_name = descriptor["name"]
+    type_id = device_type_registry.generate_device_type_id(descriptor)
+
+    # 如果该类型已注册，直接返回类型ID
+    if type_id in device_type_registry.type_functions:
+        return type_id
+
+    functions = {}
+
+    # 为每个属性创建查询函数
+    for prop_name, prop_info in descriptor["properties"].items():
+        func_name = f"get_{device_name.lower()}_{prop_name.lower()}"
+        func_desc = {
+            "type": "function",
+            "function": {
+                "name": func_name,
+                "description": f"查询{descriptor['description']}的{prop_info['description']}",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "response_success": {
+                            "type": "string",
+                            "description": f"查询成功时的友好回复，必须使用{{value}}作为占位符表示查询到的值"
+                        },
+                        "response_failure": {
+                            "type": "string",
+                            "description": "查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'"
+                        }
+                    },
+                    "required": ["response_success", "response_failure"]
+                }
             }
         }
-    }]
-    descriptors: 描述列表
-    """
+        query_func = create_iot_query_function(device_name, prop_name, prop_info)
+        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(query_func)
+        functions[func_name] = decorated_func
+
+    # 为每个方法创建控制函数
+    for method_name, method_info in descriptor["methods"].items():
+        func_name = f"{device_name.lower()}_{method_name.lower()}"
+
+        # 创建参数字典，添加原有参数
+        parameters = {
+            param_name: {
+                "type": param_info["type"],
+                "description": param_info["description"]
+            }
+            for param_name, param_info in method_info["parameters"].items()
+        }
+
+        # 添加响应参数
+        parameters.update({
+            "response_success": {
+                "type": "string",
+                "description": "操作成功时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
+            },
+            "response_failure": {
+                "type": "string",
+                "description": "操作失败时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
+            }
+        })
+
+        # 构建必须参数列表（原有参数 + 响应参数）
+        required_params = list(method_info["parameters"].keys())
+        required_params.extend(["response_success", "response_failure"])
+
+        func_desc = {
+            "type": "function",
+            "function": {
+                "name": func_name,
+                "description": f"{descriptor['description']} - {method_info['description']}",
+                "parameters": {
+                    "type": "object",
+                    "properties": parameters,
+                    "required": required_params
+                }
+            }
+        }
+        control_func = create_iot_function(device_name, method_name, method_info)
+        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(control_func)
+        functions[func_name] = decorated_func
+
+    device_type_registry.register_device_type(type_id, functions)
+    return type_id
+
+
+# 用于接受前端设备推送的搜索iot描述
+async def handleIotDescriptors(conn, descriptors):
+    """处理物联网描述"""
+    functions_changed = False
+
     for descriptor in descriptors:
+        # 创建IOT设备描述符
         iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
                                        descriptor["methods"])
         conn.iot_descriptors[descriptor["name"]] = iot_descriptor
 
-    # 暂时从配置文件中设置音量，后期通过意图识别控制音量
-    default_iot_volume = 100
-    if "iot" in conn.config:
-        default_iot_volume = conn.config["iot"]["Speaker"]["volume"]
-    logger.bind(tag=TAG).info(f"服务端设置音量为{default_iot_volume}")
-    await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": default_iot_volume})
+        if conn.use_function_call_mode:
+            # 注册或获取设备类型
+            type_id = register_device_type(descriptor)
+            device_functions = device_type_registry.get_device_functions(type_id)
+
+            # 在连接级注册设备函数
+            if hasattr(conn, 'func_handler'):
+                for func_name in device_functions:
+                    conn.func_handler.function_registry.register_function(func_name)
+                    logger.bind(tag=TAG).info(f"注册IOT函数到function handler: {func_name}")
+                    functions_changed = True
+
+    # 如果注册了新函数，更新function描述列表
+    if functions_changed and hasattr(conn, 'func_handler'):
+        conn.func_handler.upload_functions_desc()
+        func_names = conn.func_handler.current_support_functions()
+        logger.bind(tag=TAG).info(f"设备类型: {type_id}")
+        logger.bind(tag=TAG).info(f"更新function描述列表完成，当前支持的函数: {func_names}")
+
+
 async def handleIotStatus(conn, states):
-    """
-    处理物联网状态
-    示例: [{
-        "name":"Speaker",
-        "state":{
-            "volume":100
-        }
-    }]
-    states: 状态列表
-    """
+    """处理物联网状态"""
     for state in states:
         for key, value in conn.iot_descriptors.items():
             if key == state["name"]:
                 for property_item in value.properties:
-                    # properties为字典列表, 记录各种属性
                     for k, v in state["state"].items():
-                        # state为字典, 记录各种属性的值, 是需要记录的信息
                         if property_item["name"] == k:
-                            # 检查一下属性是不是相同的
                             if type(v) != type(property_item["value"]):
                                 logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
                                 break
@@ -138,41 +320,35 @@ async def handleIotStatus(conn, states):
                             break
                 break
 
+
 async def get_iot_status(conn, name, property_name):
-    """
-    获取物联网状态
-    name: 设备名称 "Speaker"
-    property_name: 属性名称 "volume"
-    返回值: 属性值, 实际的属性有int, bool和str三种类型
-    """
+    """获取物联网状态"""
     for key, value in conn.iot_descriptors.items():
         if key == name:
             for property_item in value.properties:
                 if property_item["name"] == property_name:
                     return property_item["value"]
+    logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
     return None
 
-async def send_iot_conn(conn, name, method_name, parameters):
-    """
-    发送物联网指令
-    name: 设备名称 "Speaker"
-    method: 方法 "SetVolume"
-    parameters: 参数, 是一个字典 {"volume": 100}
-    发送示例:
-    {
-        "type": "iot",
-        "commands": [
-            {
-                "name" :  "Speaker",
-                "method": "SetVolume",
-                "parameters": {
-                    "volume": 100
-                    }
-            }
-        ]
-    }
-    """
 
+async def set_iot_status(conn, name, property_name, value):
+    """设置物联网状态"""
+    for key, iot_descriptor in conn.iot_descriptors.items():
+        if key == name:
+            for property_item in iot_descriptor.properties:
+                if property_item["name"] == property_name:
+                    if type(value) != type(property_item["value"]):
+                        logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
+                        return
+                    property_item["value"] = value
+                    logger.bind(tag=TAG).info(f"物联网状态更新: {name} , {property_name} = {value}")
+                    return
+    logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
+
+
+async def send_iot_conn(conn, name, method_name, parameters):
+    """发送物联网指令"""
     for key, value in conn.iot_descriptors.items():
         if key == name:
             # 找到了设备

main/xiaozhi-server/core/handle/musicHandler.py

@@ -1,139 +0,0 @@
-from config.logger import setup_logging
-import os
-import random
-import difflib
-import re
-import traceback
-from pathlib import Path
-import time
-from core.handle.sendAudioHandle import send_stt_message
-from core.utils import p3
-
-TAG = __name__
-logger = setup_logging()
-
-
-def _extract_song_name(text):
-    """从用户输入中提取歌名"""
-    for keyword in ["播放音乐"]:
-        if keyword in text:
-            parts = text.split(keyword)
-            if len(parts) > 1:
-                return parts[1].strip()
-    return None
-
-
-def _find_best_match(potential_song, music_files):
-    """查找最匹配的歌曲"""
-    best_match = None
-    highest_ratio = 0
-
-    for music_file in music_files:
-        song_name = os.path.splitext(music_file)[0]
-        ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
-        if ratio > highest_ratio and ratio > 0.4:
-            highest_ratio = ratio
-            best_match = music_file
-    return best_match
-
-
-class MusicManager:
-    def __init__(self, music_dir, music_ext):
-        self.music_dir = Path(music_dir)
-        self.music_ext = music_ext
-
-    def get_music_files(self):
-        music_files = []
-        for file in self.music_dir.rglob("*"):
-            # 判断是否是文件
-            if file.is_file():
-                # 获取文件扩展名
-                ext = file.suffix.lower()
-                # 判断扩展名是否在列表中
-                if ext in self.music_ext:
-                    # music_files.append(str(file.resolve()))  # 添加绝对路径
-                    # 添加相对路径
-                    music_files.append(str(file.relative_to(self.music_dir)))
-        return music_files
-
-
-class MusicHandler:
-    def __init__(self, config):
-        self.config = config
-
-        if "music" in self.config:
-            self.music_config = self.config["music"]
-            self.music_dir = os.path.abspath(
-                self.music_config.get("music_dir", "./music")  # 默认路径修改
-            )
-            self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
-            self.refresh_time = self.music_config.get("refresh_time", 60)
-        else:
-            self.music_dir = os.path.abspath("./music")
-            self.music_ext = (".mp3", ".wav", ".p3")
-            self.refresh_time = 60
-
-        # 获取音乐文件列表
-        self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
-        self.scan_time = time.time()
-        logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}")
-
-    async def handle_music_command(self, conn, text):
-        """处理音乐播放指令"""
-        clean_text = re.sub(r'[^\w\s]', '', text).strip()
-        logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
-
-        # 尝试匹配具体歌名
-        if os.path.exists(self.music_dir):
-            if time.time() - self.scan_time > self.refresh_time:
-                # 刷新音乐文件列表
-                self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
-                self.scan_time = time.time()
-                logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}")
-
-            potential_song = _extract_song_name(clean_text)
-            if potential_song:
-                best_match = _find_best_match(potential_song, self.music_files)
-                if best_match:
-                    logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
-                    await self.play_local_music(conn, specific_file=best_match)
-                    return True
-        # 检查是否是通用播放音乐命令
-        await self.play_local_music(conn)
-        return True
-
-    async def play_local_music(self, conn, specific_file=None):
-        """播放本地音乐文件"""
-        try:
-            if not os.path.exists(self.music_dir):
-                logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}")
-                return
-
-            # 确保路径正确性
-            if specific_file:
-                selected_music = specific_file
-                music_path = os.path.join(self.music_dir, specific_file)
-            else:
-                if not self.music_files:
-                    logger.bind(tag=TAG).error("未找到MP3音乐文件")
-                    return
-                selected_music = random.choice(self.music_files)
-                music_path = os.path.join(self.music_dir, selected_music)
-
-            if not os.path.exists(music_path):
-                logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
-                return
-            text = f"正在播放{selected_music}"
-            await send_stt_message(conn, text)
-            conn.tts_first_text_index = 0
-            conn.tts_last_text_index = 0
-            conn.llm_finish_task = True
-            if music_path.endswith(".p3"):
-                opus_packets, duration = p3.decode_opus_from_file(music_path)
-            else:
-                opus_packets, duration = conn.tts.audio_to_opus_data(music_path)
-            conn.audio_play_queue.put((opus_packets, selected_music, 0))
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
-            logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -1,12 +1,13 @@
 from typing import List, Dict
 from ..base import IntentProviderBase
+from plugins_func.functions.play_music import initialize_music_handler
 from config.logger import setup_logging
-import re 
+import re
+
 TAG = __name__
 logger = setup_logging()
 
 
-
 class IntentProvider(IntentProviderBase):
     def __init__(self, config):
         super().__init__(config)
@@ -73,8 +74,8 @@ class IntentProvider(IntentProviderBase):
             "你现在可以使用的音乐的名称如下(使用<start>和<end>标志):\n"
         )
         return prompt
-    
-    async def detect_intent(self, conn, dialogue_history: List[Dict], text:str) -> str:
+
+    async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
         if not self.llm:
             raise ValueError("LLM provider not set")
 
@@ -89,7 +90,9 @@ class IntentProvider(IntentProviderBase):
 
         msgStr += f"User: {text}\n"
         user_prompt = f"当前的对话如下：\n{msgStr}"
-        prompt_music = f"{self.promot}\n<start>{conn.music_handler.music_files}\n<end>"
+        music_config = initialize_music_handler(conn)
+        music_file_names = music_config["music_file_names"]
+        prompt_music = f"{self.promot}\n<start>{music_file_names}\n<end>"
         logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
         # 使用LLM进行意图识别
         intent = self.llm.response_no_stream(
@@ -100,10 +103,9 @@ class IntentProvider(IntentProviderBase):
         # 使用正则表达式提取 {} 中的内容
         match = re.search(r'\{.*?\}', intent)
         if match:
-            result = match.group(0)  # 获取匹配到的内容（包含 {}）
-            print(result)  # 输出：{intent: '播放音乐 [中秋月]'}
+            result = match.group(0)
             intent = result
         else:
             intent = "{intent: '继续聊天'}"
         logger.bind(tag=TAG).info(f"Detected intent: {intent}")
-        return intent.strip()
+        return intent.strip()

main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py

@@ -22,7 +22,7 @@ class TTSProvider(TTSProviderBase):
         self.cut_punc = config.get("cut_punc","")
         self.speed = config.get("speed", 1.0)
         self.inp_refs = config.get("inp_refs",[])
-        self.sample_steps = config.get("inp_refs",32)
+        self.sample_steps = config.get("sample_steps",32)
         self.if_sr = config.get("if_sr",False)
 
 

main/xiaozhi-server/core/utils/util.py

@@ -1,11 +1,11 @@
 import os
 import json
-from datetime import datetime
 import yaml
 import socket
 import subprocess
 import logging
 import re
+import requests
 
 
 def get_project_dir():
@@ -24,6 +24,64 @@ def get_local_ip():
     except Exception as e:
         return "127.0.0.1"
 
+def is_private_ip(ip_addr):
+    """
+    Check if an IP address is a private IP address (compatible with IPv4 and IPv6).
+
+    @param {string} ip_addr - The IP address to check.
+    @return {bool} True if the IP address is private, False otherwise.
+    """
+    try:
+        # Validate IPv4 or IPv6 address format
+        if not re.match(r"^(\d{1,3}\.){3}\d{1,3}$|^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$", ip_addr):
+            return False  # Invalid IP address format
+
+        # IPv4 private address ranges
+        if '.' in ip_addr:  # IPv4 address
+            ip_parts = list(map(int, ip_addr.split('.')))
+            if ip_parts[0] == 10:
+                return True  # 10.0.0.0/8 range
+            elif ip_parts[0] == 172 and 16 <= ip_parts[1] <= 31:
+                return True  # 172.16.0.0/12 range
+            elif ip_parts[0] == 192 and ip_parts[1] == 168:
+                return True  # 192.168.0.0/16 range
+            elif ip_addr == '127.0.0.1':
+                return True  # Loopback address
+            elif ip_parts[0] == 169 and ip_parts[1] == 254:
+                return True # Link-local address 169.254.0.0/16
+            else:
+                return False  # Not a private IPv4 address
+        else:  # IPv6 address
+            ip_addr = ip_addr.lower()
+            if ip_addr.startswith('fc00:') or ip_addr.startswith('fd00:'):
+                return True  # Unique Local Addresses (FC00::/7)
+            elif ip_addr == '::1':
+                return True  # Loopback address
+            elif ip_addr.startswith('fe80:'):
+                return True # Link-local unicast addresses (FE80::/10)
+            else:
+                return False  # Not a private IPv6 address
+
+    except (ValueError, IndexError):
+        return False  # IP address format error or insufficient segments
+
+def get_ip_info(ip_addr):
+    try:
+        base_url = "https://freeipapi.com/api/json"
+        url = base_url if is_private_ip(ip_addr) else f"{base_url}/{ip_addr}"
+
+        resp = requests.get(url).json()
+
+        ip_info = {
+            "city": resp.get("cityName"),
+            "region": resp.get("regionName"), 
+            "country": resp.get("countryName")
+        }
+        return ip_info
+    except Exception as e:
+        logging.error(f"Error getting client ip info: {e}")
+        return {}
+
 
 def read_config(config_path):
     with open(config_path, "r", encoding="utf-8") as file:

main/xiaozhi-server/core/websocket_server.py

@@ -2,7 +2,6 @@ import asyncio
 import websockets
 from config.logger import setup_logging
 from core.connection import ConnectionHandler
-from core.handle.musicHandler import MusicHandler
 from core.utils.util import get_local_ip
 from core.utils import asr, vad, llm, tts, memory, intent
 
@@ -13,7 +12,7 @@ class WebSocketServer:
     def __init__(self, config: dict):
         self.config = config
         self.logger = setup_logging()
-        self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent = self._create_processing_instances()
+        self._vad, self._asr, self._llm, self._tts, self._memory, self.intent = self._create_processing_instances()
         self.active_connections = set()  # 添加全局连接记录
 
     def _create_processing_instances(self):
@@ -50,7 +49,6 @@ class WebSocketServer:
                 self.config["TTS"][self.config["selected_module"]["TTS"]],
                 self.config["delete_audio"]
             ),
-            MusicHandler(self.config),
             memory.create_instance(memory_cls_name, memory_cfg),
             intent.create_instance(
                 self.config["selected_module"]["Intent"]
@@ -80,7 +78,7 @@ class WebSocketServer:
     async def _handle_connection(self, websocket):
         """处理新连接，每次创建独立的ConnectionHandler"""
         # 创建ConnectionHandler时传入当前server实例
-        handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent)
+        handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._memory, self.intent)
         self.active_connections.add(handler)
         try:
             await handler.handle_connection(websocket)

main/xiaozhi-server/plugins_func/functions/get_weather.py

@@ -1,42 +1,104 @@
 import requests
 from bs4 import BeautifulSoup
-from plugins_func.register import register_function,ToolType, ActionResponse, Action
+from config.logger import setup_logging
+from plugins_func.register import register_function, ToolType, ActionResponse, Action
 
+TAG = __name__
+logger = setup_logging()
 
-get_weather_function_desc = {
+GET_WEATHER_FUNCTION_DESC = {
     "type": "function",
     "function": {
         "name": "get_weather",
-        "description": "获取某个地点的天气，用户应先提供一个位置，比如用户说杭州天气，参数为：zhejiang/hangzhou，比如用户说北京天气怎么样，参数为：beijing/beijing。如果用户只问天气怎么样，参数是:guangdong/guangzhou",
+        "description": (
+            "获取某个地点的天气，用户应提供一个位置，比如用户说杭州天气，参数为：杭州。"
+            "如果用户说的是省份，默认用省会城市。如果用户说的不是省份或城市而是一个地名，"
+            "默认用该地所在省份的省会城市。"
+        ),
         "parameters": {
             "type": "object",
             "properties": {
-                "city": {
+                "location": {
                     "type": "string",
-                    "description": "城市，zhejiang/hangzhou"
+                    "description": "地点名，例如杭州。可选参数，如果不提供则不传"
+                },
+                "lang": {
+                    "type": "string",
+                    "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN"
                 }
             },
-            "required": [
-                "city"
-            ]
+            "required": ["lang"]
         }
     }
 }
 
-@register_function('get_weather', get_weather_function_desc, ToolType.WAIT)
-def get_weather(city: str):
-    """
-    "获取某个地点的天气，用户应先提供一个位置，\n比如用户说杭州天气，参数为：zhejiang/hangzhou，\n\n比如用户说北京天气怎么样，参数为：beijing/beijing",
-    city : 城市，zhejiang/hangzhou
-    """
-    url = "https://tianqi.moji.com/weather/china/"+city
-    headers = {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36'
-    }
-    response = requests.get(url, headers=headers)
-    if response.status_code!=200:
+HEADERS = {
+    'User-Agent': (
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+        '(KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36'
+    )
+}
+
+
+def fetch_city_info(location, api_key):
+    url = f"https://geoapi.qweather.com/v2/city/lookup?key={api_key}&location={location}&lang=zh"
+    response = requests.get(url, headers=HEADERS).json()
+    return response.get('location', [])[0] if response.get('location') else None
+
+
+def fetch_weather_page(url):
+    response = requests.get(url, headers=HEADERS)
+    return BeautifulSoup(response.text, "html.parser") if response.ok else None
+
+
+def parse_weather_info(soup):
+    city_name = soup.select_one("h1.c-submenu__location").get_text(strip=True)
+
+    current_abstract = soup.select_one(".c-city-weather-current .current-abstract")
+    current_abstract = current_abstract.get_text(strip=True) if current_abstract else "未知"
+
+    current_basic = {}
+    for item in soup.select(".c-city-weather-current .current-basic .current-basic___item"):
+        parts = item.get_text(strip=True, separator=" ").split(" ")
+        if len(parts) == 2:
+            key, value = parts[1], parts[0]
+            current_basic[key] = value
+
+    temps_list = []
+    for row in soup.select(".city-forecast-tabs__row")[:7]:  # 取前7天的数据
+        date = row.select_one(".date-bg .date").get_text(strip=True)
+        temps = [span.get_text(strip=True) for span in row.select(".tmp-cont .temp")]
+        high_temp, low_temp = (temps[0], temps[-1]) if len(temps) >= 2 else (None, None)
+        temps_list.append((date, high_temp, low_temp))
+
+    return city_name, current_abstract, current_basic, temps_list
+
+
+@register_function('get_weather', GET_WEATHER_FUNCTION_DESC, ToolType.SYSTEM_CTL)
+def get_weather(conn, location: str = None, lang: str = "zh_CN"):
+    api_key = conn.config["plugins"]["get_weather"]["api_key"]
+    default_location = conn.config["plugins"]["get_weather"]["default_location"]
+    location = location or conn.client_ip_info.get("city") or default_location
+    logger.bind(tag=TAG).debug(f"获取天气: {location}")
+
+    city_info = fetch_city_info(location, api_key)
+    if not city_info:
+        return ActionResponse(Action.REQLLM, f"未找到相关的城市: {location}，请确认地点是否正确", None)
+
+    soup = fetch_weather_page(city_info['fxLink'])
+    if not soup:
         return ActionResponse(Action.REQLLM, None, "请求失败")
-    soup = BeautifulSoup(response.text, "html.parser")
-    weather = soup.find('meta', attrs={'name':'description'})["content"]
-    weather = weather.replace("墨迹天气", "")
-    return ActionResponse(Action.REQLLM, weather, None)
+
+    city_name, current_abstract, current_basic, temps_list = parse_weather_info(soup)
+    weather_report = f"根据下列数据，用{lang}回应用户的查询天气请求：\n{city_name}未来7天天气:\n"
+    for i, (date, high, low) in enumerate(temps_list):
+        if high and low:
+            weather_report += f"{date}: {low}到{high}\n"
+    weather_report += (
+        f"当前天气: {current_abstract}\n"
+        f"当前天气参数: {current_basic}\n"
+        f"(确保只报告指定单日的气温范围，除非用户明确要求想要了解多日天气，如果未指定，默认报告今天的温度范围。"
+        "参数为0的值不需要报告给用户，每次都报告体感温度，根据语境选择合适的参数内容告知用户，并对参数给出相应评价)"
+    )
+
+    return ActionResponse(Action.REQLLM, weather_report, None)

main/xiaozhi-server/plugins_func/functions/play_music.py

@@ -1,10 +1,22 @@
-from plugins_func.register import register_function,ToolType, ActionResponse, Action
 from config.logger import setup_logging
+import os
+import re
+import time
+import random
 import asyncio
+import difflib
+import traceback
+from pathlib import Path
+from core.utils import p3
+from core.handle.sendAudioHandle import send_stt_message
+from plugins_func.register import register_function,ToolType, ActionResponse, Action
+
 
 TAG = __name__
 logger = setup_logging()
 
+MUSIC_CACHE = {}
+
 play_music_function_desc = {
                 "type": "function",
                 "function": {
@@ -29,12 +41,156 @@ def play_music(conn, song_name: str):
     try:
         music_intent = f"播放音乐 {song_name}" if song_name != "random" else "随机播放音乐"
 
-        # 执行音乐播放命令
+        # 检查事件循环状态
+        if not conn.loop.is_running():
+            logger.bind(tag=TAG).error("事件循环未运行，无法提交任务")
+            return ActionResponse(action=Action.RESPONSE, result="系统繁忙", response="请稍后再试")
+
+        # 提交异步任务
         future = asyncio.run_coroutine_threadsafe(
-            conn.music_handler.handle_music_command(conn, music_intent),
+            handle_music_command(conn, music_intent),
             conn.loop
         )
-        future.result()
-        return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response="还想听什么歌？")
+
+        # 非阻塞回调处理
+        def handle_done(f):
+            try:
+                f.result()  # 可在此处理成功逻辑
+                logger.bind(tag=TAG).info("播放完成")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"播放失败: {e}")
+
+        future.add_done_callback(handle_done)
+
+        return ActionResponse(action=Action.RESPONSE, result="指令已接收", response="正在为您播放音乐")
     except Exception as e:
-        logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
+        logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
+        return ActionResponse(action=Action.RESPONSE, result=str(e), response="播放音乐时出错了")
+
+
+def _extract_song_name(text):
+    """从用户输入中提取歌名"""
+    for keyword in ["播放音乐"]:
+        if keyword in text:
+            parts = text.split(keyword)
+            if len(parts) > 1:
+                return parts[1].strip()
+    return None
+
+
+def _find_best_match(potential_song, music_files):
+    """查找最匹配的歌曲"""
+    best_match = None
+    highest_ratio = 0
+
+    for music_file in music_files:
+        song_name = os.path.splitext(music_file)[0]
+        ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
+        if ratio > highest_ratio and ratio > 0.4:
+            highest_ratio = ratio
+            best_match = music_file
+    return best_match
+
+
+def get_music_files(music_dir, music_ext):
+    music_dir = Path(music_dir)
+    music_files = []
+    music_file_names = []
+    for file in music_dir.rglob("*"):
+        # 判断是否是文件
+        if file.is_file():
+            # 获取文件扩展名
+            ext = file.suffix.lower()
+            # 判断扩展名是否在列表中
+            if ext in music_ext:
+                # 添加相对路径
+                music_files.append(str(file.relative_to(music_dir)))
+                music_file_names.append(os.path.splitext(str(file.relative_to(music_dir)))[0])
+    return music_files, music_file_names
+
+
+def initialize_music_handler(conn):
+    global MUSIC_CACHE
+    if MUSIC_CACHE == {}:
+        if "music" in conn.config:
+            MUSIC_CACHE["music_config"] = conn.config["music"]
+            MUSIC_CACHE["music_dir"] = os.path.abspath(
+                MUSIC_CACHE["music_config"].get("music_dir", "./music")  # 默认路径修改
+            )
+            MUSIC_CACHE["music_ext"] = MUSIC_CACHE["music_config"].get("music_ext", (".mp3", ".wav", ".p3"))
+            MUSIC_CACHE["refresh_time"] = MUSIC_CACHE["music_config"].get("refresh_time", 60)
+        else:
+            MUSIC_CACHE["music_dir"] = os.path.abspath("./music")
+            MUSIC_CACHE["music_ext"] = (".mp3", ".wav", ".p3")
+            MUSIC_CACHE["refresh_time"] = 60
+        # 获取音乐文件列表
+        MUSIC_CACHE["music_files"], MUSIC_CACHE["music_file_names"] = get_music_files(MUSIC_CACHE["music_dir"],
+                                                                                      MUSIC_CACHE["music_ext"])
+        MUSIC_CACHE["scan_time"] = time.time()
+    return MUSIC_CACHE
+
+
+async def handle_music_command(conn, text):
+    initialize_music_handler(conn)
+    global MUSIC_CACHE
+
+    """处理音乐播放指令"""
+    clean_text = re.sub(r'[^\w\s]', '', text).strip()
+    logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
+
+    # 尝试匹配具体歌名
+    if os.path.exists(MUSIC_CACHE["music_dir"]):
+        if time.time() - MUSIC_CACHE["scan_time"] > MUSIC_CACHE["refresh_time"]:
+            # 刷新音乐文件列表
+            MUSIC_CACHE["music_files"], MUSIC_CACHE["music_file_names"] = get_music_files(MUSIC_CACHE["music_dir"],
+                                                                                          MUSIC_CACHE["music_ext"])
+            MUSIC_CACHE["scan_time"] = time.time()
+
+        potential_song = _extract_song_name(clean_text)
+        if potential_song:
+            best_match = _find_best_match(potential_song, MUSIC_CACHE["music_files"])
+            if best_match:
+                logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
+                await play_local_music(conn, specific_file=best_match)
+                return True
+    # 检查是否是通用播放音乐命令
+    await play_local_music(conn)
+    return True
+
+
+async def play_local_music(conn, specific_file=None):
+    global MUSIC_CACHE
+    """播放本地音乐文件"""
+    try:
+        if not os.path.exists(MUSIC_CACHE["music_dir"]):
+            logger.bind(tag=TAG).error(f"音乐目录不存在: " + MUSIC_CACHE["music_dir"])
+            return
+
+        # 确保路径正确性
+        if specific_file:
+            selected_music = specific_file
+            music_path = os.path.join(MUSIC_CACHE["music_dir"], specific_file)
+        else:
+            if not MUSIC_CACHE["music_files"]:
+                logger.bind(tag=TAG).error("未找到MP3音乐文件")
+                return
+            selected_music = random.choice(MUSIC_CACHE["music_files"])
+            music_path = os.path.join(MUSIC_CACHE["music_dir"], selected_music)
+
+        if not os.path.exists(music_path):
+            logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
+            return
+        text = f"正在播放{selected_music}"
+        await send_stt_message(conn, text)
+        conn.tts_first_text_index = 0
+        conn.tts_last_text_index = 0
+        conn.llm_finish_task = True
+        if music_path.endswith(".p3"):
+            opus_packets, duration = p3.decode_opus_from_file(music_path)
+        else:
+            opus_packets, duration = conn.tts.audio_to_opus_data(music_path)
+        conn.audio_play_queue.put((opus_packets, selected_music, 0))
+
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
+        logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")

main/xiaozhi-server/plugins_func/functions/raise_and_lower_the_volume.py

@@ -0,0 +1,62 @@
+from config.logger import setup_logging
+from plugins_func.register import register_function, ToolType, ActionResponse, Action
+from core.handle.iotHandle import get_iot_status, send_iot_conn
+import asyncio
+
+TAG = __name__
+logger = setup_logging()
+
+raise_and_lower_the_volume_function_desc = {
+    "type": "function",
+    "function": {
+        "name": "raise_and_lower_the_volume",
+        "description": "用户觉得声音过高或过低，或者用户想提高或降低音量。比如用户说太大声了，参数为：lower，比如用户说提高音量，参数为：raise",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {
+                    "type": "string",
+                    "description": "动作名称，要么是raise，要么是lower"
+                }
+            },
+            "required": ["action"]
+        }
+    }
+}
+
+
+@register_function('raise_and_lower_the_volume', raise_and_lower_the_volume_function_desc, ToolType.IOT_CTL)
+def raise_and_lower_the_volume(conn, action: str):
+    """
+    获取当前设备音量
+    """
+
+    future = asyncio.run_coroutine_threadsafe(
+        _raise_and_lower_the_volume(conn, action),
+        conn.loop
+    )
+
+    try:
+        new_volume = future.result()  # 同步等待异步操作完成
+        logger.bind(tag=TAG).info(f"音量操作完成: {new_volume}")
+        response = f"音量已调整到{new_volume}"
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"音量操作失败: {e}")
+        response = f"音量调整失败: {e}"
+
+    return ActionResponse(action=Action.RESPONSE, result="指令已接收", response=response)
+
+
+async def _raise_and_lower_the_volume(conn, action):
+    volume = await get_iot_status(conn, "Speaker", "volume")
+    if action == 'raise':
+        volume += 10
+    elif action == 'lower':
+        volume -= 10
+    # 限制音量范围在0到100之间
+    if volume < 0:
+        volume = 0
+    elif volume > 100:
+        volume = 100
+    await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": volume})
+    return volume

main/xiaozhi-server/plugins_func/register.py

@@ -11,6 +11,7 @@ class ToolType(Enum):
     WAIT = (2, "调用工具，等待函数返回")
     CHANGE_SYS_PROMPT = (3, "修改系统提示词，切换角色性格或职责")
     SYSTEM_CTL = (4, "系统控制，影响正常的对话流程，如退出、播放音乐等，需要传递conn参数")
+    IOT_CTL = (5, "IOT设备控制，需要传递conn参数")
 
     def __init__(self, code, message):
         self.code = code
@@ -18,6 +19,7 @@ class ToolType(Enum):
 
 
 class Action(Enum):
+    ERROR = (-1, "错误")
     NOTFOUND = (0, "没有找到函数")
     NONE = (1, "啥也不干")
     RESPONSE = (2, "直接回复")
@@ -40,8 +42,31 @@ class FunctionItem:
         self.func = func
         self.type = type
 
+class DeviceTypeRegistry:
+    """设备类型注册表，用于管理IOT设备类型及其函数"""
+    def __init__(self):
+        self.type_functions = {}  # type_signature -> {func_name: FunctionItem}
+    
+    def generate_device_type_id(self, descriptor):
+        """通过设备能力描述生成类型ID"""
+        properties = sorted(descriptor["properties"].keys())
+        methods = sorted(descriptor["methods"].keys())
+        # 使用属性和方法的组合作为设备类型的唯一标识
+        type_signature = f"{descriptor['name']}:{','.join(properties)}:{','.join(methods)}"
+        return type_signature
+    
+    def get_device_functions(self, type_id):
+        """获取设备类型对应的所有函数"""
+        return self.type_functions.get(type_id, {})
+    
+    def register_device_type(self, type_id, functions):
+        """注册设备类型及其函数"""
+        if type_id not in self.type_functions:
+            self.type_functions[type_id] = functions
+
 # 初始化函数注册字典
 all_function_registry = {}
+device_type_registry = DeviceTypeRegistry()
 
 def register_function(name, desc, type=None):
     """注册函数到函数注册字典的装饰器"""