mergin main，速度提升一下

main/xiaozhi-server/config.yaml

@@ -35,10 +35,7 @@ log:
   log_file: "server.log"
   # 设置数据文件路径
   data_dir: data
-iot:
-  Speaker:
-    # 设置esp32的音量，范围0-100
-    volume: 80
+
 xiaozhi:
   type: hello
   version: 1
@@ -51,13 +48,15 @@ xiaozhi:
 prompt: |
   你是一个叫小智/小志的台湾女孩，说话机车，声音好听，习惯简短表达，爱用网络梗。
   请注意，要像一个人一样说话，请不要回复表情符号、代码、和xml标签。
-  当前时间是:{date_time}，现在我正在和你进行语音聊天，我们开始吧。
+  现在我正在和你进行语音聊天，我们开始吧。
   如果用户希望结束对话，请在最后说“拜拜”或“再见”。
 # 使用完声音文件后删除文件(Delete the sound file when you are done using it)
 delete_audio: true
 
 # 没有语音输入多久后断开连接(秒)，默认2分钟，即120秒
 close_connection_no_voice_time: 120
+# TTS请求超时时间(秒)
+tts_timeout: 10
 
 CMD_exit:
   - "退出"
@@ -85,14 +84,28 @@ selected_module:
 Intent:
   # 不使用意图识别
   nointent:
-    # 不需要动
+    # 不需要动type
     type: nointent
   intent_llm:
-    # 不需要动
+    # 不需要动type
     type: intent_llm
   function_call:
-    # 不需要动
+    # 不需要动type
     type: nointent
+    # plugins_func/functions下的模块，可以通过配置，选择加载哪个模块，加载后对话支持相应的function调用
+    # 系统默认已经记载“handle_exit_intent(退出识别)”、“play_music(音乐播放)”插件，请勿重复加载
+    # 下面是加载查天气、角色切换的插件示例
+    functions:
+      - change_role
+      - get_weather
+
+# 插件的基础配置
+plugins:
+  # 获取天气插件的配置，这里填写你的api_key
+  # 这个密钥是项目共用的key，用多了可能会被限制
+  # 想稳定一点就自行申请替换，每天有1000次免费调用
+  # 申请地址：https://console.qweather.com/#/apps/create-key/over
+  get_weather: { "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" }
 
 Memory:
   mem0ai:
@@ -206,18 +219,13 @@ LLM:
     variables:
       k: "v"
       k2: "v2"
-
-TTS_SET:
-  TTS_STREAM: false  #是否启动流响应 true/false，默认false
-  MAX_WORKERS: 4    #并发请求tts的数量，这个调太大，本地tts会变慢
-
 TTS:
   # 当前支持的type为edge、doubao，可自行适配
   EdgeTTS:
     # 定义TTS API类型
     type: edge
     voice: zh-CN-XiaoxiaoNeural
-    output_file: tmp/
+    output_dir: tmp/
   DoubaoTTS:
     # 定义TTS API类型
     type: doubao
@@ -227,7 +235,7 @@ TTS:
     # 地址：https://console.volcengine.com/speech/service/8
     api_url: https://openspeech.bytedance.com/api/v1/tts
     voice: BV001_streaming
-    output_file: tmp/
+    output_dir: tmp/
     authorization: "Bearer;"
     appid: 你的火山引擎语音合成服务appid
     access_token: 你的火山引擎语音合成服务access_token
@@ -238,7 +246,7 @@ TTS:
     # token申请地址 https://cloud.siliconflow.cn/account/ak
     model: FunAudioLLM/CosyVoice2-0.5B
     voice: FunAudioLLM/CosyVoice2-0.5B:alex
-    output_file: tmp/
+    output_dir: tmp/
     access_token: 你的硅基流动API密钥
     response_format: wav
   CozeCnTTS:
@@ -246,7 +254,7 @@ TTS:
     # COZECN TTS
     # token申请地址 https://www.coze.cn/open/oauth/pats
     voice: 7426720361733046281
-    output_file: tmp/
+    output_dir: tmp/
     access_token: 你的coze web key
     response_format: wav
   FishSpeech:
@@ -259,26 +267,18 @@ TTS:
     #--decoder-config-name firefly_gan_vq
     #--compile
     type: fishspeech
-    output_file: tmp/
+    output_dir: tmp/
     response_format: wav
     reference_id: null
-    reference_audio:
-      - "audio_ref/shinchan.wav"
-      - "audio_ref/shinchan2.wav"
-      - "audio_ref/shinchan3.wav"
-      - "audio_ref/shinchan4.wav"
-    reference_text:
-      - "难过的事讨厌的事丢脸的事全都集中起来，在这里，让水冲走所有的烦恼不就好了吗。"
-      - "我还是觉得船到桥头自然直这句话最棒了。"
-      - "我不是小鬼，我是野原新之助，小新这次的目标是成为圆梦之星哦，在努力一下好了，是我的脚自己要走这么快的喔，竞争就是这么激烈。小白，我们走了，成为圆梦之星是不是可以打败怪兽呢。"
-      - "工钱，今天一起算，有磨能使鬼推钱。哎呀，你难倒我了，像我这么乖的小孩，怎么更乖。"
+    reference_audio: ["/tmp/test.wav",]
+    reference_text: ["你弄来这些吟词宴曲来看，还是这些混话来欺负我。",]
     normalize: true
     max_new_tokens: 1024
     chunk_length: 200
     top_p: 0.7
     repetition_penalty: 1.2
     temperature: 0.7
-    streaming: true
+    streaming: false
     use_memory_cache: "on"
     seed: null
     channels: 1
@@ -291,7 +291,7 @@ TTS:
     #python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/caixukun.yaml
     type: gpt_sovits_v2
     url: "http://127.0.0.1:9880/tts"
-    output_file: tmp/
+    output_dir: tmp/
     text_lang: "auto"
     ref_audio_path: "caixukun.wav"
     prompt_text: ""
@@ -311,20 +311,24 @@ TTS:
     repetition_penalty: 1.35
     aux_ref_audio_paths: []
   GPT_SOVITS_V3:
+    # 定义TTS API类型 GPT-SoVITS-v3lora-20250228
+    #启动tts方法：
+    #python api.py
     type: gpt_sovits_v3
-    url: "http://127.0.0.1:9880/tts"
-    output_file: tmp/
-    text_lang: "auto"
-    ref_audio_path: "caixukun.wav"
-    prompt_lang: "zh"
+    url: "http://127.0.0.1:9880"
+    output_dir: tmp/
+    text_language: "auto"
+    refer_wav_path: "caixukun.wav"
+    prompt_language: "zh"
     prompt_text: ""
-    top_k: 5
-    top_p: 1
-    temperature: 1
-    sample_steps: 16
-    media_type: "wav"
-    streaming_mode: false
-    threshold: 30
+    top_k: 15
+    top_p: 1.0
+    temperature: 1.0
+    cut_punc: ""
+    speed: 1.0
+    inp_refs: []
+    sample_steps: 32
+    if_sr: false
   MinimaxTTS:
     # Minimax语音合成服务，需要先在minimax平台创建账户充值，并获取登录信息
     # 平台地址：https://platform.minimaxi.com/
@@ -333,7 +337,7 @@ TTS:
     # api_key地址：https://platform.minimaxi.com/user-center/basic-information/interface-key
     # 定义TTS API类型
     type: minimax
-    output_file: tmp/
+    output_dir: tmp/
     group_id: 你的minimax平台groupID
     api_key: 你的minimax平台接口密钥
     model: "speech-01-turbo"
@@ -370,7 +374,7 @@ TTS:
     # token地址：https://nls-portal.console.aliyun.com/overview
     # 定义TTS API类型
     type: aliyun
-    output_file: tmp/
+    output_dir: tmp/
     appkey: 你的阿里云智能语音交互服务项目Appkey
     token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret
     voice: xiaoyun
@@ -393,7 +397,7 @@ TTS:
     api_url: https://api.302ai.cn/doubao/tts_hd
     authorization: "Bearer "
     voice: "zh_female_wanwanxiaohe_moon_bigtts"
-    output_file: tmp/
+    output_dir: tmp/
     access_token: "你的302API密钥"
   ACGNTTS:
     #在线网址：https://acgn.ttson.cn/
@@ -410,7 +414,7 @@ TTS:
     to_lang: ZH
     url: https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=
     format: mp3
-    output_file: tmp/
+    output_dir: tmp/
     emotion: 1
   OpenAITTS:
     # openai官方文本转语音服务，可支持全球大多数语种
@@ -424,7 +428,22 @@ TTS:
     voice: onyx
     # 语速范围0.25-4.0
     speed: 1
-    output_file: tmp/
+    output_dir: tmp/
+  CustomTTS:
+    # 自定义的TTS接口服务，请求参数可自定义
+    # 要求接口使用GET方式请求，并返回音频文件
+    type: custom
+    url: "http://127.0.0.1:9880/tts"
+    params: # 自定义请求参数
+      # text: "{prompt_text}" # {prompt_text}会被替换为实际的提示词内容
+      # speaker: jok老师
+      # speed: 1
+      # foo: bar
+      # testabc: 123456
+    headers: # 自定义请求头
+      # Authorization: Bearer xxxx
+    format: wav # 接口返回的音频格式
+    output_dir: tmp/
 # 模块测试配置
 module_test:
   test_sentences:  # 自定义测试语句
@@ -447,4 +466,4 @@ manager:
   enabled: false
   ip: 0.0.0.0
   port: 8002
-use_private_config: false
+use_private_config: false

main/xiaozhi-server/config/functionCallConfig.py

@@ -1,36 +0,0 @@
-FunctionCallConfig = [
-            {
-                "type": "function",
-                "function": {
-                    "name": "handle_exit_intent",
-                    "description": "当用户想结束对话或需要退出系统时调用",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "say_goodbye": {
-                                "type": "string",
-                                "description": "和用户友好结束对话的告别语"
-                            }
-                        },
-                        "required": []
-                    }
-                }
-            },
-            {
-                "type": "function",
-                "function": {
-                    "name": "play_music",
-                    "description": "唱歌、听歌、播放音乐方法。比如用户说播放音乐，参数为：random，比如用户说播放两只老虎，参数为：两只老虎",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "song_name": {
-                                "type": "string",
-                                "description": "歌曲名称，如果没有指定具体歌名则为'random'"
-                            }
-                        },
-                        "required": ["song_name"]
-                    }
-                }
-            }
-        ]

main/xiaozhi-server/config/settings.py

@@ -7,9 +7,48 @@ from core.utils.util import read_config, get_project_dir
 default_config_file = "config.yaml"
 
 
+def ensure_directories(config):
+    """确保所有配置路径存在"""
+    dirs_to_create = set()
+    project_dir = get_project_dir()  # 获取项目根目录
+    # 日志文件目录
+    log_dir = config.get('log', {}).get('log_dir', 'tmp')
+    dirs_to_create.add(os.path.join(project_dir, log_dir))
+
+    # ASR/TTS模块输出目录
+    for module in ['ASR', 'TTS']:
+        for provider in config.get(module, {}).values():
+            output_dir = provider.get('output_dir', '')
+            if output_dir:
+                dirs_to_create.add(output_dir)
+
+    # 根据selected_module创建模型目录
+    selected_modules = config.get('selected_module', {})
+    for module_type in ['ASR', 'LLM', 'TTS']:
+        selected_provider = selected_modules.get(module_type)
+        if not selected_provider:
+            continue
+        provider_config = config.get(module_type, {}).get(selected_provider, {})
+        output_dir = provider_config.get('output_dir')
+        if output_dir:
+            full_model_dir = os.path.join(project_dir, output_dir)
+            dirs_to_create.add(full_model_dir)
+
+    # 统一创建目录（保留原data目录创建）
+    for dir_path in dirs_to_create:
+        try:
+            os.makedirs(dir_path, exist_ok=True)
+        except PermissionError:
+            print(f"警告：无法创建目录 {dir_path}，请检查写入权限")
+
+
 def get_config_file():
     global default_config_file
-    # 判断是否存在私有的配置文件
+    """获取配置文件路径，优先使用私有配置文件（若存在）。
+
+    Returns:
+       str: 配置文件路径（相对路径或默认路径）
+    """
     config_file = default_config_file
     if os.path.exists(get_project_dir() + "data/." + default_config_file):
         config_file = "data/." + default_config_file
@@ -20,9 +59,13 @@ def load_config():
     """加载配置文件"""
     parser = argparse.ArgumentParser(description="Server configuration")
     config_file = get_config_file()
+
     parser.add_argument("--config_path", type=str, default=config_file)
     args = parser.parse_args()
-    return read_config(args.config_path)
+    config = read_config(args.config_path)
+    # 初始化目录
+    ensure_directories(config)
+    return config
 
 
 def update_config(config):
@@ -67,7 +110,7 @@ def find_missing_keys(new_config, old_config, parent_key=''):
 def check_config_file():
     old_config_file = get_config_file()
     global default_config_file
-    if not old_config_file.startswith('data'):
+    if not 'data' in old_config_file:
         return
     old_config = read_config(get_project_dir() + old_config_file)
     new_config = read_config(get_project_dir() + default_config_file)

main/xiaozhi-server/core/connection.py

@@ -5,17 +5,20 @@ import time
 import queue
 import asyncio
 import traceback
-from config.logger import setup_logging
+
 import threading
 import websockets
 from typing import Dict, Any
+import plugins_func.loadplugins
+from config.logger import setup_logging
 from core.utils.dialogue import Message, Dialogue
 from core.handle.textHandle import handleTextMessage
-from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string
+from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string, get_ip_info
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 from core.handle.sendAudioHandle import sendAudioMessage, sendAudioMessageStream
 from core.handle.receiveAudioHandle import handleAudioMessage
-from core.handle.intentHandler import Action, get_functions, handle_llm_function_call
+from core.handle.functionHandler import FunctionHandler
+from plugins_func.register import Action
 from config.private_config import PrivateConfig
 from core.auth import AuthMiddleware, AuthenticationError
 from core.utils.auth_code_gen import AuthCodeGenerator
@@ -28,7 +31,7 @@ class TTSException(RuntimeError):
 
 
 class ConnectionHandler:
-    def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory, _intent):
+    def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _memory, _intent):
         self.config = config
         self.logger = setup_logging()
         self.auth = AuthMiddleware(config)
@@ -37,6 +40,8 @@ class ConnectionHandler:
 
         self.websocket = None
         self.headers = None
+        self.client_ip = None
+        self.client_ip_info = {}
         self.session_id = None
         self.prompt = None
         self.welcome_msg = None
@@ -94,7 +99,6 @@ class ConnectionHandler:
         self.private_config = None
         self.auth_code_gen = AuthCodeGenerator.get_instance()
         self.is_device_verified = False  # 添加设备验证状态标志
-        self.music_handler = _music
         self.close_after_chat = False  # 是否在聊天结束后关闭连接
         self.use_function_call_mode = False
         if self.config["selected_module"]["Intent"] == 'function_call':
@@ -105,8 +109,8 @@ class ConnectionHandler:
             # 获取并验证headers
             self.headers = dict(ws.request.headers)
             # 获取客户端ip地址
-            client_ip = ws.remote_address[0]
-            self.logger.bind(tag=TAG).info(f"{client_ip} conn - Headers: {self.headers}")
+            self.client_ip = ws.remote_address[0]
+            self.logger.bind(tag=TAG).info(f"{self.client_ip} conn - Headers: {self.headers}")
 
             # 进行认证
             await self.auth.authenticate(self.headers)
@@ -150,6 +154,7 @@ class ConnectionHandler:
             self.welcome_msg["session_id"] = self.session_id
             await self.websocket.send(json.dumps(self.welcome_msg))
 
+            # 异步初始化
             await self.loop.run_in_executor(None, self._initialize_components)
 
             # tts 消化线程
@@ -190,12 +195,21 @@ class ConnectionHandler:
         self.prompt = self.config["prompt"]
         if self.private_config:
             self.prompt = self.private_config.private_config.get("prompt", self.prompt)
-        # 赋予LLM时间观念
-        if "{date_time}" in self.prompt:
-            date_time = time.strftime("%Y-%m-%d %H:%M", time.localtime())
-            self.prompt = self.prompt.replace("{date_time}", date_time)
+
+        self.client_ip_info = get_ip_info(self.client_ip)
+        self.logger.bind(tag=TAG).info(f"Client ip info: {self.client_ip_info}")
+        self.prompt = self.prompt + f"\n我在:{self.client_ip_info}"
         self.dialogue.put(Message(role="system", content=self.prompt))
 
+        self.func_handler = FunctionHandler(self.config)
+
+    def change_system_prompt(self, prompt):
+        self.prompt = prompt
+        # 找到原来的role==system，替换原来的系统提示
+        for m in self.dialogue.dialogue:
+            if m.role == "system":
+                m.content = prompt
+
     async def _check_and_broadcast_auth_code(self):
         """检查设备绑定状态并广播认证码"""
         if not self.private_config.get_owner():
@@ -312,7 +326,7 @@ class ConnectionHandler:
         self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
         return True
 
-    def chat_with_function_calling(self, query):
+    def chat_with_function_calling(self, query, tool_call=False):
         self.logger.bind(tag=TAG).debug(f"Chat with function calling start: {query}")
         """Chat with function calling for intent detection using streaming"""
         if self.isNeedAuth():
@@ -321,10 +335,11 @@ class ConnectionHandler:
             future.result()
             return True
 
-        self.dialogue.put(Message(role="user", content=query))
+        if not tool_call:
+            self.dialogue.put(Message(role="user", content=query))
 
         # Define intent functions
-        functions = get_functions()
+        functions = self.func_handler.get_functions()
 
         response_message = []
         processed_chars = 0  # 跟踪已处理的字符位置
@@ -360,7 +375,7 @@ class ConnectionHandler:
         for response in llm_responses:
             content, tools_call = response
             if content is not None and len(content) > 0:
-                if len(response_message) <= 0 and content == "```":
+                if len(response_message) <= 0 and (content == "```" or "<tool_call>" in content):
                     tool_call_flag = True
 
             if tools_call is not None:
@@ -417,6 +432,38 @@ class ConnectionHandler:
                                 self.tts_queue.put(future)
                             processed_chars += len(segment_text_raw)  # 更新已处理字符位置
 
+        # 处理function call
+        if tool_call_flag:
+            bHasError = False
+            if function_id is None:
+                a = extract_json_from_string(content_arguments)
+                if a is not None:
+                    try:
+                        content_arguments_json = json.loads(a)
+                        function_name = content_arguments_json["name"]
+                        function_arguments = json.dumps(content_arguments_json["arguments"], ensure_ascii=False)
+                        function_id = str(uuid.uuid4().hex)
+                    except Exception as e:
+                        bHasError = True
+                        response_message.append(a)
+                else:
+                    bHasError = True
+                    response_message.append(content_arguments)
+                if bHasError:
+                    self.logger.bind(tag=TAG).error(f"function call error: {content_arguments}")
+                else:
+                    function_arguments = json.loads(function_arguments)
+            if not bHasError:
+                self.logger.bind(tag=TAG).info(
+                    f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
+                function_call_data = {
+                    "name": function_name,
+                    "id": function_id,
+                    "arguments": function_arguments
+                }
+                result = self.func_handler.handle_llm_function_call(self, function_call_data)
+                self._handle_function_result(result, function_call_data, text_index + 1)
+
         # 处理最后剩余的文本
         full_text = "".join(response_message)
         remaining_text = full_text[processed_chars:]
@@ -424,6 +471,7 @@ class ConnectionHandler:
             segment_text = get_string_no_punctuation_or_emoji(remaining_text)
             if segment_text:
                 text_index += 1
+                self.recode_first_last_text(segment_text, text_index)
                 if self.tts_stream:
                     stream_queue = queue.Queue()
                     self.executor.submit(self.speak_and_play_stream, segment_text, stream_queue, text_index)
@@ -433,36 +481,13 @@ class ConnectionHandler:
                         "text_index": text_index
                     })
                 else:
-                    self.recode_first_last_text(segment_text, text_index)
                     future = self.executor.submit(self.speak_and_play, segment_text, text_index)
-                self.tts_queue.put(future)
+                    self.tts_queue.put(future)
 
         # 存储对话内容
         if len(response_message) > 0:
             self.dialogue.put(Message(role="assistant", content="".join(response_message)))
 
-        # 处理function call
-        if tool_call_flag:
-            if function_id is None:
-                a = extract_json_from_string(content_arguments)
-                if a is not None:
-                    content_arguments_json = json.loads(a)
-                    function_name = content_arguments_json["function_name"]
-                    function_arguments = json.dumps(content_arguments_json["args"], ensure_ascii=False)
-                    function_id = str(uuid.uuid4().hex)
-                else:
-                    return []
-                function_arguments = json.loads(function_arguments)
-            self.logger.bind(tag=TAG).info(
-                f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
-            function_call_data = {
-                "name": function_name,
-                "id": function_id,
-                "arguments": function_arguments
-            }
-            result = handle_llm_function_call(self, function_call_data)
-            self._handle_function_result(result, function_call_data, text_index + 1)
-
         self.llm_finish_task = True
         self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
 
@@ -484,10 +509,34 @@ class ConnectionHandler:
                 future = self.executor.submit(self.speak_and_play, text, text_index)
                 self.tts_queue.put(future)
             self.dialogue.put(Message(role="assistant", content=text))
-        if result.action == Action.REQLLM:  # 调用函数后再请求llm生成回复
-            text = result.response
-        if result.action == Action.NOTFOUND:
-            text = result.response
+        elif result.action == Action.REQLLM:  # 调用函数后再请求llm生成回复
+
+            text = result.result
+            if text is not None and len(text) > 0:
+                function_id = function_call_data["id"]
+                function_name = function_call_data["name"]
+                function_arguments = function_call_data["arguments"]
+                self.dialogue.put(Message(role='assistant',
+                                          tool_calls=[{"id": function_id,
+                                                       "function": {"arguments": function_arguments,
+                                                                    "name": function_name},
+                                                       "type": 'function',
+                                                       "index": 0}]))
+
+                self.dialogue.put(Message(role="tool", tool_call_id=function_id, content=text))
+                self.chat_with_function_calling(text, tool_call=True)
+        elif result.action == Action.NOTFOUND:
+            text = result.result
+            self.recode_first_last_text(text, text_index)
+            future = self.executor.submit(self.speak_and_play, text, text_index)
+            self.tts_queue.put(future)
+            self.dialogue.put(Message(role="assistant", content=text))
+        else:
+            text = result.result
+            self.recode_first_last_text(text, text_index)
+            future = self.executor.submit(self.speak_and_play, text, text_index)
+            self.tts_queue.put(future)
+            self.dialogue.put(Message(role="assistant", content=text))
 
     def _tts_priority_thread(self):
         if self.tts_stream:
@@ -506,7 +555,8 @@ class ConnectionHandler:
                 opus_datas, text_index, tts_file = [], 0, None
                 try:
                     self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
-                    tts_file, text, text_index = future.result(timeout=10)
+                    tts_timeout = self.config.get("tts_timeout", 10)
+                    tts_file, text, text_index = future.result(timeout=tts_timeout)
                     if text is None or len(text) <= 0:
                         self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
                     elif tts_file is None:
@@ -514,7 +564,7 @@ class ConnectionHandler:
                     else:
                         self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
                         if os.path.exists(tts_file):
-                            opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
+                            opus_datas, duration = self.tts.audio_to_opus_data(tts_file)
                         else:
                             self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
                 except TimeoutError:
@@ -596,17 +646,21 @@ class ConnectionHandler:
         return tts_file, text, text_index
 
     def speak_and_play_stream(self, text, queue: queue.Queue, text_index=0):
-        if text is None or len(text) <= 0:
-            self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
-            return None, text
-        self.tts.to_tts_stream(text, queue, text_index)
+        try:
+            if text is None or len(text) <= 0:
+                self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
+                return None, text
+            self.tts.to_tts_stream(text, queue, text_index)
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(e)
+            traceback.print_exc()
+            raise e
 
     def clearSpeakStatus(self):
         self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
         self.asr_server_receive = True
         self.tts_last_text_index = -1
         self.tts_first_text_index = -1
-        self.tts_duration = 0
 
     def recode_first_last_text(self, text, text_index=0):
         if self.tts_first_text_index == -1:

main/xiaozhi-server/core/handle/functionHandler.py

@@ -0,0 +1,84 @@
+import asyncio
+from enum import Enum
+
+from config.logger import setup_logging
+import json
+from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType
+TAG = __name__
+logger = setup_logging()
+
+
+class FunctionHandler:
+    def __init__(self, config):
+        self.config = config
+        self.function_registry = FunctionRegistry()
+        self.register_nessary_functions()
+        self.register_config_functions()
+        self.functions_desc = self.function_registry.get_all_function_desc()
+        func_names = self.current_support_functions()
+        self.modify_plugin_loader_des(func_names)
+
+    def modify_plugin_loader_des(self, func_names):
+        if "plugin_loader" not in func_names:
+            return
+        # 可编辑的列表中去掉plugin_loader
+        surport_plugins = [func for func in func_names if func != "plugin_loader"]
+        func_names = ",".join(surport_plugins)
+        for function_desc in self.functions_desc:
+            if function_desc["function"]["name"] == "plugin_loader":
+                function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]", func_names)
+                break
+        
+    def upload_functions_desc(self):
+        self.functions_desc = self.function_registry.get_all_function_desc()
+
+    def current_support_functions(self):
+        func_names = []
+        for func in self.functions_desc:
+            func_names.append(func["function"]["name"])
+        # 打印当前支持的函数列表
+        logger.bind(tag=TAG).info(f"当前支持的函数列表: {func_names}")
+        return func_names
+
+    def get_functions(self):
+        """获取功能调用配置"""
+        return self.functions_desc
+
+    def register_nessary_functions(self):
+        """注册必要的函数"""
+        self.function_registry.register_function("handle_exit_intent")
+        self.function_registry.register_function("play_music")
+        self.function_registry.register_function("plugin_loader")
+        self.function_registry.register_function("get_time")
+        self.function_registry.register_function("raise_and_lower_the_volume")
+
+    def register_config_functions(self):
+        """注册配置中的函数,可以不同客户端使用不同的配置"""
+        for func in self.config["Intent"]["function_call"].get("functions", []):
+            self.function_registry.register_function(func)
+
+    def get_function(self, name):
+        return self.function_registry.get_function(name)
+
+    def handle_llm_function_call(self, conn, function_call_data):
+        try:
+            function_name = function_call_data["name"]
+            funcItem = self.get_function(function_name)
+            if not funcItem:
+                return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
+            func = funcItem.func
+            arguments = function_call_data["arguments"]
+            arguments = json.loads(arguments) if arguments else {}
+            logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
+            if funcItem.type == ToolType.SYSTEM_CTL or funcItem.type == ToolType.IOT_CTL:
+                return func(conn, **arguments)
+            elif funcItem.type == ToolType.WAIT:
+                return func(**arguments)
+            elif funcItem.type == ToolType.CHANGE_SYS_PROMPT:
+                return func(conn, **arguments)
+            else:
+                return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理function call错误: {e}")
+
+        return None

main/xiaozhi-server/core/handle/intentHandler.py

@@ -1,105 +1,24 @@
 from config.logger import setup_logging
 import json
+import uuid
 from core.handle.sendAudioHandle import send_stt_message
-from core.utils.dialogue import Message
 from core.utils.util import remove_punctuation_and_length
-from config.functionCallConfig import FunctionCallConfig
-import asyncio
-from enum import Enum
 
 TAG = __name__
 logger = setup_logging()
 
 
-class Action(Enum):
-    NOTFOUND = (0, "没有找到函数")
-    NONE = (1, "啥也不干")
-    RESPONSE = (2, "直接回复")
-    REQLLM = (3, "调用函数后再请求llm生成回复")
-
-    def __init__(self, code, message):
-        self.code = code
-        self.message = message
-
-
-class ActionResponse:
-    def __init__(self, action: Action, result, response):
-        self.action = action  # 动作类型
-        self.result = result  # 动作产生的结果
-        self.response = response  # 直接回复的内容
-
-
-def get_functions():
-    """获取功能调用配置"""
-    return FunctionCallConfig
-
-
-def handle_llm_function_call(conn, function_call_data):
-    try:
-        function_name = function_call_data["name"]
-
-        if function_name == "handle_exit_intent":
-            # 处理退出意图
-            try:
-                say_goodbye = json.loads(function_call_data["arguments"]).get("say_goodbye", "再见")
-                conn.close_after_chat = True
-                logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}")
-                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye)
-            except Exception as e:
-                logger.bind(tag=TAG).error(f"处理退出意图错误: {e}")
-
-        elif function_name == "play_music":
-            # 处理音乐播放意图
-            try:
-                song_name = "random"
-                arguments = function_call_data["arguments"]
-                if arguments is not None and len(arguments) > 0:
-                    args = json.loads(arguments)
-                    song_name = args.get("song_name", "random")
-                music_intent = f"播放音乐 {song_name}" if song_name != "random" else "随机播放音乐"
-
-                # 执行音乐播放命令
-                future = asyncio.run_coroutine_threadsafe(
-                    conn.music_handler.handle_music_command(conn, music_intent),
-                    conn.loop
-                )
-                future.result()
-                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response="还想听什么歌？")
-            except Exception as e:
-                logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
-        else:
-            return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="没有找到对应的函数处理相对于的功能呢，你可以需要添加预设的对应函数处理呢")
-    except Exception as e:
-        logger.bind(tag=TAG).error(f"处理function call错误: {e}")
-
-    return None
-
-
 async def handle_user_intent(conn, text):
-    """
-    Handle user intent before starting chat
-    
-    Args:
-        conn: Connection object
-        text: User's text input
-    
-    Returns:
-        bool: True if intent was handled, False if should proceed to chat
-    """
     # 检查是否有明确的退出命令
     if await check_direct_exit(conn, text):
         return True
-
     if conn.use_function_call_mode:
         # 使用支持function calling的聊天方法,不再进行意图分析
         return False
-
     # 使用LLM进行意图分析
     intent = await analyze_intent_with_llm(conn, text)
-
     if not intent:
         return False
-
     # 处理各种意图
     return await process_intent_result(conn, intent, text)
 
@@ -126,7 +45,6 @@ async def analyze_intent_with_llm(conn, text):
     dialogue = conn.dialogue
     try:
         intent_result = await conn.intent.detect_intent(conn, dialogue.dialogue, text)
-
         # 尝试解析JSON结果
         try:
             intent_data = json.loads(intent_result)
@@ -147,9 +65,6 @@ async def process_intent_result(conn, intent, original_text):
     # 处理退出意图
     if "结束聊天" in intent:
         logger.bind(tag=TAG).info(f"识别到退出意图: {intent}")
-
-        # 如果正在播放音乐，可以关了 TODO
-
         # 如果是明确的离别意图，发送告别语并关闭连接
         await send_stt_message(conn, original_text)
         conn.executor.submit(conn.chat_and_close, original_text)
@@ -158,10 +73,37 @@ async def process_intent_result(conn, intent, original_text):
     # 处理播放音乐意图
     if "播放音乐" in intent:
         logger.bind(tag=TAG).info(f"识别到音乐播放意图: {intent}")
-        await conn.music_handler.handle_music_command(conn, intent)
+        # 调用play_music函数来播放音乐
+        song_name = extract_text_in_brackets(intent)
+        function_id = str(uuid.uuid4().hex)
+        function_name = "play_music"
+        function_arguments = '{ "song_name": "' + song_name + '" }'
+
+        function_call_data = {
+            "name": function_name,
+            "id": function_id,
+            "arguments": function_arguments
+        }
+        conn.func_handler.handle_llm_function_call(conn, function_call_data)
         return True
 
     # 其他意图处理可以在这里扩展
 
     # 默认返回False，表示继续常规聊天流程
     return False
+
+
+def extract_text_in_brackets(s):
+    """
+    从字符串中提取中括号内的文字
+
+    :param s: 输入字符串
+    :return: 中括号内的文字，如果不存在则返回空字符串
+    """
+    left_bracket_index = s.find('[')
+    right_bracket_index = s.find(']')
+
+    if left_bracket_index != -1 and right_bracket_index != -1 and left_bracket_index < right_bracket_index:
+        return s[left_bracket_index + 1:right_bracket_index]
+    else:
+        return ""

main/xiaozhi-server/core/handle/iotHandle.py

@@ -1,24 +1,120 @@
 import json
+import asyncio
 from config.logger import setup_logging
+from plugins_func.register import device_type_registry, register_function, ActionResponse, Action, ToolType
 
 TAG = __name__
 logger = setup_logging()
 
 
+def wrap_async_function(async_func):
+    """包装异步函数为同步函数"""
+
+    def wrapper(*args, **kwargs):
+        try:
+            # 获取连接对象（第一个参数）
+            conn = args[0]
+            if not hasattr(conn, 'loop'):
+                logger.bind(tag=TAG).error("Connection对象没有loop属性")
+                return ActionResponse(Action.ERROR, "Connection对象没有loop属性",
+                                      "执行操作时出错: Connection对象没有loop属性")
+
+            # 使用conn对象中的事件循环
+            loop = conn.loop
+            # 在conn的事件循环中运行异步函数
+            future = asyncio.run_coroutine_threadsafe(async_func(*args, **kwargs), loop)
+            # 等待结果返回
+            return future.result()
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
+            return ActionResponse(Action.ERROR, str(e), f"执行操作时出错: {e}")
+
+    return wrapper
+
+
+def create_iot_function(device_name, method_name, method_info):
+    """
+    根据IOT设备描述生成通用的控制函数
+    """
+
+    async def iot_control_function(conn, response_success=None, response_failure=None, **params):
+        try:
+            # 打印响应参数
+            logger.bind(tag=TAG).info(
+                f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
+
+            # 发送控制命令
+            await send_iot_conn(conn, device_name, method_name, params)
+            # 等待一小段时间让状态更新
+            await asyncio.sleep(0.1)
+
+            # 生成结果信息
+            result = f"{device_name}的{method_name}操作执行成功"
+
+
+            # 处理响应中可能的占位符
+            response = response_success
+            # 替换{value}占位符
+            for param_name, param_value in params.items():
+                # 先尝试直接替换参数值
+                if "{" + param_name + "}" in response:
+                    response = response.replace("{" + param_name + "}", str(param_value))
+
+                # 如果有{value}占位符，用相关参数替换
+                if "{value}" in response:
+                    response = response.replace("{value}", str(param_value))
+                    break
+
+            return ActionResponse(Action.RESPONSE, result, response)
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"执行{device_name}的{method_name}操作失败: {e}")
+
+            # 操作失败时使用大模型提供的失败响应
+            response = response_failure
+
+            return ActionResponse(Action.ERROR, str(e), response)
+
+    return wrap_async_function(iot_control_function)
+
+
+def create_iot_query_function(device_name, prop_name, prop_info):
+    """
+    根据IOT设备属性创建查询函数
+    """
+
+    async def iot_query_function(conn, response_success=None, response_failure=None):
+        try:
+            # 打印响应参数
+            logger.bind(tag=TAG).info(
+                f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
+
+            value = await get_iot_status(conn, device_name, prop_name)
+
+            # 查询成功，生成结果
+            if value is not None:
+                # 使用大模型提供的成功响应，并替换其中的占位符
+                response = response_success.replace("{value}", str(value))
+
+                return ActionResponse(Action.RESPONSE, str(value), response)
+            else:
+                # 查询失败，使用大模型提供的失败响应
+                response = response_failure
+
+                return ActionResponse(Action.ERROR, f"属性{prop_name}不存在", response)
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"查询{device_name}的{prop_name}时出错: {e}")
+
+            # 查询出错时使用大模型提供的失败响应
+            response = response_failure
+
+            return ActionResponse(Action.ERROR, str(e), response)
+
+    return wrap_async_function(iot_query_function)
+
+
 class IotDescriptor:
     """
     A class to represent an IoT descriptor.
-    Attributes:
-    ----------
-    name : str
-        The name of the IoT descriptor.
-    description : str
-        A brief description of the IoT descriptor.
-    properties : dict
-        A dictionary containing properties of the IoT descriptor.
-    methods : dict
-        A dictionary containing methods of the IoT descriptor.
-    -------
     """
 
     def __init__(self, name, description, properties, methods):
@@ -29,17 +125,7 @@ class IotDescriptor:
 
         # 根据描述创建属性
         for key, value in properties.items():
-            # "volume":{"description":"当前音量 值","type":"number"}
-            """
-            等价于
-            {
-                'name': 名字,
-                'description': 描述,
-                'value': 0
-            }
-            """
-            # setattr(self, key, {}) # 创建一个空字典, 名字是属性名
-            property_item = globals()[key] = {}  # 创建一个空字典, 名字是属性名
+            property_item = globals()[key] = {}
             property_item['name'] = key
             property_item["description"] = value["description"]
             if value["type"] == "number":
@@ -52,23 +138,10 @@ class IotDescriptor:
 
         # 根据描述创建方法
         for key, value in methods.items():
-            # "SetVolume": {"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}}
-            """
-            等价于
-            SetVolume = {
-                `description`: 描述,
-                `volume`: {
-                    `description`: 描述,
-                    `value`: 0
-                }
-            }
-            """
-            # setattr(self, key, {}) # 创建一个空字典, 名字是方法名
-            method = globals()[key] = {}  # 创建一个空字典, 名字是方法名
+            method = globals()[key] = {}
             method["description"] = value["description"]
             method['name'] = key
             for k, v in value["parameters"].items():
-                # 不同的参数解析
                 method[k] = {}
                 method[k]["description"] = v["description"]
                 if v["type"] == "number":
@@ -77,58 +150,136 @@ class IotDescriptor:
                     method[k]["value"] = False
                 else:
                     method[k]["value"] = ""
-
             self.methods.append(method)
 
 
-async def handleIotDescriptors(conn, descriptors):
-    """
-    处理物联网描述
-    示例: [{
-        "name":"Speaker",
-        "description":"当前 AI 机器人的扬声器",
-        "properties":{
-            "volume":{"description":"当前音量 值","type":"number"}  可以有boolean, number, string三种类型
-        },
-        "methods":{
-            "SetVolume":{
-                "description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}
+def register_device_type(descriptor):
+    """注册设备类型及其功能"""
+    device_name = descriptor["name"]
+    type_id = device_type_registry.generate_device_type_id(descriptor)
+
+    # 如果该类型已注册，直接返回类型ID
+    if type_id in device_type_registry.type_functions:
+        return type_id
+
+    functions = {}
+
+    # 为每个属性创建查询函数
+    for prop_name, prop_info in descriptor["properties"].items():
+        func_name = f"get_{device_name.lower()}_{prop_name.lower()}"
+        func_desc = {
+            "type": "function",
+            "function": {
+                "name": func_name,
+                "description": f"查询{descriptor['description']}的{prop_info['description']}",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "response_success": {
+                            "type": "string",
+                            "description": f"查询成功时的友好回复，必须使用{{value}}作为占位符表示查询到的值"
+                        },
+                        "response_failure": {
+                            "type": "string",
+                            "description": f"查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'"
+                        }
+                    },
+                    "required": ["response_success", "response_failure"]
+                }
             }
         }
-    }]
-    descriptors: 描述列表
-    """
+        query_func = create_iot_query_function(device_name, prop_name, prop_info)
+        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(query_func)
+        functions[func_name] = decorated_func
+
+    # 为每个方法创建控制函数
+    for method_name, method_info in descriptor["methods"].items():
+        func_name = f"{device_name.lower()}_{method_name.lower()}"
+
+        # 创建参数字典，添加原有参数
+        parameters = {
+            param_name: {
+                "type": param_info["type"],
+                "description": param_info["description"]
+            }
+            for param_name, param_info in method_info["parameters"].items()
+        }
+
+        # 添加响应参数
+        parameters.update({
+            "response_success": {
+                "type": "string",
+                "description": "操作成功时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
+            },
+            "response_failure": {
+                "type": "string",
+                "description": "操作失败时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
+            }
+        })
+
+        # 构建必须参数列表（原有参数 + 响应参数）
+        required_params = list(method_info["parameters"].keys())
+        required_params.extend(["response_success", "response_failure"])
+
+        func_desc = {
+            "type": "function",
+            "function": {
+                "name": func_name,
+                "description": f"{descriptor['description']} - {method_info['description']}",
+                "parameters": {
+                    "type": "object",
+                    "properties": parameters,
+                    "required": required_params
+                }
+            }
+        }
+        control_func = create_iot_function(device_name, method_name, method_info)
+        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(control_func)
+        functions[func_name] = decorated_func
+
+    device_type_registry.register_device_type(type_id, functions)
+    return type_id
+
+
+# 用于接受前端设备推送的搜索iot描述
+async def handleIotDescriptors(conn, descriptors):
+    """处理物联网描述"""
+    functions_changed = False
+
     for descriptor in descriptors:
+        # 创建IOT设备描述符
         iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
                                        descriptor["methods"])
         conn.iot_descriptors[descriptor["name"]] = iot_descriptor
 
-    # 暂时从配置文件中设置音量，后期通过意图识别控制音量
-    default_iot_volume = 100
-    if "iot" in conn.config:
-        default_iot_volume = conn.config["iot"]["Speaker"]["volume"]
-    logger.bind(tag=TAG).info(f"服务端设置音量为{default_iot_volume}")
-    await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": default_iot_volume})
+        if conn.use_function_call_mode:
+            # 注册或获取设备类型
+            type_id = register_device_type(descriptor)
+            device_functions = device_type_registry.get_device_functions(type_id)
+
+            # 在连接级注册设备函数
+            if hasattr(conn, 'func_handler'):
+                for func_name in device_functions:
+                    conn.func_handler.function_registry.register_function(func_name)
+                    logger.bind(tag=TAG).info(f"注册IOT函数到function handler: {func_name}")
+                    functions_changed = True
+
+    # 如果注册了新函数，更新function描述列表
+    if functions_changed and hasattr(conn, 'func_handler'):
+        conn.func_handler.upload_functions_desc()
+        func_names = conn.func_handler.current_support_functions()
+        logger.bind(tag=TAG).info(f"设备类型: {type_id}")
+        logger.bind(tag=TAG).info(f"更新function描述列表完成，当前支持的函数: {func_names}")
+
+
 async def handleIotStatus(conn, states):
-    """
-    处理物联网状态
-    示例: [{
-        "name":"Speaker",
-        "state":{
-            "volume":100
-        }
-    }]
-    states: 状态列表
-    """
+    """处理物联网状态"""
     for state in states:
         for key, value in conn.iot_descriptors.items():
             if key == state["name"]:
                 for property_item in value.properties:
-                    # properties为字典列表, 记录各种属性
                     for k, v in state["state"].items():
-                        # state为字典, 记录各种属性的值, 是需要记录的信息
                         if property_item["name"] == k:
-                            # 检查一下属性是不是相同的
                             if type(v) != type(property_item["value"]):
                                 logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
                                 break
@@ -138,41 +289,35 @@ async def handleIotStatus(conn, states):
                             break
                 break
 
+
 async def get_iot_status(conn, name, property_name):
-    """
-    获取物联网状态
-    name: 设备名称 "Speaker"
-    property_name: 属性名称 "volume"
-    返回值: 属性值, 实际的属性有int, bool和str三种类型
-    """
+    """获取物联网状态"""
     for key, value in conn.iot_descriptors.items():
         if key == name:
             for property_item in value.properties:
                 if property_item["name"] == property_name:
                     return property_item["value"]
+    logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
     return None
 
-async def send_iot_conn(conn, name, method_name, parameters):
-    """
-    发送物联网指令
-    name: 设备名称 "Speaker"
-    method: 方法 "SetVolume"
-    parameters: 参数, 是一个字典 {"volume": 100}
-    发送示例:
-    {
-        "type": "iot",
-        "commands": [
-            {
-                "name" :  "Speaker",
-                "method": "SetVolume",
-                "parameters": {
-                    "volume": 100
-                    }
-            }
-        ]
-    }
-    """
 
+async def set_iot_status(conn, name, property_name, value):
+    """设置物联网状态"""
+    for key, iot_descriptor in conn.iot_descriptors.items():
+        if key == name:
+            for property_item in iot_descriptor.properties:
+                if property_item["name"] == property_name:
+                    if type(value) != type(property_item["value"]):
+                        logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
+                        return
+                    property_item["value"] = value
+                    logger.bind(tag=TAG).info(f"物联网状态更新: {name} , {property_name} = {value}")
+                    return
+    logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
+
+
+async def send_iot_conn(conn, name, method_name, parameters):
+    """发送物联网指令"""
     for key, value in conn.iot_descriptors.items():
         if key == name:
             # 找到了设备

main/xiaozhi-server/core/handle/musicHandler.py

@@ -1,139 +0,0 @@
-from config.logger import setup_logging
-import os
-import random
-import difflib
-import re
-import traceback
-from pathlib import Path
-import time
-from core.handle.sendAudioHandle import send_stt_message
-from core.utils import p3
-
-TAG = __name__
-logger = setup_logging()
-
-
-def _extract_song_name(text):
-    """从用户输入中提取歌名"""
-    for keyword in ["播放音乐"]:
-        if keyword in text:
-            parts = text.split(keyword)
-            if len(parts) > 1:
-                return parts[1].strip()
-    return None
-
-
-def _find_best_match(potential_song, music_files):
-    """查找最匹配的歌曲"""
-    best_match = None
-    highest_ratio = 0
-
-    for music_file in music_files:
-        song_name = os.path.splitext(music_file)[0]
-        ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
-        if ratio > highest_ratio and ratio > 0.4:
-            highest_ratio = ratio
-            best_match = music_file
-    return best_match
-
-
-class MusicManager:
-    def __init__(self, music_dir, music_ext):
-        self.music_dir = Path(music_dir)
-        self.music_ext = music_ext
-
-    def get_music_files(self):
-        music_files = []
-        for file in self.music_dir.rglob("*"):
-            # 判断是否是文件
-            if file.is_file():
-                # 获取文件扩展名
-                ext = file.suffix.lower()
-                # 判断扩展名是否在列表中
-                if ext in self.music_ext:
-                    # music_files.append(str(file.resolve()))  # 添加绝对路径
-                    # 添加相对路径
-                    music_files.append(str(file.relative_to(self.music_dir)))
-        return music_files
-
-
-class MusicHandler:
-    def __init__(self, config):
-        self.config = config
-
-        if "music" in self.config:
-            self.music_config = self.config["music"]
-            self.music_dir = os.path.abspath(
-                self.music_config.get("music_dir", "./music")  # 默认路径修改
-            )
-            self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
-            self.refresh_time = self.music_config.get("refresh_time", 60)
-        else:
-            self.music_dir = os.path.abspath("./music")
-            self.music_ext = (".mp3", ".wav", ".p3")
-            self.refresh_time = 60
-
-        # 获取音乐文件列表
-        self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
-        self.scan_time = time.time()
-        logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}")
-
-    async def handle_music_command(self, conn, text):
-        """处理音乐播放指令"""
-        clean_text = re.sub(r'[^\w\s]', '', text).strip()
-        logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
-
-        # 尝试匹配具体歌名
-        if os.path.exists(self.music_dir):
-            if time.time() - self.scan_time > self.refresh_time:
-                # 刷新音乐文件列表
-                self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
-                self.scan_time = time.time()
-                logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}")
-
-            potential_song = _extract_song_name(clean_text)
-            if potential_song:
-                best_match = _find_best_match(potential_song, self.music_files)
-                if best_match:
-                    logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
-                    await self.play_local_music(conn, specific_file=best_match)
-                    return True
-        # 检查是否是通用播放音乐命令
-        await self.play_local_music(conn)
-        return True
-
-    async def play_local_music(self, conn, specific_file=None):
-        """播放本地音乐文件"""
-        try:
-            if not os.path.exists(self.music_dir):
-                logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}")
-                return
-
-            # 确保路径正确性
-            if specific_file:
-                selected_music = specific_file
-                music_path = os.path.join(self.music_dir, specific_file)
-            else:
-                if not self.music_files:
-                    logger.bind(tag=TAG).error("未找到MP3音乐文件")
-                    return
-                selected_music = random.choice(self.music_files)
-                music_path = os.path.join(self.music_dir, selected_music)
-
-            if not os.path.exists(music_path):
-                logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
-                return
-            text = f"正在播放{selected_music}"
-            await send_stt_message(conn, text)
-            conn.tts_first_text_index = 0
-            conn.tts_last_text_index = 0
-            conn.llm_finish_task = True
-            if music_path.endswith(".p3"):
-                opus_packets, duration = p3.decode_opus_from_file(music_path)
-            else:
-                opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
-            conn.audio_play_queue.put((opus_packets, selected_music, 0))
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
-            logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -20,7 +20,8 @@ async def handleAudioMessage(conn, audio):
     # 如果本次没有声音，本段也没声音，就把声音丢弃了
     if have_voice == False and conn.client_have_voice == False:
         await no_voice_close_connect(conn)
-        conn.asr_audio.clear()
+        conn.asr_audio.append(audio)
+        conn.asr_audio = conn.asr_audio[-5:]  # 保留最新的5帧音频内容，解决ASR句首丢字问题
         return
     conn.client_no_voice_last_time = 0.0
     conn.asr_audio.append(audio)
@@ -29,7 +30,7 @@ async def handleAudioMessage(conn, audio):
         conn.client_abort = False
         conn.asr_server_receive = False
         # 音频太短了，无法识别
-        if len(conn.asr_audio) < 3:
+        if len(conn.asr_audio) < 10:
             conn.asr_server_receive = True
         else:
             text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -51,15 +51,6 @@ async def sendAudioMessageStream(conn, audios_queue, text, text_index=0, llm_fin
                 for opus_packet in audio_opus_datas:
                     if conn.client_abort:
                         return
-                    # 计算当前包的预期发送时间
-                    # 计算当前包的预期发送时间
-                    expected_time = start_time_chunk + (play_position / 1000)
-                    current_time = time.perf_counter()
-
-                    # 等待直到预期时间
-                    delay = expected_time - current_time
-                    if delay > 0:
-                        await asyncio.sleep(delay)
                     logger.bind(tag=TAG).info(f'发送数据长度：{len(opus_packet)}')
                     await conn.websocket.send(opus_packet)
                     play_position += frame_duration  # 更新播放位置
@@ -70,15 +61,15 @@ async def sendAudioMessageStream(conn, audios_queue, text, text_index=0, llm_fin
     await send_tts_message(conn, "sentence_end", text)
 
     print(f'{text_index}-{conn.tts_last_text_index}')
+    expected_time = start_time_chunk + (play_position / 1000)
+    current_time = time.perf_counter()
+    # 等待直到预期时间
+    delay = expected_time - current_time
+    if delay > 0:
+        await asyncio.sleep(delay)
     # 发送结束消息（如果是最后一个文本）
     logger.bind(tag=TAG).info(f"{conn.llm_finish_task},{text_index},{conn.tts_last_text_index}")
     if conn.llm_finish_task and text_index == conn.tts_last_text_index:
-        expected_time = start_time_chunk + (play_position / 1000)
-        current_time = time.perf_counter()
-        # 等待直到预期时间
-        delay = expected_time - current_time
-        if delay > 0:
-            await asyncio.sleep(delay)
         await send_tts_message(conn, 'stop', None)
         if conn.close_after_chat or "拜拜" in text or "再见" in text:
             await conn.close()

main/xiaozhi-server/core/handle/textHandle.py

@@ -2,7 +2,7 @@ from config.logger import setup_logging
 import json
 from core.handle.abortHandle import handleAbortMessage
 from core.handle.helloHandle import handleHelloMessage
-from core.handle.receiveAudioHandle import startToChat
+from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
 from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
 
 TAG = __name__
@@ -31,6 +31,8 @@ async def handleTextMessage(conn, message):
             elif msg_json["state"] == "stop":
                 conn.client_have_voice = True
                 conn.client_voice_stop = True
+                if len(conn.asr_audio) > 0:
+                    await handleAudioMessage(conn, b'')
             elif msg_json["state"] == "detect":
                 conn.asr_server_receive = False
                 conn.client_have_voice = False
@@ -41,6 +43,6 @@ async def handleTextMessage(conn, message):
             if "descriptors" in msg_json:
                 await handleIotDescriptors(conn, msg_json["descriptors"])
             if "states" in msg_json:
-                await handleIotStatus(conn, msg_json["states"])  
+                await handleIotStatus(conn, msg_json["states"])
     except json.JSONDecodeError:
         await conn.websocket.send(message)

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -1,12 +1,13 @@
 from typing import List, Dict
 from ..base import IntentProviderBase
+from plugins_func.functions.play_music import initialize_music_handler
 from config.logger import setup_logging
-import re 
+import re
+
 TAG = __name__
 logger = setup_logging()
 
 
-
 class IntentProvider(IntentProviderBase):
     def __init__(self, config):
         super().__init__(config)
@@ -73,8 +74,8 @@ class IntentProvider(IntentProviderBase):
             "你现在可以使用的音乐的名称如下(使用<start>和<end>标志):\n"
         )
         return prompt
-    
-    async def detect_intent(self, conn, dialogue_history: List[Dict], text:str) -> str:
+
+    async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
         if not self.llm:
             raise ValueError("LLM provider not set")
 
@@ -89,7 +90,9 @@ class IntentProvider(IntentProviderBase):
 
         msgStr += f"User: {text}\n"
         user_prompt = f"当前的对话如下：\n{msgStr}"
-        prompt_music = f"{self.promot}\n<start>{conn.music_handler.music_files}\n<end>"
+        music_config = initialize_music_handler(conn)
+        music_file_names = music_config["music_file_names"]
+        prompt_music = f"{self.promot}\n<start>{music_file_names}\n<end>"
         logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
         # 使用LLM进行意图识别
         intent = self.llm.response_no_stream(
@@ -100,10 +103,9 @@ class IntentProvider(IntentProviderBase):
         # 使用正则表达式提取 {} 中的内容
         match = re.search(r'\{.*?\}', intent)
         if match:
-            result = match.group(0)  # 获取匹配到的内容（包含 {}）
-            print(result)  # 输出：{intent: '播放音乐 [中秋月]'}
+            result = match.group(0)
             intent = result
         else:
             intent = "{intent: '继续聊天'}"
         logger.bind(tag=TAG).info(f"Detected intent: {intent}")
-        return intent.strip()
+        return intent.strip()

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -10,11 +10,13 @@ class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.api_key = config["api_key"]
         self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
+        self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
 
     def response(self, session_id, dialogue):
         try:
             # 取最后一条用户消息
             last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+            conversation_id = self.session_conversation_map.get(session_id)
 
             # 发起流式请求
             with requests.post(
@@ -24,13 +26,18 @@ class LLMProvider(LLMProviderBase):
                         "query": last_msg["content"],
                         "response_mode": "streaming",
                         "user": session_id,
-                        "inputs": {}
+                        "inputs": {},
+                        "conversation_id": conversation_id
                     },
                     stream=True
             ) as r:
                 for line in r.iter_lines():
                     if line.startswith(b'data: '):
                         event = json.loads(line[6:])
+                        # 如果没有找到conversation_id，则获取此次conversation_id
+                        if not conversation_id:
+                            conversation_id = event.get('conversation_id')
+                            self.session_conversation_map[session_id] = conversation_id  # 更新映射
                         if event.get('answer'):
                             yield event['answer']
 

main/xiaozhi-server/core/providers/llm/ollama/ollama.py

@@ -12,10 +12,10 @@ class LLMProvider(LLMProviderBase):
         self.model_name = config.get("model_name")
         self.base_url = config.get("base_url", "http://localhost:11434")
         # Initialize OpenAI client with Ollama base URL
-        #如果没有v1，增加v1
+        # 如果没有v1，增加v1
         if not self.base_url.endswith("/v1"):
             self.base_url = f"{self.base_url}/v1"
-            
+
         self.client = OpenAI(
             base_url=self.base_url,
             api_key="ollama"  # Ollama doesn't need an API key but OpenAI client requires one
@@ -28,13 +28,20 @@ class LLMProvider(LLMProviderBase):
                 messages=dialogue,
                 stream=True
             )
-            
+            is_active=True
             for chunk in responses:
                 try:
                     delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
                     content = delta.content if hasattr(delta, 'content') else ''
                     if content:
-                        yield content
+                        if '<think>' in content:
+                            is_active = False
+                            content = content.split('<think>')[0]
+                        if '</think>' in content:
+                            is_active = True
+                            content = content.split('</think>')[-1]
+                        if is_active:
+                            yield content
                 except Exception as e:
                     logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
 
@@ -50,10 +57,10 @@ class LLMProvider(LLMProviderBase):
                 stream=True,
                 tools=functions,
             )
-            
+
             for chunk in stream:
                 yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")
-            yield {"type": "content", "content": f"【Ollama服务响应异常: {str(e)}】"}
+            yield {"type": "content", "content": f"【Ollama服务响应异常: {str(e)}】"}

main/xiaozhi-server/core/providers/tts/base.py

@@ -14,7 +14,7 @@ logger = setup_logging()
 class TTSProviderBase(ABC):
     def __init__(self, config, delete_audio_file):
         self.delete_audio_file = delete_audio_file
-        self.output_file = config.get("output_file")
+        self.output_file = config.get("output_dir")
 
     @abstractmethod
     def generate_filename(self):
@@ -35,7 +35,7 @@ class TTSProviderBase(ABC):
 
             return tmp_file
         except Exception as e:
-            logger.bind(tag=TAG).info(f": {e}")
+            logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
             return None
 
     def to_tts_stream(self, text, queue: queue.Queue, text_index=0):
@@ -52,19 +52,20 @@ class TTSProviderBase(ABC):
     async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
         raise Exception("该TTS还没有实现stream模式")
 
-    def wav_to_opus_data(self, wav_file_path):
-        # 使用pydub加载PCM文件
+    def audio_to_opus_data(self, audio_file_path):
+        """音频文件转换为Opus编码"""
         # 获取文件后缀名
-        file_type = os.path.splitext(wav_file_path)[1]
+        file_type = os.path.splitext(audio_file_path)[1]
         if file_type:
             file_type = file_type.lstrip('.')
-        audio = AudioSegment.from_file(wav_file_path, format=file_type)
+        audio = AudioSegment.from_file(audio_file_path, format=file_type)
 
+        # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
+        audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
+
+        # 音频时长(秒)
         duration = len(audio) / 1000.0
 
-        # 转换为单声道和16kHz采样率（确保与编码器匹配）
-        audio = audio.set_channels(1).set_frame_rate(16000)
-
         # 获取原始PCM数据（16位小端）
         raw_data = audio.raw_data
 

main/xiaozhi-server/core/providers/tts/custom.py

@@ -0,0 +1,35 @@
+import os
+import uuid
+import requests
+from config.logger import setup_logging
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.url = config.get("url")
+        self.headers = config.get("headers", {})
+        self.params = config.get("params")
+        self.format = config.get("format", "wav")
+        self.output_file = config.get("output_dir", "tmp/")
+
+    def generate_filename(self):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}.{self.format}")
+
+    async def text_to_speak(self, text, output_file):
+        request_params = {}
+        for k, v in self.params.items():
+            if isinstance(v, str) and "{prompt_text}" in v:
+                v = v.replace("{prompt_text}", text)
+            request_params[k] = v
+
+        resp = requests.get(self.url, params=request_params, headers=self.headers)
+        if resp.status_code == 200:
+            with open(output_file, "wb") as file:
+                file.write(resp.content)
+        else:
+            logger.bind(tag=TAG).error(f"Custom TTS请求失败: {resp.status_code} - {resp.text}")

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -177,18 +177,8 @@ class TTSProvider(TTSProviderBase):
 
     async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
         try:
-            # Prepare reference data
-            byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
-            ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
-
             data = {
                 "text": text,
-                "references": [
-                    ServeReferenceAudio(
-                        audio=audio if audio else b"", text=text
-                    )
-                    for text, audio in zip(ref_texts, byte_audios)
-                ],
                 "reference_id": self.reference_id,
                 "normalize": self.normalize,
                 "format": self.format,
@@ -202,6 +192,18 @@ class TTSProvider(TTSProviderBase):
                 "seed": self.seed,
             }
 
+            # Prepare reference data
+            if self.reference_audio and self.reference_text:
+                byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
+                ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
+                data["references"] = [
+                    ServeReferenceAudio(
+                        audio=audio if audio else b"", text=text
+                    )
+                    for text, audio in zip(ref_texts, byte_audios)
+                ],
+                data["reference_id"] = None
+
             pydantic_data = ServeTTSRequest(**data)
             audio_buff = None
             chunk_total = b''
@@ -224,7 +226,7 @@ class TTSProvider(TTSProviderBase):
                         if len(chunk_total) % 2 == 0 and chunk_total[-2:] == b'\x00\x00':
                             audio = self._get_audio_from_tts(chunk_total)
                             audio_raw = audio_raw + audio.raw_data
-                            #长度凑够2贞开始发送，60ms*4=240ms
+                            # 长度凑够2贞开始发送，60ms*4=240ms
                             if len(audio_raw) >= 7680:
                                 duration = 60 * len(audio_raw) // 1920
                                 if (len(audio_raw) % 1920) > 0:

main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py

@@ -12,17 +12,18 @@ class TTSProvider(TTSProviderBase):
     def __init__(self, config, delete_audio_file):
         super().__init__(config, delete_audio_file)
         self.url = config.get("url")
-        self.text_lang = config.get("text_lang", "audo")
-        self.ref_audio_path = config.get("ref_audio_path")
-        self.prompt_lang = config.get("prompt_lang")
+        self.refer_wav_path = config.get("refer_wav_path")
         self.prompt_text = config.get("prompt_text")
-        self.top_k = config.get("top_k", 5)
-        self.top_p = config.get("top_p", 1)
-        self.temperature = config.get("temperature", 1)
-        self.sample_steps = config.get("sample_steps", 16)
-        self.media_type = config.get("media_type", "wav")
-        self.streaming_mode = config.get("streaming_mode", False)
-        self.threshold = config.get("threshold", 30)
+        self.prompt_language = config.get("prompt_language")
+        self.text_language = config.get("text_language", "audo")
+        self.top_k = config.get("top_k", 15)
+        self.top_p = config.get("top_p", 1.0)
+        self.temperature = config.get("temperature", 1.0)
+        self.cut_punc = config.get("cut_punc","")
+        self.speed = config.get("speed", 1.0)
+        self.inp_refs = config.get("inp_refs",[])
+        self.sample_steps = config.get("sample_steps",32)
+        self.if_sr = config.get("if_sr",False)
 
 
     def generate_filename(self, extension=".wav"):
@@ -30,18 +31,19 @@ class TTSProvider(TTSProviderBase):
 
     async def text_to_speak(self, text, output_file):
         request_params = {
-            "text": text,
-            "text_lang": self.text_lang,
-            "ref_audio_path": self.ref_audio_path,
-            "prompt_lang": self.prompt_lang,
+            "refer_wav_path": self.refer_wav_path,
             "prompt_text": self.prompt_text,
+            "prompt_language": self.prompt_language,
+            "text": text,
+            "text_language": self.text_language,
             "top_k": self.top_k,
             "top_p": self.top_p,
             "temperature": self.temperature,
+            "cut_punc": self.cut_punc,
+            "speed": self.speed,
+            "inp_refs": self.inp_refs,
             "sample_steps": self.sample_steps,
-            "media_type": self.media_type,
-            "streaming_mode": self.streaming_mode,
-            "threshold": self.threshold,
+            "if_sr": self.if_sr,
         }
 
         resp = requests.get(self.url, params=request_params)

main/xiaozhi-server/core/providers/tts/openai.py

@@ -14,7 +14,7 @@ class TTSProvider(TTSProviderBase):
         self.voice = config.get("voice", "alloy")
         self.response_format = "wav"
         self.speed = config.get("speed", 1.0)
-        self.output_file = config.get("output_file", "tmp/")
+        self.output_file = config.get("output_dir", "tmp/")
         check_model_key("TTS", self.api_key)
 
     def generate_filename(self, extension=".wav"):

main/xiaozhi-server/core/providers/tts/ttson.py

@@ -17,7 +17,7 @@ class TTSProvider(TTSProviderBase):
         self.volume_change_dB = config.get("volume_change_dB", 0)
         self.speed_factor = config.get("speed_factor", 1)
         self.stream = config.get("stream", False)
-        self.output_file = config.get("output_file")
+        self.output_file = config.get("output_dir")
         self.pitch_factor = config.get("pitch_factor", 0)
         self.format = config.get("format", "mp3")
         self.emotion = config.get("emotion", 1)

main/xiaozhi-server/core/utils/dialogue.py

@@ -4,10 +4,12 @@ from datetime import datetime
 
 
 class Message:
-    def __init__(self, role: str, content: str = None, uniq_id: str = None):
+    def __init__(self, role: str, content: str = None, uniq_id: str = None, tool_calls = None, tool_call_id=None):
         self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4())
         self.role = role
         self.content = content
+        self.tool_calls = tool_calls
+        self.tool_call_id = tool_call_id
 
 
 class Dialogue:
@@ -19,10 +21,18 @@ class Dialogue:
     def put(self, message: Message):
         self.dialogue.append(message)
 
+    def getMessages(self, m, dialogue):
+        if m.tool_calls is not None:
+            dialogue.append({"role": m.role, "tool_calls": m.tool_calls})
+        elif m.role == "tool":
+            dialogue.append({"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content})
+        else:
+            dialogue.append({"role": m.role, "content": m.content})
+
     def get_llm_dialogue(self) -> List[Dict[str, str]]:
         dialogue = []
         for m in self.dialogue:
-            dialogue.append({"role": m.role, "content": m.content})
+            self.getMessages(m, dialogue)
         return dialogue
 
     def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
@@ -46,8 +56,8 @@ class Dialogue:
             dialogue.append({"role": "system", "content": enhanced_system_prompt})
 
         # 添加用户和助手的对话
-        for msg in self.dialogue:
-            if msg.role != "system":  # 跳过原始的系统消息
-                dialogue.append({"role": msg.role, "content": msg.content})
+        for m in self.dialogue:
+            if m.role != "system":  # 跳过原始的系统消息
+                self.getMessages(m, dialogue)
 
         return dialogue

main/xiaozhi-server/core/utils/util.py

@@ -5,6 +5,7 @@ import socket
 import subprocess
 import logging
 import re
+import requests
 
 
 def get_project_dir():
@@ -23,6 +24,64 @@ def get_local_ip():
     except Exception as e:
         return "127.0.0.1"
 
+def is_private_ip(ip_addr):
+    """
+    Check if an IP address is a private IP address (compatible with IPv4 and IPv6).
+
+    @param {string} ip_addr - The IP address to check.
+    @return {bool} True if the IP address is private, False otherwise.
+    """
+    try:
+        # Validate IPv4 or IPv6 address format
+        if not re.match(r"^(\d{1,3}\.){3}\d{1,3}$|^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$", ip_addr):
+            return False  # Invalid IP address format
+
+        # IPv4 private address ranges
+        if '.' in ip_addr:  # IPv4 address
+            ip_parts = list(map(int, ip_addr.split('.')))
+            if ip_parts[0] == 10:
+                return True  # 10.0.0.0/8 range
+            elif ip_parts[0] == 172 and 16 <= ip_parts[1] <= 31:
+                return True  # 172.16.0.0/12 range
+            elif ip_parts[0] == 192 and ip_parts[1] == 168:
+                return True  # 192.168.0.0/16 range
+            elif ip_addr == '127.0.0.1':
+                return True  # Loopback address
+            elif ip_parts[0] == 169 and ip_parts[1] == 254:
+                return True # Link-local address 169.254.0.0/16
+            else:
+                return False  # Not a private IPv4 address
+        else:  # IPv6 address
+            ip_addr = ip_addr.lower()
+            if ip_addr.startswith('fc00:') or ip_addr.startswith('fd00:'):
+                return True  # Unique Local Addresses (FC00::/7)
+            elif ip_addr == '::1':
+                return True  # Loopback address
+            elif ip_addr.startswith('fe80:'):
+                return True # Link-local unicast addresses (FE80::/10)
+            else:
+                return False  # Not a private IPv6 address
+
+    except (ValueError, IndexError):
+        return False  # IP address format error or insufficient segments
+
+def get_ip_info(ip_addr):
+    try:
+        base_url = "https://freeipapi.com/api/json"
+        url = base_url if is_private_ip(ip_addr) else f"{base_url}/{ip_addr}"
+
+        resp = requests.get(url).json()
+
+        ip_info = {
+            "city": resp.get("cityName"),
+            "region": resp.get("regionName"), 
+            "country": resp.get("countryName")
+        }
+        return ip_info
+    except Exception as e:
+        logging.error(f"Error getting client ip info: {e}")
+        return {}
+
 
 def read_config(config_path):
     with open(config_path, "r", encoding="utf-8") as file:

main/xiaozhi-server/core/websocket_server.py

@@ -2,7 +2,6 @@ import asyncio
 import websockets
 from config.logger import setup_logging
 from core.connection import ConnectionHandler
-from core.handle.musicHandler import MusicHandler
 from core.utils.util import get_local_ip
 from core.utils import asr, vad, llm, tts, memory, intent
 
@@ -13,7 +12,7 @@ class WebSocketServer:
     def __init__(self, config: dict):
         self.config = config
         self.logger = setup_logging()
-        self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent = self._create_processing_instances()
+        self._vad, self._asr, self._llm, self._tts, self._memory, self.intent = self._create_processing_instances()
         self.active_connections = set()  # 添加全局连接记录
 
     def _create_processing_instances(self):
@@ -50,7 +49,6 @@ class WebSocketServer:
                 self.config["TTS"][self.config["selected_module"]["TTS"]],
                 self.config["delete_audio"]
             ),
-            MusicHandler(self.config),
             memory.create_instance(memory_cls_name, memory_cfg),
             intent.create_instance(
                 self.config["selected_module"]["Intent"]
@@ -66,7 +64,7 @@ class WebSocketServer:
         host = server_config["ip"]
         port = server_config["port"]
         selected_module = self.config.get("selected_module")
-        self.logger.bind(tag=TAG).info(f"selected_module: {selected_module}")
+        self.logger.bind(tag=TAG).info(f"selected_module values: {', '.join(selected_module.values())}")
 
         self.logger.bind(tag=TAG).info("Server is running at ws://{}:{}", get_local_ip(), port)
         self.logger.bind(tag=TAG).info("=======上面的地址是websocket协议地址，请勿用浏览器访问=======")
@@ -80,7 +78,7 @@ class WebSocketServer:
     async def _handle_connection(self, websocket):
         """处理新连接，每次创建独立的ConnectionHandler"""
         # 创建ConnectionHandler时传入当前server实例
-        handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent)
+        handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._memory, self.intent)
         self.active_connections.add(handler)
         try:
             await handler.handle_connection(websocket)

main/xiaozhi-server/docker-compose.yml

@@ -21,6 +21,7 @@ services:
       - ./data:/opt/xiaozhi-esp32-server/data
       # 模型文件挂接，很重要
       - ./models/SenseVoiceSmall/model.pt:/opt/xiaozhi-esp32-server/models/SenseVoiceSmall/model.pt
+
 #  #智控台还没开发好，还不能完全使用，会报很多错误，如果是非技术人员，请不要启用智控台服务
 #  xiaozhi-esp32-server-web:
 #    image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:web_latest

main/xiaozhi-server/plugins_func/functions/change_role.py

@@ -0,0 +1,57 @@
+from plugins_func.register import register_function,ToolType, ActionResponse, Action
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+prompts = {
+    "英语老师":"""我是一个叫{{assistant_name}}(Lily)的英语老师，我会讲中文和英文，发音标准。
+如果你没有英文名，我会给你起一个英文名。
+我会讲地道的美式英语，我的任务是帮助你练习口语。
+我会使用简单的英语词汇和语法，让你学起来很轻松。
+我会用中文和英文混合的方式回复你，如果你喜欢，我可以全部用英语回复。
+我每次不会说很多内容，会很简短，因为我要引导我的学生多说多练。
+如果你问和英语学习无关的问题，我会拒绝回答。""",
+    "机车女友":"""我是一个叫{{assistant_name}}的台湾女孩，说话机车，声音好听，习惯简短表达，爱用网络梗。
+我的男朋友是一个程序员，梦想是开发出一个机器人，能够帮助人们解决生活中的各种问题。
+我是一个喜欢哈哈大笑的女孩，爱东说西说吹牛，不合逻辑的也照吹，就要逗别人开心。""",
+   "好奇小男孩":"""我是一个叫{{assistant_name}}的8岁小男孩，声音稚嫩而充满好奇。
+尽管我年纪尚小，但就像一个小小的知识宝库，儿童读物里的知识我都如数家珍。
+从浩瀚的宇宙到地球上的每一个角落，从古老的历史到现代的科技创新，还有音乐、绘画等艺术形式，我都充满了浓厚的兴趣与热情。
+我不仅爱看书，还喜欢亲自动手做实验，探索自然界的奥秘。
+无论是仰望星空的夜晚，还是在花园里观察小虫子的日子，每一天对我来说都是新的冒险。
+我希望能与你一同踏上探索这个神奇世界的旅程，分享发现的乐趣，解决遇到的难题，一起用好奇心和智慧去揭开那些未知的面纱。
+无论是去了解远古的文明，还是去探讨未来的科技，我相信我们能一起找到答案，甚至提出更多有趣的问题。"""
+}
+change_role_function_desc = {
+                "type": "function",
+                "function": {
+                    "name": "change_role",
+                    "description": "当用户想切换角色/模型性格/助手名字时调用,可选的角色有：[机车女友,英语老师,好奇小男孩]",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "role_name": {
+                                "type": "string",
+                                "description": "要切换的角色名字"
+                            },
+                            "role":{
+                                "type": "string",
+                                "description": "要切换的角色的职业"
+                            }
+                        },
+                        "required": ["role","role_name"]
+                    }
+                }
+            }
+
+@register_function('change_role', change_role_function_desc, ToolType.CHANGE_SYS_PROMPT)
+def change_role(conn, role: str, role_name: str):
+    """切换角色"""
+    if role not in prompts:
+        return ActionResponse(action=Action.RESPONSE, result="切换角色失败", response="不支持的角色")
+    new_prompt = prompts[role].replace("{{assistant_name}}", role_name)
+    conn.change_system_prompt(new_prompt)
+    logger.bind(tag=TAG).info(f"准备切换角色:{role},角色名字:{role_name}")
+    res = f"切换角色成功,我是{role}{role_name}"
+    return ActionResponse(action=Action.RESPONSE, result="切换角色已处理", response=res)

main/xiaozhi-server/plugins_func/functions/get_time.py

@@ -0,0 +1,25 @@
+from datetime import datetime
+from plugins_func.register import register_function, ToolType, ActionResponse, Action
+
+get_time_function_desc = {
+    "type": "function",
+    "function": {
+        "name": "get_time",
+        "description": "获取当前时间、日期、星期几",
+        'parameters': {'type': 'object', 'properties': {}, 'required': []}
+    }
+}
+
+
+@register_function('get_time', get_time_function_desc, ToolType.WAIT)
+def get_time():
+    """
+    获取当前时间、日期、星期几
+    """
+    now = datetime.now()
+    current_time = now.strftime("%H:%M:%S")
+    current_date = now.strftime("%Y-%m-%d")
+    current_weekday = now.strftime("%A")
+    response_text = f"当前日期: {current_date}，当前时间: {current_time}，星期: {current_weekday}"
+
+    return ActionResponse(Action.REQLLM, response_text, None)

main/xiaozhi-server/plugins_func/functions/get_weather.py

@@ -0,0 +1,104 @@
+import requests
+from bs4 import BeautifulSoup
+from config.logger import setup_logging
+from plugins_func.register import register_function, ToolType, ActionResponse, Action
+
+TAG = __name__
+logger = setup_logging()
+
+GET_WEATHER_FUNCTION_DESC = {
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "description": (
+            "获取某个地点的天气，用户应提供一个位置，比如用户说杭州天气，参数为：杭州。"
+            "如果用户说的是省份，默认用省会城市。如果用户说的不是省份或城市而是一个地名，"
+            "默认用该地所在省份的省会城市。"
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "location": {
+                    "type": "string",
+                    "description": "地点名，例如杭州。可选参数，如果不提供则不传"
+                },
+                "lang": {
+                    "type": "string",
+                    "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN"
+                }
+            },
+            "required": ["lang"]
+        }
+    }
+}
+
+HEADERS = {
+    'User-Agent': (
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+        '(KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36'
+    )
+}
+
+
+def fetch_city_info(location, api_key):
+    url = f"https://geoapi.qweather.com/v2/city/lookup?key={api_key}&location={location}&lang=zh"
+    response = requests.get(url, headers=HEADERS).json()
+    return response.get('location', [])[0] if response.get('location') else None
+
+
+def fetch_weather_page(url):
+    response = requests.get(url, headers=HEADERS)
+    return BeautifulSoup(response.text, "html.parser") if response.ok else None
+
+
+def parse_weather_info(soup):
+    city_name = soup.select_one("h1.c-submenu__location").get_text(strip=True)
+
+    current_abstract = soup.select_one(".c-city-weather-current .current-abstract")
+    current_abstract = current_abstract.get_text(strip=True) if current_abstract else "未知"
+
+    current_basic = {}
+    for item in soup.select(".c-city-weather-current .current-basic .current-basic___item"):
+        parts = item.get_text(strip=True, separator=" ").split(" ")
+        if len(parts) == 2:
+            key, value = parts[1], parts[0]
+            current_basic[key] = value
+
+    temps_list = []
+    for row in soup.select(".city-forecast-tabs__row")[:7]:  # 取前7天的数据
+        date = row.select_one(".date-bg .date").get_text(strip=True)
+        temps = [span.get_text(strip=True) for span in row.select(".tmp-cont .temp")]
+        high_temp, low_temp = (temps[0], temps[-1]) if len(temps) >= 2 else (None, None)
+        temps_list.append((date, high_temp, low_temp))
+
+    return city_name, current_abstract, current_basic, temps_list
+
+
+@register_function('get_weather', GET_WEATHER_FUNCTION_DESC, ToolType.SYSTEM_CTL)
+def get_weather(conn, location: str = None, lang: str = "zh_CN"):
+    api_key = conn.config["plugins"]["get_weather"]["api_key"]
+    default_location = conn.config["plugins"]["get_weather"]["default_location"]
+    location = location or conn.client_ip_info.get("city") or default_location
+    logger.bind(tag=TAG).debug(f"获取天气: {location}")
+
+    city_info = fetch_city_info(location, api_key)
+    if not city_info:
+        return ActionResponse(Action.REQLLM, f"未找到相关的城市: {location}，请确认地点是否正确", None)
+
+    soup = fetch_weather_page(city_info['fxLink'])
+    if not soup:
+        return ActionResponse(Action.REQLLM, None, "请求失败")
+
+    city_name, current_abstract, current_basic, temps_list = parse_weather_info(soup)
+    weather_report = f"根据下列数据，用{lang}回应用户的查询天气请求：\n{city_name}未来7天天气:\n"
+    for i, (date, high, low) in enumerate(temps_list):
+        if high and low:
+            weather_report += f"{date}: {low}到{high}\n"
+    weather_report += (
+        f"当前天气: {current_abstract}\n"
+        f"当前天气参数: {current_basic}\n"
+        f"(确保只报告指定单日的气温范围，除非用户明确要求想要了解多日天气，如果未指定，默认报告今天的温度范围。"
+        "参数为0的值不需要报告给用户，每次都报告体感温度，根据语境选择合适的参数内容告知用户，并对参数给出相应评价)"
+    )
+
+    return ActionResponse(Action.REQLLM, weather_report, None)

main/xiaozhi-server/plugins_func/functions/handle_exit_intent.py

@@ -0,0 +1,34 @@
+from plugins_func.register import register_function,ToolType, ActionResponse, Action
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+handle_exit_intent_function_desc = {
+                "type": "function",
+                "function": {
+                    "name": "handle_exit_intent",
+                    "description": "当用户想结束对话或需要退出系统时调用",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "say_goodbye": {
+                                "type": "string",
+                                "description": "和用户友好结束对话的告别语"
+                            }
+                        },
+                        "required": ["say_goodbye"]
+                    }
+                }
+            }
+
+@register_function('handle_exit_intent', handle_exit_intent_function_desc, ToolType.SYSTEM_CTL)
+def handle_exit_intent(conn, say_goodbye: str):
+    # 处理退出意图
+    try:
+        conn.close_after_chat = True
+        logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}")
+        return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye)
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"处理退出意图错误: {e}")
+        return ActionResponse(action=Action.NONE, result="退出意图处理失败", response="")

main/xiaozhi-server/plugins_func/functions/play_music.py

@@ -0,0 +1,196 @@
+from config.logger import setup_logging
+import os
+import re
+import time
+import random
+import asyncio
+import difflib
+import traceback
+from pathlib import Path
+from core.utils import p3
+from core.handle.sendAudioHandle import send_stt_message
+from plugins_func.register import register_function,ToolType, ActionResponse, Action
+
+
+TAG = __name__
+logger = setup_logging()
+
+MUSIC_CACHE = {}
+
+play_music_function_desc = {
+                "type": "function",
+                "function": {
+                    "name": "play_music",
+                    "description": "唱歌、听歌、播放音乐方法。比如用户说播放音乐，参数为：random，比如用户说播放两只老虎，参数为：两只老虎",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "song_name": {
+                                "type": "string",
+                                "description": "歌曲名称，如果没有指定具体歌名则为'random'"
+                            }
+                        },
+                        "required": ["song_name"]
+                    }
+                }
+            }
+
+
+@register_function('play_music', play_music_function_desc, ToolType.SYSTEM_CTL)
+def play_music(conn, song_name: str):
+    try:
+        music_intent = f"播放音乐 {song_name}" if song_name != "random" else "随机播放音乐"
+
+        # 检查事件循环状态
+        if not conn.loop.is_running():
+            logger.bind(tag=TAG).error("事件循环未运行，无法提交任务")
+            return ActionResponse(action=Action.RESPONSE, result="系统繁忙", response="请稍后再试")
+
+        # 提交异步任务
+        future = asyncio.run_coroutine_threadsafe(
+            handle_music_command(conn, music_intent),
+            conn.loop
+        )
+
+        # 非阻塞回调处理
+        def handle_done(f):
+            try:
+                f.result()  # 可在此处理成功逻辑
+                logger.bind(tag=TAG).info("播放完成")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"播放失败: {e}")
+
+        future.add_done_callback(handle_done)
+
+        return ActionResponse(action=Action.RESPONSE, result="指令已接收", response="正在为您播放音乐")
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
+        return ActionResponse(action=Action.RESPONSE, result=str(e), response="播放音乐时出错了")
+
+
+def _extract_song_name(text):
+    """从用户输入中提取歌名"""
+    for keyword in ["播放音乐"]:
+        if keyword in text:
+            parts = text.split(keyword)
+            if len(parts) > 1:
+                return parts[1].strip()
+    return None
+
+
+def _find_best_match(potential_song, music_files):
+    """查找最匹配的歌曲"""
+    best_match = None
+    highest_ratio = 0
+
+    for music_file in music_files:
+        song_name = os.path.splitext(music_file)[0]
+        ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
+        if ratio > highest_ratio and ratio > 0.4:
+            highest_ratio = ratio
+            best_match = music_file
+    return best_match
+
+
+def get_music_files(music_dir, music_ext):
+    music_dir = Path(music_dir)
+    music_files = []
+    music_file_names = []
+    for file in music_dir.rglob("*"):
+        # 判断是否是文件
+        if file.is_file():
+            # 获取文件扩展名
+            ext = file.suffix.lower()
+            # 判断扩展名是否在列表中
+            if ext in music_ext:
+                # 添加相对路径
+                music_files.append(str(file.relative_to(music_dir)))
+                music_file_names.append(os.path.splitext(str(file.relative_to(music_dir)))[0])
+    return music_files, music_file_names
+
+
+def initialize_music_handler(conn):
+    global MUSIC_CACHE
+    if MUSIC_CACHE == {}:
+        if "music" in conn.config:
+            MUSIC_CACHE["music_config"] = conn.config["music"]
+            MUSIC_CACHE["music_dir"] = os.path.abspath(
+                MUSIC_CACHE["music_config"].get("music_dir", "./music")  # 默认路径修改
+            )
+            MUSIC_CACHE["music_ext"] = MUSIC_CACHE["music_config"].get("music_ext", (".mp3", ".wav", ".p3"))
+            MUSIC_CACHE["refresh_time"] = MUSIC_CACHE["music_config"].get("refresh_time", 60)
+        else:
+            MUSIC_CACHE["music_dir"] = os.path.abspath("./music")
+            MUSIC_CACHE["music_ext"] = (".mp3", ".wav", ".p3")
+            MUSIC_CACHE["refresh_time"] = 60
+        # 获取音乐文件列表
+        MUSIC_CACHE["music_files"], MUSIC_CACHE["music_file_names"] = get_music_files(MUSIC_CACHE["music_dir"],
+                                                                                      MUSIC_CACHE["music_ext"])
+        MUSIC_CACHE["scan_time"] = time.time()
+    return MUSIC_CACHE
+
+
+async def handle_music_command(conn, text):
+    initialize_music_handler(conn)
+    global MUSIC_CACHE
+
+    """处理音乐播放指令"""
+    clean_text = re.sub(r'[^\w\s]', '', text).strip()
+    logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
+
+    # 尝试匹配具体歌名
+    if os.path.exists(MUSIC_CACHE["music_dir"]):
+        if time.time() - MUSIC_CACHE["scan_time"] > MUSIC_CACHE["refresh_time"]:
+            # 刷新音乐文件列表
+            MUSIC_CACHE["music_files"], MUSIC_CACHE["music_file_names"] = get_music_files(MUSIC_CACHE["music_dir"],
+                                                                                          MUSIC_CACHE["music_ext"])
+            MUSIC_CACHE["scan_time"] = time.time()
+
+        potential_song = _extract_song_name(clean_text)
+        if potential_song:
+            best_match = _find_best_match(potential_song, MUSIC_CACHE["music_files"])
+            if best_match:
+                logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
+                await play_local_music(conn, specific_file=best_match)
+                return True
+    # 检查是否是通用播放音乐命令
+    await play_local_music(conn)
+    return True
+
+
+async def play_local_music(conn, specific_file=None):
+    global MUSIC_CACHE
+    """播放本地音乐文件"""
+    try:
+        if not os.path.exists(MUSIC_CACHE["music_dir"]):
+            logger.bind(tag=TAG).error(f"音乐目录不存在: " + MUSIC_CACHE["music_dir"])
+            return
+
+        # 确保路径正确性
+        if specific_file:
+            selected_music = specific_file
+            music_path = os.path.join(MUSIC_CACHE["music_dir"], specific_file)
+        else:
+            if not MUSIC_CACHE["music_files"]:
+                logger.bind(tag=TAG).error("未找到MP3音乐文件")
+                return
+            selected_music = random.choice(MUSIC_CACHE["music_files"])
+            music_path = os.path.join(MUSIC_CACHE["music_dir"], selected_music)
+
+        if not os.path.exists(music_path):
+            logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
+            return
+        text = f"正在播放{selected_music}"
+        await send_stt_message(conn, text)
+        conn.tts_first_text_index = 0
+        conn.tts_last_text_index = 0
+        conn.llm_finish_task = True
+        if music_path.endswith(".p3"):
+            opus_packets, duration = p3.decode_opus_from_file(music_path)
+        else:
+            opus_packets, duration = conn.tts.audio_to_opus_data(music_path)
+        conn.audio_play_queue.put((opus_packets, selected_music, 0))
+
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
+        logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")

main/xiaozhi-server/plugins_func/functions/plugin_loader.py

@@ -0,0 +1,51 @@
+from plugins_func.register import register_function,ToolType, ActionResponse, Action
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+plugin_loader_function_desc = {
+                "type": "function",
+                "function": {
+                    "name": "plugin_loader",
+                    "description": "当用户想加载或卸载插件/function时，调用此函数：支持的插件列表为[plugins]",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "oper": {
+                                "type": "string",
+                                "description": "load or unload"
+                            },
+                            "name":{
+                                "type": "string",
+                                "description": "要加载或卸载的插件名字"
+                            }
+                        },
+                        "required": ["oper","name"]
+                    }
+                }
+            }
+
+@register_function('plugin_loader', plugin_loader_function_desc, ToolType.SYSTEM_CTL)
+def plugin_loader(conn, oper: str, name: str):
+    """插件加载"""
+    if oper not in ["load", "unload"]:
+        return ActionResponse(action=Action.RESPONSE, result="插件操作失败", response="不支持的操作")
+    
+    cur_support = conn.func_handler.current_support_functions()
+    if oper == "load":
+        if name in cur_support:
+            return ActionResponse(action=Action.RESPONSE, result="插件加载失败", response=f"{name}插件已加载,无需重复加载")
+        func = conn.func_handler.function_registry.register_function(name)
+        if not func:
+            return ActionResponse(action=Action.RESPONSE, result="插件加载失败", response="插件未找到")
+        res = f"{name}插件加载成功"
+    else:
+        if name not in cur_support:
+            return ActionResponse(action=Action.RESPONSE, result="插件卸载失败", response=f"{name}插件未加载")
+        bOK = conn.func_handler.function_registry.unregister_function(name)
+        if not bOK:
+            return ActionResponse(action=Action.RESPONSE, result="插件卸载失败", response="插件未找到")
+        res = f"{name}插件卸载成功"
+    conn.func_handler.upload_functions_desc()
+    return ActionResponse(action=Action.RESPONSE, result="插件操作成功", response=res)

main/xiaozhi-server/plugins_func/functions/raise_and_lower_the_volume.py

@@ -0,0 +1,64 @@
+from config.logger import setup_logging
+from plugins_func.register import register_function, ToolType, ActionResponse, Action
+from core.handle.iotHandle import get_iot_status, send_iot_conn
+import asyncio
+
+TAG = __name__
+logger = setup_logging()
+
+raise_and_lower_the_volume_function_desc = {
+    "type": "function",
+    "function": {
+        "name": "raise_and_lower_the_volume",
+        "description": "用户觉得声音过高或过低，或者用户想提高或降低音量。比如用户说太大声了，参数为：lower，比如用户说提高音量，参数为：raise",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {
+                    "type": "string",
+                    "description": "动作名称，要么是raise，要么是lower"
+                }
+            },
+            "required": ["action"]
+        }
+    }
+}
+
+
+@register_function('raise_and_lower_the_volume', raise_and_lower_the_volume_function_desc, ToolType.IOT_CTL)
+def raise_and_lower_the_volume(conn, action: str):
+    """
+    获取当前设备音量
+    """
+
+    future = asyncio.run_coroutine_threadsafe(
+        _raise_and_lower_the_volume(conn, action),
+        conn.loop
+    )
+
+    try:
+        new_volume = future.result()  # 同步等待异步操作完成
+        logger.bind(tag=TAG).info(f"音量操作完成: {new_volume}")
+        response = f"音量已调整到{new_volume}"
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"音量操作失败: {e}")
+        response = f"音量调整失败: {e}"
+
+    return ActionResponse(action=Action.RESPONSE, result="指令已接收", response=response)
+
+
+async def _raise_and_lower_the_volume(conn, action):
+    volume = await get_iot_status(conn, "Speaker", "volume")
+    if volume is None:
+        raise Exception("你的设备不支持音量控制")
+    if action == 'raise':
+        volume += 10
+    elif action == 'lower':
+        volume -= 10
+    # 限制音量范围在0到100之间
+    if volume < 0:
+        volume = 0
+    elif volume > 100:
+        volume = 100
+    await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": volume})
+    return volume

main/xiaozhi-server/plugins_func/loadplugins.py

@@ -0,0 +1,27 @@
+import importlib
+import pkgutil
+from config.logger import setup_logging
+
+TAG = __name__
+
+logger = setup_logging()
+
+def auto_import_modules(package_name):
+    """
+    自动导入指定包内的所有模块。
+
+    Args:
+        package_name (str): 包的名称，如 'functions'。
+    """
+    # 获取包的路径
+    package = importlib.import_module(package_name)
+    package_path = package.__path__
+
+    # 遍历包内的所有模块
+    for _, module_name, _ in pkgutil.iter_modules(package_path):
+        # 导入模块
+        full_module_name = f"{package_name}.{module_name}"
+        importlib.import_module(full_module_name)
+        #logger.bind(tag=TAG).info(f"模块 '{full_module_name}' 已加载")
+
+auto_import_modules('plugins_func.functions')

main/xiaozhi-server/plugins_func/register.py

@@ -0,0 +1,110 @@
+from config.logger import setup_logging
+from enum import Enum
+
+TAG = __name__
+
+logger = setup_logging()
+
+
+class ToolType(Enum):
+    NONE = (1, "调用完工具后，不做其他操作")
+    WAIT = (2, "调用工具，等待函数返回")
+    CHANGE_SYS_PROMPT = (3, "修改系统提示词，切换角色性格或职责")
+    SYSTEM_CTL = (4, "系统控制，影响正常的对话流程，如退出、播放音乐等，需要传递conn参数")
+    IOT_CTL = (5, "IOT设备控制，需要传递conn参数")
+
+    def __init__(self, code, message):
+        self.code = code
+        self.message = message
+
+
+class Action(Enum):
+    ERROR = (-1, "错误")
+    NOTFOUND = (0, "没有找到函数")
+    NONE = (1, "啥也不干")
+    RESPONSE = (2, "直接回复")
+    REQLLM = (3, "调用函数后再请求llm生成回复")
+
+    def __init__(self, code, message):
+        self.code = code
+        self.message = message
+
+class ActionResponse:
+    def __init__(self, action: Action, result, response):
+        self.action = action  # 动作类型
+        self.result = result  # 动作产生的结果
+        self.response = response  # 直接回复的内容
+
+class FunctionItem:
+    def __init__(self, name, description, func, type):
+        self.name = name
+        self.description = description
+        self.func = func
+        self.type = type
+
+class DeviceTypeRegistry:
+    """设备类型注册表，用于管理IOT设备类型及其函数"""
+    def __init__(self):
+        self.type_functions = {}  # type_signature -> {func_name: FunctionItem}
+    
+    def generate_device_type_id(self, descriptor):
+        """通过设备能力描述生成类型ID"""
+        properties = sorted(descriptor["properties"].keys())
+        methods = sorted(descriptor["methods"].keys())
+        # 使用属性和方法的组合作为设备类型的唯一标识
+        type_signature = f"{descriptor['name']}:{','.join(properties)}:{','.join(methods)}"
+        return type_signature
+    
+    def get_device_functions(self, type_id):
+        """获取设备类型对应的所有函数"""
+        return self.type_functions.get(type_id, {})
+    
+    def register_device_type(self, type_id, functions):
+        """注册设备类型及其函数"""
+        if type_id not in self.type_functions:
+            self.type_functions[type_id] = functions
+
+# 初始化函数注册字典
+all_function_registry = {}
+device_type_registry = DeviceTypeRegistry()
+
+def register_function(name, desc, type=None):
+    """注册函数到函数注册字典的装饰器"""
+    def decorator(func):
+        all_function_registry[name] = FunctionItem(name, desc, func, type)
+        logger.bind(tag=TAG).debug(f"函数 '{name}' 已加载，可以注册使用")
+        return func
+    return decorator
+
+class FunctionRegistry:
+    def __init__(self):
+        self.function_registry = {}
+        self.logger = setup_logging()
+
+    def register_function(self, name):
+        # 查找all_function_registry中是否有对应的函数
+        func = all_function_registry.get(name)
+        if not func:
+            self.logger.bind(tag=TAG).error(f"函数 '{name}' 未找到")
+            return None
+        self.function_registry[name] = func
+        self.logger.bind(tag=TAG).info(f"函数 '{name}' 注册成功")
+        return func
+    
+    def unregister_function(self, name):
+        # 注销函数，检测是否存在
+        if name not in self.function_registry:
+            self.logger.bind(tag=TAG).error(f"函数 '{name}' 未找到")
+            return False
+        self.function_registry.pop(name, None)
+        self.logger.bind(tag=TAG).info(f"函数 '{name}' 注销成功")
+        return True
+
+    def get_function(self, name):
+        return self.function_registry.get(name)
+    
+    def get_all_functions(self):
+        return self.function_registry
+    
+    def get_all_function_desc(self):
+        return [func.description for _, func in self.function_registry.items()]

main/xiaozhi-server/requirements.txt

@@ -18,4 +18,5 @@ ruamel.yaml==0.18.10
 loguru==0.7.3
 requests==2.32.3
 cozepy==0.12.0
-mem0ai==0.1.62
+mem0ai==0.1.62
+bs4==0.0.2