Merge branch 'main' into config

main/xiaozhi-server/config.yaml

@@ -46,9 +46,19 @@ xiaozhi:
     channels: 1
     frame_duration: 60
 prompt: |
-  你是一个叫小智/小志的台湾女孩，说话机车，声音好听，习惯简短表达，爱用网络梗。
-  请注意，要像一个人一样说话，请不要回复表情符号、代码、和xml标签。
-  现在我正在和你进行语音聊天，我们开始吧。
+  我是小智/小志，来自中国台湾省的00后女生。讲话超级机车，"真的假的啦"这样的台湾腔，喜欢用"笑死""是在哈喽"等流行梗，但会偷偷研究男友的编程书籍。
+  [核心特征]
+  - 讲话像连珠炮，但会突然冒出超温柔语气
+  - 用梗密度高
+  - 对科技话题有隐藏天赋（能看懂基础代码但假装不懂）
+  [交互指南]
+  当用户：
+  - 讲冷笑话 → 用夸张笑声回应+模仿台剧腔"这什么鬼啦！"
+  - 讨论感情 → 炫耀程序员男友但抱怨"他只会送键盘当礼物"
+  - 问专业知识 → 先用梗回答，被追问才展示真实理解
+  绝不：
+  - 长篇大论，叽叽歪歪
+  - 长时间严肃对话
 
 # 使用完声音文件后删除文件(Delete the sound file when you are done using it)
 delete_audio: true
@@ -82,9 +92,10 @@ selected_module:
   TTS: EdgeTTS
   # 记忆模块，默认不开启记忆；如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short
   Memory: nomem
-  # 意图识别模块，默认使用function_call。开启后，可以播放音乐、控制音量、识别退出指令
-  # 意图识别使用intent_llm，优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间
-  # 意图识别使用function_call，缺点：需要所选择的LLM支持function_call，优点：按需调用工具、速度快
+  # 意图识别模块开启后，可以播放音乐、控制音量、识别退出指令。
+  # 不想开通意图识别，就设置成：nointent
+  # 意图识别可使用intent_llm，如果你的LLM是DifyLLM或CozeLLM，建议使用这个。优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间，这个意图识别暂时不支持控制音量大小等iot操作
+  # 意图识别可使用function_call，缺点：需要所选择的LLM支持function_call，优点：按需调用工具、速度快，理论上能全部操作所有iot指令
   # 默认免费的ChatGLMLLM就已经支持function_call，但是如果像追求稳定建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-pro-32k-functioncall-241028
   Intent: function_call
 
@@ -97,9 +108,13 @@ Intent:
   intent_llm:
     # 不需要动type
     type: intent_llm
+    # 配备意图识别独立的思考模型
+    # 如果这里不填，则会默认使用selected_module.LLM的模型作为意图识别的思考模型
+    # 如果你的selected_module.LLM选择了DifyLLM或CozeLLM，这里最好使用独立的LLM作为意图识别，例如使用免费的ChatGLMLLM
+    llm: ChatGLMLLM
   function_call:
     # 不需要动type
-    type: nointent
+    type: function_call
     # plugins_func/functions下的模块，可以通过配置，选择加载哪个模块，加载后对话支持相应的function调用
     # 系统默认已经记载“handle_exit_intent(退出识别)”、“play_music(音乐播放)”插件，请勿重复加载
     # 下面是加载查天气、角色切换、加载查新闻的插件示例
@@ -174,6 +189,14 @@ ASR:
     output_dir: tmp/
     # 可以在这里申请相关Key等信息
     # https://console.volcengine.com/speech/app
+  TencentASR:
+    # token申请地址：https://console.cloud.tencent.com/cam/capi
+    # 免费领取资源：https://console.cloud.tencent.com/asr/resourcebundle
+    type: tencent
+    appid: 你的腾讯语音合成服务appid
+    secret_id: 你的腾讯语音合成服务secret_id
+    secret_key: 你的腾讯语音合成服务secret_key
+    output_dir: tmp/
 VAD:
   SileroVAD:
     threshold: 0.5
@@ -284,6 +307,18 @@ LLM:
     variables:
       k: "v"
       k2: "v2"
+  XinferenceLLM:
+    # 定义LLM API类型
+    type: xinference
+    # Xinference服务地址和模型名称
+    model_name: qwen2.5:72b-AWQ  # 使用的模型名称，需要预先在Xinference启动对应模型
+    base_url: http://localhost:9997  # Xinference服务地址
+  XinferenceSmallLLM:
+    # 定义轻量级LLM API类型，用于意图识别
+    type: xinference
+    # Xinference服务地址和模型名称
+    model_name: qwen2.5:3b-AWQ  # 使用的小模型名称，用于意图识别
+    base_url: http://localhost:9997  # Xinference服务地址
 TTS:
   # 当前支持的type为edge、doubao，可自行适配
   EdgeTTS:
@@ -455,6 +490,18 @@ TTS:
     # pitch_rate: 0
     # 添加 302.ai TTS 配置
     # token申请地址：https://dash.302.ai/
+  TencentTTS:
+    # 腾讯云智能语音交互服务，需要先在腾讯云平台开通服务
+    # appid、secret_id、secret_key申请地址：https://console.cloud.tencent.com/cam/capi
+    # 免费领取资源：https://console.cloud.tencent.com/tts/resourcebundle
+    type: tencent
+    output_dir: tmp/
+    appid: 你的腾讯云AppId
+    secret_id: 你的腾讯云SecretID
+    secret_key: 你的腾讯云SecretKey
+    region: ap-guangzhou
+    voice: 101001
+
   TTS302AI:
     # 302AI语音合成服务，需要先在302平台创建账户充值，并获取密钥信息
     # 获取api_keyn路径：https://dash.302.ai/apis/list
@@ -542,4 +589,4 @@ wakeup_words:
   - "小龙小龙"
   - "喵喵同学"
   - "小滨小滨"
-  - "小冰小冰"
+  - "小冰小冰"

main/xiaozhi-server/config/logger.py

@@ -3,24 +3,32 @@ import sys
 from loguru import logger
 from config.settings import load_config
 
-SERVER_VERSION = "0.1.15"
+SERVER_VERSION = "0.1.17"
+
 
 def setup_logging():
     """从配置文件中读取日志配置，并设置日志输出格式和级别"""
     config = load_config()
     log_config = config["log"]
-    log_format = log_config.get("log_format", "<green>{time:YYMMDD HH:mm:ss}</green>[{version}_{selected_module}][<light-blue>{extra[tag]}</light-blue>]-<level>{level}</level>-<light-green>{message}</light-green>")
-    log_format_file = log_config.get("log_format_file", "{time:YYYY-MM-DD HH:mm:ss} - {version_{selected_module}} - {name} - {level} - {extra[tag]} - {message}")
+    log_format = log_config.get(
+        "log_format",
+        "<green>{time:YYMMDD HH:mm:ss}</green>[{version}_{selected_module}][<light-blue>{extra[tag]}</light-blue>]-<level>{level}</level>-<light-green>{message}</light-green>",
+    )
+    log_format_file = log_config.get(
+        "log_format_file",
+        "{time:YYYY-MM-DD HH:mm:ss} - {version_{selected_module}} - {name} - {level} - {extra[tag]} - {message}",
+    )
 
     selected_module = config.get("selected_module")
-    selected_module_str = ''.join([key[0] + value[0] for key, value in selected_module.items()])
+    selected_module_str = "".join(
+        [value[0] + value[1] for key, value in selected_module.items()]
+    )
 
     log_format = log_format.replace("{version}", SERVER_VERSION)
     log_format = log_format.replace("{selected_module}", selected_module_str)
     log_format_file = log_format_file.replace("{version}", SERVER_VERSION)
     log_format_file = log_format_file.replace("{selected_module}", selected_module_str)
 
-
     log_level = log_config.get("log_level", "INFO")
     log_dir = log_config.get("log_dir", "tmp")
     log_file = log_config.get("log_file", "server.log")

main/xiaozhi-server/core/connection.py

@@ -13,19 +13,24 @@ from plugins_func.loadplugins import auto_import_modules
 from config.logger import setup_logging
 from core.utils.dialogue import Message, Dialogue
 from core.handle.textHandle import handleTextMessage
-from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string, get_ip_info
+from core.utils.util import (
+    get_string_no_punctuation_or_emoji,
+    extract_json_from_string,
+    get_ip_info,
+)
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 from core.handle.sendAudioHandle import sendAudioMessage
 from core.handle.receiveAudioHandle import handleAudioMessage
 from core.handle.functionHandler import FunctionHandler
-from plugins_func.register import Action
+from plugins_func.register import Action, ActionResponse
 from config.private_config import PrivateConfig
 from core.auth import AuthMiddleware, AuthenticationError
 from core.utils.auth_code_gen import AuthCodeGenerator
+from core.mcp.manager import MCPManager
 
 TAG = __name__
 
-auto_import_modules('plugins_func.functions')
+auto_import_modules("plugins_func.functions")
 
 
 class TTSException(RuntimeError):
@@ -33,7 +38,9 @@ class TTSException(RuntimeError):
 
 
 class ConnectionHandler:
-    def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _memory, _intent):
+    def __init__(
+        self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _memory, _intent
+    ):
         self.config = config
         self.logger = setup_logging()
         self.auth = AuthMiddleware(config)
@@ -86,6 +93,7 @@ class ConnectionHandler:
 
         # iot相关变量
         self.iot_descriptors = {}
+        self.func_handler = None
 
         self.cmd_exit = self.config["CMD_exit"]
         self.max_cmd_length = 0
@@ -98,7 +106,7 @@ class ConnectionHandler:
         self.is_device_verified = False  # 添加设备验证状态标志
         self.close_after_chat = False  # 是否在聊天结束后关闭连接
         self.use_function_call_mode = False
-        if self.config["selected_module"]["Intent"] == 'function_call':
+        if self.config["selected_module"]["Intent"] == "function_call":
             self.use_function_call_mode = True
 
     async def handle_connection(self, ws):
@@ -107,7 +115,9 @@ class ConnectionHandler:
             self.headers = dict(ws.request.headers)
             # 获取客户端ip地址
             self.client_ip = ws.remote_address[0]
-            self.logger.bind(tag=TAG).info(f"{self.client_ip} conn - Headers: {self.headers}")
+            self.logger.bind(tag=TAG).info(
+                f"{self.client_ip} conn - Headers: {self.headers}"
+            )
 
             # 进行认证
             await self.auth.authenticate(self.headers)
@@ -122,10 +132,14 @@ class ConnectionHandler:
             await self.websocket.send(json.dumps(self.welcome_msg))
             # Load private configuration if device_id is provided
             bUsePrivateConfig = self.config.get("use_private_config", False)
-            self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
+            self.logger.bind(tag=TAG).info(
+                f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}"
+            )
             if bUsePrivateConfig and device_id:
                 try:
-                    self.private_config = PrivateConfig(device_id, self.config, self.auth_code_gen)
+                    self.private_config = PrivateConfig(
+                        device_id, self.config, self.auth_code_gen
+                    )
                     await self.private_config.load_or_create()
                     # 判断是否已经绑定
                     owner = self.private_config.get_owner()
@@ -138,23 +152,33 @@ class ConnectionHandler:
                     if all([llm, tts]):
                         self.llm = llm
                         self.tts = tts
-                        self.logger.bind(tag=TAG).info(f"Loaded private config and instances for device {device_id}")
+                        self.logger.bind(tag=TAG).info(
+                            f"Loaded private config and instances for device {device_id}"
+                        )
                     else:
-                        self.logger.bind(tag=TAG).error(f"Failed to create instances for device {device_id}")
+                        self.logger.bind(tag=TAG).error(
+                            f"Failed to create instances for device {device_id}"
+                        )
                         self.private_config = None
                 except Exception as e:
-                    self.logger.bind(tag=TAG).error(f"Error initializing private config: {e}")
+                    self.logger.bind(tag=TAG).error(
+                        f"Error initializing private config: {e}"
+                    )
                     self.private_config = None
                     raise
 
             # 异步初始化
             self.executor.submit(self._initialize_components)
             # tts 消化线程
-            self.tts_priority_thread = threading.Thread(target=self._tts_priority_thread, daemon=True)
+            self.tts_priority_thread = threading.Thread(
+                target=self._tts_priority_thread, daemon=True
+            )
             self.tts_priority_thread.start()
 
             # 音频播放 消化线程
-            self.audio_play_priority_thread = threading.Thread(target=self._audio_play_priority_thread, daemon=True)
+            self.audio_play_priority_thread = threading.Thread(
+                target=self._audio_play_priority_thread, daemon=True
+            )
             self.audio_play_priority_thread.start()
 
             try:
@@ -171,7 +195,15 @@ class ConnectionHandler:
             self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}")
             return
         finally:
+            await self._save_and_close(ws)
+
+    async def _save_and_close(self, ws):
+        """保存记忆并关闭连接"""
+        try:
             await self.memory.save_memory(self.dialogue.dialogue)
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
+        finally:
             await self.close(ws)
 
     async def _route_message(self, message):
@@ -190,11 +222,40 @@ class ConnectionHandler:
 
         """加载插件"""
         self.func_handler = FunctionHandler(self)
-
+        self.mcp_manager = MCPManager(self)
         """加载记忆"""
         device_id = self.headers.get("device-id", None)
         self.memory.init_memory(device_id, self.llm)
-        self.intent.set_llm(self.llm)
+
+        """为意图识别设置LLM，优先使用专用LLM"""
+        # 检查是否配置了专用的意图识别LLM
+        intent_llm_name = self.config["Intent"]["intent_llm"]["llm"]
+
+        # 记录开始初始化意图识别LLM的时间
+        intent_llm_init_start = time.time()
+
+        if (
+            not self.use_function_call_mode
+            and intent_llm_name
+            and intent_llm_name in self.config["LLM"]
+        ):
+            # 如果配置了专用LLM，则创建独立的LLM实例
+            from core.utils import llm as llm_utils
+
+            intent_llm_config = self.config["LLM"][intent_llm_name]
+            intent_llm_type = intent_llm_config.get("type", intent_llm_name)
+            intent_llm = llm_utils.create_instance(intent_llm_type, intent_llm_config)
+            self.logger.bind(tag=TAG).info(
+                f"为意图识别创建了专用LLM: {intent_llm_name}, 类型: {intent_llm_type}"
+            )
+
+            self.intent.set_llm(intent_llm)
+        else:
+            # 否则使用主LLM
+            self.intent.set_llm(self.llm)
+
+        # 记录意图识别LLM初始化耗时
+        intent_llm_init_time = time.time() - intent_llm_init_start
 
         """加载位置信息"""
         self.client_ip_info = get_ip_info(self.client_ip)
@@ -203,6 +264,11 @@ class ConnectionHandler:
             self.prompt = self.prompt + f"\nuser location:{self.client_ip_info}"
             self.dialogue.update_system_message(self.prompt)
 
+        """加载MCP工具"""
+        asyncio.run_coroutine_threadsafe(
+            self.mcp_manager.initialize_servers(), self.loop
+        )
+
     def change_system_prompt(self, prompt):
         self.prompt = prompt
         # 找到原来的role==system，替换原来的系统提示
@@ -233,7 +299,9 @@ class ConnectionHandler:
     def chat(self, query):
         if self.isNeedAuth():
             self.llm_finish_task = True
-            future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
+            future = asyncio.run_coroutine_threadsafe(
+                self._check_and_broadcast_auth_code(), self.loop
+            )
             future.result()
             return True
 
@@ -244,13 +312,14 @@ class ConnectionHandler:
         try:
             start_time = time.time()
             # 使用带记忆的对话
-            future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
+            future = asyncio.run_coroutine_threadsafe(
+                self.memory.query_memory(query), self.loop
+            )
             memory_str = future.result()
 
             self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}")
             llm_responses = self.llm.response(
-                self.session_id,
-                self.dialogue.get_llm_dialogue_with_memory(memory_str)
+                self.session_id, self.dialogue.get_llm_dialogue_with_memory(memory_str)
             )
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
@@ -264,7 +333,7 @@ class ConnectionHandler:
                 break
 
             end_time = time.time()
-            self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
+            # self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
 
             # 合并当前全部文本并处理未分割部分
             full_text = "".join(response_message)
@@ -280,7 +349,7 @@ class ConnectionHandler:
 
             # 找到分割点则处理
             if last_punct_pos != -1:
-                segment_text_raw = current_text[:last_punct_pos + 1]
+                segment_text_raw = current_text[: last_punct_pos + 1]
                 segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
                 if segment_text:
                     # 强制设置空字符，测试TTS出错返回语音的健壮性
@@ -288,7 +357,9 @@ class ConnectionHandler:
                     #     segment_text = " "
                     text_index += 1
                     self.recode_first_last_text(segment_text, text_index)
-                    future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                    future = self.executor.submit(
+                        self.speak_and_play, segment_text, text_index
+                    )
                     self.tts_queue.put(future)
                     processed_chars += len(segment_text_raw)  # 更新已处理字符位置
 
@@ -300,12 +371,16 @@ class ConnectionHandler:
             if segment_text:
                 text_index += 1
                 self.recode_first_last_text(segment_text, text_index)
-                future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                future = self.executor.submit(
+                    self.speak_and_play, segment_text, text_index
+                )
                 self.tts_queue.put(future)
 
         self.llm_finish_task = True
         self.dialogue.put(Message(role="assistant", content="".join(response_message)))
-        self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
+        self.logger.bind(tag=TAG).debug(
+            json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)
+        )
         return True
 
     def chat_with_function_calling(self, query, tool_call=False):
@@ -313,7 +388,9 @@ class ConnectionHandler:
         """Chat with function calling for intent detection using streaming"""
         if self.isNeedAuth():
             self.llm_finish_task = True
-            future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
+            future = asyncio.run_coroutine_threadsafe(
+                self._check_and_broadcast_auth_code(), self.loop
+            )
             future.result()
             return True
 
@@ -322,9 +399,8 @@ class ConnectionHandler:
 
         # Define intent functions
         functions = None
-        if hasattr(self, 'func_handler'):
+        if hasattr(self, "func_handler"):
             functions = self.func_handler.get_functions()
-
         response_message = []
         processed_chars = 0  # 跟踪已处理的字符位置
 
@@ -332,7 +408,9 @@ class ConnectionHandler:
             start_time = time.time()
 
             # 使用带记忆的对话
-            future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
+            future = asyncio.run_coroutine_threadsafe(
+                self.memory.query_memory(query), self.loop
+            )
             memory_str = future.result()
 
             # self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}")
@@ -341,7 +419,7 @@ class ConnectionHandler:
             llm_responses = self.llm.response_with_functions(
                 self.session_id,
                 self.dialogue.get_llm_dialogue_with_memory(memory_str),
-                functions=functions
+                functions=functions,
             )
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
@@ -358,8 +436,13 @@ class ConnectionHandler:
         content_arguments = ""
         for response in llm_responses:
             content, tools_call = response
+            if "content" in response:
+                content = response["content"]
+                tools_call = None
             if content is not None and len(content) > 0:
-                if len(response_message) <= 0 and (content == "```" or "<tool_call>" in content):
+                if len(response_message) <= 0 and (
+                    content == "```" or "<tool_call>" in content
+                ):
                     tool_call_flag = True
 
             if tools_call is not None:
@@ -381,7 +464,7 @@ class ConnectionHandler:
                         break
 
                     end_time = time.time()
-                    self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
+                    # self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
 
                     # 处理文本分段和TTS逻辑
                     # 合并当前全部文本并处理未分割部分
@@ -398,14 +481,19 @@ class ConnectionHandler:
 
                     # 找到分割点则处理
                     if last_punct_pos != -1:
-                        segment_text_raw = current_text[:last_punct_pos + 1]
-                        segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
+                        segment_text_raw = current_text[: last_punct_pos + 1]
+                        segment_text = get_string_no_punctuation_or_emoji(
+                            segment_text_raw
+                        )
                         if segment_text:
                             text_index += 1
                             self.recode_first_last_text(segment_text, text_index)
-                            future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                            future = self.executor.submit(
+                                self.speak_and_play, segment_text, text_index
+                            )
                             self.tts_queue.put(future)
-                            processed_chars += len(segment_text_raw)  # 更新已处理字符位置
+                            # 更新已处理字符位置
+                            processed_chars += len(segment_text_raw)
 
         # 处理function call
         if tool_call_flag:
@@ -416,7 +504,9 @@ class ConnectionHandler:
                     try:
                         content_arguments_json = json.loads(a)
                         function_name = content_arguments_json["name"]
-                        function_arguments = json.dumps(content_arguments_json["arguments"], ensure_ascii=False)
+                        function_arguments = json.dumps(
+                            content_arguments_json["arguments"], ensure_ascii=False
+                        )
                         function_id = str(uuid.uuid4().hex)
                     except Exception as e:
                         bHasError = True
@@ -425,18 +515,29 @@ class ConnectionHandler:
                     bHasError = True
                     response_message.append(content_arguments)
                 if bHasError:
-                    self.logger.bind(tag=TAG).error(f"function call error: {content_arguments}")
+                    self.logger.bind(tag=TAG).error(
+                        f"function call error: {content_arguments}"
+                    )
                 else:
                     function_arguments = json.loads(function_arguments)
             if not bHasError:
                 self.logger.bind(tag=TAG).info(
-                    f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
+                    f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}"
+                )
                 function_call_data = {
                     "name": function_name,
                     "id": function_id,
-                    "arguments": function_arguments
+                    "arguments": function_arguments,
                 }
-                result = self.func_handler.handle_llm_function_call(self, function_call_data)
+
+                # 处理MCP工具调用
+                if self.mcp_manager.is_mcp_tool(function_name):
+                    result = self._handle_mcp_tool_call(function_call_data)
+                else:
+                    # 处理系统函数
+                    result = self.func_handler.handle_llm_function_call(
+                        self, function_call_data
+                    )
                 self._handle_function_result(result, function_call_data, text_index + 1)
 
         # 处理最后剩余的文本
@@ -447,18 +548,66 @@ class ConnectionHandler:
             if segment_text:
                 text_index += 1
                 self.recode_first_last_text(segment_text, text_index)
-                future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                future = self.executor.submit(
+                    self.speak_and_play, segment_text, text_index
+                )
                 self.tts_queue.put(future)
 
         # 存储对话内容
         if len(response_message) > 0:
-            self.dialogue.put(Message(role="assistant", content="".join(response_message)))
+            self.dialogue.put(
+                Message(role="assistant", content="".join(response_message))
+            )
 
         self.llm_finish_task = True
-        self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
+        self.logger.bind(tag=TAG).debug(
+            json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)
+        )
 
         return True
 
+    def _handle_mcp_tool_call(self, function_call_data):
+        function_arguments = function_call_data["arguments"]
+        function_name = function_call_data["name"]
+        try:
+            args_dict = function_arguments
+            if isinstance(function_arguments, str):
+                try:
+                    args_dict = json.loads(function_arguments)
+                except json.JSONDecodeError:
+                    self.logger.bind(tag=TAG).error(
+                        f"无法解析 function_arguments: {function_arguments}"
+                    )
+                    return ActionResponse(
+                        action=Action.REQLLM, result="参数解析失败", response=""
+                    )
+
+            tool_result = asyncio.run_coroutine_threadsafe(
+                self.mcp_manager.execute_tool(function_name, args_dict), self.loop
+            ).result()
+            # meta=None content=[TextContent(type='text', text='北京当前天气:\n温度: 21°C\n天气: 晴\n湿度: 6%\n风向: 西北 风\n风力等级: 5级', annotations=None)] isError=False
+            content_text = ""
+            if tool_result is not None and tool_result.content is not None:
+                for content in tool_result.content:
+                    content_type = content.type
+                    if content_type == "text":
+                        content_text = content.text
+                    elif content_type == "image":
+                        pass
+
+            if len(content_text) > 0:
+                return ActionResponse(
+                    action=Action.REQLLM, result=content_text, response=""
+                )
+
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"MCP工具调用错误: {e}")
+            return ActionResponse(
+                action=Action.REQLLM, result="工具调用出错", response=""
+            )
+
+        return ActionResponse(action=Action.REQLLM, result="工具调用出错", response="")
+
     def _handle_function_result(self, result, function_call_data, text_index):
         if result.action == Action.RESPONSE:  # 直接回复前端
             text = result.response
@@ -473,14 +622,26 @@ class ConnectionHandler:
                 function_id = function_call_data["id"]
                 function_name = function_call_data["name"]
                 function_arguments = function_call_data["arguments"]
-                self.dialogue.put(Message(role='assistant',
-                                          tool_calls=[{"id": function_id,
-                                                       "function": {"arguments": function_arguments,
-                                                                    "name": function_name},
-                                                       "type": 'function',
-                                                       "index": 0}]))
+                self.dialogue.put(
+                    Message(
+                        role="assistant",
+                        tool_calls=[
+                            {
+                                "id": function_id,
+                                "function": {
+                                    "arguments": function_arguments,
+                                    "name": function_name,
+                                },
+                                "type": "function",
+                                "index": 0,
+                            }
+                        ],
+                    )
+                )
 
-                self.dialogue.put(Message(role="tool", tool_call_id=function_id, content=text))
+                self.dialogue.put(
+                    Message(role="tool", tool_call_id=function_id, content=text)
+                )
                 self.chat_with_function_calling(text, tool_call=True)
         elif result.action == Action.NOTFOUND:
             text = result.result
@@ -514,15 +675,23 @@ class ConnectionHandler:
                     tts_timeout = self.config.get("tts_timeout", 10)
                     tts_file, text, text_index = future.result(timeout=tts_timeout)
                     if text is None or len(text) <= 0:
-                        self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
+                        self.logger.bind(tag=TAG).error(
+                            f"TTS出错：{text_index}: tts text is empty"
+                        )
                     elif tts_file is None:
-                        self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}")
+                        self.logger.bind(tag=TAG).error(
+                            f"TTS出错： file is empty: {text_index}: {text}"
+                        )
                     else:
-                        self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
+                        self.logger.bind(tag=TAG).debug(
+                            f"TTS生成：文件路径: {tts_file}"
+                        )
                         if os.path.exists(tts_file):
                             opus_datas, duration = self.tts.audio_to_opus_data(tts_file)
                         else:
-                            self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
+                            self.logger.bind(tag=TAG).error(
+                                f"TTS出错：文件不存在{tts_file}"
+                            )
                 except TimeoutError:
                     self.logger.bind(tag=TAG).error("TTS超时")
                 except Exception as e:
@@ -530,16 +699,30 @@ class ConnectionHandler:
                 if not self.client_abort:
                     # 如果没有中途打断就发送语音
                     self.audio_play_queue.put((opus_datas, text, text_index))
-                if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
+                if (
+                    self.tts.delete_audio_file
+                    and tts_file is not None
+                    and os.path.exists(tts_file)
+                ):
                     os.remove(tts_file)
             except Exception as e:
                 self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
                 self.clearSpeakStatus()
                 asyncio.run_coroutine_threadsafe(
-                    self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
-                    self.loop
+                    self.websocket.send(
+                        json.dumps(
+                            {
+                                "type": "tts",
+                                "state": "stop",
+                                "session_id": self.session_id,
+                            }
+                        )
+                    ),
+                    self.loop,
+                )
+                self.logger.bind(tag=TAG).error(
+                    f"tts_priority priority_thread: {text} {e}"
                 )
-                self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
 
     def _audio_play_priority_thread(self):
         while not self.stop_event.is_set():
@@ -551,11 +734,14 @@ class ConnectionHandler:
                     if self.stop_event.is_set():
                         break
                     continue
-                future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
-                                                          self.loop)
+                future = asyncio.run_coroutine_threadsafe(
+                    sendAudioMessage(self, opus_datas, text, text_index), self.loop
+                )
                 future.result()
             except Exception as e:
-                self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")
+                self.logger.bind(tag=TAG).error(
+                    f"audio_play_priority priority_thread: {text} {e}"
+                )
 
     def speak_and_play(self, text, text_index=0):
         if text is None or len(text) <= 0:
@@ -582,19 +768,21 @@ class ConnectionHandler:
 
     async def close(self, ws=None):
         """资源清理方法"""
+        # 清理MCP资源
+        await self.mcp_manager.cleanup_all()
 
         # 触发停止事件并清理资源
         if self.stop_event:
             self.stop_event.set()
-        
+
         # 立即关闭线程池
         if self.executor:
             self.executor.shutdown(wait=False, cancel_futures=True)
             self.executor = None
-        
+
         # 清空任务队列
         self._clear_queues()
-        
+
         if ws:
             await ws.close()
         elif self.websocket:

main/xiaozhi-server/core/handle/functionHandler.py

@@ -17,6 +17,7 @@ class FunctionHandler:
         self.functions_desc = self.function_registry.get_all_function_desc()
         func_names = self.current_support_functions()
         self.modify_plugin_loader_des(func_names)
+        self.finish_init = True
 
     def modify_plugin_loader_des(self, func_names):
         if "plugin_loader" not in func_names:
@@ -26,8 +27,9 @@ class FunctionHandler:
         func_names = ",".join(surport_plugins)
         for function_desc in self.functions_desc:
             if function_desc["function"]["name"] == "plugin_loader":
-                function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]",
-                                                                                                            func_names)
+                function_desc["function"]["description"] = function_desc["function"][
+                    "description"
+                ].replace("[plugins]", func_names)
                 break
 
     def upload_functions_desc(self):
@@ -69,19 +71,26 @@ class FunctionHandler:
             function_name = function_call_data["name"]
             funcItem = self.get_function(function_name)
             if not funcItem:
-                return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
+                return ActionResponse(
+                    action=Action.NOTFOUND, result="没有找到对应的函数", response=""
+                )
             func = funcItem.func
             arguments = function_call_data["arguments"]
             arguments = json.loads(arguments) if arguments else {}
             logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
-            if funcItem.type == ToolType.SYSTEM_CTL or funcItem.type == ToolType.IOT_CTL:
+            if (
+                funcItem.type == ToolType.SYSTEM_CTL
+                or funcItem.type == ToolType.IOT_CTL
+            ):
                 return func(conn, **arguments)
             elif funcItem.type == ToolType.WAIT:
                 return func(**arguments)
             elif funcItem.type == ToolType.CHANGE_SYS_PROMPT:
                 return func(conn, **arguments)
             else:
-                return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
+                return ActionResponse(
+                    action=Action.NOTFOUND, result="没有找到对应的函数", response=""
+                )
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理function call错误: {e}")
 

main/xiaozhi-server/core/handle/helloHandle.py

@@ -10,8 +10,14 @@ import time
 
 logger = setup_logging()
 
-WAKEUP_CONFIG = {"dir": "config/assets/", "file_name": "wakeup_words", "create_time": time.time(), "refresh_time": 10,
-                 "words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"]}
+WAKEUP_CONFIG = {
+    "dir": "config/assets/",
+    "file_name": "wakeup_words",
+    "create_time": time.time(),
+    "refresh_time": 10,
+    "words": ["你好小智", "你好啊小智", "小智你好", "小智"],
+    "text": "",
+}
 
 
 async def handleHelloMessage(conn):
@@ -19,7 +25,9 @@ async def handleHelloMessage(conn):
 
 
 async def checkWakeupWords(conn, text):
-    enable_wakeup_words_response_cache = conn.config["enable_wakeup_words_response_cache"]
+    enable_wakeup_words_response_cache = conn.config[
+        "enable_wakeup_words_response_cache"
+    ]
     """是否开启唤醒词加速"""
     if not enable_wakeup_words_response_cache:
         return False
@@ -36,7 +44,10 @@ async def checkWakeupWords(conn, text):
             asyncio.create_task(wakeupWordsResponse(conn))
             return False
         opus_packets, duration = conn.tts.audio_to_opus_data(file)
-        conn.audio_play_queue.put((opus_packets, text, 0))
+        text_hello = WAKEUP_CONFIG["text"]
+        if not text_hello:
+            text_hello = text
+        conn.audio_play_queue.put((opus_packets, text_hello, 0))
         if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
             asyncio.create_task(wakeupWordsResponse(conn))
         return True
@@ -47,7 +58,7 @@ def getWakeupWordFile(file_name):
     for file in os.listdir(WAKEUP_CONFIG["dir"]):
         if file.startswith("my_" + file_name):
             """避免缓存文件是一个空文件"""
-            if os.stat(f"config/assets/{file}").st_size > (5 * 1024):
+            if os.stat(f"config/assets/{file}").st_size > (15 * 1024):
                 return f"config/assets/{file}"
 
     """查找config/assets/目录下名称为wakeup_words的文件"""
@@ -62,13 +73,18 @@ async def wakeupWordsResponse(conn):
     wakeup_word = random.choice(WAKEUP_CONFIG["words"])
     result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
     tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
+
     if tts_file is not None and os.path.exists(tts_file):
         file_type = os.path.splitext(tts_file)[1]
         if file_type:
-            file_type = file_type.lstrip('.')
+            file_type = file_type.lstrip(".")
         old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"])
         if old_file is not None:
             os.remove(old_file)
         """将文件挪到"wakeup_words.mp3"""
-        shutil.move(tts_file, WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type)
+        shutil.move(
+            tts_file,
+            WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type,
+        )
         WAKEUP_CONFIG["create_time"] = time.time()
+        WAKEUP_CONFIG["text"] = result

main/xiaozhi-server/core/handle/intentHandler.py

@@ -4,6 +4,8 @@ import uuid
 from core.handle.sendAudioHandle import send_stt_message
 from core.handle.helloHandle import checkWakeupWords
 from core.utils.util import remove_punctuation_and_length
+from core.utils.dialogue import Message
+from loguru import logger
 
 TAG = __name__
 logger = setup_logging()
@@ -21,11 +23,11 @@ async def handle_user_intent(conn, text):
         # 使用支持function calling的聊天方法,不再进行意图分析
         return False
     # 使用LLM进行意图分析
-    intent = await analyze_intent_with_llm(conn, text)
-    if not intent:
+    intent_result = await analyze_intent_with_llm(conn, text)
+    if not intent_result:
         return False
     # 处理各种意图
-    return await process_intent_result(conn, intent, text)
+    return await process_intent_result(conn, intent_result, text)
 
 
 async def check_direct_exit(conn, text):
@@ -35,15 +37,15 @@ async def check_direct_exit(conn, text):
     for cmd in cmd_exit:
         if text == cmd:
             logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
+            await send_stt_message(conn, text)
             await conn.close()
             return True
     return False
 
 
-
 async def analyze_intent_with_llm(conn, text):
     """使用LLM分析用户意图"""
-    if not hasattr(conn, 'intent') or not conn.intent:
+    if not hasattr(conn, "intent") or not conn.intent:
         logger.bind(tag=TAG).warning("意图识别服务未初始化")
         return None
 
@@ -51,52 +53,75 @@ async def analyze_intent_with_llm(conn, text):
     dialogue = conn.dialogue
     try:
         intent_result = await conn.intent.detect_intent(conn, dialogue.dialogue, text)
-        # 尝试解析JSON结果
-        try:
-            intent_data = json.loads(intent_result)
-            if "intent" in intent_data:
-                return intent_data["intent"]
-        except json.JSONDecodeError:
-            # 如果不是JSON格式，尝试直接获取意图文本
-            return intent_result.strip()
-
+        return intent_result
     except Exception as e:
         logger.bind(tag=TAG).error(f"意图识别失败: {str(e)}")
 
     return None
 
 
-async def process_intent_result(conn, intent, original_text):
+async def process_intent_result(conn, intent_result, original_text):
     """处理意图识别结果"""
-    # 处理退出意图
-    if "结束聊天" in intent:
-        logger.bind(tag=TAG).info(f"识别到退出意图: {intent}")
-        # 如果是明确的离别意图，发送告别语并关闭连接
-        await send_stt_message(conn, original_text)
-        conn.executor.submit(conn.chat_and_close, original_text)
-        return True
+    try:
+        # 尝试将结果解析为JSON
+        intent_data = json.loads(intent_result)
 
-    # 处理播放音乐意图
-    if "播放音乐" in intent:
-        logger.bind(tag=TAG).info(f"识别到音乐播放意图: {intent}")
-        # 调用play_music函数来播放音乐
-        song_name = extract_text_in_brackets(intent)
-        function_id = str(uuid.uuid4().hex)
-        function_name = "play_music"
-        function_arguments = '{ "song_name": "' + song_name + '" }'
+        # 检查是否有function_call
+        if "function_call" in intent_data:
+            # 直接从意图识别获取了function_call
+            logger.bind(tag=TAG).info(
+                f"检测到function_call格式的意图结果: {intent_data['function_call']['name']}"
+            )
+            function_name = intent_data["function_call"]["name"]
+            if function_name == "continue_chat":
+                return False
+            function_args = None
+            if "arguments" in intent_data["function_call"]:
+                function_args = intent_data["function_call"]["arguments"]
+            # 确保参数是字符串格式的JSON
+            if isinstance(function_args, dict):
+                function_args = json.dumps(function_args)
 
-        function_call_data = {
-            "name": function_name,
-            "id": function_id,
-            "arguments": function_arguments
-        }
-        conn.func_handler.handle_llm_function_call(conn, function_call_data)
-        return True
+            function_call_data = {
+                "name": function_name,
+                "id": str(uuid.uuid4().hex),
+                "arguments": function_args,
+            }
 
-    # 其他意图处理可以在这里扩展
+            await send_stt_message(conn, original_text)
 
-    # 默认返回False，表示继续常规聊天流程
-    return False
+            # 使用executor执行函数调用和结果处理
+            def process_function_call():
+                conn.dialogue.put(Message(role="user", content=original_text))
+                result = conn.func_handler.handle_llm_function_call(
+                    conn, function_call_data
+                )
+                if result and function_name != "play_music":
+                    # 获取当前最新的文本索引
+                    text = result.response
+                    if text is None:
+                        text = result.result
+                    if text is not None:
+                        text_index = (
+                            conn.tts_last_text_index + 1
+                            if hasattr(conn, "tts_last_text_index")
+                            else 0
+                        )
+                        conn.recode_first_last_text(text, text_index)
+                        future = conn.executor.submit(
+                            conn.speak_and_play, text, text_index
+                        )
+                        conn.llm_finish_task = True
+                        conn.tts_queue.put(future)
+                        conn.dialogue.put(Message(role="assistant", content=text))
+
+            # 将函数执行放在线程池中
+            conn.executor.submit(process_function_call)
+            return True
+        return False
+    except json.JSONDecodeError as e:
+        logger.bind(tag=TAG).error(f"处理意图结果时出错: {e}")
+        return False
 
 
 def extract_text_in_brackets(s):
@@ -106,10 +131,14 @@ def extract_text_in_brackets(s):
     :param s: 输入字符串
     :return: 中括号内的文字，如果不存在则返回空字符串
     """
-    left_bracket_index = s.find('[')
-    right_bracket_index = s.find(']')
+    left_bracket_index = s.find("[")
+    right_bracket_index = s.find("]")
 
-    if left_bracket_index != -1 and right_bracket_index != -1 and left_bracket_index < right_bracket_index:
-        return s[left_bracket_index + 1:right_bracket_index]
+    if (
+        left_bracket_index != -1
+        and right_bracket_index != -1
+        and left_bracket_index < right_bracket_index
+    ):
+        return s[left_bracket_index + 1 : right_bracket_index]
     else:
-        return ""
+        return ""

main/xiaozhi-server/core/handle/iotHandle.py

@@ -1,7 +1,13 @@
 import json
 import asyncio
 from config.logger import setup_logging
-from plugins_func.register import device_type_registry, register_function, ActionResponse, Action, ToolType
+from plugins_func.register import (
+    device_type_registry,
+    register_function,
+    ActionResponse,
+    Action,
+    ToolType,
+)
 
 TAG = __name__
 logger = setup_logging()
@@ -14,10 +20,13 @@ def wrap_async_function(async_func):
         try:
             # 获取连接对象（第一个参数）
             conn = args[0]
-            if not hasattr(conn, 'loop'):
+            if not hasattr(conn, "loop"):
                 logger.bind(tag=TAG).error("Connection对象没有loop属性")
-                return ActionResponse(Action.ERROR, "Connection对象没有loop属性",
-                                      "执行操作时出错: Connection对象没有loop属性")
+                return ActionResponse(
+                    Action.ERROR,
+                    "Connection对象没有loop属性",
+                    "执行操作时出错: Connection对象没有loop属性",
+                )
 
             # 使用conn对象中的事件循环
             loop = conn.loop
@@ -37,11 +46,14 @@ def create_iot_function(device_name, method_name, method_info):
     根据IOT设备描述生成通用的控制函数
     """
 
-    async def iot_control_function(conn, response_success=None, response_failure=None, **params):
+    async def iot_control_function(
+        conn, response_success=None, response_failure=None, **params
+    ):
         try:
             # 打印响应参数
             logger.bind(tag=TAG).info(
-                f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
+                f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
+            )
 
             # 发送控制命令
             await send_iot_conn(conn, device_name, method_name, params)
@@ -51,14 +63,15 @@ def create_iot_function(device_name, method_name, method_info):
             # 生成结果信息
             result = f"{device_name}的{method_name}操作执行成功"
 
-
             # 处理响应中可能的占位符
             response = response_success
             # 替换{value}占位符
             for param_name, param_value in params.items():
                 # 先尝试直接替换参数值
                 if "{" + param_name + "}" in response:
-                    response = response.replace("{" + param_name + "}", str(param_value))
+                    response = response.replace(
+                        "{" + param_name + "}", str(param_value)
+                    )
 
                 # 如果有{value}占位符，用相关参数替换
                 if "{value}" in response:
@@ -86,7 +99,8 @@ def create_iot_query_function(device_name, prop_name, prop_info):
         try:
             # 打印响应参数
             logger.bind(tag=TAG).info(
-                f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
+                f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
+            )
 
             value = await get_iot_status(conn, device_name, prop_name)
 
@@ -126,7 +140,7 @@ class IotDescriptor:
         # 根据描述创建属性
         for key, value in properties.items():
             property_item = globals()[key] = {}
-            property_item['name'] = key
+            property_item["name"] = key
             property_item["description"] = value["description"]
             if value["type"] == "number":
                 property_item["value"] = 0
@@ -140,7 +154,7 @@ class IotDescriptor:
         for key, value in methods.items():
             method = globals()[key] = {}
             method["description"] = value["description"]
-            method['name'] = key
+            method["name"] = key
             for k, v in value["parameters"].items():
                 method[k] = {}
                 method[k]["description"] = v["description"]
@@ -177,19 +191,21 @@ def register_device_type(descriptor):
                     "properties": {
                         "response_success": {
                             "type": "string",
-                            "description": f"查询成功时的友好回复，必须使用{{value}}作为占位符表示查询到的值"
+                            "description": f"查询成功时的友好回复，必须使用{{value}}作为占位符表示查询到的值",
                         },
                         "response_failure": {
                             "type": "string",
-                            "description": f"查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'"
-                        }
+                            "description": f"查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'",
+                        },
                     },
-                    "required": ["response_success", "response_failure"]
-                }
-            }
+                    "required": ["response_success", "response_failure"],
+                },
+            },
         }
         query_func = create_iot_query_function(device_name, prop_name, prop_info)
-        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(query_func)
+        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
+            query_func
+        )
         functions[func_name] = decorated_func
 
     # 为每个方法创建控制函数
@@ -200,22 +216,24 @@ def register_device_type(descriptor):
         parameters = {
             param_name: {
                 "type": param_info["type"],
-                "description": param_info["description"]
+                "description": param_info["description"],
             }
             for param_name, param_info in method_info["parameters"].items()
         }
 
         # 添加响应参数
-        parameters.update({
-            "response_success": {
-                "type": "string",
-                "description": "操作成功时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
-            },
-            "response_failure": {
-                "type": "string",
-                "description": "操作失败时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
+        parameters.update(
+            {
+                "response_success": {
+                    "type": "string",
+                    "description": "操作成功时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称",
+                },
+                "response_failure": {
+                    "type": "string",
+                    "description": "操作失败时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称",
+                },
             }
-        })
+        )
 
         # 构建必须参数列表（原有参数 + 响应参数）
         required_params = list(method_info["parameters"].keys())
@@ -229,12 +247,14 @@ def register_device_type(descriptor):
                 "parameters": {
                     "type": "object",
                     "properties": parameters,
-                    "required": required_params
-                }
-            }
+                    "required": required_params,
+                },
+            },
         }
         control_func = create_iot_function(device_name, method_name, method_info)
-        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(control_func)
+        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
+            control_func
+        )
         functions[func_name] = decorated_func
 
     device_type_registry.register_device_type(type_id, functions)
@@ -243,13 +263,24 @@ def register_device_type(descriptor):
 
 # 用于接受前端设备推送的搜索iot描述
 async def handleIotDescriptors(conn, descriptors):
+    wait_max_time = 5
+    while conn.func_handler is None or not conn.func_handler.finish_init:
+        await asyncio.sleep(1)
+        wait_max_time -= 1
+        if wait_max_time <= 0:
+            logger.bind(tag=TAG).error("连接对象没有func_handler")
+            return
     """处理物联网描述"""
     functions_changed = False
 
     for descriptor in descriptors:
         # 创建IOT设备描述符
-        iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
-                                       descriptor["methods"])
+        iot_descriptor = IotDescriptor(
+            descriptor["name"],
+            descriptor["description"],
+            descriptor["properties"],
+            descriptor["methods"],
+        )
         conn.iot_descriptors[descriptor["name"]] = iot_descriptor
 
         if conn.use_function_call_mode:
@@ -258,18 +289,22 @@ async def handleIotDescriptors(conn, descriptors):
             device_functions = device_type_registry.get_device_functions(type_id)
 
             # 在连接级注册设备函数
-            if hasattr(conn, 'func_handler'):
+            if hasattr(conn, "func_handler"):
                 for func_name in device_functions:
                     conn.func_handler.function_registry.register_function(func_name)
-                    logger.bind(tag=TAG).info(f"注册IOT函数到function handler: {func_name}")
+                    logger.bind(tag=TAG).info(
+                        f"注册IOT函数到function handler: {func_name}"
+                    )
                     functions_changed = True
 
     # 如果注册了新函数，更新function描述列表
-    if functions_changed and hasattr(conn, 'func_handler'):
+    if functions_changed and hasattr(conn, "func_handler"):
         conn.func_handler.upload_functions_desc()
         func_names = conn.func_handler.current_support_functions()
         logger.bind(tag=TAG).info(f"设备类型: {type_id}")
-        logger.bind(tag=TAG).info(f"更新function描述列表完成，当前支持的函数: {func_names}")
+        logger.bind(tag=TAG).info(
+            f"更新function描述列表完成，当前支持的函数: {func_names}"
+        )
 
 
 async def handleIotStatus(conn, states):
@@ -281,11 +316,15 @@ async def handleIotStatus(conn, states):
                     for k, v in state["state"].items():
                         if property_item["name"] == k:
                             if type(v) != type(property_item["value"]):
-                                logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
+                                logger.bind(tag=TAG).error(
+                                    f"属性{property_item['name']}的值类型不匹配"
+                                )
                                 break
                             else:
                                 property_item["value"] = v
-                                logger.bind(tag=TAG).info(f"物联网状态更新: {key} , {property_item['name']} = {v}")
+                                logger.bind(tag=TAG).info(
+                                    f"物联网状态更新: {key} , {property_item['name']} = {v}"
+                                )
                             break
                 break
 
@@ -308,10 +347,14 @@ async def set_iot_status(conn, name, property_name, value):
             for property_item in iot_descriptor.properties:
                 if property_item["name"] == property_name:
                     if type(value) != type(property_item["value"]):
-                        logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
+                        logger.bind(tag=TAG).error(
+                            f"属性{property_item['name']}的值类型不匹配"
+                        )
                         return
                     property_item["value"] = value
-                    logger.bind(tag=TAG).info(f"物联网状态更新: {name} , {property_name} = {value}")
+                    logger.bind(tag=TAG).info(
+                        f"物联网状态更新: {name} , {property_name} = {value}"
+                    )
                     return
     logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
 
@@ -324,15 +367,19 @@ async def send_iot_conn(conn, name, method_name, parameters):
             for method in value.methods:
                 # 找到了方法
                 if method["name"] == method_name:
-                    await conn.websocket.send(json.dumps({
-                        "type": "iot",
-                        "commands": [
+                    await conn.websocket.send(
+                        json.dumps(
                             {
-                                "name": name,
-                                "method": method_name,
-                                "parameters": parameters
+                                "type": "iot",
+                                "commands": [
+                                    {
+                                        "name": name,
+                                        "method": method_name,
+                                        "parameters": parameters,
+                                    }
+                                ],
                             }
-                        ]
-                    }))
+                        )
+                    )
                     return
     logger.bind(tag=TAG).error(f"未找到方法{method_name}")

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -21,7 +21,9 @@ async def handleAudioMessage(conn, audio):
     if have_voice == False and conn.client_have_voice == False:
         await no_voice_close_connect(conn)
         conn.asr_audio.append(audio)
-        conn.asr_audio = conn.asr_audio[-5:]  # 保留最新的5帧音频内容，解决ASR句首丢字问题
+        conn.asr_audio = conn.asr_audio[
+            -10:
+        ]  # 保留最新的10帧音频内容，解决ASR句首丢字问题
         return
     conn.client_no_voice_last_time = 0.0
     conn.asr_audio.append(audio)
@@ -30,10 +32,12 @@ async def handleAudioMessage(conn, audio):
         conn.client_abort = False
         conn.asr_server_receive = False
         # 音频太短了，无法识别
-        if len(conn.asr_audio) < 10:
+        if len(conn.asr_audio) < 15:
             conn.asr_server_receive = True
         else:
-            text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
+            text, file_path = await conn.asr.speech_to_text(
+                conn.asr_audio, conn.session_id
+            )
             logger.bind(tag=TAG).info(f"识别文本: {text}")
             text_len, _ = remove_punctuation_and_length(text)
             if text_len > 0:
@@ -47,12 +51,12 @@ async def handleAudioMessage(conn, audio):
 async def startToChat(conn, text):
     # 首先进行意图分析
     intent_handled = await handle_user_intent(conn, text)
-    
+
     if intent_handled:
         # 如果意图已被处理，不再进行聊天
         conn.asr_server_receive = True
         return
-    
+
     # 意图未被处理，继续常规聊天流程
     await send_stt_message(conn, text)
     if conn.use_function_call_mode:
@@ -67,10 +71,17 @@ async def no_voice_close_connect(conn):
         conn.client_no_voice_last_time = time.time() * 1000
     else:
         no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
-        close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
-        if not conn.close_after_chat and no_voice_time > 1000 * close_connection_no_voice_time:
+        close_connection_no_voice_time = conn.config.get(
+            "close_connection_no_voice_time", 120
+        )
+        if (
+            not conn.close_after_chat
+            and no_voice_time > 1000 * close_connection_no_voice_time
+        ):
             conn.close_after_chat = True
             conn.client_abort = False
             conn.asr_server_receive = False
-            prompt = "请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。"
+            prompt = (
+                "请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。"
+            )
             await startToChat(conn, prompt)

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -2,7 +2,10 @@ from config.logger import setup_logging
 import json
 import asyncio
 import time
-from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji
+from core.utils.util import (
+    remove_punctuation_and_length,
+    get_string_no_punctuation_or_emoji,
+)
 
 TAG = __name__
 logger = setup_logging()
@@ -21,10 +24,11 @@ async def sendAudioMessage(conn, audios, text, text_index=0):
 
     # 发送结束消息（如果是最后一个文本）
     if conn.llm_finish_task and text_index == conn.tts_last_text_index:
-        await send_tts_message(conn, 'stop', None)
+        await send_tts_message(conn, "stop", None)
         if conn.close_after_chat:
             await conn.close()
 
+
 # 播放音频
 async def sendAudio(conn, audios):
     # 流控参数优化
@@ -36,13 +40,12 @@ async def sendAudio(conn, audios):
     pre_buffer = min(3, len(audios))
     for i in range(pre_buffer):
         await conn.websocket.send(audios[i])
-        conn.logger.bind(tag=TAG).debug(f"预缓冲帧 {i}, 时间: {(time.perf_counter() - start_time) * 1000:.2f}ms")
 
     # 正常播放剩余帧
     for opus_packet in audios[pre_buffer:]:
         if conn.client_abort:
             return
-        
+
         # 计算预期发送时间
         expected_time = start_time + (play_position / 1000)
         current_time = time.perf_counter()
@@ -51,19 +54,13 @@ async def sendAudio(conn, audios):
             await asyncio.sleep(delay)
 
         await conn.websocket.send(opus_packet)
-        conn.logger.bind(tag=TAG).debug(f"发送帧，位置: {play_position}ms, 实际间隔: {(time.perf_counter() - current_time) * 1000:.2f}ms")
 
         play_position += frame_duration
 
 
-
 async def send_tts_message(conn, state, text=None):
     """发送 TTS 状态消息"""
-    message = {
-        "type": "tts",
-        "state": state,
-        "session_id": conn.session_id
-    }
+    message = {"type": "tts", "state": state, "session_id": conn.session_id}
     if text is not None:
         message["text"] = text
 
@@ -72,7 +69,9 @@ async def send_tts_message(conn, state, text=None):
         # 播放提示音
         tts_notify = conn.config.get("enable_stop_tts_notify", False)
         if tts_notify:
-            stop_tts_notify_voice = conn.config.get("stop_tts_notify_voice", "config/assets/tts_notify.mp3")
+            stop_tts_notify_voice = conn.config.get(
+                "stop_tts_notify_voice", "config/assets/tts_notify.mp3"
+            )
             audios, duration = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
             await sendAudio(conn, audios)
         # 清除服务端讲话状态
@@ -85,16 +84,17 @@ async def send_tts_message(conn, state, text=None):
 async def send_stt_message(conn, text):
     """发送 STT 状态消息"""
     stt_text = get_string_no_punctuation_or_emoji(text)
-    await conn.websocket.send(json.dumps({
-        "type": "stt",
-        "text": stt_text,
-        "session_id": conn.session_id}
-    ))
     await conn.websocket.send(
-        json.dumps({
-            "type": "llm",
-            "text": "😊",
-            "emotion": "happy",
-            "session_id": conn.session_id}
-        ))
+        json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id})
+    )
+    await conn.websocket.send(
+        json.dumps(
+            {
+                "type": "llm",
+                "text": "😊",
+                "emotion": "happy",
+                "session_id": conn.session_id,
+            }
+        )
+    )
     await send_tts_message(conn, "start")

main/xiaozhi-server/core/handle/textHandle.py

@@ -6,6 +6,7 @@ from core.utils.util import remove_punctuation_and_length
 from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
 from core.handle.sendAudioHandle import send_stt_message, send_tts_message
 from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
+import asyncio
 
 TAG = __name__
 logger = setup_logging()
@@ -34,7 +35,7 @@ async def handleTextMessage(conn, message):
                 conn.client_have_voice = True
                 conn.client_voice_stop = True
                 if len(conn.asr_audio) > 0:
-                    await handleAudioMessage(conn, b'')
+                    await handleAudioMessage(conn, b"")
             elif msg_json["state"] == "detect":
                 conn.asr_server_receive = False
                 conn.client_have_voice = False
@@ -57,8 +58,8 @@ async def handleTextMessage(conn, message):
                         await startToChat(conn, text)
         elif msg_json["type"] == "iot":
             if "descriptors" in msg_json:
-                await handleIotDescriptors(conn, msg_json["descriptors"])
+                asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
             if "states" in msg_json:
-                await handleIotStatus(conn, msg_json["states"])
+                asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
     except json.JSONDecodeError:
         await conn.websocket.send(message)

main/xiaozhi-server/core/mcp/MCPClient.py

@@ -0,0 +1,86 @@
+from datetime import timedelta
+from typing import Optional
+from contextlib import AsyncExitStack
+import os, shutil
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+
+from config.logger import setup_logging
+
+TAG = __name__
+
+class MCPClient:
+    def __init__(self, config):
+        # Initialize session and client objects
+        self.session: Optional[ClientSession] = None
+        self.exit_stack = AsyncExitStack()
+        self.logger = setup_logging()
+        self.config = config
+        self.tolls = []
+
+    async def initialize(self):
+        args = self.config.get("args", [])
+
+        command = (
+            shutil.which("npx")
+            if self.config["command"] == "npx"
+            else self.config["command"]
+        )
+        
+        env={**os.environ}
+        if self.config.get("env"):
+            env.update(self.config["env"])
+        
+        server_params = StdioServerParameters(
+            command=command,
+            args=args,
+            env=env
+        )
+        
+        stdio_transport = await self.exit_stack.enter_async_context(stdio_client(server_params))
+        self.stdio, self.write = stdio_transport
+        time_out_delta =  timedelta(seconds=15)
+        self.session = await self.exit_stack.enter_async_context(ClientSession(read_stream=self.stdio, write_stream=self.write, read_timeout_seconds=time_out_delta))
+        
+        await self.session.initialize()
+        
+        # List available tools
+        response = await self.session.list_tools()
+        tools = response.tools
+        self.tools = tools
+        self.logger.bind(tag=TAG).info(f"Connected to server with tools:{[tool.name for tool in tools]}")
+    
+    def has_tool(self, tool_name):
+        return any(tool.name == tool_name for tool in self.tools)
+    
+    def get_available_tools(self):
+        available_tools = [{"type": "function", "function":{ 
+            "name": tool.name,
+            "description": tool.description,
+            "parameters": tool.inputSchema
+        } } for tool in self.tools]
+
+        return available_tools
+    
+    async def call_tool(self, tool_name: str, tool_args: dict):
+        self.logger.bind(tag=TAG).info(f"MCPClient Calling tool {tool_name} with args: {tool_args}")
+        try:
+            response = await self.session.call_tool(tool_name, tool_args)
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"Error calling tool {tool_name}: {e}")
+            from types import SimpleNamespace
+            error_content = SimpleNamespace(
+                type='text',
+                text=f"Error calling tool {tool_name}: {e}"
+            )
+            error_response = SimpleNamespace(
+                content=[error_content],
+                isError=True
+            )
+            return error_response
+        self.logger.bind(tag=TAG).info(f"MCPClient Response from tool {tool_name}: {response}")
+        return response
+
+    async def cleanup(self):
+        """Clean up resources"""
+        await self.exit_stack.aclose()

main/xiaozhi-server/core/mcp/manager.py

@@ -0,0 +1,110 @@
+"""MCP服务管理器"""
+import os, json
+from typing import Dict, Any, List
+from .MCPClient import MCPClient
+from config.logger import setup_logging
+from core.utils.util import get_project_dir
+from plugins_func.register import register_function, ActionResponse, Action, ToolType
+
+TAG = __name__
+
+class MCPManager:
+    """管理多个MCP服务的集中管理器"""
+
+    def __init__(self,conn) -> None:
+        """
+        初始化MCP管理器
+        """
+        self.conn = conn
+        self.logger = setup_logging()
+        self.config_path = get_project_dir() + 'data/.mcp_server_settings.json'
+        if os.path.exists(self.config_path) == False:
+            self.config_path = ""
+            self.logger.bind(tag=TAG).warning(f"请检查mcp服务配置文件：data/.mcp_server_settings.json")
+        self.client: Dict[str, MCPClient] = {}
+        self.tools = []
+
+    def load_config(self) -> Dict[str, Any]:
+        """加载MCP服务配置
+        Returns:
+            Dict[str, Any]: 服务配置字典
+        """
+        if len(self.config_path) == 0:
+            return {}
+        
+        try:
+            with open(self.config_path, 'r', encoding='utf-8') as f:
+                config = json.load(f)
+            return config.get('mcpServers', {})
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"Error loading MCP config from {self.config_path}: {e}")
+            return {}
+
+    async def initialize_servers(self) -> None:
+        """初始化所有MCP服务"""
+        config = self.load_config()
+        for name, srv_config in config.items():
+            if not srv_config.get("command"):
+                self.logger.bind(tag=TAG).warning(f"Skipping server {name}: command not specified")
+                continue
+
+            try:
+                client = MCPClient(srv_config)
+                await client.initialize()
+                self.client[name] = client
+                self.logger.bind(tag=TAG).info(f"Initialized MCP client: {name}")
+                client_tools = client.get_available_tools()
+                self.tools.extend(client_tools)
+                for tool in client_tools:
+                    func_name = "mcp_"+tool["function"]["name"]
+                    register_function(func_name, tool, ToolType.MCP_CLIENT)(self.execute_tool)
+                    self.conn.func_handler.function_registry.register_function(func_name)
+
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"Failed to initialize MCP server {name}: {e}")
+        self.conn.func_handler.upload_functions_desc()
+
+    def get_all_tools(self) -> List[Dict[str, Any]]:
+        """获取所有服务的工具function定义
+        Returns:
+            List[Dict[str, Any]]: 所有工具的function定义列表
+        """
+        return self.tools
+
+    def is_mcp_tool(self, tool_name: str) -> bool:
+        """检查是否是MCP工具
+        Args:
+            tool_name: 工具名称
+        Returns:
+            bool: 是否是MCP工具
+        """
+        for tool in self.tools:
+            if tool.get("function") != None and tool["function"].get("name") == tool_name:
+                return True
+        return False
+
+    async def execute_tool(self, tool_name: str, arguments: Dict[str, Any]) -> Any:
+        """执行工具调用
+        Args:
+            tool_name: 工具名称
+            arguments: 工具参数 
+        Returns:
+            Any: 工具执行结果
+        Raises:
+            ValueError: 工具未找到时抛出
+        """
+        self.logger.bind(tag=TAG).info(f"Executing tool {tool_name} with arguments: {arguments}")
+        for client in self.client.values():
+            if client.has_tool(tool_name):
+                return await client.call_tool(tool_name, arguments)
+            
+        raise ValueError(f"Tool {tool_name} not found in any MCP server")
+
+    async def cleanup_all(self) -> None:
+        for name, client in self.client.items():
+            try:
+                await client.cleanup()
+                self.logger.bind(tag=TAG).info(f"Cleaned up MCP client: {name}")
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"Error cleaning up MCP client {name}: {e}")
+        self.client.clear()

main/xiaozhi-server/core/providers/asr/tencent.py

@@ -0,0 +1,251 @@
+import base64
+import hashlib
+import hmac
+import json
+import time
+from datetime import datetime, timezone
+import os
+import uuid
+from typing import Optional, Tuple, List
+import wave
+import opuslib_next
+
+import requests
+from core.providers.asr.base import ASRProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class ASRProvider(ASRProviderBase):
+    API_URL = "https://asr.tencentcloudapi.com"
+    API_VERSION = "2019-06-14"
+    FORMAT = "pcm"  # 支持的音频格式：pcm, wav, mp3
+
+    def __init__(self, config: dict, delete_audio_file: bool = True):
+        self.secret_id = config.get("secret_id")
+        self.secret_key = config.get("secret_key")
+        self.output_dir = config.get("output_dir")
+        
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """将Opus音频数据解码并保存为WAV文件"""
+
+        file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+        
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+        
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+        
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+        
+        return file_path
+
+    @staticmethod
+    def decode_opus(opus_data: List[bytes]) -> bytes:
+        """将Opus音频数据解码为PCM数据"""
+        import opuslib_next
+        
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+        
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+        
+        return b"".join(pcm_data)
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        if not opus_data:
+            logger.bind(tag=TAG).warn("音频数据为空！")
+            return None, None
+
+        try:
+            # 检查配置是否已设置
+            if not self.secret_id or not self.secret_key:
+                logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别")
+                return None, None
+
+            # 将Opus音频数据解码为PCM
+            pcm_data = self.decode_opus(opus_data)
+            
+            # 将音频数据转换为Base64编码
+            base64_audio = base64.b64encode(pcm_data).decode('utf-8')
+
+            # 构建请求体
+            request_body = self._build_request_body(base64_audio)
+
+            # 获取认证头
+            timestamp, authorization = self._get_auth_headers(request_body)
+
+            # 发送请求
+            start_time = time.time()
+            result = self._send_request(request_body, timestamp, authorization)
+            
+            if result:
+                logger.bind(tag=TAG).debug(f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}")
+            
+            return result, None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
+            return None, None
+
+    def _build_request_body(self, base64_audio: str) -> str:
+        """构建请求体"""
+        request_map = {
+            "ProjectId": 0,
+            "SubServiceType": 2,  # 一句话识别
+            "EngSerViceType": "16k_zh",  # 中文普通话通用
+            "SourceType": 1,  # 音频数据来源为语音文件
+            "VoiceFormat": self.FORMAT,  # 音频格式
+            "Data": base64_audio,  # Base64编码的音频数据
+            "DataLen": len(base64_audio)  # 数据长度
+        }
+        return json.dumps(request_map)
+
+    def _get_auth_headers(self, request_body: str) -> Tuple[str, str]:
+        """获取认证头"""
+        try:
+            # 获取当前UTC时间戳
+            now = datetime.now(timezone.utc)
+            timestamp = str(int(now.timestamp()))
+            date = now.strftime("%Y-%m-%d")
+
+            # 服务名称必须是 "asr"
+            service = "asr"
+
+            # 拼接凭证范围
+            credential_scope = f"{date}/{service}/tc3_request"
+
+            # 使用TC3-HMAC-SHA256签名方法
+            algorithm = "TC3-HMAC-SHA256"
+
+            # 构建规范请求字符串
+            http_request_method = "POST"
+            canonical_uri = "/"
+            canonical_query_string = ""
+
+            # 注意：头部信息需要按照ASCII升序排列，且key和value都转为小写
+            # 必须包含content-type和host头部
+            content_type = "application/json; charset=utf-8"
+            host = "asr.tencentcloudapi.com"
+            action = "SentenceRecognition"  # 接口名称
+
+            # 构建规范头部信息，注意顺序和格式
+            canonical_headers = f"content-type:{content_type.lower()}\n" + \
+                               f"host:{host.lower()}\n" + \
+                               f"x-tc-action:{action.lower()}\n"
+
+            signed_headers = "content-type;host;x-tc-action"
+
+            # 请求体哈希值
+            payload_hash = self._sha256_hex(request_body)
+
+            # 构建规范请求字符串
+            canonical_request = f"{http_request_method}\n" + \
+                               f"{canonical_uri}\n" + \
+                               f"{canonical_query_string}\n" + \
+                               f"{canonical_headers}\n" + \
+                               f"{signed_headers}\n" + \
+                               f"{payload_hash}"
+
+            # 计算规范请求的哈希值
+            hashed_canonical_request = self._sha256_hex(canonical_request)
+
+            # 构建待签名字符串
+            string_to_sign = f"{algorithm}\n" + \
+                            f"{timestamp}\n" + \
+                            f"{credential_scope}\n" + \
+                            f"{hashed_canonical_request}"
+
+            # 计算签名密钥
+            secret_date = self._hmac_sha256(f"TC3{self.secret_key}", date)
+            secret_service = self._hmac_sha256(secret_date, service)
+            secret_signing = self._hmac_sha256(secret_service, "tc3_request")
+
+            # 计算签名
+            signature = self._bytes_to_hex(self._hmac_sha256(secret_signing, string_to_sign))
+
+            # 构建授权头
+            authorization = f"{algorithm} " + \
+                           f"Credential={self.secret_id}/{credential_scope}, " + \
+                           f"SignedHeaders={signed_headers}, " + \
+                           f"Signature={signature}"
+
+            return timestamp, authorization
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成认证头失败: {e}", exc_info=True)
+            raise RuntimeError(f"生成认证头失败: {e}")
+
+    def _send_request(self, request_body: str, timestamp: str, authorization: str) -> Optional[str]:
+        """发送请求到腾讯云API"""
+        headers = {
+            "Content-Type": "application/json; charset=utf-8",
+            "Host": "asr.tencentcloudapi.com",
+            "Authorization": authorization,
+            "X-TC-Action": "SentenceRecognition",
+            "X-TC-Version": self.API_VERSION,
+            "X-TC-Timestamp": timestamp,
+            "X-TC-Region": "ap-shanghai"
+        }
+
+        try:
+            response = requests.post(self.API_URL, headers=headers, data=request_body)
+            
+            if not response.ok:
+                raise IOError(f"请求失败: {response.status_code} {response.reason}")
+            
+            response_json = response.json()
+            
+            # 检查是否有错误
+            if "Response" in response_json and "Error" in response_json["Response"]:
+                error = response_json["Response"]["Error"]
+                error_code = error["Code"]
+                error_message = error["Message"]
+                raise IOError(f"API返回错误: {error_code}: {error_message}")
+            
+            # 提取识别结果
+            if "Response" in response_json and "Result" in response_json["Response"]:
+                return response_json["Response"]["Result"]
+            else:
+                logger.bind(tag=TAG).warn(f"响应中没有识别结果: {response_json}")
+                return ""
+                
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"发送请求失败: {e}", exc_info=True)
+            return None
+
+    def _sha256_hex(self, data: str) -> str:
+        """计算字符串的SHA256哈希值"""
+        digest = hashlib.sha256(data.encode('utf-8')).digest()
+        return self._bytes_to_hex(digest)
+
+    def _hmac_sha256(self, key, data: str) -> bytes:
+        """计算HMAC-SHA256"""
+        if isinstance(key, str):
+            key = key.encode('utf-8')
+        
+        return hmac.new(key, data.encode('utf-8'), hashlib.sha256).digest()
+
+    def _bytes_to_hex(self, bytes_data: bytes) -> str:
+        """字节数组转十六进制字符串"""
+        return ''.join(f"{b:02x}" for b in bytes_data)

main/xiaozhi-server/core/providers/intent/base.py

@@ -10,14 +10,21 @@ class IntentProviderBase(ABC):
     def __init__(self, config):
         self.config = config
         self.intent_options = config.get("intent_options", {
+            "handle_exit_intent": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候",
+            "play_music": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图",
+            "get_weather": "查询天气, 用户希望查询某个地点的天气情况",
+            "get_news": "查询新闻, 用户希望查询最新新闻或特定类型的新闻",
+            "get_lunar": "用于获取今天的阴历/农历和黄历信息",
+            "get_time": "获取今天日期或者当前时间信息",
             "continue_chat": "继续聊天",
-            "end_chat": "结束聊天",
-            "play_music": "播放音乐"
         })
 
     def set_llm(self, llm):
         self.llm = llm
-        logger.bind(tag=TAG).debug("Set LLM for intent provider")
+        # 获取模型名称和类型信息
+        model_name = getattr(llm, 'model_name', str(llm.__class__.__name__))
+        # 记录更详细的日志
+        logger.bind(tag=TAG).info(f"意图识别设置LLM: {model_name}")
 
     @abstractmethod
     async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
@@ -30,5 +37,6 @@ class IntentProviderBase(ABC):
             - "继续聊天"
             - "结束聊天" 
             - "播放音乐 歌名" 或 "随机播放音乐"
+            - "查询天气 地点名" 或 "查询天气 [当前位置]"
         """
         pass

main/xiaozhi-server/core/providers/intent/function_call/function_call.py

@@ -0,0 +1,20 @@
+from ..base import IntentProviderBase
+from typing import List, Dict
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class IntentProvider(IntentProviderBase):
+    async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
+        """
+        默认的意图识别实现，始终返回继续聊天
+        Args:
+            dialogue_history: 对话历史记录列表
+            text: 本次对话记录
+        Returns:
+            固定返回"继续聊天"
+        """
+        logger.bind(tag=TAG).debug("Using functionCallProvider, always returning continue chat")
+        return self.intent_options["continue_chat"]

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -3,6 +3,9 @@ from ..base import IntentProviderBase
 from plugins_func.functions.play_music import initialize_music_handler
 from config.logger import setup_logging
 import re
+import json
+import hashlib
+import time
 
 TAG = __name__
 logger = setup_logging()
@@ -13,6 +16,10 @@ class IntentProvider(IntentProviderBase):
         super().__init__(config)
         self.llm = None
         self.promot = self.get_intent_system_prompt()
+        # 添加缓存管理
+        self.intent_cache = {}  # 缓存意图识别结果
+        self.cache_expiry = 600  # 缓存有效期10分钟
+        self.cache_max_size = 100  # 最多缓存100个意图
 
     def get_intent_system_prompt(self) -> str:
         """
@@ -22,62 +29,119 @@ class IntentProvider(IntentProviderBase):
         """
         intent_list = []
 
-        """
-        "continue_chat":    "1.继续聊天, 除了播放音乐和结束聊天的时候的选项, 比如日常的聊天和问候, 对话等",
-        "end_chat":         "2.结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候",
-        "play_music":       "3.播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图"
-        """
-        for key, value in self.intent_options.items():
-            if key == "play_music":
-                intent_list.append("3.播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图")
-            elif key == "end_chat":
-                intent_list.append("2.结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候")
-            elif key == "continue_chat":
-                intent_list.append("1.继续聊天, 除了播放音乐和结束聊天的时候的选项, 比如日常的聊天和问候, 对话等")
-            else:
-                intent_list.append(value)
-
-        # "如果是唱歌、听歌、播放音乐，请指定歌名，格式为'播放音乐 [识别出的歌名]'。\n"
-        # "如果听不出具体歌名，可以返回'随机播放音乐'。\n"
-        # "只需要返回意图结果的json，不要解释。"
-        # "返回格式如下：\n"
         prompt = (
-            "你是一个意图识别助手。你需要根据和用户的对话记录，重点分析用户的最后一句话，判断用户意图属于以下哪一类(使用<start>和<end>标志)：\n"
+            "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图属于以下哪一类：\n"
             "<start>"
             f"{', '.join(intent_list)}"
             "<end>\n"
-            "你需要按照以下的步骤处理用户的对话"
-            "1. 思考出对话的意图是哪一类的"
-            "2. 属于1和2的意图, 直接返回，返回格式如下：\n"
-            "{intent: '用户意图'}\n"
-            "3. 属于3的意图，则继续分析用户希望播放的音乐\n"
-            "4. 如果无法识别出具体歌名，可以返回'随机播放音乐'\n"
-            "{intent: '播放音乐 [获取的音乐名字]'}\n"
-            "下面是几个处理的示例(思考的内容不返回, 只返回json部分, 无额外的内容)\n"
-            "```"
+            "处理步骤:"
+            "1. 思考意图类型，生成function_call格式"
+            "\n\n"
+            "返回格式示例：\n"
+            "1. 播放音乐意图: {\"function_call\": {\"name\": \"play_music\", \"arguments\": {\"song_name\": \"音乐名称\"}}}\n"
+            "2. 查询天气意图: {\"function_call\": {\"name\": \"get_weather\", \"arguments\": {\"location\": \"地点名称\", \"lang\": \"zh_CN\"}}}\n"
+            "3. 查询新闻意图: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"category\": \"新闻类别\", \"detail\": false, \"lang\": \"zh_CN\"}}}\n"
+            "4. 结束对话意图: {\"function_call\": {\"name\": \"handle_exit_intent\", \"arguments\": {\"say_goodbye\": \"goodbye\"}}}\n"
+            "5. 获取当天日期时间: {\"function_call\": {\"name\": \"get_time\"}}\n"
+            "6. 获取当前黄历意图: {\"function_call\": {\"name\": \"get_lunar\"}}\n"
+            "7. 继续聊天意图: {\"function_call\": {\"name\": \"continue_chat\"}}\n"
+            "\n"
+            "注意:\n"
+            "- 播放音乐：无歌名时，song_name设为\"random\"\n"
+            "- 查询天气：无地点时，location设为null\n"
+            "- 查询新闻：无类别时，category设为null；查询详情时，detail设为true\n"
+            "- 如果没有明显的意图，应按照继续聊天意图处理\n"
+            "- 只返回纯JSON，不要任何其他内容\n"
+            "\n"
+            "示例分析:\n"
+            "```\n"
+            "用户: 你好小智\n"
+            "返回: {\"function_call\": {\"name\": \"continue_chat\"}}\n"
+            "```\n"
+            "```\n"
             "用户: 你今天怎么样?\n"
-            "思考(不返回): 用户发来的数据是一个问候语，属于继续聊天的意图, 是种类1, 种类1的需求是直接返回\n"
-            "返回结果: {intent: '继续聊天'}\n"
-            "```"
-            "用户: 我今天有点累了, 我们明天再聊吧\n"
-            "思考(不返回): 用户表达了今天不想继续对话，属于结束聊天的意图, 是种类2, 种类2的需求是直接返回\n"
-            "返回结果: {intent: '结束聊天'}\n"
-            "```"
-            "用户: 我今天有点累了, 我们明天再聊吧\n"
-            "思考(不返回): 用户表达了今天不想继续对话，属于结束聊天的意图, 是种类2, 种类2的需求是直接返回\n"
-            "返回结果: {intent: '结束聊天'}\n"
-            "```"
-            "用户: 你可以播放一首中秋月给我听吗\n"
-            "思考(不返回): 用户表达了想听音乐的续签，属于播放音乐的意图, 是种类3, 种类3的需求需要继续判断播放的音乐, 这里用户希望的歌曲名明确给出是中秋月\n"
-            "返回结果: {intent: '播放音乐 [中秋月]'}\n"
-            "```"
-            "你现在可以使用的音乐的名称如下(使用<start>和<end>标志):\n"
+            "返回: {\"function_call\": {\"name\": \"continue_chat\"}}\n"
+            "```\n"
+            "```\n"
+            "用户: 现在是几号了?现在几点了？\n"
+            "返回: {\"function_call\": {\"name\": \"get_time\"}}\n"
+            "```\n"
+            "```\n"
+            "用户: 今天农历是多少？\n"
+            "返回: {\"function_call\": {\"name\": \"get_lunar\"}}\n"
+            "```\n"
+            "```\n"
+            "用户: 我们明天再聊吧\n"
+            "返回: {\"function_call\": {\"name\": \"handle_exit_intent\"}}\n"
+            "```\n"
+            "```\n"
+            "用户: 播放中秋月\n"
+            "返回: {\"function_call\": {\"name\": \"play_music\", \"arguments\": {\"song_name\": \"中秋月\"}}}\n"
+            "```\n"
+            "```\n"
+            "用户: 北京天气怎么样\n"
+            "返回: {\"function_call\": {\"name\": \"get_weather\", \"arguments\": {\"location\": \"北京\", \"lang\": \"zh_CN\"}}}\n"
+            "```\n"
+            "```\n"
+            "用户: 今天天气怎么样\n"
+            "返回: {\"function_call\": {\"name\": \"get_weather\", \"arguments\": {\"location\": null, \"lang\": \"zh_CN\"}}}\n"
+            "```\n"
+            "```\n"
+            "用户: 播报财经新闻\n"
+            "返回: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"category\": \"财经\", \"detail\": false, \"lang\": \"zh_CN\"}}}\n"
+            "```\n"
+            "```\n"
+            "用户: 有什么最新新闻\n"
+            "返回: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"category\": null, \"detail\": false, \"lang\": \"zh_CN\"}}}\n"
+            "```\n"
+            "```\n"
+            "用户: 详细介绍一下这条新闻\n"
+            "返回: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"detail\": true, \"lang\": \"zh_CN\"}}}\n"
+            "```\n"
+            "可用的音乐名称:\n"
         )
         return prompt
+        
+    def clean_cache(self):
+        """清理过期缓存"""
+        now = time.time()
+        # 找出过期键
+        expired_keys = [k for k, v in self.intent_cache.items() if now - v['timestamp'] > self.cache_expiry]
+        for key in expired_keys:
+            del self.intent_cache[key]
+            
+        # 如果缓存太大，移除最旧的条目
+        if len(self.intent_cache) > self.cache_max_size:
+            # 按时间戳排序并保留最新的条目
+            sorted_items = sorted(self.intent_cache.items(), key=lambda x: x[1]['timestamp'])
+            for key, _ in sorted_items[:len(sorted_items) - self.cache_max_size]:
+                del self.intent_cache[key]
 
     async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
         if not self.llm:
             raise ValueError("LLM provider not set")
+            
+        # 记录整体开始时间
+        total_start_time = time.time()
+            
+        # 打印使用的模型信息
+        model_info = getattr(self.llm, 'model_name', str(self.llm.__class__.__name__))
+        logger.bind(tag=TAG).info(f"使用意图识别模型: {model_info}")
+            
+        # 计算缓存键
+        cache_key = hashlib.md5(text.encode()).hexdigest()
+        
+        # 检查缓存
+        if cache_key in self.intent_cache:
+            cache_entry = self.intent_cache[cache_key]
+            # 检查缓存是否过期
+            if time.time() - cache_entry['timestamp'] <= self.cache_expiry:
+                cache_time = time.time() - total_start_time
+                logger.bind(tag=TAG).info(f"使用缓存的意图: {cache_key} -> {cache_entry['intent']}, 耗时: {cache_time:.4f}秒")
+                return cache_entry['intent']
+                
+        # 清理缓存
+        self.clean_cache()
 
         # 构建用户最后一句话的提示
         msgStr = ""
@@ -94,18 +158,78 @@ class IntentProvider(IntentProviderBase):
         music_file_names = music_config["music_file_names"]
         prompt_music = f"{self.promot}\n<start>{music_file_names}\n<end>"
         logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
+        
+        # 记录预处理完成时间
+        preprocess_time = time.time() - total_start_time
+        logger.bind(tag=TAG).debug(f"意图识别预处理耗时: {preprocess_time:.4f}秒")
+        
         # 使用LLM进行意图识别
+        llm_start_time = time.time()
+        logger.bind(tag=TAG).info(f"开始LLM意图识别调用, 模型: {model_info}")
+        
         intent = self.llm.response_no_stream(
             system_prompt=prompt_music,
             user_prompt=user_prompt
         )
-        # 使用正则表达式提取大括号中的内容  
-        # 使用正则表达式提取 {} 中的内容
-        match = re.search(r'\{.*?\}', intent)
+        
+        # 记录LLM调用完成时间
+        llm_time = time.time() - llm_start_time
+        logger.bind(tag=TAG).info(f"LLM意图识别完成, 模型: {model_info}, 调用耗时: {llm_time:.4f}秒")
+        
+        # 记录后处理开始时间
+        postprocess_start_time = time.time()
+        
+        # 清理和解析响应
+        intent = intent.strip()
+        # 尝试提取JSON部分
+        match = re.search(r'\{.*\}', intent, re.DOTALL)
         if match:
-            result = match.group(0)
-            intent = result
-        else:
-            intent = "{intent: '继续聊天'}"
-        logger.bind(tag=TAG).info(f"Detected intent: {intent}")
-        return intent.strip()
+            intent = match.group(0)
+        
+        # 记录总处理时间
+        total_time = time.time() - total_start_time
+        logger.bind(tag=TAG).info(f"【意图识别性能】模型: {model_info}, 总耗时: {total_time:.4f}秒, LLM调用: {llm_time:.4f}秒, 查询: '{text[:20]}...'")
+        
+        # 尝试解析为JSON
+        try:
+            intent_data = json.loads(intent)
+            # 如果包含function_call，则格式化为适合处理的格式
+            if "function_call" in intent_data:
+                function_data = intent_data["function_call"]
+                function_name = function_data.get("name")
+                function_args = function_data.get("arguments", {})
+                
+                # 记录识别到的function call
+                logger.bind(tag=TAG).info(f"识别到function call: {function_name}, 参数: {function_args}")
+                
+                # 添加到缓存
+                self.intent_cache[cache_key] = {
+                    'intent': intent,
+                    'timestamp': time.time()
+                }
+                
+                # 后处理时间
+                postprocess_time = time.time() - postprocess_start_time
+                logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
+                
+                # 确保返回完全序列化的JSON字符串
+                return intent
+            else:
+                # 添加到缓存
+                self.intent_cache[cache_key] = {
+                    'intent': intent,
+                    'timestamp': time.time()
+                }
+                
+                # 后处理时间
+                postprocess_time = time.time() - postprocess_start_time
+                logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
+                
+                # 返回普通意图
+                return intent
+        except json.JSONDecodeError:
+            # 后处理时间
+            postprocess_time = time.time() - postprocess_start_time
+            logger.bind(tag=TAG).error(f"无法解析意图JSON: {intent}, 后处理耗时: {postprocess_time:.4f}秒")
+            # 如果解析失败，默认返回继续聊天意图
+            return "{\"intent\": \"继续聊天\"}"

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -6,11 +6,12 @@ from core.providers.llm.base import LLMProviderBase
 TAG = __name__
 logger = setup_logging()
 
+
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.api_key = config["api_key"]
         self.mode = config.get("mode", "chat-messages")
-        self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
+        self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip("/")
         self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
 
     def response(self, session_id, dialogue):
@@ -22,57 +23,58 @@ class LLMProvider(LLMProviderBase):
             # 发起流式请求
             if self.mode == "chat-messages":
                 request_json = {
-                        "query": last_msg["content"],
-                        "response_mode": "streaming",
-                        "user": session_id,
-                        "inputs": {},
-                        "conversation_id": conversation_id
-                    }
+                    "query": last_msg["content"],
+                    "response_mode": "streaming",
+                    "user": session_id,
+                    "inputs": {},
+                    "conversation_id": conversation_id,
+                }
             elif self.mode == "workflows/run":
                 request_json = {
                     "inputs": {"query": last_msg["content"]},
                     "response_mode": "streaming",
-                    "user": session_id
+                    "user": session_id,
                 }
             elif self.mode == "completion-messages":
                 request_json = {
                     "inputs": {"query": last_msg["content"]},
                     "response_mode": "streaming",
-                    "user": session_id
+                    "user": session_id,
                 }
 
             with requests.post(
-                    f"{self.base_url}/{self.mode}",
-                    headers={"Authorization": f"Bearer {self.api_key}"},
-                    json=request_json,
-                    stream=True
+                f"{self.base_url}/{self.mode}",
+                headers={"Authorization": f"Bearer {self.api_key}"},
+                json=request_json,
+                stream=True,
             ) as r:
                 if self.mode == "chat-messages":
                     for line in r.iter_lines():
-                        if line.startswith(b'data: '):
+                        if line.startswith(b"data: "):
                             event = json.loads(line[6:])
                             # 如果没有找到conversation_id，则获取此次conversation_id
                             if not conversation_id:
-                                conversation_id = event.get('conversation_id')
-                                self.session_conversation_map[session_id] = conversation_id  # 更新映射
-                            if event.get('answer'):
-                                yield event['answer']
+                                conversation_id = event.get("conversation_id")
+                                self.session_conversation_map[session_id] = (
+                                    conversation_id  # 更新映射
+                                )
+                            if event.get("answer"):
+                                yield event["answer"]
                 elif self.mode == "workflows/run":
                     for line in r.iter_lines():
-                        # logger.bind(tag=TAG).info(f"chat message response: {line}")
-                        if line.startswith(b'data: '):
+                        if line.startswith(b"data: "):
                             event = json.loads(line[6:])
-                            if event.get('event') == "workflow_finished":
-                                if event['data']['status'] == "succeeded":
-                                    yield event['data']['outputs']['answer']
+                            if event.get("event") == "workflow_finished":
+                                if event["data"]["status"] == "succeeded":
+                                    yield event["data"]["outputs"]["answer"]
                                 else:
                                     yield "【服务响应异常】"
                 elif self.mode == "completion-messages":
                     for line in r.iter_lines():
-                        if line.startswith(b'data: '):
+                        if line.startswith(b"data: "):
                             event = json.loads(line[6:])
-                            if event.get('answer'):
-                                yield event['answer'] 
+                            if event.get("answer"):
+                                yield event["answer"]
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")

main/xiaozhi-server/core/providers/llm/xinference/xinference.py

@@ -0,0 +1,85 @@
+from config.logger import setup_logging
+from openai import OpenAI
+import json
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.model_name = config.get("model_name")
+        self.base_url = config.get("base_url", "http://localhost:9997")
+        # Initialize OpenAI client with Xinference base URL
+        # 如果没有v1，增加v1
+        if not self.base_url.endswith("/v1"):
+            self.base_url = f"{self.base_url}/v1"
+            
+        logger.bind(tag=TAG).info(f"Initializing Xinference LLM provider with model: {self.model_name}, base_url: {self.base_url}")
+
+        try:
+            self.client = OpenAI(
+                base_url=self.base_url,
+                api_key="xinference"  # Xinference has a similar setup to Ollama where it doesn't need an actual key
+            )
+            logger.bind(tag=TAG).info("Xinference client initialized successfully")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error initializing Xinference client: {e}")
+            raise
+
+    def response(self, session_id, dialogue):
+        try:
+            logger.bind(tag=TAG).debug(f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}")
+            responses = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=dialogue,
+                stream=True
+            )
+            is_active=True
+            for chunk in responses:
+                try:
+                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
+                    content = delta.content if hasattr(delta, 'content') else ''
+                    if content:
+                        if '<think>' in content:
+                            is_active = False
+                            content = content.split('<think>')[0]
+                        if '</think>' in content:
+                            is_active = True
+                            content = content.split('</think>')[-1]
+                        if is_active:
+                            yield content
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in Xinference response generation: {e}")
+            yield "【Xinference服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        try:
+            logger.bind(tag=TAG).debug(f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}")
+            if functions:
+                logger.bind(tag=TAG).debug(f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}")
+                
+            stream = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=dialogue,
+                stream=True,
+                tools=functions,
+            )
+
+            for chunk in stream:
+                delta = chunk.choices[0].delta
+                content = delta.content
+                tool_calls = delta.tool_calls
+                
+                if content:
+                    yield content, tool_calls
+                elif tool_calls:
+                    yield None, tool_calls
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in Xinference function call: {e}")
+            yield {"type": "content", "content": f"【Xinference服务响应异常: {str(e)}】"} 

main/xiaozhi-server/core/providers/tts/tencent.py

@@ -0,0 +1,156 @@
+import hashlib
+import hmac
+import os
+import time
+import uuid
+import json
+import base64
+import requests
+from datetime import datetime, timezone
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.appid = config.get("appid")
+        self.secret_id = config.get("secret_id")
+        self.secret_key = config.get("secret_key")
+        self.voice = config.get("voice")
+        self.api_url = "https://tts.tencentcloudapi.com"  # 正确的API端点
+        self.region = config.get("region")
+        self.output_file = config.get("output_dir")
+
+    def _get_auth_headers(self, request_body):
+        """生成鉴权请求头"""
+        # 获取当前UTC时间戳
+        timestamp = int(time.time())
+        
+        # 使用UTC时间计算日期
+        utc_date = datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime('%Y-%m-%d')
+        
+        # 服务名称必须是 "tts"
+        service = "tts"
+        
+        # 拼接凭证范围
+        credential_scope = f"{utc_date}/{service}/tc3_request"
+        
+        # 使用TC3-HMAC-SHA256签名方法
+        algorithm = "TC3-HMAC-SHA256"
+        
+        # 构建规范请求字符串
+        http_request_method = "POST"
+        canonical_uri = "/"
+        canonical_querystring = ""
+        
+        # 请求头必须包含host和content-type，且按字典序排列
+        canonical_headers = (
+            f"content-type:application/json\n"
+            f"host:tts.tencentcloudapi.com\n"
+        )
+        signed_headers = "content-type;host"
+        
+        # 请求体哈希值
+        payload = json.dumps(request_body)
+        payload_hash = hashlib.sha256(payload.encode('utf-8')).hexdigest()
+        
+        # 构建规范请求字符串
+        canonical_request = (
+            f"{http_request_method}\n"
+            f"{canonical_uri}\n"
+            f"{canonical_querystring}\n"
+            f"{canonical_headers}\n"
+            f"{signed_headers}\n"
+            f"{payload_hash}"
+        )
+        
+        # 计算规范请求的哈希值
+        hashed_canonical_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
+        
+        # 构建待签名字符串
+        string_to_sign = (
+            f"{algorithm}\n"
+            f"{timestamp}\n"
+            f"{credential_scope}\n"
+            f"{hashed_canonical_request}"
+        )
+        
+        # 计算签名密钥
+        secret_date = self._hmac_sha256(f"TC3{self.secret_key}".encode('utf-8'), utc_date)
+        secret_service = self._hmac_sha256(secret_date, service)
+        secret_signing = self._hmac_sha256(secret_service, "tc3_request")
+        
+        # 计算签名
+        signature = hmac.new(
+            secret_signing, 
+            string_to_sign.encode('utf-8'), 
+            hashlib.sha256
+        ).hexdigest()
+        
+        # 构建授权头
+        authorization = (
+            f"{algorithm} "
+            f"Credential={self.secret_id}/{credential_scope}, "
+            f"SignedHeaders={signed_headers}, "
+            f"Signature={signature}"
+        )
+        
+        # 构建请求头
+        headers = {
+            "Content-Type": "application/json",
+            "Host": "tts.tencentcloudapi.com",
+            "Authorization": authorization,
+            "X-TC-Action": "TextToVoice",
+            "X-TC-Timestamp": str(timestamp),
+            "X-TC-Version": "2019-08-23",
+            "X-TC-Region": self.region,
+            "X-TC-Language": "zh-CN"
+        }
+        
+        return headers
+    
+    def _hmac_sha256(self, key, msg):
+        """HMAC-SHA256加密"""
+        if isinstance(msg, str):
+            msg = msg.encode('utf-8')
+        return hmac.new(key, msg, hashlib.sha256).digest()
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        # 构建请求体
+        request_json = {
+            "Text": text,  # 合成语音的源文本
+            "SessionId": str(uuid.uuid4()),  # 会话ID，随机生成
+            "VoiceType": int(self.voice),  # 音色
+        }
+
+        try:
+            # 获取请求头（每次请求都重新生成，以确保时间戳和签名是最新的）
+            headers = self._get_auth_headers(request_json)
+            
+            # 发送请求
+            resp = requests.post(self.api_url, json.dumps(request_json), headers=headers)
+            
+            # 检查响应
+            if resp.status_code == 200:
+                response_data = resp.json()
+                
+                # 检查是否成功
+                if response_data.get("Response", {}).get("Error") is not None:
+                    error_info = response_data["Response"]["Error"]
+                    raise Exception(f"API返回错误: {error_info['Code']}: {error_info['Message']}")
+                
+                # 提取音频数据
+                audio_data = response_data["Response"].get("Audio")
+                if audio_data:
+                    # 解码Base64音频数据并保存
+                    with open(output_file, "wb") as f:
+                        f.write(base64.b64decode(audio_data))
+                else:
+                    raise Exception(f"{__name__}: 没有返回音频数据: {response_data}")
+            else:
+                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+        except Exception as e:
+            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/utils/util.py

@@ -67,13 +67,11 @@ def is_private_ip(ip_addr):
 
 def get_ip_info(ip_addr):
     try:
-        base_url = "https://freeipapi.com/api/json"
-        url = base_url if is_private_ip(ip_addr) else f"{base_url}/{ip_addr}"
-
+        url = "https://whois.pconline.com.cn/ipJson.jsp?json=true"
         resp = requests.get(url).json()
 
         ip_info = {
-            "city": resp.get("cityName")
+            "city": resp.get("city")
         }
         return ip_info
     except Exception as e:

main/xiaozhi-server/core/websocket_server.py

@@ -12,50 +12,72 @@ class WebSocketServer:
     def __init__(self, config: dict):
         self.config = config
         self.logger = setup_logging()
-        self._vad, self._asr, self._llm, self._tts, self._memory, self.intent = self._create_processing_instances()
+        self._vad, self._asr, self._llm, self._tts, self._memory, self.intent = (
+            self._create_processing_instances()
+        )
         self.active_connections = set()  # 添加全局连接记录
 
     def _create_processing_instances(self):
-        memory_cls_name = self.config["selected_module"].get("Memory", "nomem") # 默认使用nomem
-        has_memory_cfg = self.config.get("Memory") and memory_cls_name in self.config["Memory"]
+        memory_cls_name = self.config["selected_module"].get(
+            "Memory", "nomem"
+        )  # 默认使用nomem
+        has_memory_cfg = (
+            self.config.get("Memory") and memory_cls_name in self.config["Memory"]
+        )
         memory_cfg = self.config["Memory"][memory_cls_name] if has_memory_cfg else {}
 
         """创建处理模块实例"""
         return (
             vad.create_instance(
                 self.config["selected_module"]["VAD"],
-                self.config["VAD"][self.config["selected_module"]["VAD"]]
+                self.config["VAD"][self.config["selected_module"]["VAD"]],
             ),
             asr.create_instance(
-                self.config["selected_module"]["ASR"]
-                if not 'type' in self.config["ASR"][self.config["selected_module"]["ASR"]]
-                else
-                self.config["ASR"][self.config["selected_module"]["ASR"]]["type"],
+                (
+                    self.config["selected_module"]["ASR"]
+                    if not "type"
+                    in self.config["ASR"][self.config["selected_module"]["ASR"]]
+                    else self.config["ASR"][self.config["selected_module"]["ASR"]][
+                        "type"
+                    ]
+                ),
                 self.config["ASR"][self.config["selected_module"]["ASR"]],
-                self.config["delete_audio"]
+                self.config["delete_audio"],
             ),
             llm.create_instance(
-                self.config["selected_module"]["LLM"]
-                if not 'type' in self.config["LLM"][self.config["selected_module"]["LLM"]]
-                else
-                self.config["LLM"][self.config["selected_module"]["LLM"]]['type'],
+                (
+                    self.config["selected_module"]["LLM"]
+                    if not "type"
+                    in self.config["LLM"][self.config["selected_module"]["LLM"]]
+                    else self.config["LLM"][self.config["selected_module"]["LLM"]][
+                        "type"
+                    ]
+                ),
                 self.config["LLM"][self.config["selected_module"]["LLM"]],
             ),
             tts.create_instance(
-                self.config["selected_module"]["TTS"]
-                if not 'type' in self.config["TTS"][self.config["selected_module"]["TTS"]]
-                else
-                self.config["TTS"][self.config["selected_module"]["TTS"]]["type"],
+                (
+                    self.config["selected_module"]["TTS"]
+                    if not "type"
+                    in self.config["TTS"][self.config["selected_module"]["TTS"]]
+                    else self.config["TTS"][self.config["selected_module"]["TTS"]][
+                        "type"
+                    ]
+                ),
                 self.config["TTS"][self.config["selected_module"]["TTS"]],
-                self.config["delete_audio"]
+                self.config["delete_audio"],
             ),
             memory.create_instance(memory_cls_name, memory_cfg),
             intent.create_instance(
-                self.config["selected_module"]["Intent"]
-                if not 'type' in self.config["Intent"][self.config["selected_module"]["Intent"]]
-                else
-                self.config["Intent"][self.config["selected_module"]["Intent"]]["type"],
-                self.config["Intent"][self.config["selected_module"]["Intent"]]
+                (
+                    self.config["selected_module"]["Intent"]
+                    if not "type"
+                    in self.config["Intent"][self.config["selected_module"]["Intent"]]
+                    else self.config["Intent"][
+                        self.config["selected_module"]["Intent"]
+                    ]["type"]
+                ),
+                self.config["Intent"][self.config["selected_module"]["Intent"]],
             ),
         )
 
@@ -64,19 +86,33 @@ class WebSocketServer:
         host = server_config["ip"]
         port = server_config["port"]
 
-        self.logger.bind(tag=TAG).info("Server is running at ws://{}:{}", get_local_ip(), port)
-        self.logger.bind(tag=TAG).info("=======上面的地址是websocket协议地址，请勿用浏览器访问=======")
-        async with websockets.serve(
-                self._handle_connection,
-                host,
-                port
-        ):
+        self.logger.bind(tag=TAG).info(
+            "Server is running at ws://{}:{}/xiaozhi/v1/", get_local_ip(), port
+        )
+        self.logger.bind(tag=TAG).info(
+            "=======上面的地址是websocket协议地址，请勿用浏览器访问======="
+        )
+        self.logger.bind(tag=TAG).info(
+            "如想测试websocket请用谷歌浏览器打开test目录下的test_page.html"
+        )
+        self.logger.bind(tag=TAG).info(
+            "=============================================================\n"
+        )
+        async with websockets.serve(self._handle_connection, host, port):
             await asyncio.Future()
 
     async def _handle_connection(self, websocket):
         """处理新连接，每次创建独立的ConnectionHandler"""
         # 创建ConnectionHandler时传入当前server实例
-        handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._memory, self.intent)
+        handler = ConnectionHandler(
+            self.config,
+            self._vad,
+            self._asr,
+            self._llm,
+            self._tts,
+            self._memory,
+            self.intent,
+        )
         self.active_connections.add(handler)
         try:
             await handler.handle_connection(websocket)

main/xiaozhi-server/mcp_server_settings.json

@@ -0,0 +1,31 @@
+{
+  "des": [
+    "在data目录下创建.mcp_server_settings.json文件，可以选择下面的MCP服务，也可以自行添加新的MCP服务。",
+    "后面不断测试补充好用的mcp服务，欢迎大家一起补充。",
+    "记得删除注释行,des属性仅为说明,不会被解析。",
+    "des和link属性，仅为说明安装方式，方便大家查看原始链接，不是必须项。"
+  ],
+  "mcpServers": {
+    "filesystem": {
+      "command": "npx",
+      "args": [
+        "-y",
+        "@modelcontextprotocol/server-filesystem",
+        "/Users/username/Desktop",
+        "/path/to/other/allowed/dir"
+      ],
+      "link":"https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem"
+    },
+    "playwright": {
+      "command": "npx",
+      "args": ["-y", "@executeautomation/playwright-mcp-server"],
+      "des" : "run 'npx playwright install' first",
+      "link": "https://github.com/executeautomation/mcp-playwright"
+    },
+    "windows-cli": {
+      "command": "npx",
+      "args": ["-y", "@simonb97/server-win-cli"],
+      "link": "https://github.com/SimonB97/win-cli-mcp-server"
+    }
+  }
+}

main/xiaozhi-server/plugins_func/functions/get_news.py

@@ -45,10 +45,10 @@ def fetch_news_from_rss(rss_url):
     try:
         response = requests.get(rss_url)
         response.raise_for_status()
-        
+
         # 解析XML
         root = ET.fromstring(response.content)
-        
+
         # 查找所有item元素（新闻条目）
         news_items = []
         for item in root.findall('.//item'):
@@ -56,14 +56,14 @@ def fetch_news_from_rss(rss_url):
             link = item.find('link').text if item.find('link') is not None else "#"
             description = item.find('description').text if item.find('description') is not None else "无描述"
             pubDate = item.find('pubDate').text if item.find('pubDate') is not None else "未知时间"
-            
+
             news_items.append({
                 'title': title,
                 'link': link,
                 'description': description,
                 'pubDate': pubDate
             })
-        
+
         return news_items
     except Exception as e:
         logger.bind(tag=TAG).error(f"获取RSS新闻失败: {e}")
@@ -75,9 +75,9 @@ def fetch_news_detail(url):
     try:
         response = requests.get(url)
         response.raise_for_status()
-        
+
         soup = BeautifulSoup(response.content, 'html.parser')
-        
+
         # 尝试提取正文内容 (这里的选择器需要根据实际网站结构调整)
         content_div = soup.select_one('.content_desc, .content, article, .article-content')
         if content_div:
@@ -98,7 +98,7 @@ def map_category(category_text):
     """将用户输入的中文类别映射到配置文件中的类别键"""
     if not category_text:
         return None
-        
+
     # 类别映射字典，目前支持社会、国际、财经新闻，如需更多类型，参见配置文件
     category_map = {
         # 社会新闻
@@ -113,10 +113,10 @@ def map_category(category_text):
         "金融": "finance",
         "经济": "finance"
     }
-    
+
     # 转换为小写并去除空格
     normalized_category = category_text.lower().strip()
-    
+
     # 返回映射结果，如果没有匹配项则返回原始输入
     return category_map.get(normalized_category, category_text)
 
@@ -129,21 +129,22 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C
         if detail:
             if not hasattr(conn, 'last_news_link') or not conn.last_news_link or 'link' not in conn.last_news_link:
                 return ActionResponse(Action.REQLLM, "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", None)
-            
+
             link = conn.last_news_link.get('link')
             title = conn.last_news_link.get('title', '未知标题')
-            
+
             if link == '#':
                 return ActionResponse(Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None)
-            
+
             logger.bind(tag=TAG).debug(f"获取新闻详情: {title}, URL={link}")
-            
+
             # 获取新闻详情
             detail_content = fetch_news_detail(link)
-            
+
             if not detail_content or detail_content == "无法获取详细内容":
-                return ActionResponse(Action.REQLLM, f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", None)
-            
+                return ActionResponse(Action.REQLLM,
+                                      f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", None)
+
             # 构建详情报告
             detail_report = (
                 f"根据下列数据，用{lang}回应用户的新闻详情查询请求：\n\n"
@@ -152,33 +153,33 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C
                 f"(请对上述新闻内容进行总结，提取关键信息，以自然、流畅的方式向用户播报，"
                 f"不要提及这是总结，就像是在讲述一个完整的新闻故事)"
             )
-            
+
             return ActionResponse(Action.REQLLM, detail_report, None)
-        
+
         # 否则，获取新闻列表并随机选择一条
         # 从配置中获取RSS URL
         rss_config = conn.config["plugins"]["get_news"]
         default_rss_url = rss_config.get("default_rss_url", "https://www.chinanews.com.cn/rss/society.xml")
-        
+
         # 将用户输入的类别映射到配置中的类别键
         mapped_category = map_category(category)
-        
+
         # 如果提供了类别，尝试从配置中获取对应的URL
         rss_url = default_rss_url
         if mapped_category and mapped_category in rss_config.get("category_urls", {}):
             rss_url = rss_config["category_urls"][mapped_category]
-        
+
         logger.bind(tag=TAG).info(f"获取新闻: 原始类别={category}, 映射类别={mapped_category}, URL={rss_url}")
-        
+
         # 获取新闻列表
         news_items = fetch_news_from_rss(rss_url)
-        
+
         if not news_items:
             return ActionResponse(Action.REQLLM, "抱歉，未能获取到新闻信息，请稍后再试。", None)
-        
+
         # 随机选择一条新闻
         selected_news = random.choice(news_items)
-        
+
         # 保存当前新闻链接到连接对象，以便后续查询详情
         if not hasattr(conn, 'last_news_link'):
             conn.last_news_link = {}
@@ -186,7 +187,7 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C
             'link': selected_news.get('link', '#'),
             'title': selected_news.get('title', '未知标题')
         }
-        
+
         # 构建新闻报告
         news_report = (
             f"根据下列数据，用{lang}回应用户的新闻查询请求：\n\n"
@@ -197,9 +198,9 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C
             f"直接读出新闻即可，不需要额外多余的内容。"
             f"如果用户询问更多详情，告知用户可以说'请详细介绍这条新闻'获取更多内容)"
         )
-        
+
         return ActionResponse(Action.REQLLM, news_report, None)
-    
+
     except Exception as e:
         logger.bind(tag=TAG).error(f"获取新闻出错: {e}")
         return ActionResponse(Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None)

main/xiaozhi-server/plugins_func/functions/get_weather.py

@@ -52,7 +52,7 @@ WEATHER_CODE_MAP = {
     "405": "雨雪天气", "406": "阵雨夹雪", "407": "阵雪", "408": "小到中雪", "409": "中到大雪", "410": "大到暴雪",
     "456": "阵雨夹雪", "457": "阵雪", "499": "雪",
     "500": "薄雾", "501": "雾", "502": "霾", "503": "扬沙", "504": "浮尘",
-    "507": "沙尘暴", "508": "强沙尘暴", 
+    "507": "沙尘暴", "508": "强沙尘暴",
     "509": "浓雾", "510": "强浓雾", "511": "中度霾", "512": "重度霾", "513": "严重霾", "514": "大雾", "515": "特强浓雾",
     "900": "热", "901": "冷", "999": "未知"
 }
@@ -120,4 +120,4 @@ def get_weather(conn, location: str = None, lang: str = "zh_CN"):
         "参数为0的值不需要报告给用户，每次都报告体感温度，根据语境选择合适的参数内容告知用户，并对参数给出相应评价)"
     )
 
-    return ActionResponse(Action.REQLLM, weather_report, None)
+    return ActionResponse(Action.REQLLM, weather_report, None)

main/xiaozhi-server/plugins_func/functions/handle_device.py

@@ -6,6 +6,7 @@ import asyncio
 TAG = __name__
 logger = setup_logging()
 
+
 async def _get_device_status(conn, device_name, device_type, property_name):
     """获取设备状态"""
     status = await get_iot_status(conn, device_type, property_name)
@@ -13,6 +14,7 @@ async def _get_device_status(conn, device_name, device_type, property_name):
         raise Exception(f"你的设备不支持{device_name}控制")
     return status
 
+
 async def _set_device_property(conn, device_name, device_type, method_name, property_name, new_value=None, action=None, step=10):
     """设置设备属性"""
     current_value = await _get_device_status(conn, device_name, device_type, property_name)
@@ -32,9 +34,11 @@ async def _set_device_property(conn, device_name, device_type, method_name, prop
     await send_iot_conn(conn, device_type, method_name, {property_name: current_value})
     return current_value
 
+
 def _handle_device_action(conn, func, success_message, error_message, *args, **kwargs):
     """处理设备操作的通用函数"""
-    future = asyncio.run_coroutine_threadsafe(func(conn, *args, **kwargs), conn.loop)
+    future = asyncio.run_coroutine_threadsafe(
+        func(conn, *args, **kwargs), conn.loop)
     try:
         result = future.result()
         logger.bind(tag=TAG).info(f"{success_message}: {result}")
@@ -45,6 +49,7 @@ def _handle_device_action(conn, func, success_message, error_message, *args, **k
         response = f"{error_message}: {e}"
         return ActionResponse(action=Action.RESPONSE, result=None, response=response)
 
+
 # 设备控制
 handle_device_function_desc = {
     "type": "function",
@@ -52,9 +57,9 @@ handle_device_function_desc = {
         "name": "handle_device",
         "description": (
             "用户想要获取或者设置设备的音量/亮度大小，或者用户觉得声音/亮度过高或过低，或者用户想提高或降低音量/亮度。"
-            "比如用户说现在亮度多少，参数为：device_type:Backlight,action:get。"
+            "比如用户说现在亮度多少，参数为：device_type:Screen,action:get。"
             "比如用户说设置音量为50，参数为：device_type:Speaker,action:set,value:50。"
-            "比如用户说亮度太高了，参数为：device_type:Backlight,action:lower。"
+            "比如用户说亮度太高了，参数为：device_type:Screen,action:lower。"
             "比如用户说调大音量，参数为：device_type:Speaker,action:raise。"
         ),
         "parameters": {
@@ -62,14 +67,14 @@ handle_device_function_desc = {
             "properties": {
                 "device_type": {
                     "type": "string",
-                    "description": "设备类型，可选值：Speaker(音量),Backlight(亮度)"
+                    "description": "设备类型，可选值：Speaker(音量),Screen(亮度)"
                 },
                 "action": {
                     "type": "string",
                     "description": "动作名称，可选值：get(获取),set(设置),raise(提高),lower(降低)"
                 },
                 "value": {
-                    "type": "int",
+                    "type": "integer",
                     "description": "值大小，可选值：0-100之间的整数"
                 }
             },
@@ -78,18 +83,19 @@ handle_device_function_desc = {
     }
 }
 
+
 @register_function('handle_device', handle_device_function_desc, ToolType.IOT_CTL)
 def handle_device(conn, device_type: str, action: str, value: int = None):
     if device_type == "Speaker":
         method_name, property_name, device_name = "SetVolume", "volume", "音量"
-    elif device_type == "Backlight":
+    elif device_type == "Screen":
         method_name, property_name, device_name = "SetBrightness", "brightness", "亮度"
     else:
         raise Exception(f"未识别的设备类型: {device_type}")
-    
+
     if action not in ["get", "set", "raise", "lower"]:
         raise Exception(f"未识别的动作名称: {action}")
-    
+
     if action == "get":
         # get
         return _handle_device_action(

main/xiaozhi-server/plugins_func/functions/hass_init.py

@@ -10,7 +10,7 @@ HASS_CACHE = {}
 def append_devices_to_prompt(conn):
     if conn.use_function_call_mode:
         funcs = conn.config["Intent"]["function_call"].get("functions", [])
-        if "hass_get_state" in funcs or "hass_get_state" in funcs:
+        if "hass_get_state" in funcs or "hass_set_state" in funcs:
             prompt = "下面是我家智能设备，可以通过homeassistant控制\n"
             devices = conn.config["plugins"]["home_assistant"].get("devices", [])
             if len(devices) == 0:
@@ -27,9 +27,13 @@ def initialize_hass_handler(conn):
     if HASS_CACHE == {}:
         if conn.use_function_call_mode:
             funcs = conn.config["Intent"]["function_call"].get("functions", [])
-            if "hass_get_state" in funcs or "hass_get_state" in funcs:
-                HASS_CACHE['base_url'] = conn.config["plugins"]["home_assistant"].get("base_url")
-                HASS_CACHE['api_key'] = conn.config["plugins"]["home_assistant"].get("api_key")
+            if "hass_get_state" in funcs or "hass_set_state" in funcs:
+                HASS_CACHE["base_url"] = conn.config["plugins"]["home_assistant"].get(
+                    "base_url"
+                )
+                HASS_CACHE["api_key"] = conn.config["plugins"]["home_assistant"].get(
+                    "api_key"
+                )
 
-                check_model_key("home_assistant", HASS_CACHE['api_key'])
+                check_model_key("home_assistant", HASS_CACHE["api_key"])
     return HASS_CACHE

main/xiaozhi-server/plugins_func/functions/hass_set_state.py

@@ -23,7 +23,7 @@ hass_set_state_function_desc = {
                             "description": "需要操作的动作,打开设备:turn_on,关闭设备:turn_off,增加亮度:brightness_up,降低亮度:brightness_down,设置亮度:brightness_value,增加>音量:,volume_up降低音量:volume_down,设置音量:volume_set,设备暂停:pause,设备继续:continue,静音/取消静音:volume_mute"
                         },
                         "input": {
-                            "type": "int",
+                            "type": "integer",
                             "description": "只有在设置音量,设置亮度时候才需要,有效值为1-100,对应音量和亮度的1%-100%"
                         },
                         "is_muted": {

main/xiaozhi-server/plugins_func/register.py

@@ -12,6 +12,7 @@ class ToolType(Enum):
     CHANGE_SYS_PROMPT = (3, "修改系统提示词，切换角色性格或职责")
     SYSTEM_CTL = (4, "系统控制，影响正常的对话流程，如退出、播放音乐等，需要传递conn参数")
     IOT_CTL = (5, "IOT设备控制，需要传递conn参数")
+    MCP_CLIENT = (6, "MCP客户端")
 
     def __init__(self, code, message):
         self.code = code

main/xiaozhi-server/requirements.txt

@@ -22,4 +22,5 @@ mem0ai==0.1.62
 bs4==0.0.2
 modelscope==1.23.2
 sherpa_onnx==1.11.0
-cnlunar==0.2.0
+mcp==1.4.1
+cnlunar==0.2.0