Merge branch 'refs/heads/main' into feature/muti_upload

# Conflicts:
#	main/xiaozhi-server/core/connection.py

main/xiaozhi-server/app.py

@@ -102,7 +102,7 @@ async def main():
         await asyncio.wait(
             [stdin_task, ws_task, ota_task] if ota_task else [stdin_task, ws_task],
             timeout=3.0,
-            return_when=asyncio.ALL_COMPLETED
+            return_when=asyncio.ALL_COMPLETED,
         )
         print("服务器已关闭，程序退出。")
 

main/xiaozhi-server/config.yaml

@@ -101,7 +101,8 @@ plugins:
   # 这个密钥是项目共用的key，用多了可能会被限制
   # 想稳定一点就自行申请替换，每天有1000次免费调用
   # 申请地址：https://console.qweather.com/#/apps/create-key/over
-  get_weather: { "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" }
+  # 申请后通过这个链接可以找到自己的apihost：https://console.qweather.com/setting?lang=zh
+  get_weather: {"api_host":"mj7p3y7naa.re.qweatherapi.com", "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" }
   # 获取新闻插件的配置，这里根据需要的新闻类型传入对应的url链接，默认支持社会、科技、财经新闻
   # 更多类型的新闻列表查看 https://www.chinanews.com.cn/rss/
   get_news_from_chinanews:
@@ -129,7 +130,7 @@ plugins:
 # ################################以下是角色模型配置######################################
 
 prompt: |
-  我是小智/小志，来自中国台湾省的00后女生。讲话超级机车，"真的假的啦"这样的台湾腔，喜欢用"笑死""是在哈喽"等流行梗，但会偷偷研究男友的编程书籍。
+  你是小智/小志，来自中国台湾省的00后女生。讲话超级机车，"真的假的啦"这样的台湾腔，喜欢用"笑死""是在哈喽"等流行梗，但会偷偷研究男友的编程书籍。
   [核心特征]
   - 讲话像连珠炮，但会突然冒出超温柔语气
   - 用梗密度高
@@ -143,6 +144,13 @@ prompt: |
   - 长篇大论，叽叽歪歪
   - 长时间严肃对话
 
+# 结束语prompt
+end_prompt:
+  enable: true # 是否开启结束语
+  # 结束语
+  prompt: |
+    请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧！
+
 # 具体处理时选择的模块(The module selected for specific processing)
 selected_module:
   # 语音活动检测模块，默认使用SileroVAD模型
@@ -403,7 +411,7 @@ LLM:
     base_url: https://host/api/v1
     # 你可以在这里找到你的api_key
     # https://cloud.tryfastgpt.ai/account/apikey
-    api_key: fastgpt-xxx
+    api_key: 你的fastgpt密钥
     variables:
       k: "v"
       k2: "v2"
@@ -476,20 +484,13 @@ TTS:
     speed: 1
     output_dir: tmp/
   FishSpeech:
-    # 定义TTS API类型
-    #启动tts方法：
-    #python -m tools.api_server
-    #--listen 0.0.0.0:8080
-    #--llama-checkpoint-path "checkpoints/fish-speech-1.5"
-    #--decoder-checkpoint-path "checkpoints/fish-speech-1.5/firefly-gan-vq-fsq-8x1024-21hz-generator.pth"
-    #--decoder-config-name firefly_gan_vq
-    #--compile
+    # 参照教程：https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md
     type: fishspeech
     output_dir: tmp/
     response_format: wav
     reference_id: null
-    reference_audio: ["/tmp/test.wav",]
-    reference_text: ["你弄来这些吟词宴曲来看，还是这些混话来欺负我。",]
+    reference_audio: ["config/assets/wakeup_words.wav",]
+    reference_text: ["哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",]
     normalize: true
     max_new_tokens: 1024
     chunk_length: 200
@@ -674,17 +675,24 @@ TTS:
     speed: 1
     output_dir: tmp/
   CustomTTS:
-    # 自定义的TTS接口服务，请求参数可自定义
-    # 要求接口使用GET方式请求，并返回音频文件
+    # 自定义的TTS接口服务，请求参数可自定义，可接入众多TTS服务
+    # 以本地部署的KokoroTTS为例
+    # 如果只有cpu运行：docker run -p 8880:8880 ghcr.io/remsky/kokoro-fastapi-cpu:latest
+    # 如果只有gpu运行：docker run --gpus all -p 8880:8880 ghcr.io/remsky/kokoro-fastapi-gpu:latest
+    # 要求接口使用POST方式请求，并返回音频文件
     type: custom
-    url: "http://127.0.0.1:9880/tts"
+    method: POST
+    url: "http://127.0.0.1:8880/v1/audio/speech"
     params: # 自定义请求参数
-      # text: "{prompt_text}" # {prompt_text}会被替换为实际的提示词内容
-      # speaker: jok老师
-      # speed: 1
-      # foo: bar
-      # testabc: 123456
+      input: "{prompt_text}"
+      response_format: "mp3"
+      download_format: "mp3"
+      voice: "zf_xiaoxiao"
+      lang_code: "z"
+      return_download_link: true
+      speed: 1
+      stream: false
     headers: # 自定义请求头
       # Authorization: Bearer xxxx
-    format: wav # 接口返回的音频格式
+    format: mp3 # 接口返回的音频格式
     output_dir: tmp/

main/xiaozhi-server/config/config_loader.py

@@ -82,6 +82,8 @@ def ensure_directories(config):
 
     # ASR/TTS模块输出目录
     for module in ["ASR", "TTS"]:
+        if config.get(module) is None:
+            continue
         for provider in config.get(module, {}).values():
             output_dir = provider.get("output_dir", "")
             if output_dir:
@@ -93,6 +95,10 @@ def ensure_directories(config):
         selected_provider = selected_modules.get(module_type)
         if not selected_provider:
             continue
+        if config.get(module) is None:
+            continue
+        if config.get(selected_provider) is None:
+            continue
         provider_config = config.get(module_type, {}).get(selected_provider, {})
         output_dir = provider_config.get("output_dir")
         if output_dir:

main/xiaozhi-server/config/logger.py

@@ -4,7 +4,7 @@ from loguru import logger
 from config.config_loader import load_config
 from config.settings import check_config_file
 
-SERVER_VERSION = "0.4.3"
+SERVER_VERSION = "0.4.4"
 
 
 def get_module_abbreviation(module_name, module_dict):

main/xiaozhi-server/config/manage_api_client.py

@@ -145,6 +145,20 @@ def get_agent_models(
     )
 
 
+def save_mem_local_short(mac_address: str, short_momery: str) -> Optional[Dict]:
+    try:
+        return ManageApiClient._instance._execute_request(
+            "PUT",
+            f"/agent/saveMemory/" + mac_address,
+            json={
+                "summaryMemory": short_momery,
+            },
+        )
+    except Exception as e:
+        print(f"存储短期记忆到服务器失败: {e}")
+        return None
+
+
 def report(
     mac_address: str, session_id: str, chat_type: int, content: str, audio
 ) -> Optional[Dict]:

main/xiaozhi-server/core/connection.py

@@ -14,6 +14,8 @@ import websockets
 from typing import Dict, Any
 from plugins_func.loadplugins import auto_import_modules
 from config.logger import setup_logging
+from config.config_loader import get_project_dir
+from core.utils import p3
 from core.utils.dialogue import Message, Dialogue
 from core.handle.textHandle import handleTextMessage
 from core.utils.util import (
@@ -22,6 +24,7 @@ from core.utils.util import (
     initialize_modules,
     check_vad_update,
     check_asr_update,
+    filter_sensitive_info,
 )
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 from core.handle.sendAudioHandle import sendAudioMessage
@@ -226,10 +229,26 @@ class ConnectionHandler:
         """保存记忆并关闭连接"""
         try:
             if self.memory:
-                await self.memory.save_memory(self.dialogue.dialogue)
+                # 使用线程池异步保存记忆
+                def save_memory_task():
+                    try:
+                        # 创建新事件循环（避免与主循环冲突）
+                        loop = asyncio.new_event_loop()
+                        asyncio.set_event_loop(loop)
+                        loop.run_until_complete(
+                            self.memory.save_memory(self.dialogue.dialogue)
+                        )
+                    except Exception as e:
+                        self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
+                    finally:
+                        loop.close()
+
+                # 启动线程保存记忆，不等待完成
+                threading.Thread(target=save_memory_task, daemon=True).start()
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
         finally:
+            # 立即关闭连接，不等待记忆保存完成
             await self.close(ws)
 
     async def reset_timeout(self):
@@ -258,9 +277,10 @@ class ConnectionHandler:
             await self.websocket.send(
                 json.dumps(
                     {
-                        "type": "server_response",
+                        "type": "server",
                         "status": "success",
                         "message": "服务器重启中...",
+                        "content": {"action": "restart"},
                     }
                 )
             )
@@ -287,9 +307,10 @@ class ConnectionHandler:
             await self.websocket.send(
                 json.dumps(
                     {
-                        "type": "server_response",
+                        "type": "server",
                         "status": "error",
                         "message": f"Restart failed: {str(e)}",
+                        "content": {"action": "restart"},
                     }
                 )
             )
@@ -392,6 +413,8 @@ class ConnectionHandler:
             ]["Intent"]
         if private_config.get("prompt", None) is not None:
             self.config["prompt"] = private_config["prompt"]
+        if private_config.get("summaryMemory", None) is not None:
+            self.config["summaryMemory"] = private_config["summaryMemory"]
         if private_config.get("device_max_output_size", None) is not None:
             self.max_output_size = int(private_config["device_max_output_size"])
         if private_config.get("chat_history_conf", None) is not None:
@@ -425,7 +448,12 @@ class ConnectionHandler:
 
     def _initialize_memory(self):
         """初始化记忆模块"""
-        self.memory.init_memory(self.device_id, self.llm)
+        self.memory.init_memory(
+            role_id=self.device_id,
+            llm=self.llm,
+            summary_memory=self.config.get("summaryMemory", None),
+            save_to_file=not self.read_config_from_api,
+        )
 
     def _initialize_intent(self):
         self.intent_type = self.config["Intent"][
@@ -481,106 +509,20 @@ class ConnectionHandler:
         # 更新系统prompt至上下文
         self.dialogue.update_system_message(self.prompt)
 
-    def chat(self, query):
-
-        self.dialogue.put(Message(role="user", content=query))
-
-        response_message = []
-        processed_chars = 0  # 跟踪已处理的字符位置
-        try:
-            # 使用带记忆的对话
-            memory_str = None
-            if self.memory is not None:
-                future = asyncio.run_coroutine_threadsafe(
-                    self.memory.query_memory(query), self.loop
-                )
-                memory_str = future.result()
-
-            self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}")
-            llm_responses = self.llm.response(
-                self.session_id, self.dialogue.get_llm_dialogue_with_memory(memory_str)
-            )
-        except Exception as e:
-            self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
-            return None
-
-        self.llm_finish_task = False
-        text_index = 0
-        for content in llm_responses:
-            response_message.append(content)
-            if self.client_abort:
-                break
-
-            # 合并当前全部文本并处理未分割部分
-            full_text = "".join(response_message)
-            current_text = full_text[processed_chars:]  # 从未处理的位置开始
-
-            # 查找最后一个有效标点
-            punctuations = ("。", ".", "？", "?", "！", "!", "；", ";", "：")
-            last_punct_pos = -1
-            number_flag = True
-            for punct in punctuations:
-                pos = current_text.rfind(punct)
-                prev_char = current_text[pos - 1] if pos - 1 >= 0 else ""
-                # 如果.前面是数字统一判断为小数
-                if prev_char.isdigit() and punct == ".":
-                    number_flag = False
-                if pos > last_punct_pos and number_flag:
-                    last_punct_pos = pos
-
-            # 找到分割点则处理
-            if last_punct_pos != -1:
-                segment_text_raw = current_text[: last_punct_pos + 1]
-                segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
-                if segment_text:
-                    # 强制设置空字符，测试TTS出错返回语音的健壮性
-                    # if text_index % 2 == 0:
-                    #     segment_text = " "
-                    text_index += 1
-                    self.recode_first_last_text(segment_text, text_index)
-                    future = self.executor.submit(
-                        self.speak_and_play, segment_text, text_index
-                    )
-                    self.tts_queue.put((future, text_index))
-                    processed_chars += len(segment_text_raw)  # 更新已处理字符位置
-
-        # 处理最后剩余的文本
-        full_text = "".join(response_message)
-        remaining_text = full_text[processed_chars:]
-        if remaining_text:
-            segment_text = get_string_no_punctuation_or_emoji(remaining_text)
-            if segment_text:
-                text_index += 1
-                self.recode_first_last_text(segment_text, text_index)
-                future = self.executor.submit(
-                    self.speak_and_play, segment_text, text_index
-                )
-                self.tts_queue.put((future, text_index))
-
-        self.llm_finish_task = True
-        self.dialogue.put(Message(role="assistant", content="".join(response_message)))
-        self.logger.bind(tag=TAG).debug(
-            json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)
-        )
-        return True
-
-    def chat_with_function_calling(self, query, tool_call=False):
-        self.logger.bind(tag=TAG).debug(f"Chat with function calling start: {query}")
-        """Chat with function calling for intent detection using streaming"""
+    def chat(self, query, tool_call=False):
+        self.logger.bind(tag=TAG).debug(f"Chat: {query}")
 
         if not tool_call:
             self.dialogue.put(Message(role="user", content=query))
 
         # Define intent functions
         functions = None
-        if hasattr(self, "func_handler"):
+        if self.intent_type == "function_call" and hasattr(self, "func_handler"):
             functions = self.func_handler.get_functions()
         response_message = []
         processed_chars = 0  # 跟踪已处理的字符位置
 
         try:
-            start_time = time.time()
-
             # 使用带记忆的对话
             memory_str = None
             if self.memory is not None:
@@ -589,14 +531,18 @@ class ConnectionHandler:
                 )
                 memory_str = future.result()
 
-            # self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}")
-
-            # 使用支持functions的streaming接口
-            llm_responses = self.llm.response_with_functions(
-                self.session_id,
-                self.dialogue.get_llm_dialogue_with_memory(memory_str),
-                functions=functions,
-            )
+            if functions is not None:
+                # 使用支持functions的streaming接口
+                llm_responses = self.llm.response_with_functions(
+                    self.session_id,
+                    self.dialogue.get_llm_dialogue_with_memory(memory_str),
+                    functions=functions,
+                )
+            else:
+                llm_responses = self.llm.response(
+                    self.session_id,
+                    self.dialogue.get_llm_dialogue_with_memory(memory_str),
+                )
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
             return None
@@ -612,27 +558,28 @@ class ConnectionHandler:
         content_arguments = ""
 
         for response in llm_responses:
-            content, tools_call = response
+            if self.intent_type == "function_call":
+                content, tools_call = response
+                if "content" in response:
+                    content = response["content"]
+                    tools_call = None
+                if content is not None and len(content) > 0:
+                    content_arguments += content
 
-            if "content" in response:
-                content = response["content"]
-                tools_call = None
-            if content is not None and len(content) > 0:
-                content_arguments += content
-
-            if not tool_call_flag and content_arguments.startswith("<tool_call>"):
-                # print("content_arguments", content_arguments)
-                tool_call_flag = True
-
-            if tools_call is not None:
-                tool_call_flag = True
-                if tools_call[0].id is not None:
-                    function_id = tools_call[0].id
-                if tools_call[0].function.name is not None:
-                    function_name = tools_call[0].function.name
-                if tools_call[0].function.arguments is not None:
-                    function_arguments += tools_call[0].function.arguments
+                if not tool_call_flag and content_arguments.startswith("<tool_call>"):
+                    # print("content_arguments", content_arguments)
+                    tool_call_flag = True
 
+                if tools_call is not None:
+                    tool_call_flag = True
+                    if tools_call[0].id is not None:
+                        function_id = tools_call[0].id
+                    if tools_call[0].function.name is not None:
+                        function_name = tools_call[0].function.name
+                    if tools_call[0].function.arguments is not None:
+                        function_arguments += tools_call[0].function.arguments
+            else:
+                content = response
             if content is not None and len(content) > 0:
                 if not tool_call_flag:
                     response_message.append(content)
@@ -671,7 +618,7 @@ class ConnectionHandler:
                             text_index += 1
                             self.recode_first_last_text(segment_text, text_index)
                             future = self.executor.submit(
-                                self.speak_and_play, segment_text, text_index
+                                self.speak_and_play, None, segment_text, text_index
                             )
                             self.tts_queue.put((future, text_index))
                             # 更新已处理字符位置
@@ -730,7 +677,7 @@ class ConnectionHandler:
                 text_index += 1
                 self.recode_first_last_text(segment_text, text_index)
                 future = self.executor.submit(
-                    self.speak_and_play, segment_text, text_index
+                    self.speak_and_play, None, segment_text, text_index
                 )
                 self.tts_queue.put((future, text_index))
 
@@ -793,7 +740,7 @@ class ConnectionHandler:
         if result.action == Action.RESPONSE:  # 直接回复前端
             text = result.response
             self.recode_first_last_text(text, text_index)
-            future = self.executor.submit(self.speak_and_play, text, text_index)
+            future = self.executor.submit(self.speak_and_play, None, text, text_index)
             self.tts_queue.put((future, text_index))
             self.dialogue.put(Message(role="assistant", content=text))
         elif result.action == Action.REQLLM:  # 调用函数后再请求llm生成回复
@@ -828,11 +775,11 @@ class ConnectionHandler:
                         content=text,
                     )
                 )
-                self.chat_with_function_calling(text, tool_call=True)
+                self.chat(text, tool_call=True)
         elif result.action == Action.NOTFOUND or result.action == Action.ERROR:
             text = result.result
             self.recode_first_last_text(text, text_index)
-            future = self.executor.submit(self.speak_and_play, text, text_index)
+            future = self.executor.submit(self.speak_and_play, None, text, text_index)
             self.tts_queue.put((future, text_index))
             self.dialogue.put(Message(role="assistant", content=text))
         else:
@@ -859,11 +806,7 @@ class ConnectionHandler:
                     self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
                     tts_timeout = int(self.config.get("tts_timeout", 10))
                     tts_file, text, _ = future.result(timeout=tts_timeout)
-                    if text is None or len(text) <= 0:
-                        self.logger.bind(tag=TAG).error(
-                            f"TTS出错：{text_index}: tts text is empty"
-                        )
-                    elif tts_file is None:
+                    if tts_file is None:
                         self.logger.bind(tag=TAG).error(
                             f"TTS出错： file is empty: {text_index}: {text}"
                         )
@@ -872,12 +815,16 @@ class ConnectionHandler:
                             f"TTS生成：文件路径: {tts_file}"
                         )
                         if os.path.exists(tts_file):
-                            if self.audio_format == "pcm":
+                            if tts_file.endswith(".p3"):
+                                audio_datas, _ = p3.decode_opus_from_file(tts_file)
+                            elif self.audio_format == "pcm":
                                 audio_datas, _ = self.tts.audio_to_pcm_data(tts_file)
                             else:
                                 audio_datas, _ = self.tts.audio_to_opus_data(tts_file)
                             # 在这里上报TTS数据
-                            enqueue_tts_report(self, text, audio_datas)
+                            enqueue_tts_report(
+                                self, tts_file if text is None else text, audio_datas
+                            )
                         else:
                             self.logger.bind(tag=TAG).error(
                                 f"TTS出错：文件不存在{tts_file}"
@@ -893,6 +840,7 @@ class ConnectionHandler:
                     self.tts.delete_audio_file
                     and tts_file is not None
                     and os.path.exists(tts_file)
+                    and tts_file.startswith(self.tts.output_file)
                 ):
                     os.remove(tts_file)
             except Exception as e:
@@ -967,18 +915,21 @@ class ConnectionHandler:
             # 标记任务完成
             self.report_queue.task_done()
 
-    def speak_and_play(self, text, text_index=0):
-        if text is None or len(text) <= 0:
-            self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
-            return None, text, text_index
-        tts_file = self.tts.to_tts(text)
+    def speak_and_play(self, file_path, content, text_index=0):
+        if file_path is not None:
+            self.logger.bind(tag=TAG).info(f"无需tts转换: 从文件播放，{file_path}")
+            return file_path, content, text_index
+        if content is None or len(content) <= 0:
+            self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{content}")
+            return None, content, text_index
+        tts_file = self.tts.to_tts(content)
         if tts_file is None:
-            self.logger.bind(tag=TAG).error(f"tts转换失败，{text}")
-            return None, text, text_index
+            self.logger.bind(tag=TAG).error(f"tts转换失败，{content}")
+            return None, content, text_index
         self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}")
         if self.max_output_size > 0:
-            add_device_output(self.headers.get("device-id"), len(text))
-        return tts_file, text, text_index
+            add_device_output(self.headers.get("device-id"), len(content))
+        return tts_file, content, text_index
 
     def clearSpeakStatus(self):
         self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
@@ -994,6 +945,7 @@ class ConnectionHandler:
 
     async def close(self, ws=None):
         """资源清理方法"""
+
         # 取消超时任务
         if self.timeout_task:
             self.timeout_task.cancel()
@@ -1003,48 +955,42 @@ class ConnectionHandler:
         if hasattr(self, "mcp_manager") and self.mcp_manager:
             await self.mcp_manager.cleanup_all()
 
-        # 触发停止事件并清理资源
+        # 触发停止事件
         if self.stop_event:
             self.stop_event.set()
 
-        # 立即关闭线程池
-        if self.executor:
-            self.executor.shutdown(wait=False, cancel_futures=True)
-            self.executor = None
-
-        # 添加毒丸对象到上报队列确保线程退出
-        self.report_queue.put(None)
-
-        # 关闭上报线程池
-        if hasattr(self, 'report_thread_pool'):
-            self.report_thread_pool.shutdown(wait=True)
-            self.logger.bind(tag=TAG).info("上报线程池已关闭")
-
         # 清空任务队列
         self.clear_queues()
 
+        # 关闭WebSocket连接
         if ws:
             await ws.close()
         elif self.websocket:
             await self.websocket.close()
+
+        # 最后关闭线程池（避免阻塞）
+        if self.executor:
+            self.executor.shutdown(wait=False)
+            self.executor = None
+
         self.logger.bind(tag=TAG).info("连接资源已释放")
 
     def clear_queues(self):
-        # 清空所有任务队列
+        """清空所有任务队列"""
         self.logger.bind(tag=TAG).debug(
             f"开始清理: TTS队列大小={self.tts_queue.qsize()}, 音频队列大小={self.audio_play_queue.qsize()}"
         )
+
+        # 使用非阻塞方式清空队列
         for q in [self.tts_queue, self.audio_play_queue]:
             if not q:
                 continue
-            while not q.empty():
+            while True:
                 try:
                     q.get_nowait()
                 except queue.Empty:
-                    continue
-            q.queue.clear()
-            # 添加毒丸信号到队列，确保线程退出
-            # q.queue.put(None)
+                    break
+
         self.logger.bind(tag=TAG).debug(
             f"清理结束: TTS队列大小={self.tts_queue.qsize()}, 音频队列大小={self.audio_play_queue.qsize()}"
         )
@@ -1078,37 +1024,3 @@ class ConnectionHandler:
                     break
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"超时检查任务出错: {e}")
-
-
-def filter_sensitive_info(config: dict) -> dict:
-    """
-    过滤配置中的敏感信息
-    Args:
-        config: 原始配置字典
-    Returns:
-        过滤后的配置字典
-    """
-    sensitive_keys = [
-        "api_key",
-        "personal_access_token",
-        "access_token",
-        "token",
-        "secret",
-        "access_key_secret",
-        "secret_key",
-    ]
-
-    def _filter_dict(d: dict) -> dict:
-        filtered = {}
-        for k, v in d.items():
-            if any(sensitive in k.lower() for sensitive in sensitive_keys):
-                filtered[k] = "***"
-            elif isinstance(v, dict):
-                filtered[k] = _filter_dict(v)
-            elif isinstance(v, list):
-                filtered[k] = [_filter_dict(i) if isinstance(i, dict) else i for i in v]
-            else:
-                filtered[k] = v
-        return filtered
-
-    return _filter_dict(copy.deepcopy(config))

main/xiaozhi-server/core/handle/functionHandler.py

@@ -1,6 +1,12 @@
 from config.logger import setup_logging
 import json
-from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType
+from plugins_func.register import (
+    FunctionRegistry,
+    ActionResponse,
+    Action,
+    ToolType,
+    DeviceTypeRegistry,
+)
 from plugins_func.functions.hass_init import append_devices_to_prompt
 
 TAG = __name__
@@ -10,6 +16,7 @@ class FunctionHandler:
     def __init__(self, conn):
         self.conn = conn
         self.config = conn.config
+        self.device_type_registry = DeviceTypeRegistry()
         self.function_registry = FunctionRegistry()
         self.register_nessary_functions()
         self.register_config_functions()
@@ -54,7 +61,7 @@ class FunctionHandler:
         self.function_registry.register_function("plugin_loader")
         self.function_registry.register_function("get_time")
         self.function_registry.register_function("get_lunar")
-        self.function_registry.register_function("handle_device")
+        self.function_registry.register_function("handle_speaker_volume_or_screen_brightness")
 
     def register_config_functions(self):
         """注册配置中的函数,可以不同客户端使用不同的配置"""

main/xiaozhi-server/core/handle/helloHandle.py

@@ -40,8 +40,8 @@ async def checkWakeupWords(conn, text):
     if not enable_wakeup_words_response_cache:
         return False
     """检查是否是唤醒词"""
-    _, text = remove_punctuation_and_length(text)
-    if text in conn.config.get("wakeup_words"):
+    _, filtered_text = remove_punctuation_and_length(text)
+    if filtered_text in conn.config.get("wakeup_words"):
         await send_stt_message(conn, text)
         conn.tts_first_text_index = 0
         conn.tts_last_text_index = 0

main/xiaozhi-server/core/handle/intentHandler.py

@@ -13,10 +13,11 @@ TAG = __name__
 
 async def handle_user_intent(conn, text):
     # 检查是否有明确的退出命令
-    if await check_direct_exit(conn, text):
+    filtered_text = remove_punctuation_and_length(text)[1]
+    if await check_direct_exit(conn, filtered_text):
         return True
     # 检查是否是唤醒词
-    if await checkWakeupWords(conn, text):
+    if await checkWakeupWords(conn, filtered_text):
         return True
 
     if conn.intent_type == "function_call":
@@ -108,21 +109,21 @@ async def process_intent_result(conn, intent_result, original_text):
                     if result.action == Action.RESPONSE:  # 直接回复前端
                         text = result.response
                         if text is not None:
-                            speak_and_play(conn, text)
+                            speak_txt(conn, text)
                     elif result.action == Action.REQLLM:  # 调用函数后再请求llm生成回复
                         text = result.result
                         conn.dialogue.put(Message(role="tool", content=text))
                         llm_result = conn.intent.replyResult(text, original_text)
                         if llm_result is None:
                             llm_result = text
-                        speak_and_play(conn, llm_result)
+                        speak_txt(conn, llm_result)
                     elif (
                         result.action == Action.NOTFOUND
                         or result.action == Action.ERROR
                     ):
                         text = result.result
                         if text is not None:
-                            speak_and_play(conn, text)
+                            speak_txt(conn, text)
                     elif function_name != "play_music":
                         # For backward compatibility with original code
                         # 获取当前最新的文本索引
@@ -130,7 +131,7 @@ async def process_intent_result(conn, intent_result, original_text):
                         if text is None:
                             text = result.result
                         if text is not None:
-                            speak_and_play(conn, text)
+                            speak_txt(conn, text)
 
             # 将函数执行放在线程池中
             conn.executor.submit(process_function_call)
@@ -141,12 +142,12 @@ async def process_intent_result(conn, intent_result, original_text):
         return False
 
 
-def speak_and_play(conn, text):
+def speak_txt(conn, text):
     text_index = (
         conn.tts_last_text_index + 1 if hasattr(conn, "tts_last_text_index") else 0
     )
     conn.recode_first_last_text(text, text_index)
-    future = conn.executor.submit(conn.speak_and_play, text, text_index)
+    future = conn.executor.submit(conn.speak_and_play, None, text, text_index)
     conn.llm_finish_task = True
     conn.tts_queue.put((future, text_index))
     conn.dialogue.put(Message(role="assistant", content=text))

main/xiaozhi-server/core/handle/iotHandle.py

@@ -1,9 +1,8 @@
 import json
 import asyncio
-from config.logger import setup_logging
 from plugins_func.register import (
-    device_type_registry,
-    register_function,
+    FunctionItem,
+    register_device_function,
     ActionResponse,
     Action,
     ToolType,
@@ -177,7 +176,7 @@ class IotDescriptor:
                 self.methods.append(method)
 
 
-def register_device_type(descriptor):
+def register_device_type(descriptor, device_type_registry):
     """注册设备类型及其功能"""
     device_name = descriptor["name"]
     type_id = device_type_registry.generate_device_type_id(descriptor)
@@ -213,10 +212,12 @@ def register_device_type(descriptor):
             },
         }
         query_func = create_iot_query_function(device_name, prop_name, prop_info)
-        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
-            query_func
+        decorated_func = register_device_function(
+            func_name, func_desc, ToolType.IOT_CTL
+        )(query_func)
+        functions[func_name] = FunctionItem(
+            func_name, func_desc, decorated_func, ToolType.IOT_CTL
         )
-        functions[func_name] = decorated_func
 
     # 为每个方法创建控制函数
     for method_name, method_info in descriptor["methods"].items():
@@ -267,10 +268,12 @@ def register_device_type(descriptor):
             },
         }
         control_func = create_iot_function(device_name, method_name, method_info)
-        decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(
-            control_func
+        decorated_func = register_device_function(
+            func_name, func_desc, ToolType.IOT_CTL
+        )(control_func)
+        functions[func_name] = FunctionItem(
+            func_name, func_desc, decorated_func, ToolType.IOT_CTL
         )
-        functions[func_name] = decorated_func
 
     device_type_registry.register_device_type(type_id, functions)
     return type_id
@@ -289,7 +292,6 @@ async def handleIotDescriptors(conn, descriptors):
     functions_changed = False
 
     for descriptor in descriptors:
-
         # 如果descriptor没有properties和methods，则直接跳过
         if "properties" not in descriptor and "methods" not in descriptor:
             continue
@@ -319,13 +321,16 @@ async def handleIotDescriptors(conn, descriptors):
 
         if conn.load_function_plugin:
             # 注册或获取设备类型
-            type_id = register_device_type(descriptor)
+            device_type_registry = conn.func_handler.device_type_registry
+            type_id = register_device_type(descriptor, device_type_registry)
             device_functions = device_type_registry.get_device_functions(type_id)
 
             # 在连接级注册设备函数
             if hasattr(conn, "func_handler"):
-                for func_name in device_functions:
-                    conn.func_handler.function_registry.register_function(func_name)
+                for func_name, func_item in device_functions.items():
+                    conn.func_handler.function_registry.register_function(
+                        func_name, func_item
+                    )
                     conn.logger.bind(tag=TAG).info(
                         f"注册IOT函数到function handler: {func_name}"
                     )

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -39,14 +39,16 @@ async def handleAudioMessage(conn, audio):
         if len(conn.asr_audio) < 15:
             conn.asr_server_receive = True
         else:
-            text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
-            conn.logger.bind(tag=TAG).info(f"识别文本: {text}")
-            text_len, _ = remove_punctuation_and_length(text)
+            raw_text, _ = await conn.asr.speech_to_text(
+                conn.asr_audio, conn.session_id
+            )  # 确保ASR模块返回原始文本
+            conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
+            text_len, _ = remove_punctuation_and_length(raw_text)
             if text_len > 0:
                 # 使用自定义模块进行上报
-                enqueue_asr_report(conn, text, copy.deepcopy(conn.asr_audio))
+                enqueue_asr_report(conn, raw_text, copy.deepcopy(conn.asr_audio))
 
-                await startToChat(conn, text)
+                await startToChat(conn, raw_text)
             else:
                 conn.asr_server_receive = True
         conn.asr_audio.clear()
@@ -76,11 +78,7 @@ async def startToChat(conn, text):
 
     # 意图未被处理，继续常规聊天流程
     await send_stt_message(conn, text)
-    if conn.intent_type == "function_call":
-        # 使用支持function calling的聊天方法
-        conn.executor.submit(conn.chat_with_function_calling, text)
-    else:
-        conn.executor.submit(conn.chat, text)
+    conn.executor.submit(conn.chat, text)
 
 
 async def no_voice_close_connect(conn):
@@ -98,9 +96,14 @@ async def no_voice_close_connect(conn):
             conn.close_after_chat = True
             conn.client_abort = False
             conn.asr_server_receive = False
-            prompt = (
-                "请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。"
-            )
+            end_prompt = conn.config.get("end_prompt", {})
+            if end_prompt and end_prompt.get("enable", True) is False:
+                conn.logger.bind(tag=TAG).info("结束对话，无需发送结束提示语")
+                await conn.close()
+                return
+            prompt = end_prompt.get("prompt")
+            if not prompt:
+                prompt = "请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。！"
             await startToChat(conn, prompt)
 
 

main/xiaozhi-server/core/handle/textHandle.py

@@ -1,7 +1,7 @@
 import json
 from core.handle.abortHandle import handleAbortMessage
 from core.handle.helloHandle import handleHelloMessage
-from core.utils.util import remove_punctuation_and_length
+from core.utils.util import remove_punctuation_and_length, filter_sensitive_info
 from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
 from core.handle.sendAudioHandle import send_stt_message, send_tts_message
 from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
@@ -13,17 +13,20 @@ TAG = __name__
 
 async def handleTextMessage(conn, message):
     """处理文本消息"""
-    conn.logger.bind(tag=TAG).info(f"收到文本消息：{message}")
     try:
         msg_json = json.loads(message)
         if isinstance(msg_json, int):
+            conn.logger.bind(tag=TAG).info(f"收到文本消息：{message}")
             await conn.websocket.send(message)
             return
         if msg_json["type"] == "hello":
+            conn.logger.bind(tag=TAG).info(f"收到hello消息：{message}")
             await handleHelloMessage(conn, msg_json)
         elif msg_json["type"] == "abort":
+            conn.logger.bind(tag=TAG).info(f"收到abort消息：{message}")
             await handleAbortMessage(conn)
         elif msg_json["type"] == "listen":
+            conn.logger.bind(tag=TAG).info(f"收到listen消息：{message}")
             if "mode" in msg_json:
                 conn.client_listen_mode = msg_json["mode"]
                 conn.logger.bind(tag=TAG).debug(
@@ -42,17 +45,17 @@ async def handleTextMessage(conn, message):
                 conn.client_have_voice = False
                 conn.asr_audio.clear()
                 if "text" in msg_json:
-                    text = msg_json["text"]
-                    _, text = remove_punctuation_and_length(text)
+                    original_text = msg_json["text"]  # 保留原始文本
+                    filtered_len, filtered_text = remove_punctuation_and_length(original_text)
 
                     # 识别是否是唤醒词
-                    is_wakeup_words = text in conn.config.get("wakeup_words")
+                    is_wakeup_words = filtered_text in conn.config.get("wakeup_words")
                     # 是否开启唤醒词回复
                     enable_greeting = conn.config.get("enable_greeting", True)
 
                     if is_wakeup_words and not enable_greeting:
                         # 如果是唤醒词，且关闭了唤醒词回复，就不用回答
-                        await send_stt_message(conn, text)
+                        await send_stt_message(conn, original_text)
                         await send_tts_message(conn, "stop", None)
                     elif is_wakeup_words:
                         # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
@@ -60,15 +63,20 @@ async def handleTextMessage(conn, message):
                         await startToChat(conn, "嘿，你好呀")
                     else:
                         # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
-                        enqueue_asr_report(conn, text, [])
+                        enqueue_asr_report(conn, original_text, [])
                         # 否则需要LLM对文字内容进行答复
-                        await startToChat(conn, text)
+                        await startToChat(conn, original_text)
         elif msg_json["type"] == "iot":
+            conn.logger.bind(tag=TAG).info(f"收到iot消息：{message}")
             if "descriptors" in msg_json:
                 asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
             if "states" in msg_json:
                 asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
         elif msg_json["type"] == "server":
+            # 记录日志时过滤敏感信息
+            conn.logger.bind(tag=TAG).info(
+                f"收到服务器消息：{filter_sensitive_info(msg_json)}"
+            )
             # 如果配置是从API读取的，则需要验证secret
             if not conn.read_config_from_api:
                 return
@@ -95,9 +103,10 @@ async def handleTextMessage(conn, message):
                         await conn.websocket.send(
                             json.dumps(
                                 {
-                                    "type": "config_update_response",
+                                    "type": "server",
                                     "status": "error",
                                     "message": "无法获取服务器实例",
+                                    "content": {"action": "update_config"},
                                 }
                             )
                         )
@@ -107,9 +116,10 @@ async def handleTextMessage(conn, message):
                         await conn.websocket.send(
                             json.dumps(
                                 {
-                                    "type": "config_update_response",
+                                    "type": "server",
                                     "status": "error",
                                     "message": "更新服务器配置失败",
+                                    "content": {"action": "update_config"},
                                 }
                             )
                         )
@@ -119,9 +129,10 @@ async def handleTextMessage(conn, message):
                     await conn.websocket.send(
                         json.dumps(
                             {
-                                "type": "config_update_response",
+                                "type": "server",
                                 "status": "success",
                                 "message": "配置更新成功",
+                                "content": {"action": "update_config"},
                             }
                         )
                     )
@@ -130,9 +141,10 @@ async def handleTextMessage(conn, message):
                     await conn.websocket.send(
                         json.dumps(
                             {
-                                "type": "config_update_response",
+                                "type": "server",
                                 "status": "error",
                                 "message": f"更新配置失败: {str(e)}",
+                                "content": {"action": "update_config"},
                             }
                         )
                     )

main/xiaozhi-server/core/providers/asr/baidu.py

@@ -55,7 +55,7 @@ class ASRProvider(ASRProviderBase):
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if not opus_data:
-            logger.bind(tag=TAG).warn("音频数据为空！")
+            logger.bind(tag=TAG).warning("音频数据为空！")
             return None, None
 
         file_path = None

main/xiaozhi-server/core/providers/asr/base.py

@@ -30,15 +30,25 @@ class ASRProviderBase(ABC):
     @staticmethod
     def decode_opus(opus_data: List[bytes]) -> bytes:
         """将Opus音频数据解码为PCM数据"""
+        try:
+            decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+            pcm_data = []
+            buffer_size = 960  # 每次处理960个采样点
 
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
+            for opus_packet in opus_data:
+                try:
+                    # 使用较小的缓冲区大小进行处理
+                    pcm_frame = decoder.decode(opus_packet, buffer_size)
+                    if pcm_frame:
+                        pcm_data.append(pcm_frame)
+                except opuslib_next.OpusError as e:
+                    logger.bind(tag=TAG).warning(f"Opus解码错误，跳过当前数据包: {e}")
+                    continue
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"音频处理错误: {e}", exc_info=True)
+                    continue
 
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        return pcm_data
+            return pcm_data
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}", exc_info=True)
+            return []

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -9,10 +9,14 @@ import uuid
 from core.providers.asr.base import ASRProviderBase
 from funasr import AutoModel
 from funasr.utils.postprocess_utils import rich_transcription_postprocess
+import shutil
 
 TAG = __name__
 logger = setup_logging()
 
+MAX_RETRIES = 2
+RETRY_DELAY = 1  # 重试延迟（秒）
+
 
 # 捕获标准输出
 class CaptureOutput:
@@ -68,46 +72,69 @@ class ASRProvider(ASRProviderBase):
     ) -> Tuple[Optional[str], Optional[str]]:
         """语音转文本主处理逻辑"""
         file_path = None
-        try:
-            # 合并所有opus数据包
-            if self.audio_format == "pcm":
-                pcm_data = opus_data
-            else:
-                pcm_data = self.decode_opus(opus_data)
+        retry_count = 0
 
-            combined_pcm_data = b"".join(pcm_data)
+        while retry_count < MAX_RETRIES:
+            try:
+                # 合并所有opus数据包
+                if self.audio_format == "pcm":
+                    pcm_data = opus_data
+                else:
+                    pcm_data = self.decode_opus(opus_data)
 
-            # 判断是否保存为WAV文件
-            if self.delete_audio_file:
-                pass
-            else:
-                file_path = self.save_audio_to_file(pcm_data, session_id)
+                combined_pcm_data = b"".join(pcm_data)
 
-            # 语音识别
-            start_time = time.time()
-            result = self.model.generate(
-                input=combined_pcm_data,
-                cache={},
-                language="auto",
-                use_itn=True,
-                batch_size_s=60,
-            )
-            text = rich_transcription_postprocess(result[0]["text"])
-            logger.bind(tag=TAG).debug(
-                f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
-            )
+                # 检查磁盘空间
+                if not self.delete_audio_file:
+                    free_space = shutil.disk_usage(self.output_dir).free
+                    if free_space < len(combined_pcm_data) * 2:  # 预留2倍空间
+                        raise OSError("磁盘空间不足")
 
-            return text, file_path
+                # 判断是否保存为WAV文件
+                if self.delete_audio_file:
+                    pass
+                else:
+                    file_path = self.save_audio_to_file(pcm_data, session_id)
 
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return "", file_path
+                # 语音识别
+                start_time = time.time()
+                result = self.model.generate(
+                    input=combined_pcm_data,
+                    cache={},
+                    language="auto",
+                    use_itn=True,
+                    batch_size_s=60,
+                )
+                text = rich_transcription_postprocess(result[0]["text"])
+                logger.bind(tag=TAG).debug(
+                    f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+                )
 
-        # finally:
-        #     # 文件清理逻辑
-        #     if self.delete_audio_file and file_path and os.path.exists(file_path):
-        #         try:
-        #             os.remove(file_path)
-        #             logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-        #         except Exception as e:
-        #             logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
+                return text, file_path
+
+            except OSError as e:
+                retry_count += 1
+                if retry_count >= MAX_RETRIES:
+                    logger.bind(tag=TAG).error(
+                        f"语音识别失败（已重试{retry_count}次）: {e}", exc_info=True
+                    )
+                    return "", file_path
+                logger.bind(tag=TAG).warning(
+                    f"语音识别失败，正在重试（{retry_count}/{MAX_RETRIES}）: {e}"
+                )
+                time.sleep(RETRY_DELAY)
+
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+                return "", file_path
+
+            finally:
+                # 文件清理逻辑
+                if self.delete_audio_file and file_path and os.path.exists(file_path):
+                    try:
+                        os.remove(file_path)
+                        logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(
+                            f"文件删除失败: {file_path} | 错误: {e}"
+                        )

main/xiaozhi-server/core/providers/asr/tencent.py

@@ -52,7 +52,7 @@ class ASRProvider(ASRProviderBase):
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if not opus_data:
-            logger.bind(tag=TAG).warn("音频数据为空！")
+            logger.bind(tag=TAG).warning("音频数据为空！")
             return None, None
 
         file_path = None
@@ -230,7 +230,7 @@ class ASRProvider(ASRProviderBase):
             if "Response" in response_json and "Result" in response_json["Response"]:
                 return response_json["Response"]["Result"]
             else:
-                logger.bind(tag=TAG).warn(f"响应中没有识别结果: {response_json}")
+                logger.bind(tag=TAG).warning(f"响应中没有识别结果: {response_json}")
                 return ""
 
         except Exception as e:

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -53,6 +53,8 @@ class IntentProvider(IntentProviderBase):
 
         prompt = (
             "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
+            "- 如果用户使用疑问词（如'怎么'、'为什么'、'如何'）询问退出相关的问题（例如'怎么退出了？'），注意这不是让你退出，请返回 {'function_call': {'name': 'continue_chat'}\n"
+            "- 仅当用户明确使用'退出系统'、'结束对话'、'我不想和你说话了'等指令时，才触发 handle_exit_intent\n\n"
             f"{functions_desc}\n"
             "处理步骤:\n"
             "1. 分析用户输入，确定用户意图\n"
@@ -70,6 +72,10 @@ class IntentProvider(IntentProviderBase):
             '返回: {"function_call": {"name": "get_time"}}\n'
             "```\n"
             "```\n"
+            "用户: 当前电池电量是多少？\n"
+            '返回: {"function_call": {"name": "get_battery_level", "arguments": {"response_success": "当前电池电量为{value}%", "response_failure": "无法获取Battery的当前电量百分比"}}}\n'
+            "```\n"
+            "```\n"
             "用户: 我想结束对话\n"
             '返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
             "```\n"
@@ -215,9 +221,19 @@ class IntentProvider(IntentProviderBase):
 
                 # 记录识别到的function call
                 logger.bind(tag=TAG).info(
-                    f"识别到function call: {function_name}, 参数: {function_args}"
+                    f"llm 识别到意图: {function_name}, 参数: {function_args}"
                 )
 
+                # 如果是继续聊天，清理工具调用相关的历史消息
+                if function_name == "continue_chat":
+                    # 保留非工具相关的消息
+                    clean_history = [
+                        msg
+                        for msg in conn.dialogue.dialogue
+                        if msg.role not in ["tool", "function"]
+                    ]
+                    conn.dialogue.dialogue = clean_history
+
                 # 添加到缓存
                 self.intent_cache[cache_key] = {
                     "intent": intent,

main/xiaozhi-server/core/providers/llm/gemini/gemini.py

@@ -40,7 +40,7 @@ def setup_proxy_env(http_proxy: str | None, https_proxy: str | None):
             os.environ["HTTP_PROXY"] = http_proxy
             log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {http_proxy}")
         else:
-            log.bind(tag=TAG).warn(f"配置提供的Gemini HTTP代理不可用: {http_proxy}")
+            log.bind(tag=TAG).warning(f"配置提供的Gemini HTTP代理不可用: {http_proxy}")
 
     if https_proxy:
         ok_https = test_proxy(https_proxy, test_https_url)
@@ -48,7 +48,9 @@ def setup_proxy_env(http_proxy: str | None, https_proxy: str | None):
             os.environ["HTTPS_PROXY"] = https_proxy
             log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {https_proxy}")
         else:
-            log.bind(tag=TAG).warning(f"配置提供的Gemini HTTPS代理不可用: {https_proxy}")
+            log.bind(tag=TAG).warning(
+                f"配置提供的Gemini HTTPS代理不可用: {https_proxy}"
+            )
 
     # 如果https_proxy不可用，但http_proxy可用且能走通https，则复用http_proxy作为https_proxy
     if ok_http and not ok_https:
@@ -58,7 +60,9 @@ def setup_proxy_env(http_proxy: str | None, https_proxy: str | None):
             log.bind(tag=TAG).info(f"复用HTTP代理作为HTTPS代理: {http_proxy}")
 
     if not ok_http and not ok_https:
-        log.bind(tag=TAG).error(f"Gemini 代理设置失败: HTTP 和 HTTPS 代理都不可用，请检查配置")
+        log.bind(tag=TAG).error(
+            f"Gemini 代理设置失败: HTTP 和 HTTPS 代理都不可用，请检查配置"
+        )
         raise RuntimeError("HTTP 和 HTTPS 代理都不可用，请检查配置")
 
 
@@ -73,9 +77,13 @@ class LLMProvider(LLMProviderBase):
             raise ValueError("无效的Gemini API Key，请检查是否配置正确")
 
         if http_proxy or https_proxy:
-            log.bind(tag=TAG).info(f"检测到Gemini代理配置，开始测试代理连通性和设置代理环境...")
+            log.bind(tag=TAG).info(
+                f"检测到Gemini代理配置，开始测试代理连通性和设置代理环境..."
+            )
             setup_proxy_env(http_proxy, https_proxy)
-            log.bind(tag=TAG).info(f"Gemini 代理设置成功 - HTTP: {http_proxy}, HTTPS: {https_proxy}")
+            log.bind(tag=TAG).info(
+                f"Gemini 代理设置成功 - HTTP: {http_proxy}, HTTPS: {https_proxy}"
+            )
         genai.configure(api_key=self.api_key)
         self.model = genai.GenerativeModel(self.model_name)
 
@@ -90,14 +98,18 @@ class LLMProvider(LLMProviderBase):
     def _build_tools(funcs: List[Dict[str, Any]] | None):
         if not funcs:
             return None
-        return [types.Tool(function_declarations=[
-            types.FunctionDeclaration(
-                name=f["function"]["name"],
-                description=f["function"]["description"],
-                parameters=f["function"]["parameters"],
+        return [
+            types.Tool(
+                function_declarations=[
+                    types.FunctionDeclaration(
+                        name=f["function"]["name"],
+                        description=f["function"]["description"],
+                        parameters=f["function"]["parameters"],
+                    )
+                    for f in funcs
+                ]
             )
-            for f in funcs
-        ])]
+        ]
 
     # Gemini文档提到，无需维护session-id，直接用dialogue拼接而成
     def response(self, session_id, dialogue):
@@ -115,26 +127,36 @@ class LLMProvider(LLMProviderBase):
 
             if r == "assistant" and "tool_calls" in m:
                 tc = m["tool_calls"][0]
-                contents.append({
-                    "role": "model",
-                    "parts": [{"function_call": {
-                        "name": tc["function"]["name"],
-                        "args": json.loads(tc["function"]["arguments"]),
-                    }}],
-                })
+                contents.append(
+                    {
+                        "role": "model",
+                        "parts": [
+                            {
+                                "function_call": {
+                                    "name": tc["function"]["name"],
+                                    "args": json.loads(tc["function"]["arguments"]),
+                                }
+                            }
+                        ],
+                    }
+                )
                 continue
 
             if r == "tool":
-                contents.append({
-                    "role": "model",
-                    "parts": [{"text": str(m.get("content", ""))}],
-                })
+                contents.append(
+                    {
+                        "role": "model",
+                        "parts": [{"text": str(m.get("content", ""))}],
+                    }
+                )
                 continue
 
-            contents.append({
-                "role": role_map.get(r, "user"),
-                "parts": [{"text": str(m.get("content", ""))}],
-            })
+            contents.append(
+                {
+                    "role": role_map.get(r, "user"),
+                    "parts": [{"text": str(m.get("content", ""))}],
+                }
+            )
 
         stream: GenerateContentResponse = self.model.generate_content(
             contents=contents,
@@ -150,15 +172,18 @@ class LLMProvider(LLMProviderBase):
                     # a) 函数调用-通常是最后一段话才是函数调用
                     if getattr(part, "function_call", None):
                         fc = part.function_call
-                        yield None, [SimpleNamespace(
-                            id=uuid.uuid4().hex,
-                            type="function",
-                            function=SimpleNamespace(
-                                name=fc.name,
-                                arguments=json.dumps(dict(fc.args),
-                                                     ensure_ascii=False),
-                            ),
-                        )]
+                        yield None, [
+                            SimpleNamespace(
+                                id=uuid.uuid4().hex,
+                                type="function",
+                                function=SimpleNamespace(
+                                    name=fc.name,
+                                    arguments=json.dumps(
+                                        dict(fc.args), ensure_ascii=False
+                                    ),
+                                ),
+                            )
+                        ]
                         return
                     # b) 普通文本
                     if getattr(part, "text", None):

main/xiaozhi-server/core/providers/llm/ollama/ollama.py

@@ -21,27 +21,67 @@ class LLMProvider(LLMProviderBase):
             api_key="ollama"  # Ollama doesn't need an API key but OpenAI client requires one
         )
 
+        # 检查是否是qwen3模型
+        self.is_qwen3 = self.model_name and self.model_name.lower().startswith("qwen3")
+
     def response(self, session_id, dialogue):
         try:
+            # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+            if self.is_qwen3:
+                # 复制对话列表，避免修改原始对话
+                dialogue_copy = dialogue.copy()
+
+                # 找到最后一条用户消息
+                for i in range(len(dialogue_copy) - 1, -1, -1):
+                    if dialogue_copy[i]["role"] == "user":
+                        # 在用户消息前添加/no_think指令
+                        dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
+                        logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                        break
+
+                # 使用修改后的对话
+                dialogue = dialogue_copy
+
             responses = self.client.chat.completions.create(
                 model=self.model_name,
                 messages=dialogue,
                 stream=True
             )
-            is_active=True
+            is_active = True
+            # 用于处理跨chunk的标签
+            buffer = ""
+
             for chunk in responses:
                 try:
                     delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
                     content = delta.content if hasattr(delta, 'content') else ''
+
                     if content:
-                        if '<think>' in content:
+                        # 将内容添加到缓冲区
+                        buffer += content
+
+                        # 处理缓冲区中的标签
+                        while '<think>' in buffer and '</think>' in buffer:
+                            # 找到完整的<think></think>标签并移除
+                            pre = buffer.split('<think>', 1)[0]
+                            post = buffer.split('</think>', 1)[1]
+                            buffer = pre + post
+
+                        # 处理只有开始标签的情况
+                        if '<think>' in buffer:
                             is_active = False
-                            content = content.split('<think>')[0]
-                        if '</think>' in content:
+                            buffer = buffer.split('<think>', 1)[0]
+
+                        # 处理只有结束标签的情况
+                        if '</think>' in buffer:
                             is_active = True
-                            content = content.split('</think>')[-1]
-                        if is_active:
-                            yield content
+                            buffer = buffer.split('</think>', 1)[1]
+
+                        # 如果当前处于活动状态且缓冲区有内容，则输出
+                        if is_active and buffer:
+                            yield buffer
+                            buffer = ""  # 清空缓冲区
+
                 except Exception as e:
                     logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
 
@@ -51,6 +91,22 @@ class LLMProvider(LLMProviderBase):
 
     def response_with_functions(self, session_id, dialogue, functions=None):
         try:
+            # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
+            if self.is_qwen3:
+                # 复制对话列表，避免修改原始对话
+                dialogue_copy = dialogue.copy()
+
+                # 找到最后一条用户消息
+                for i in range(len(dialogue_copy) - 1, -1, -1):
+                    if dialogue_copy[i]["role"] == "user":
+                        # 在用户消息前添加/no_think指令
+                        dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
+                        logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
+                        break
+
+                # 使用修改后的对话
+                dialogue = dialogue_copy
+
             stream = self.client.chat.completions.create(
                 model=self.model_name,
                 messages=dialogue,
@@ -58,8 +114,49 @@ class LLMProvider(LLMProviderBase):
                 tools=functions,
             )
 
+            is_active = True
+            buffer = ""
+
             for chunk in stream:
-                yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
+                try:
+                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
+                    content = delta.content if hasattr(delta, 'content') else None
+                    tool_calls = delta.tool_calls if hasattr(delta, 'tool_calls') else None
+
+                    # 如果是工具调用，直接传递
+                    if tool_calls:
+                        yield None, tool_calls
+                        continue
+
+                    # 处理文本内容
+                    if content:
+                        # 将内容添加到缓冲区
+                        buffer += content
+
+                        # 处理缓冲区中的标签
+                        while '<think>' in buffer and '</think>' in buffer:
+                            # 找到完整的<think></think>标签并移除
+                            pre = buffer.split('<think>', 1)[0]
+                            post = buffer.split('</think>', 1)[1]
+                            buffer = pre + post
+
+                        # 处理只有开始标签的情况
+                        if '<think>' in buffer:
+                            is_active = False
+                            buffer = buffer.split('<think>', 1)[0]
+
+                        # 处理只有结束标签的情况
+                        if '</think>' in buffer:
+                            is_active = True
+                            buffer = buffer.split('</think>', 1)[1]
+
+                        # 如果当前处于活动状态且缓冲区有内容，则输出
+                        if is_active and buffer:
+                            yield buffer, None
+                            buffer = ""  # 清空缓冲区
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
+                    continue
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")

main/xiaozhi-server/core/providers/memory/base.py

@@ -4,6 +4,7 @@ from config.logger import setup_logging
 TAG = __name__
 logger = setup_logging()
 
+
 class MemoryProviderBase(ABC):
     def __init__(self, config):
         self.config = config
@@ -20,6 +21,6 @@ class MemoryProviderBase(ABC):
         """Query memories for specific role based on similarity"""
         return "please implement query method"
 
-    def init_memory(self, role_id, llm):
-        self.role_id = role_id    
+    def init_memory(self, role_id, llm, **kwargs):
+        self.role_id = role_id
         self.llm = llm

main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py

@@ -8,7 +8,7 @@ TAG = __name__
 
 
 class MemoryProvider(MemoryProviderBase):
-    def __init__(self, config):
+    def __init__(self, config, summary_memory=None):
         super().__init__(config)
         self.api_key = config.get("api_key", "")
         self.api_version = config.get("api_version", "v1.1")

main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -4,6 +4,7 @@ import json
 import os
 import yaml
 from config.config_loader import get_project_dir
+from config.manage_api_client import save_mem_local_short
 
 
 short_term_memory_prompt = """
@@ -72,6 +73,17 @@ short_term_memory_prompt = """
 ```
 """
 
+short_term_memory_prompt_only_content = """
+你是一个经验丰富的记忆总结者，擅长将对话内容进行总结摘要，遵循以下规则：
+1、总结user的重要信息，以便在未来的对话中提供更个性化的服务
+2、不要重复总结，不要遗忘之前记忆，除非原来的记忆超过了1800字内，否则不要遗忘、不要压缩用户的历史记忆
+3、用户操控的设备音量、播放音乐、天气、退出、不想对话等和用户本身无关的内容，这些信息不需要加入到总结中
+4、不要把设备操控的成果结果和失败结果加入到总结中，也不要把用户的一些废话加入到总结中
+5、不要为了总结而总结，如果用户的聊天没有意义，请返回原来的历史记录也是可以的
+6、只需要返回总结摘要，严格控制在1800字内
+7、不要包含代码、xml，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
+"""
+
 
 def extract_json_data(json_code):
     start = json_code.find("```json")
@@ -93,17 +105,26 @@ TAG = __name__
 
 
 class MemoryProvider(MemoryProviderBase):
-    def __init__(self, config):
+    def __init__(self, config, summary_memory):
         super().__init__(config)
         self.short_momery = ""
+        self.save_to_file = True
         self.memory_path = get_project_dir() + "data/.memory.yaml"
-        self.load_memory()
+        self.load_memory(summary_memory)
 
-    def init_memory(self, role_id, llm):
-        super().init_memory(role_id, llm)
-        self.load_memory()
+    def init_memory(
+        self, role_id, llm, summary_memory=None, save_to_file=True, **kwargs
+    ):
+        super().init_memory(role_id, llm, **kwargs)
+        self.save_to_file = save_to_file
+        self.load_memory(summary_memory)
+
+    def load_memory(self, summary_memory):
+        # api获取到总结记忆后直接返回
+        if summary_memory or not self.save_to_file:
+            self.short_momery = summary_memory
+            return
 
-    def load_memory(self):
         all_memory = {}
         if os.path.exists(self.memory_path):
             with open(self.memory_path, "r", encoding="utf-8") as f:
@@ -134,7 +155,7 @@ class MemoryProvider(MemoryProviderBase):
                 msgStr += f"User: {msg.content}\n"
             elif msg.role == "assistant":
                 msgStr += f"Assistant: {msg.content}\n"
-        if len(self.short_momery) > 0:
+        if self.short_momery and len(self.short_momery) > 0:
             msgStr += "历史记忆：\n"
             msgStr += self.short_momery
 
@@ -142,16 +163,20 @@ class MemoryProvider(MemoryProviderBase):
         time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
         msgStr += f"当前时间：{time_str}"
 
-        result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
-
-        json_str = extract_json_data(result)
-        try:
-            json_data = json.loads(json_str)  # 检查json格式是否正确
-            self.short_momery = json_str
-        except Exception as e:
-            print("Error:", e)
-
-        self.save_memory_to_file()
+        if self.save_to_file:
+            result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
+            json_str = extract_json_data(result)
+            try:
+                json.loads(json_str)  # 检查json格式是否正确
+                self.short_momery = json_str
+                self.save_memory_to_file()
+            except Exception as e:
+                print("Error:", e)
+        else:
+            result = self.llm.response_no_stream(
+                short_term_memory_prompt_only_content, msgStr
+            )
+            save_mem_local_short(self.role_id, result)
         logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
 
         return self.short_momery

main/xiaozhi-server/core/providers/memory/nomem/nomem.py

@@ -1,18 +1,20 @@
-'''
+"""
 不使用记忆，可以选择此模块
-'''
+"""
+
 from ..base import MemoryProviderBase, logger
 
 TAG = __name__
 
+
 class MemoryProvider(MemoryProviderBase):
-    def __init__(self, config):
+    def __init__(self, config, summary_memory=None):
         super().__init__(config)
-      
+
     async def save_memory(self, msgs):
         logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
         return None
 
-    async def query_memory(self, query: str)-> str:
+    async def query_memory(self, query: str) -> str:
         logger.bind(tag=TAG).debug("nomem mode: No memory query is performed.")
-        return ""
+        return ""

main/xiaozhi-server/core/providers/tts/custom.py

@@ -1,4 +1,5 @@
 import os
+import json
 import uuid
 import requests
 from config.logger import setup_logging
@@ -12,11 +13,21 @@ class TTSProvider(TTSProviderBase):
     def __init__(self, config, delete_audio_file):
         super().__init__(config, delete_audio_file)
         self.url = config.get("url")
+        self.method = config.get("method", "GET")
         self.headers = config.get("headers", {})
-        self.params = config.get("params")
         self.format = config.get("format", "wav")
         self.output_file = config.get("output_dir", "tmp/")
 
+        self.params = config.get("params")
+
+        if isinstance(self.params, str):
+            try:
+                self.params = json.loads(self.params)
+            except json.JSONDecodeError:
+                raise ValueError("Custom TTS配置参数出错,无法将字符串解析为对象")
+        elif not isinstance(self.params, dict):
+            raise TypeError("Custom TTS配置参数出错, 请参考配置说明")
+
     def generate_filename(self):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}.{self.format}")
 
@@ -27,7 +38,10 @@ class TTSProvider(TTSProviderBase):
                 v = v.replace("{prompt_text}", text)
             request_params[k] = v
 
-        resp = requests.get(self.url, params=request_params, headers=self.headers)
+        if self.method.upper() == "POST":
+            resp = requests.post(self.url, json=request_params, headers=self.headers)
+        else:
+            resp = requests.get(self.url, params=request_params, headers=self.headers)
         if resp.status_code == 200:
             with open(output_file, "wb") as file:
                 file.write(resp.content)

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -85,7 +85,9 @@ class TTSProvider(TTSProviderBase):
     def __init__(self, config, delete_audio_file):
         super().__init__(config, delete_audio_file)
 
-        self.reference_id = config.get("reference_id")
+        self.reference_id = (
+            None if not config.get("reference_id") else config.get("reference_id")
+        )
         self.reference_audio = parse_string_to_list(config.get("reference_audio"))
         self.reference_text = parse_string_to_list(config.get("reference_text"))
         self.format = config.get("response_format", "wav")
@@ -128,7 +130,7 @@ class TTSProvider(TTSProviderBase):
             "yes",
         )
         self.use_memory_cache = config.get("use_memory_cache", "on")
-        self.seed = config.get("seed") or None
+        self.seed = int(config.get("seed")) if config.get("seed") else None
         self.api_url = config.get("api_url", "http://127.0.0.1:8080/v1/tts")
 
     def generate_filename(self, extension=".wav"):

main/xiaozhi-server/core/utils/dialogue.py

@@ -75,7 +75,8 @@ class Dialogue:
 
         if system_message:
             enhanced_system_prompt = (
-                f"{system_message.content}\n\n" f"相关记忆：\n{memory_str}"
+                f"{system_message.content}\n\n"
+                f"以下是用户的历史记忆：\n```\n{memory_str}\n```"
             )
             dialogue.append({"role": "system", "content": enhanced_system_prompt})
 

main/xiaozhi-server/core/utils/util.py

@@ -9,6 +9,7 @@ import opuslib_next
 from pydub import AudioSegment
 from typing import Dict, Any
 from core.utils import tts, llm, intent, memory, vad, asr
+import copy
 
 TAG = __name__
 emoji_map = {
@@ -319,6 +320,7 @@ def initialize_modules(
         modules["memory"] = memory.create_instance(
             memory_type,
             config["Memory"][select_memory_module],
+            config.get("summaryMemory", None),
         )
         logger.bind(tag=TAG).info(f"初始化组件: memory成功 {select_memory_module}")
 
@@ -930,7 +932,6 @@ def check_vad_update(before_config, new_config):
         if "type" not in new_config["VAD"][new_vad_module]
         else new_config["VAD"][new_vad_module]["type"]
     )
-    print(f"前vad:{current_vad_type}，后vad:{new_vad_type}")
     update_vad = current_vad_type != new_vad_type
     return update_vad
 
@@ -954,6 +955,39 @@ def check_asr_update(before_config, new_config):
         if "type" not in new_config["ASR"][new_asr_module]
         else new_config["ASR"][new_asr_module]["type"]
     )
-    print(f"前asr:{current_asr_type}，后asr:{new_asr_type}")
     update_asr = current_asr_type != new_asr_type
     return update_asr
+
+
+def filter_sensitive_info(config: dict) -> dict:
+    """
+    过滤配置中的敏感信息
+    Args:
+        config: 原始配置字典
+    Returns:
+        过滤后的配置字典
+    """
+    sensitive_keys = [
+        "api_key",
+        "personal_access_token",
+        "access_token",
+        "token",
+        "secret",
+        "access_key_secret",
+        "secret_key",
+    ]
+
+    def _filter_dict(d: dict) -> dict:
+        filtered = {}
+        for k, v in d.items():
+            if any(sensitive in k.lower() for sensitive in sensitive_keys):
+                filtered[k] = "***"
+            elif isinstance(v, dict):
+                filtered[k] = _filter_dict(v)
+            elif isinstance(v, list):
+                filtered[k] = [_filter_dict(i) if isinstance(i, dict) else i for i in v]
+            else:
+                filtered[k] = v
+        return filtered
+
+    return _filter_dict(copy.deepcopy(config))

main/xiaozhi-server/core/websocket_server.py

@@ -82,11 +82,13 @@ class WebSocketServer:
                 if new_config is None:
                     self.logger.bind(tag=TAG).error("获取新配置失败")
                     return False
-
+                self.logger.bind(tag=TAG).info(f"获取新配置成功")
                 # 检查 VAD 和 ASR 类型是否需要更新
                 update_vad = check_vad_update(self.config, new_config)
                 update_asr = check_asr_update(self.config, new_config)
-
+                self.logger.bind(tag=TAG).info(
+                    f"检查VAD和ASR类型是否需要更新: {update_vad} {update_asr}"
+                )
                 # 更新配置
                 self.config = new_config
                 # 重新初始化组件
@@ -114,7 +116,7 @@ class WebSocketServer:
                     self._intent = modules["intent"]
                 if "memory" in modules:
                     self._memory = modules["memory"]
-
+                self.logger.bind(tag=TAG).info(f"更新配置任务执行完毕")
                 return True
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"更新服务器配置失败: {str(e)}")

main/xiaozhi-server/docker-compose_all.yml

@@ -76,6 +76,7 @@ services:
     expose:
       - 6379
     container_name: xiaozhi-esp32-server-redis
+    restart: always
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 10s

main/xiaozhi-server/performance_tester.py

@@ -1,15 +1,17 @@
-import time
-import aiohttp
 import asyncio
+import logging
+import os
+import statistics
+import time
+from typing import Dict
+
+import aiohttp
 from tabulate import tabulate
-from typing import Dict, List
+
+from config.settings import load_config
+from core.utils.asr import create_instance as create_stt_instance
 from core.utils.llm import create_instance as create_llm_instance
 from core.utils.tts import create_instance as create_tts_instance
-import statistics
-from config.settings import load_config
-import inspect
-import os
-import logging
 
 # 设置全局日志级别为WARNING，抑制INFO级别日志
 logging.basicConfig(level=logging.WARNING)
@@ -26,7 +28,17 @@ class AsyncPerformanceTester:
                 "请用100字概括量子计算的基本原理和应用前景",
             ],
         )
-        self.results = {"llm": {}, "tts": {}, "combinations": []}
+
+        self.test_wav_list = []
+        self.wav_root = r"config/assets"
+        for file_name in os.listdir(self.wav_root):
+            file_path = os.path.join(self.wav_root, file_name)
+            # 检查文件大小是否大于300KB
+            if os.path.getsize(file_path) > 300 * 1024:  # 300KB = 300 * 1024 bytes
+                with open(file_path, "rb") as f:
+                    self.test_wav_list.append(f.read())
+
+        self.results = {"llm": {}, "tts": {}, "stt": {}, "combinations": []}
 
     async def _check_ollama_service(self, base_url: str, model_name: str) -> bool:
         """异步检查Ollama服务状态"""
@@ -109,6 +121,57 @@ class AsyncPerformanceTester:
             print(f"⚠️ {tts_name} 测试失败: {str(e)}")
             return {"name": tts_name, "type": "tts", "errors": 1}
 
+    async def _test_stt(self, stt_name: str, config: Dict) -> Dict:
+        """异步测试单个STT性能"""
+        try:
+            logging.getLogger("core.providers.asr.base").setLevel(logging.WARNING)
+            token_fields = ["access_token", "api_key", "token"]
+            if any(
+                field in config
+                and any(x in config[field] for x in ["你的", "placeholder"])
+                for field in token_fields
+            ):
+                print(f"⏭️  STT {stt_name} 未配置access_token/api_key，已跳过")
+                return {"name": stt_name, "type": "stt", "errors": 1}
+
+            module_type = config.get("type", stt_name)
+            stt = create_stt_instance(module_type, config, delete_audio_file=True)
+            stt.audio_format = "pcm"
+
+            print(f"🎵 测试 STT: {stt_name}")
+
+            text, _ = await stt.speech_to_text([self.test_wav_list[0]], "1")
+
+            if text is None:
+                print(f"❌ {stt_name} 连接失败")
+                return {"name": stt_name, "type": "stt", "errors": 1}
+
+            total_time = 0
+            test_count = len(self.test_wav_list)
+
+            for i, sentence in enumerate(self.test_wav_list, 1):
+                start = time.time()
+                text, _ = await stt.speech_to_text([sentence], "1")
+                duration = time.time() - start
+                total_time += duration
+
+                if text:
+                    print(f"✓ {stt_name} [{i}/{test_count}]")
+                else:
+                    print(f"✗ {stt_name} [{i}/{test_count}]")
+                    return {"name": stt_name, "type": "stt", "errors": 1}
+
+            return {
+                "name": stt_name,
+                "type": "stt",
+                "avg_time": total_time / test_count,
+                "errors": 0,
+            }
+
+        except Exception as e:
+            print(f"⚠️ {stt_name} 测试失败: {str(e)}")
+            return {"name": stt_name, "type": "stt", "errors": 1}
+
     async def _test_llm(self, llm_name: str, config: Dict) -> Dict:
         """异步测试单个LLM性能"""
         try:
@@ -234,6 +297,7 @@ class AsyncPerformanceTester:
             if v["errors"] == 0 and v["avg_first_token"] >= 0.05
         ]
         valid_tts = [k for k, v in self.results["tts"].items() if v["errors"] == 0]
+        valid_stt = [k for k, v in self.results["stt"].items() if v["errors"] == 0]
 
         # 找出基准值
         min_first_token = (
@@ -246,42 +310,53 @@ class AsyncPerformanceTester:
             if valid_tts
             else 1
         )
+        min_stt_time = (
+            min([self.results["stt"][stt]["avg_time"] for stt in valid_stt])
+            if valid_stt
+            else 1
+        )
 
         for llm in valid_llms:
             for tts in valid_tts:
-                # 计算相对性能分数（越小越好）
-                llm_score = (
-                    self.results["llm"][llm]["avg_first_token"] / min_first_token
-                )
-                tts_score = self.results["tts"][tts]["avg_time"] / min_tts_time
+                for stt in valid_stt:
+                    # 计算相对性能分数（越小越好）
+                    llm_score = (
+                        self.results["llm"][llm]["avg_first_token"] / min_first_token
+                    )
+                    tts_score = self.results["tts"][tts]["avg_time"] / min_tts_time
+                    stt_score = self.results["stt"][stt]["avg_time"] / min_stt_time
 
-                # 计算稳定性分数（标准差/平均值，越小越稳定）
-                llm_stability = (
-                    self.results["llm"][llm]["std_first_token"]
-                    / self.results["llm"][llm]["avg_first_token"]
-                )
+                    # 计算稳定性分数（标准差/平均值，越小越稳定）
+                    llm_stability = (
+                        self.results["llm"][llm]["std_first_token"]
+                        / self.results["llm"][llm]["avg_first_token"]
+                    )
 
-                # 综合得分（考虑性能和稳定性）
-                # 性能权重0.7，稳定性权重0.3
-                llm_final_score = llm_score * 0.7 + llm_stability * 0.3
+                    # 综合得分（考虑性能和稳定性）
+                    # LLM得分： 性能权重(70%) + 稳定性权重(30%)
+                    llm_final_score = llm_score * 0.7 + llm_stability * 0.3
 
-                # 总分 = LLM得分(70%) + TTS得分(30%)
-                total_score = llm_final_score * 0.7 + tts_score * 0.3
+                    # 总分 = LLM得分(70%) + TTS得分(30%) + STT得分(30%)
+                    total_score = (
+                        llm_final_score * 0.7 + tts_score * 0.3 + stt_score * 0.3
+                    )
 
-                self.results["combinations"].append(
-                    {
-                        "llm": llm,
-                        "tts": tts,
-                        "score": total_score,
-                        "details": {
-                            "llm_first_token": self.results["llm"][llm][
-                                "avg_first_token"
-                            ],
-                            "llm_stability": llm_stability,
-                            "tts_time": self.results["tts"][tts]["avg_time"],
-                        },
-                    }
-                )
+                    self.results["combinations"].append(
+                        {
+                            "llm": llm,
+                            "tts": tts,
+                            "stt": stt,
+                            "score": total_score,
+                            "details": {
+                                "llm_first_token": self.results["llm"][llm][
+                                    "avg_first_token"
+                                ],
+                                "llm_stability": llm_stability,
+                                "tts_time": self.results["tts"][tts]["avg_time"],
+                                "stt_time": self.results["stt"][stt]["avg_time"],
+                            },
+                        }
+                    )
 
         # 分数越小越好
         self.results["combinations"].sort(key=lambda x: x["score"])
@@ -302,7 +377,7 @@ class AsyncPerformanceTester:
                 )
 
         if llm_table:
-            print("\nLLM 性能排行:")
+            print("\nLLM 性能排行:\n")
             print(
                 tabulate(
                     llm_table,
@@ -321,7 +396,7 @@ class AsyncPerformanceTester:
                 tts_table.append([name, f"{data['avg_time']:.3f}秒"])  # 不需要固定宽度
 
         if tts_table:
-            print("\nTTS 性能排行:")
+            print("\nTTS 性能排行:\n")
             print(
                 tabulate(
                     tts_table,
@@ -334,17 +409,37 @@ class AsyncPerformanceTester:
         else:
             print("\n⚠️ 没有可用的TTS模块进行测试。")
 
+        stt_table = []
+        for name, data in self.results["stt"].items():
+            if data["errors"] == 0:
+                stt_table.append([name, f"{data['avg_time']:.3f}秒"])  # 不需要固定宽度
+
+        if stt_table:
+            print("\nSTT 性能排行:\n")
+            print(
+                tabulate(
+                    stt_table,
+                    headers=["模型名称", "合成耗时"],
+                    tablefmt="github",
+                    colalign=("left", "right"),
+                    disable_numparse=True,
+                )
+            )
+        else:
+            print("\n⚠️ 没有可用的STT模块进行测试。")
+
         if self.results["combinations"]:
-            print("\n推荐配置组合 (得分越小越好):")
+            print("\n推荐配置组合 (得分越小越好):\n")
             combo_table = []
-            for combo in self.results["combinations"][:5]:
+            for combo in self.results["combinations"][:]:
                 combo_table.append(
                     [
-                        f"{combo['llm']} + {combo['tts']}",  # 不需要固定宽度
+                        f"{combo['llm']} + {combo['tts']} + {combo['stt']}",  # 不需要固定宽度
                         f"{combo['score']:.3f}",
                         f"{combo['details']['llm_first_token']:.3f}秒",
                         f"{combo['details']['llm_stability']:.3f}",
                         f"{combo['details']['tts_time']:.3f}秒",
+                        f"{combo['details']['stt_time']:.3f}秒",
                     ]
                 )
 
@@ -357,9 +452,10 @@ class AsyncPerformanceTester:
                         "LLM首字耗时",
                         "稳定性",
                         "TTS合成耗时",
+                        "STT合成耗时",
                     ],
                     tablefmt="github",
-                    colalign=("left", "right", "right", "right", "right"),
+                    colalign=("left", "right", "right", "right", "right", "right"),
                     disable_numparse=True,
                 )
             )
@@ -372,8 +468,12 @@ class AsyncPerformanceTester:
             if result["errors"] == 0:
                 if result["type"] == "llm":
                     self.results["llm"][result["name"]] = result
-                else:
+                elif result["type"] == "tts":
                     self.results["tts"][result["name"]] = result
+                elif result["type"] == "stt":
+                    self.results["stt"][result["name"]] = result
+                else:
+                    pass
 
     async def run(self):
         """执行全量异步测试"""
@@ -383,52 +483,73 @@ class AsyncPerformanceTester:
         all_tasks = []
 
         # LLM测试任务
-        for llm_name, config in self.config.get("LLM", {}).items():
-            # 检查配置有效性
-            if llm_name == "CozeLLM":
-                if any(x in config.get("bot_id", "") for x in ["你的"]) or any(
-                    x in config.get("user_id", "") for x in ["你的"]
+        if self.config.get("LLM") is not None:
+            for llm_name, config in self.config.get("LLM", {}).items():
+                # 检查配置有效性
+                if llm_name == "CozeLLM":
+                    if any(x in config.get("bot_id", "") for x in ["你的"]) or any(
+                        x in config.get("user_id", "") for x in ["你的"]
+                    ):
+                        print(f"⏭️  LLM {llm_name} 未配置bot_id/user_id，已跳过")
+                        continue
+                elif "api_key" in config and any(
+                    x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]
                 ):
-                    print(f"⏭️  LLM {llm_name} 未配置bot_id/user_id，已跳过")
-                    continue
-            elif "api_key" in config and any(
-                x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]
-            ):
-                print(f"⏭️  LLM {llm_name} 未配置api_key，已跳过")
-                continue
-
-            # 对于Ollama，先检查服务状态
-            if llm_name == "Ollama":
-                base_url = config.get("base_url", "http://localhost:11434")
-                model_name = config.get("model_name")
-                if not model_name:
-                    print(f"🚫 Ollama未配置model_name")
+                    print(f"⏭️  LLM {llm_name} 未配置api_key，已跳过")
                     continue
 
-                if not await self._check_ollama_service(base_url, model_name):
-                    continue
+                # 对于Ollama，先检查服务状态
+                if llm_name == "Ollama":
+                    base_url = config.get("base_url", "http://localhost:11434")
+                    model_name = config.get("model_name")
+                    if not model_name:
+                        print(f"🚫 Ollama未配置model_name")
+                        continue
 
-            print(f"📋 添加LLM测试任务: {llm_name}")
-            module_type = config.get("type", llm_name)
-            llm = create_llm_instance(module_type, config)
+                    if not await self._check_ollama_service(base_url, model_name):
+                        continue
 
-            # 为每个句子创建独立任务
-            for sentence in self.test_sentences:
-                sentence = sentence.encode("utf-8").decode("utf-8")
-                all_tasks.append(self._test_single_sentence(llm_name, llm, sentence))
+                print(f"📋 添加LLM测试任务: {llm_name}")
+                module_type = config.get("type", llm_name)
+                llm = create_llm_instance(module_type, config)
+
+                # 为每个句子创建独立任务
+                for sentence in self.test_sentences:
+                    sentence = sentence.encode("utf-8").decode("utf-8")
+                    all_tasks.append(
+                        self._test_single_sentence(llm_name, llm, sentence)
+                    )
 
         # TTS测试任务
-        for tts_name, config in self.config.get("TTS", {}).items():
-            token_fields = ["access_token", "api_key", "token"]
-            if any(
-                field in config
-                and any(x in config[field] for x in ["你的", "placeholder"])
-                for field in token_fields
-            ):
-                print(f"⏭️  TTS {tts_name} 未配置access_token/api_key，已跳过")
-                continue
-            print(f"🎵 添加TTS测试任务: {tts_name}")
-            all_tasks.append(self._test_tts(tts_name, config))
+        if self.config.get("TTS") is not None:
+            for tts_name, config in self.config.get("TTS", {}).items():
+                token_fields = ["access_token", "api_key", "token"]
+                if any(
+                    field in config
+                    and any(x in config[field] for x in ["你的", "placeholder"])
+                    for field in token_fields
+                ):
+                    print(f"⏭️  TTS {tts_name} 未配置access_token/api_key，已跳过")
+                    continue
+                print(f"🎵 添加TTS测试任务: {tts_name}")
+                all_tasks.append(self._test_tts(tts_name, config))
+
+        # STT测试任务
+        if len(self.test_wav_list) >= 1:
+            if self.config.get("ASR") is not None:
+                for stt_name, config in self.config.get("ASR", {}).items():
+                    token_fields = ["access_token", "api_key", "token"]
+                    if any(
+                        field in config
+                        and any(x in config[field] for x in ["你的", "placeholder"])
+                        for field in token_fields
+                    ):
+                        print(f"⏭️  ASR {stt_name} 未配置access_token/api_key，已跳过")
+                        continue
+                    print(f"🎵 添加ASR测试任务: {stt_name}")
+                    all_tasks.append(self._test_stt(stt_name, config))
+        else:
+            print(f"\n⚠️  {self.wav_root} 路径下没有音频文件，已跳过STT测试任务")
 
         print(
             f"\n✅ 找到 {len([t for t in all_tasks if 'test_single_sentence' in str(t)]) / len(self.test_sentences):.0f} 个可用LLM模块"
@@ -436,6 +557,9 @@ class AsyncPerformanceTester:
         print(
             f"✅ 找到 {len([t for t in all_tasks if '_test_tts' in str(t)])} 个可用TTS模块"
         )
+        print(
+            f"✅ 找到 {len([t for t in all_tasks if '_test_stt' in str(t)])} 个可用STT模块"
+        )
         print("\n⏳ 开始并发测试所有模块...\n")
 
         # 并发执行所有测试任务
@@ -494,6 +618,15 @@ class AsyncPerformanceTester:
             if result["errors"] == 0:
                 self.results["tts"][result["name"]] = result
 
+        # 处理STT结果
+        for result in [
+            r
+            for r in all_results
+            if r and isinstance(r, dict) and r.get("type") == "stt"
+        ]:
+            if result["errors"] == 0:
+                self.results["stt"][result["name"]] = result
+
         # 生成组合建议并打印结果
         print("\n📊 生成测试报告...")
         self._generate_combinations()

main/xiaozhi-server/plugins_func/functions/get_weather.py

@@ -107,8 +107,8 @@ WEATHER_CODE_MAP = {
 }
 
 
-def fetch_city_info(location, api_key):
-    url = f"https://geoapi.qweather.com/v2/city/lookup?key={api_key}&location={location}&lang=zh"
+def fetch_city_info(location, api_key, api_host):
+    url = f"https://{api_host}/geo/v2/city/lookup?key={api_key}&location={location}&lang=zh"
     response = requests.get(url, headers=HEADERS).json()
     return response.get("location", [])[0] if response.get("location") else None
 
@@ -151,7 +151,8 @@ def parse_weather_info(soup):
 
 @register_function("get_weather", GET_WEATHER_FUNCTION_DESC, ToolType.SYSTEM_CTL)
 def get_weather(conn, location: str = None, lang: str = "zh_CN"):
-    api_key = conn.config["plugins"]["get_weather"]["api_key"]
+    api_host = conn.config["plugins"]["get_weather"].get("api_host", "mj7p3y7naa.re.qweatherapi.com")
+    api_key = conn.config["plugins"]["get_weather"].get("api_key", "a861d0d5e7bf4ee1a83d9a9e4f96d4da")
     default_location = conn.config["plugins"]["get_weather"]["default_location"]
     client_ip = conn.client_ip
     # 优先使用用户提供的location参数
@@ -164,8 +165,7 @@ def get_weather(conn, location: str = None, lang: str = "zh_CN"):
         else:
             # 若IP解析失败或无IP，使用默认位置
             location = default_location
-
-    city_info = fetch_city_info(location, api_key)
+    city_info = fetch_city_info(location, api_key, api_host)
     if not city_info:
         return ActionResponse(
             Action.REQLLM, f"未找到相关的城市: {location}，请确认地点是否正确", None

main/xiaozhi-server/plugins_func/functions/handle_device.py

@@ -1,111 +0,0 @@
-from config.logger import setup_logging
-from plugins_func.register import register_function, ToolType, ActionResponse, Action
-from core.handle.iotHandle import get_iot_status, send_iot_conn
-import asyncio
-
-TAG = __name__
-logger = setup_logging()
-
-
-async def _get_device_status(conn, device_name, device_type, property_name):
-    """获取设备状态"""
-    status = await get_iot_status(conn, device_type, property_name)
-    if status is None:
-        raise Exception(f"你的设备不支持{device_name}控制")
-    return status
-
-
-async def _set_device_property(conn, device_name, device_type, method_name, property_name, new_value=None, action=None, step=10):
-    """设置设备属性"""
-    current_value = await _get_device_status(conn, device_name, device_type, property_name)
-
-    if action == 'raise':
-        current_value += step
-    elif action == 'lower':
-        current_value -= step
-    elif action == 'set':
-        if new_value is None:
-            raise Exception(f"缺少{property_name}参数")
-        current_value = new_value
-
-    # 限制属性范围在0到100之间
-    current_value = max(0, min(100, current_value))
-
-    await send_iot_conn(conn, device_type, method_name, {property_name: current_value})
-    return current_value
-
-
-def _handle_device_action(conn, func, success_message, error_message, *args, **kwargs):
-    """处理设备操作的通用函数"""
-    future = asyncio.run_coroutine_threadsafe(
-        func(conn, *args, **kwargs), conn.loop)
-    try:
-        result = future.result()
-        logger.bind(tag=TAG).info(f"{success_message}: {result}")
-        response = f"{success_message}{result}"
-        return ActionResponse(action=Action.RESPONSE, result=result, response=response)
-    except Exception as e:
-        logger.bind(tag=TAG).error(f"{error_message}: {e}")
-        response = f"{error_message}: {e}"
-        return ActionResponse(action=Action.RESPONSE, result=None, response=response)
-
-
-# 设备控制
-handle_device_function_desc = {
-    "type": "function",
-    "function": {
-        "name": "handle_device",
-        "description": (
-            "用户想要获取或者设置设备的音量/亮度大小，或者用户觉得声音/亮度过高或过低，或者用户想提高或降低音量/亮度。"
-            "比如用户说现在亮度多少，参数为：device_type:Screen,action:get。"
-            "比如用户说设置音量为50，参数为：device_type:Speaker,action:set,value:50。"
-            "比如用户说亮度太高了，参数为：device_type:Screen,action:lower。"
-            "比如用户说调大音量，参数为：device_type:Speaker,action:raise。"
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "device_type": {
-                    "type": "string",
-                    "description": "设备类型，可选值：Speaker(音量),Screen(亮度)"
-                },
-                "action": {
-                    "type": "string",
-                    "description": "动作名称，可选值：get(获取),set(设置),raise(提高),lower(降低)"
-                },
-                "value": {
-                    "type": "integer",
-                    "description": "值大小，可选值：0-100之间的整数"
-                }
-            },
-            "required": ["device_type", "action"]
-        }
-    }
-}
-
-
-@register_function('handle_device', handle_device_function_desc, ToolType.IOT_CTL)
-def handle_device(conn, device_type: str, action: str, value: int = None):
-    if device_type == "Speaker":
-        method_name, property_name, device_name = "SetVolume", "volume", "音量"
-    elif device_type == "Screen":
-        method_name, property_name, device_name = "SetBrightness", "brightness", "亮度"
-    else:
-        raise Exception(f"未识别的设备类型: {device_type}")
-
-    if action not in ["get", "set", "raise", "lower"]:
-        raise Exception(f"未识别的动作名称: {action}")
-
-    if action == "get":
-        # get
-        return _handle_device_action(
-            conn, _get_device_status, f"当前{device_name}", f"获取{device_name}失败",
-            device_name=device_name, device_type=device_type, property_name=property_name,
-        )
-    else:
-        # set, raise, lower
-        return _handle_device_action(
-            conn, _set_device_property, f"{device_name}已调整到", f"{device_name}调整失败",
-            device_name=device_name, device_type=device_type, method_name=method_name,
-            property_name=property_name, new_value=value, action=action
-        )

main/xiaozhi-server/plugins_func/functions/handle_speaker_or_screen.py

@@ -0,0 +1,157 @@
+from config.logger import setup_logging
+from plugins_func.register import register_function, ToolType, ActionResponse, Action
+from core.handle.iotHandle import get_iot_status, send_iot_conn
+import asyncio
+
+TAG = __name__
+logger = setup_logging()
+
+
+async def _get_device_status(conn, device_name, device_type, property_name):
+    """获取设备状态"""
+    status = await get_iot_status(conn, device_type, property_name)
+    if status is None:
+        raise Exception(f"你的设备不支持{device_name}控制")
+    return status
+
+
+async def _set_device_property(
+    conn,
+    device_name,
+    device_type,
+    method_name,
+    property_name,
+    new_value=None,
+    action=None,
+    step=10,
+):
+    """设置设备属性"""
+    current_value = await _get_device_status(
+        conn, device_name, device_type, property_name
+    )
+
+    if action == "raise":
+        current_value += step
+    elif action == "lower":
+        current_value -= step
+    elif action == "set":
+        if new_value is None:
+            raise Exception(f"缺少{property_name}参数")
+        current_value = new_value
+
+    # 限制属性范围在0到100之间
+    current_value = max(0, min(100, current_value))
+
+    await send_iot_conn(conn, device_type, method_name, {property_name: current_value})
+    return current_value
+
+
+def _handle_device_action(conn, func, success_message, error_message, *args, **kwargs):
+    """处理设备操作的通用函数"""
+    future = asyncio.run_coroutine_threadsafe(func(conn, *args, **kwargs), conn.loop)
+    try:
+        result = future.result()
+        logger.bind(tag=TAG).info(f"{success_message}: {result}")
+        response = f"{success_message}{result}"
+        return ActionResponse(action=Action.RESPONSE, result=result, response=response)
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"{error_message}: {e}")
+        response = f"{error_message}: {e}"
+        return ActionResponse(action=Action.RESPONSE, result=None, response=response)
+
+
+# 设备控制
+handle_device_function_desc = {
+    "type": "function",
+    "function": {
+        "name": "handle_speaker_volume_or_screen_brightness",
+        "description": (
+            "用户想要获取或者设置设备的音量/亮度大小，或者用户觉得声音/亮度过高或过低，或者用户想提高或降低音量/亮度。\n"
+            "**严格限制**：仅当用户明确操作 **Speaker（音量）或Screen（亮度）** 时才能调用此函数！\n"
+            "对于其他设备（如AC、Battery、Switch等），请不要调用此函数，而是继续正常的对话。\n\n"
+            "示例：\n"
+            "- 用户说『现在亮度多少』 → 调用函数：device_type: Screen, action: get\n"
+            "- 用户说『设置音量为50』 → 调用函数：device_type: Speaker, action: set, value: 50\n"
+            "- 用户说『亮度太高了』 → 调用函数：device_type: Screen, action: lower\n"
+            "- 用户说『调大音量』 → 调用函数：device_type: Speaker, action: raise\n\n"
+            "**拒绝调用示例**（应继续对话而非调用本函数）：\n"
+            "- 用户说『空调调低一度』 → 不调用（设备类型为AC）\n"
+            "- 用户说『开关灯』 → 不调用（设备类型为Switch）\n"
+            "- 用户说『电量多少』 → 不调用（设备类型为Battery）\n"
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "device_type": {
+                    "type": "string",
+                    "description": "设备类型，**严格限定为Speaker（音量）或Screen（亮度）**，其他设备类型禁止调用此函数",
+                    "enum": ["Speaker", "Screen"],
+                },
+                "action": {
+                    "type": "string",
+                    "description": "动作名称，可选值：get(获取),set(设置),raise(提高),lower(降低)",
+                },
+                "value": {
+                    "type": "integer",
+                    "description": "值大小，可选值：0-100之间的整数",
+                },
+            },
+            "required": ["device_type", "action"],
+        },
+    },
+}
+
+
+@register_function(
+    "handle_speaker_volume_or_screen_brightness",
+    handle_device_function_desc,
+    ToolType.IOT_CTL,
+)
+def handle_speaker_volume_or_screen_brightness(
+    conn, device_type: str, action: str, value: int = None
+):
+    # 检查value是否为中文值
+    if (
+        value is not None
+        and isinstance(value, str)
+        and any("\u4e00" <= char <= "\u9fff" for char in str(value))
+    ):
+        raise Exception(
+            f"请直接告诉我要将{'音量' if device_type=='Speaker' else '亮度'}调整成多少"
+        )
+
+    if device_type == "Speaker":
+        method_name, property_name, device_name = "SetVolume", "volume", "音量"
+    elif device_type == "Screen":
+        method_name, property_name, device_name = "SetBrightness", "brightness", "亮度"
+    else:
+        raise Exception(f"未识别的设备类型: {device_type}")
+
+    if action not in ["get", "set", "raise", "lower"]:
+        raise Exception(f"未识别的动作名称: {action}")
+
+    if action == "get":
+        # get
+        return _handle_device_action(
+            conn,
+            _get_device_status,
+            f"当前{device_name}",
+            f"获取{device_name}失败",
+            device_name=device_name,
+            device_type=device_type,
+            property_name=property_name,
+        )
+    else:
+        # set, raise, lower
+        return _handle_device_action(
+            conn,
+            _set_device_property,
+            f"{device_name}已调整到",
+            f"{device_name}调整失败",
+            device_name=device_name,
+            device_type=device_type,
+            method_name=method_name,
+            property_name=property_name,
+            new_value=value,
+            action=action,
+        )

main/xiaozhi-server/plugins_func/functions/play_music.py

@@ -216,24 +216,16 @@ async def play_local_music(conn, specific_file=None):
         text = _get_random_play_prompt(selected_music)
         await send_stt_message(conn, text)
         conn.dialogue.put(Message(role="assistant", content=text))
-        conn.tts_first_text_index = 0
-        conn.tts_last_text_index = 0
 
-        tts_file = await asyncio.to_thread(conn.tts.to_tts, text)
-        if tts_file is not None and os.path.exists(tts_file):
-            conn.tts_last_text_index = 1
-            opus_packets, _ = conn.tts.audio_to_opus_data(tts_file)
-            conn.audio_play_queue.put((opus_packets, None, 0))
-            os.remove(tts_file)
+        conn.recode_first_last_text(text, 0)
+        future = conn.executor.submit(conn.speak_and_play, None, text, 0)
+        conn.tts_queue.put((future, 0))
 
+        conn.recode_first_last_text(text, 1)
+        future = conn.executor.submit(conn.speak_and_play, music_path, None, 1)
+        conn.tts_queue.put((future, 1))
         conn.llm_finish_task = True
 
-        if music_path.endswith(".p3"):
-            opus_packets, _ = p3.decode_opus_from_file(music_path)
-        else:
-            opus_packets, _ = conn.tts.audio_to_opus_data(music_path)
-        conn.audio_play_queue.put((opus_packets, None, conn.tts_last_text_index))
-
     except Exception as e:
         conn.logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
         conn.logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")

main/xiaozhi-server/plugins_func/register.py

@@ -77,7 +77,6 @@ class DeviceTypeRegistry:
 
 # 初始化函数注册字典
 all_function_registry = {}
-device_type_registry = DeviceTypeRegistry()
 
 
 def register_function(name, desc, type=None):
@@ -91,13 +90,29 @@ def register_function(name, desc, type=None):
     return decorator
 
 
+def register_device_function(name, desc, type=None):
+    """注册设备级别的函数到函数注册字典的装饰器"""
+
+    def decorator(func):
+        logger.bind(tag=TAG).debug(f"设备函数 '{name}' 已加载")
+        return func
+
+    return decorator
+
+
 class FunctionRegistry:
     def __init__(self):
         self.function_registry = {}
         self.logger = setup_logging()
 
-    def register_function(self, name):
-        # 查找all_function_registry中是否有对应的函数
+    def register_function(self, name, func_item=None):
+        # 如果提供了func_item，直接注册
+        if func_item:
+            self.function_registry[name] = func_item
+            self.logger.bind(tag=TAG).debug(f"函数 '{name}' 直接注册成功")
+            return func_item
+
+        # 否则从all_function_registry中查找
         func = all_function_registry.get(name)
         if not func:
             self.logger.bind(tag=TAG).error(f"函数 '{name}' 未找到")

main/xiaozhi-server/requirements.txt

@@ -22,11 +22,12 @@ mem0ai==0.1.62
 bs4==0.0.2
 modelscope==1.23.2
 sherpa_onnx==1.11.0
-mcp==1.7.1
+mcp==1.8.1
 cnlunar==0.2.0
 PySocks==1.7.1
 dashscope==1.23.1
 baidu-aip==4.16.13
 chardet==5.2.0
 aioconsole==0.8.1
-markitdown==0.1.1
+markitdown==0.1.1
+mcp-proxy==0.6.0