Merge branch 'main' into py_tts_huoshan

main/xiaozhi-server/core/api/base_handler.py

@@ -10,7 +10,15 @@ class BaseHandler:
     def _add_cors_headers(self, response):
         """添加CORS头信息"""
         response.headers["Access-Control-Allow-Headers"] = (
-            "client-id, content-type, device-id"
+            "client-id, content-type, device-id, authorization"
         )
         response.headers["Access-Control-Allow-Credentials"] = "true"
         response.headers["Access-Control-Allow-Origin"] = "*"
+
+    async def handle_options(self, request):
+        """处理OPTIONS请求，添加CORS头信息"""
+        response = web.Response(body=b"", content_type="text/plain")
+        self._add_cors_headers(response)
+        # 添加允许的方法
+        response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
+        return response

main/xiaozhi-server/core/api/ota_handler.py

@@ -3,15 +3,46 @@ import time
 import base64
 import hashlib
 import hmac
+import os
+import re
+import glob
+from typing import Dict, List, Tuple
 from aiohttp import web
 
 from core.auth import AuthManager
-from core.utils.util import get_local_ip
+from core.utils.util import get_local_ip, get_vision_url
 from core.api.base_handler import BaseHandler
 
 TAG = __name__
 
 
+def _safe_basename(filename: str) -> str:
+    # Prevent directory traversal
+    return os.path.basename(filename)
+
+
+def _parse_version(ver: str) -> Tuple[int, ...]:
+    # conservative parser: split by non-digit, keep numeric parts
+    parts = re.findall(r"\d+", ver)
+    return tuple(int(p) for p in parts) if parts else (0,)
+
+
+def _is_higher_version(a: str, b: str) -> bool:
+    """Return True if version string a > b (semver-like numeric compare)."""
+    ta = _parse_version(a)
+    tb = _parse_version(b)
+    # compare tuple lexicographically, but allow different lengths
+    maxlen = max(len(ta), len(tb))
+    for i in range(maxlen):
+        ai = ta[i] if i < len(ta) else 0
+        bi = tb[i] if i < len(tb) else 0
+        if ai > bi:
+            return True
+        if ai < bi:
+            return False
+    return False
+
+
 class OTAHandler(BaseHandler):
     def __init__(self, config: dict):
         super().__init__(config)
@@ -23,6 +54,54 @@ class OTAHandler(BaseHandler):
         expire_seconds = auth_config.get("expire_seconds")
         self.auth = AuthManager(secret_key=secret_key, expire_seconds=expire_seconds)
 
+        # firmware storage
+        self.bin_dir = os.path.join(os.getcwd(), "data", "bin")
+        # cache structure: { 'updated_at': timestamp, 'ttl': seconds, 'files_by_model': { model: [(version, filename), ...] } }
+        self._bin_cache: Dict = {
+            "updated_at": 0,
+            "ttl": config.get("firmware_cache_ttl", 30),
+            "files_by_model": {},
+        }
+
+    def _refresh_bin_cache_if_needed(self):
+        now = int(time.time())
+        ttl = int(self._bin_cache.get("ttl", 30))
+        if now - int(
+            self._bin_cache.get("updated_at", 0)
+        ) < ttl and self._bin_cache.get("files_by_model"):
+            return
+
+        files_by_model: Dict[str, List[Tuple[str, str]]] = {}
+        try:
+            if not os.path.isdir(self.bin_dir):
+                os.makedirs(self.bin_dir, exist_ok=True)
+
+            # match files like model_1.2.3.bin (allow dots, dashes, underscores in model and version)
+            pattern = os.path.join(self.bin_dir, "*.bin")
+            for path in glob.glob(pattern):
+                fname = os.path.basename(path)
+                # filename format: {model}_{version}.bin
+                m = re.match(r"^(.+?)_([0-9][A-Za-z0-9\.\-_]*)\.bin$", fname)
+                if not m:
+                    # skip files not conforming to naming rule
+                    continue
+                model = m.group(1)
+                version = m.group(2)
+                files_by_model.setdefault(model, []).append((version, fname))
+
+            # sort versions for each model descending
+            for model, items in files_by_model.items():
+                items.sort(key=lambda it: _parse_version(it[0]), reverse=True)
+
+            self._bin_cache["files_by_model"] = files_by_model
+            self._bin_cache["updated_at"] = now
+            self.logger.bind(tag=TAG).info(
+                f"Firmware cache refreshed: {len(files_by_model)} models"
+            )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"刷新固件缓存失败: {e}")
+            # keep previous cache if any
+
     def generate_password_signature(self, content: str, secret_key: str) -> str:
         """生成MQTT密码签名
 
@@ -62,7 +141,14 @@ class OTAHandler(BaseHandler):
             return f"ws://{local_ip}:{port}/xiaozhi/v1/"
 
     async def handle_post(self, request):
-        """处理 OTA POST 请求"""
+        """处理 OTA POST 请求
+
+        This handler will:
+        - read device id/client id (as before)
+        - attempt to determine device model and current firmware version (prefer headers, fallback to body)
+        - check data/bin for newer firmware for that model
+        - if found a newer firmware, set firmware.url to the download endpoint
+        """
         try:
             data = await request.text()
             self.logger.bind(tag=TAG).debug(f"OTA请求方法: {request.method}")
@@ -81,33 +167,76 @@ class OTAHandler(BaseHandler):
             else:
                 raise Exception("OTA请求ClientID为空")
 
-            data_json = json.loads(data)
+            data_json = {}
+            try:
+                data_json = json.loads(data) if data else {}
+            except Exception:
+                data_json = {}
 
             server_config = self.config["server"]
-            port = int(server_config.get("port", 8000))
+            # Distinguish ports:
+            # - websocket_port is used to construct websocket URL (server["port"])
+            # - http_port is used to construct OTA download URLs (server["http_port"])
+            websocket_port = int(server_config.get("port", 8000))
+            http_port = int(server_config.get("http_port", 8003))
             local_ip = get_local_ip()
 
+            # Determine device model (prefer headers)
+            device_model = ""
+            # header candidates
+            for h in ("device-model", "device_model", "model"):
+                if h in request.headers:
+                    device_model = request.headers.get(h, "").strip()
+                    break
+            # body fallback
+            if not device_model:
+                try:
+                    if "board" in data_json and isinstance(data_json["board"], dict):
+                        device_model = data_json["board"].get("type", "")
+                    elif "model" in data_json:
+                        device_model = data_json.get("model", "")
+                except Exception:
+                    device_model = ""
+            if not device_model:
+                device_model = "default"
+
+            # Determine device current version (prefer headers)
+            device_version = ""
+            for h in (
+                "device-version",
+                "device_version",
+                "firmware-version",
+                "app-version",
+                "application-version",
+            ):
+                if h in request.headers:
+                    device_version = request.headers.get(h, "").strip()
+                    break
+            if not device_version:
+                try:
+                    device_version = data_json.get("application", {}).get("version", "")
+                except Exception:
+                    device_version = ""
+            if not device_version:
+                device_version = "0.0.0"
+
             return_json = {
                 "server_time": {
                     "timestamp": int(round(time.time() * 1000)),
                     "timezone_offset": server_config.get("timezone_offset", 8) * 60,
                 },
                 "firmware": {
-                    "version": data_json["application"].get("version", "1.0.0"),
+                    "version": device_version,
                     "url": "",
                 },
             }
 
+            # existing mqtt/websocket logic (unchanged)
             mqtt_gateway_endpoint = server_config.get("mqtt_gateway")
 
             if mqtt_gateway_endpoint:  # 如果配置了非空字符串
-                # 尝试从请求数据中获取设备型号
-                device_model = "default"
+                # 尝试从请求数据中获取设备型号（已解析 above）
                 try:
-                    if "device" in data_json and isinstance(data_json["device"], dict):
-                        device_model = data_json["device"].get("model", "default")
-                    elif "model" in data_json:
-                        device_model = data_json["model"]
                     group_id = f"GID_{device_model}".replace(":", "_").replace(" ", "_")
                 except Exception as e:
                     self.logger.bind(tag=TAG).error(f"获取设备型号失败: {e}")
@@ -159,20 +288,61 @@ class OTAHandler(BaseHandler):
                             token = self.auth.generate_token(client_id, device_id)
                     else:
                         token = self.auth.generate_token(client_id, device_id)
+                # NOTE: use websocket_port here
                 return_json["websocket"] = {
-                    "url": self._get_websocket_url(local_ip, port),
+                    "url": self._get_websocket_url(local_ip, websocket_port),
                     "token": token,
                 }
                 self.logger.bind(tag=TAG).info(
                     f"未配置MQTT网关，为设备 {device_id} 下发WebSocket配置"
                 )
-                self.logger.bind(tag=TAG).info(f"{return_json}")
+
+            # Now check firmware files for updates
+            try:
+                self._refresh_bin_cache_if_needed()
+                files_by_model = self._bin_cache.get("files_by_model", {})
+                candidates = files_by_model.get(device_model, [])
+
+                self.logger.bind(tag=TAG).info(
+                    f"查找型号 {device_model} 的固件，找到 {len(candidates)} 个候选"
+                )
+
+                chosen_url = ""
+                chosen_version = device_version
+
+                # candidates are sorted descending by version
+                for ver, fname in candidates:
+                    if _is_higher_version(ver, device_version):
+                        # build download url (only allow download via our download endpoint)
+                        chosen_version = ver
+                        # Use get_vision_url to get the base URL and replace the path
+                        vision_url = get_vision_url(self.config)
+                        # Replace the path from "/mcp/vision/explain" to "/xiaozhi/ota/download/{fname}"
+                        chosen_url = vision_url.replace(
+                            "/mcp/vision/explain", f"/xiaozhi/ota/download/{fname}"
+                        )
+                        break
+
+                if chosen_url:
+                    return_json["firmware"]["version"] = chosen_version
+                    return_json["firmware"]["url"] = chosen_url
+                    self.logger.bind(tag=TAG).info(
+                        f"为设备 {device_id} 下发固件 {chosen_version} [如果地址前缀有误，请检查配置文件中的server.vision_explain]-> {chosen_url} "
+                    )
+                else:
+                    self.logger.bind(tag=TAG).info(
+                        f"设备 {device_id} 固件已是最新: {device_version}"
+                    )
+
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"检查固件版本时出错: {e}")
 
             response = web.Response(
                 text=json.dumps(return_json, separators=(",", ":")),
                 content_type="application/json",
             )
         except Exception as e:
+            self.logger.bind(tag=TAG).error(f"OTA POST处理异常: {e}")
             return_json = {"success": False, "message": "request error."}
             response = web.Response(
                 text=json.dumps(return_json, separators=(",", ":")),
@@ -187,8 +357,9 @@ class OTAHandler(BaseHandler):
         try:
             server_config = self.config["server"]
             local_ip = get_local_ip()
-            port = int(server_config.get("port", 8000))
-            websocket_url = self._get_websocket_url(local_ip, port)
+            # use websocket port for websocket URL
+            websocket_port = int(server_config.get("port", 8000))
+            websocket_url = self._get_websocket_url(local_ip, websocket_port)
             message = f"OTA接口运行正常，向设备发送的websocket地址是：{websocket_url}"
             response = web.Response(text=message, content_type="text/plain")
         except Exception as e:
@@ -197,3 +368,48 @@ class OTAHandler(BaseHandler):
         finally:
             self._add_cors_headers(response)
             return response
+
+    async def handle_download(self, request):
+        """
+        下载固件接口
+        URL: /xiaozhi/ota/download/{filename}
+        - 只允许下载 data/bin 目录下的 .bin 文件
+        - filename 必须是 basename 且匹配安全的模式
+        """
+        try:
+            fname = request.match_info.get("filename", "")
+            if not fname:
+                raise web.HTTPBadRequest(text="filename required")
+
+            # sanitize
+            fname = _safe_basename(fname)
+            # pattern: allow letters, numbers, dot, underscore, dash
+            if not re.match(r"^[A-Za-z0-9\.\-_]+\.bin$", fname):
+                raise web.HTTPBadRequest(text="invalid filename")
+
+            file_path = os.path.join(self.bin_dir, fname)
+            # ensure realpath is under bin_dir
+            file_real = os.path.realpath(file_path)
+            bin_dir_real = os.path.realpath(self.bin_dir)
+            if (
+                not file_real.startswith(bin_dir_real + os.sep)
+                and file_real != bin_dir_real
+            ):
+                raise web.HTTPForbidden(text="forbidden")
+
+            if not os.path.isfile(file_real):
+                raise web.HTTPNotFound(text="file not found")
+
+            # use FileResponse to stream file
+            resp = web.FileResponse(path=file_real)
+        except web.HTTPError as e:
+            resp = e
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"固件下载异常: {e}")
+            resp = web.Response(text="download error", status=500)
+        finally:
+            try:
+                self._add_cors_headers(resp)
+            except Exception:
+                pass
+            return resp

main/xiaozhi-server/core/api/vision_handler.py

@@ -2,6 +2,7 @@ import json
 import copy
 from aiohttp import web
 from config.logger import setup_logging
+from core.api.base_handler import BaseHandler
 from core.utils.util import get_vision_url, is_valid_image_file
 from core.utils.vllm import create_instance
 from config.config_loader import get_private_config_from_api
@@ -16,10 +17,9 @@ TAG = __name__
 MAX_FILE_SIZE = 5 * 1024 * 1024
 
 
-class VisionHandler:
+class VisionHandler(BaseHandler):
     def __init__(self, config: dict):
-        self.config = config
-        self.logger = setup_logging()
+        super().__init__(config)
         # 初始化认证工具
         self.auth = AuthToken(config["server"]["auth_key"])
 
@@ -96,7 +96,7 @@ class VisionHandler:
             current_config = copy.deepcopy(self.config)
             read_config_from_api = current_config.get("read_config_from_api", False)
             if read_config_from_api:
-                current_config = get_private_config_from_api(
+                current_config = await get_private_config_from_api(
                     current_config,
                     device_id,
                     client_id,
@@ -172,11 +172,3 @@ class VisionHandler:
         finally:
             self._add_cors_headers(response)
             return response
-
-    def _add_cors_headers(self, response):
-        """添加CORS头信息"""
-        response.headers["Access-Control-Allow-Headers"] = (
-            "client-id, content-type, device-id"
-        )
-        response.headers["Access-Control-Allow-Credentials"] = "true"
-        response.headers["Access-Control-Allow-Origin"] = "*"

main/xiaozhi-server/core/connection.py

@@ -68,8 +68,12 @@ class ConnectionHandler:
         self.logger = setup_logging()
         self.server = server  # 保存server实例的引用
 
-        self.need_bind = False
-        self.bind_code = None
+        self.need_bind = False  # 是否需要绑定设备
+        self.bind_completed_event = asyncio.Event()
+        self.bind_code = None  # 绑定设备的验证码
+        self.last_bind_prompt_time = 0  # 上次播放绑定提示的时间戳(秒)
+        self.bind_prompt_interval = 60  # 绑定提示播放间隔(秒)
+
         self.read_config_from_api = self.config.get("read_config_from_api", False)
 
         self.websocket = None
@@ -88,7 +92,7 @@ class ConnectionHandler:
         self.client_listen_mode = "auto"
 
         # 线程任务相关
-        self.loop = asyncio.get_event_loop()
+        self.loop = None  # 在 handle_connection 中获取运行中的事件循环
         self.stop_event = threading.Event()
         self.executor = ThreadPoolExecutor(max_workers=5)
 
@@ -116,6 +120,7 @@ class ConnectionHandler:
         self.client_audio_buffer = bytearray()
         self.client_have_voice = False
         self.client_voice_window = deque(maxlen=5)
+        self.first_activity_time = 0.0  # 记录首次活动的时间（毫秒）
         self.last_activity_time = 0.0  # 统一的活动时间戳（毫秒）
         self.client_voice_stop = False
         self.last_is_voice = False
@@ -158,10 +163,13 @@ class ConnectionHandler:
         self.conn_from_mqtt_gateway = False
 
         # 初始化提示词管理器
-        self.prompt_manager = PromptManager(config, self.logger)
+        self.prompt_manager = PromptManager(self.config, self.logger)
 
     async def handle_connection(self, ws):
         try:
+            # 获取运行中的事件循环（必须在异步上下文中）
+            self.loop = asyncio.get_running_loop()
+
             # 获取并验证headers
             self.headers = dict(ws.request.headers)
             real_ip = self.headers.get("x-real-ip") or self.headers.get(
@@ -187,6 +195,7 @@ class ConnectionHandler:
                 self.logger.bind(tag=TAG).info("连接来自:MQTT网关")
 
             # 初始化活动时间戳
+            self.first_activity_time = time.time() * 1000
             self.last_activity_time = time.time() * 1000
 
             # 启动超时检查任务
@@ -195,10 +204,8 @@ class ConnectionHandler:
             self.welcome_msg = self.config["xiaozhi"]
             self.welcome_msg["session_id"] = self.session_id
 
-            # 获取差异化配置
-            self._initialize_private_config()
-            # 异步初始化
-            self.executor.submit(self._initialize_components)
+            # 在后台初始化配置和组件（完全不阻塞主循环）
+            asyncio.create_task(self._background_initialize())
 
             try:
                 async for message in self.websocket:
@@ -237,7 +244,9 @@ class ConnectionHandler:
                         loop = asyncio.new_event_loop()
                         asyncio.set_event_loop(loop)
                         loop.run_until_complete(
-                            self.memory.save_memory(self.dialogue.dialogue)
+                            self.memory.save_memory(
+                                self.dialogue.dialogue, self.session_id
+                            )
                         )
                     except Exception as e:
                         self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
@@ -260,8 +269,37 @@ class ConnectionHandler:
                     f"保存记忆后关闭连接失败: {close_error}"
                 )
 
+    async def _discard_message_with_bind_prompt(self):
+        """丢弃消息并检查是否需要播放绑定提示"""
+        current_time = time.time()
+        # 检查是否需要播放绑定提示
+        if current_time - self.last_bind_prompt_time >= self.bind_prompt_interval:
+            self.last_bind_prompt_time = current_time
+            # 复用现有的绑定提示逻辑
+            from core.handle.receiveAudioHandle import check_bind_device
+
+            asyncio.create_task(check_bind_device(self))
+
     async def _route_message(self, message):
         """消息路由"""
+        # 检查是否已经获取到真实的绑定状态
+        if not self.bind_completed_event.is_set():
+            # 还没有获取到真实状态，等待直到获取到真实状态或超时
+            try:
+                await asyncio.wait_for(self.bind_completed_event.wait(), timeout=1)
+            except asyncio.TimeoutError:
+                # 超时仍未获取到真实状态，丢弃消息
+                await self._discard_message_with_bind_prompt()
+                return
+
+        # 已经获取到真实状态，检查是否需要绑定
+        if self.need_bind:
+            # 需要绑定，丢弃消息
+            await self._discard_message_with_bind_prompt()
+            return
+
+        # 不需要绑定，继续处理消息
+
         if isinstance(message, str):
             await handleTextMessage(self, message)
         elif isinstance(message, bytes):
@@ -391,6 +429,15 @@ class ConnectionHandler:
 
     def _initialize_components(self):
         try:
+            if self.tts is None:
+                self.tts = self._initialize_tts()
+            # 打开语音合成通道
+            asyncio.run_coroutine_threadsafe(
+                self.tts.open_audio_channels(self), self.loop
+            )
+            if self.need_bind:
+                self.bind_completed_event.set()
+                return
             self.selected_module_str = build_module_string(
                 self.config.get("selected_module", {})
             )
@@ -414,17 +461,10 @@ class ConnectionHandler:
 
             # 初始化声纹识别
             self._initialize_voiceprint()
-
             # 打开语音识别通道
             asyncio.run_coroutine_threadsafe(
                 self.asr.open_audio_channels(self), self.loop
             )
-            if self.tts is None:
-                self.tts = self._initialize_tts()
-            # 打开语音合成通道
-            asyncio.run_coroutine_threadsafe(
-                self.tts.open_audio_channels(self), self.loop
-            )
 
             """加载记忆"""
             self._initialize_memory()
@@ -439,6 +479,7 @@ class ConnectionHandler:
             self.logger.bind(tag=TAG).error(f"实例化组件失败: {e}")
 
     def _init_prompt_enhancement(self):
+
         # 更新上下文信息
         self.prompt_manager.update_context_info(self, self.client_ip)
         enhanced_prompt = self.prompt_manager.build_enhanced_prompt(
@@ -474,7 +515,11 @@ class ConnectionHandler:
 
     def _initialize_asr(self):
         """初始化ASR"""
-        if self._asr.interface_type == InterfaceType.LOCAL:
+        if (
+            self._asr is not None
+            and hasattr(self._asr, "interface_type")
+            and self._asr.interface_type == InterfaceType.LOCAL
+        ):
             # 如果公共ASR是本地服务，则直接返回
             # 因为本地一个实例ASR，可以被多个连接共享
             asr = self._asr
@@ -501,22 +546,35 @@ class ConnectionHandler:
         except Exception as e:
             self.logger.bind(tag=TAG).warning(f"声纹识别初始化失败: {str(e)}")
 
-    def _initialize_private_config(self):
-        """如果是从配置文件获取，则进行二次实例化"""
+    async def _background_initialize(self):
+        """在后台初始化配置和组件（完全不阻塞主循环）"""
+        try:
+            # 异步获取差异化配置
+            await self._initialize_private_config_async()
+            # 在线程池中初始化组件
+            self.executor.submit(self._initialize_components)
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"后台初始化失败: {e}")
+
+    async def _initialize_private_config_async(self):
+        """从接口异步获取差异化配置（异步版本，不阻塞主循环）"""
         if not self.read_config_from_api:
+            self.need_bind = False
+            self.bind_completed_event.set()
             return
-        """从接口获取差异化的配置进行二次实例化，非全量重新实例化"""
         try:
             begin_time = time.time()
-            private_config = get_private_config_from_api(
+            private_config = await get_private_config_from_api(
                 self.config,
                 self.headers.get("device-id"),
                 self.headers.get("client-id", self.headers.get("device-id")),
             )
             private_config["delete_audio"] = bool(self.config.get("delete_audio", True))
             self.logger.bind(tag=TAG).info(
-                f"{time.time() - begin_time} 秒，获取差异化配置成功: {json.dumps(filter_sensitive_info(private_config), ensure_ascii=False)}"
+                f"{time.time() - begin_time} 秒，异步获取差异化配置成功: {json.dumps(filter_sensitive_info(private_config), ensure_ascii=False)}"
             )
+            self.need_bind = False
+            self.bind_completed_event.set()
         except DeviceNotFoundException as e:
             self.need_bind = True
             private_config = {}
@@ -526,7 +584,7 @@ class ConnectionHandler:
             private_config = {}
         except Exception as e:
             self.need_bind = True
-            self.logger.bind(tag=TAG).error(f"获取差异化配置失败: {e}")
+            self.logger.bind(tag=TAG).error(f"异步获取差异化配置失败: {e}")
             private_config = {}
 
         init_llm, init_tts, init_memory, init_intent = (
@@ -599,8 +657,14 @@ class ConnectionHandler:
             self.chat_history_conf = int(private_config["chat_history_conf"])
         if private_config.get("mcp_endpoint", None) is not None:
             self.config["mcp_endpoint"] = private_config["mcp_endpoint"]
+        if private_config.get("context_providers", None) is not None:
+            self.config["context_providers"] = private_config["context_providers"]
+
+        # 使用 run_in_executor 在线程池中执行 initialize_modules，避免阻塞主循环
         try:
-            modules = initialize_modules(
+            modules = await self.loop.run_in_executor(
+                None,  # 使用默认线程池
+                initialize_modules,
                 self.logger,
                 private_config,
                 init_vad,
@@ -744,18 +808,26 @@ class ConnectionHandler:
         force_final_answer = False  # 标记是否强制最终回答
 
         if depth >= MAX_DEPTH:
-            self.logger.bind(tag=TAG).debug(f"已达到最大工具调用深度 {MAX_DEPTH}，将强制基于现有信息回答")
+            self.logger.bind(tag=TAG).debug(
+                f"已达到最大工具调用深度 {MAX_DEPTH}，将强制基于现有信息回答"
+            )
             force_final_answer = True
             # 添加系统指令，要求 LLM 基于现有信息回答
-            self.dialogue.put(Message(
-                role="user",
-                content="[系统提示] 已达到最大工具调用次数限制，请你基于目前已经获取的所有信息，直接给出最终答案。不要再尝试调用任何工具。"
-            ))
+            self.dialogue.put(
+                Message(
+                    role="user",
+                    content="[系统提示] 已达到最大工具调用次数限制，请你基于目前已经获取的所有信息，直接给出最终答案。不要再尝试调用任何工具。",
+                )
+            )
 
         # Define intent functions
         functions = None
         # 达到最大深度时，禁用工具调用，强制 LLM 直接回答
-        if self.intent_type == "function_call" and hasattr(self, "func_handler") and not force_final_answer:
+        if (
+            self.intent_type == "function_call"
+            and hasattr(self, "func_handler")
+            and not force_final_answer
+        ):
             functions = self.func_handler.get_functions()
         response_message = []
 
@@ -844,11 +916,16 @@ class ConnectionHandler:
                 if a is not None:
                     try:
                         content_arguments_json = json.loads(a)
-                        tool_calls_list.append({
-                            "id": str(uuid.uuid4().hex),
-                            "name": content_arguments_json["name"],
-                            "arguments": json.dumps(content_arguments_json["arguments"], ensure_ascii=False)
-                        })
+                        tool_calls_list.append(
+                            {
+                                "id": str(uuid.uuid4().hex),
+                                "name": content_arguments_json["name"],
+                                "arguments": json.dumps(
+                                    content_arguments_json["arguments"],
+                                    ensure_ascii=False,
+                                ),
+                            }
+                        )
                     except Exception as e:
                         bHasError = True
                         response_message.append(a)
@@ -880,7 +957,9 @@ class ConnectionHandler:
                     )
 
                     future = asyncio.run_coroutine_threadsafe(
-                        self.func_handler.handle_llm_function_call(self, tool_call_data),
+                        self.func_handler.handle_llm_function_call(
+                            self, tool_call_data
+                        ),
                         self.loop,
                     )
                     futures_with_data.append((future, tool_call_data))
@@ -888,7 +967,7 @@ class ConnectionHandler:
                 # 等待协程结束（实际等待时长为最慢的那个）
                 tool_results = []
                 for future, tool_call_data in futures_with_data:
-                    result = future.result()  
+                    result = future.result()
                     tool_results.append((result, tool_call_data))
 
                 # 统一处理所有工具调用结果
@@ -922,7 +1001,11 @@ class ConnectionHandler:
         need_llm_tools = []
 
         for result, tool_call_data in tool_results:
-            if result.action in [Action.RESPONSE, Action.NOTFOUND, Action.ERROR]:  # 直接回复前端
+            if result.action in [
+                Action.RESPONSE,
+                Action.NOTFOUND,
+                Action.ERROR,
+            ]:  # 直接回复前端
                 text = result.response if result.response else result.result
                 self.tts.tts_one_sentence(self, ContentType.TEXT, content_detail=text)
                 self.dialogue.put(Message(role="assistant", content=text))
@@ -957,7 +1040,11 @@ class ConnectionHandler:
                     self.dialogue.put(
                         Message(
                             role="tool",
-                            tool_call_id=str(uuid.uuid4()) if tool_call_data["id"] is None else tool_call_data["id"],
+                            tool_call_id=(
+                                str(uuid.uuid4())
+                                if tool_call_data["id"] is None
+                                else tool_call_data["id"]
+                            ),
                             content=text,
                         )
                     )
@@ -990,8 +1077,8 @@ class ConnectionHandler:
     def _process_report(self, type, text, audio_data, report_time):
         """处理上报任务"""
         try:
-            # 执行上报（传入二进制数据）
-            report(self, type, text, audio_data, report_time)
+            # 执行异步上报（在事件循环中运行）
+            asyncio.run(report(self, type, text, audio_data, report_time))
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"上报处理异常: {e}")
         finally:
@@ -1082,7 +1169,6 @@ class ConnectionHandler:
                         f"关闭线程池时出错: {executor_error}"
                     )
                 self.executor = None
-
             self.logger.bind(tag=TAG).info("连接资源已释放")
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"关闭连接时出错: {e}")
@@ -1112,6 +1198,11 @@ class ConnectionHandler:
                     except queue.Empty:
                         break
 
+            # 重置音频流控器（取消后台任务并清空队列）
+            if hasattr(self, "audio_rate_controller") and self.audio_rate_controller:
+                self.audio_rate_controller.reset()
+                self.logger.bind(tag=TAG).debug("已重置音频流控器")
+
             self.logger.bind(tag=TAG).debug(
                 f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
             )
@@ -1137,13 +1228,14 @@ class ConnectionHandler:
         """检查连接超时"""
         try:
             while not self.stop_event.is_set():
+                last_activity_time = self.last_activity_time
+                if self.need_bind:
+                    last_activity_time = self.first_activity_time
+
                 # 检查是否超时（只有在时间戳已初始化的情况下）
-                if self.last_activity_time > 0.0:
+                if last_activity_time > 0.0:
                     current_time = time.time() * 1000
-                    if (
-                        current_time - self.last_activity_time
-                        > self.timeout_seconds * 1000
-                    ):
+                    if current_time - last_activity_time > self.timeout_seconds * 1000:
                         if not self.stop_event.is_set():
                             self.logger.bind(tag=TAG).info("连接超时，准备关闭")
                             # 设置停止事件，防止重复处理
@@ -1171,7 +1263,7 @@ class ConnectionHandler:
             tools_call: 新的工具调用
         """
         for tool_call in tools_call:
-            tool_index = getattr(tool_call, 'index', None)
+            tool_index = getattr(tool_call, "index", None)
             if tool_index is None:
                 if tool_call.function.name:
                     # 有 function_name，说明是新的工具调用
@@ -1189,4 +1281,4 @@ class ConnectionHandler:
             if tool_call.function.name:
                 tool_calls_list[tool_index]["name"] = tool_call.function.name
             if tool_call.function.arguments:
-                tool_calls_list[tool_index]["arguments"] += tool_call.function.arguments
+                tool_calls_list[tool_index]["arguments"] += tool_call.function.arguments

main/xiaozhi-server/core/handle/helloHandle.py

@@ -101,7 +101,7 @@ async def checkWakeupWords(conn, text):
         }
 
     # 获取音频数据
-    opus_packets = audio_to_data(response.get("file_path"))
+    opus_packets = await audio_to_data(response.get("file_path"), use_cache=False)
     # 播放唤醒词回复
     conn.client_abort = False
 

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -123,7 +123,7 @@ async def max_out_size(conn):
     text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！"
     await send_stt_message(conn, text)
     file_path = "config/assets/max_output_size.wav"
-    opus_packets = audio_to_data(file_path)
+    opus_packets = await audio_to_data(file_path)
     conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))
     conn.close_after_chat = True
 
@@ -142,7 +142,7 @@ async def check_bind_device(conn):
 
         # 播放提示音
         music_path = "config/assets/bind_code.wav"
-        opus_packets = audio_to_data(music_path)
+        opus_packets = await audio_to_data(music_path)
         conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, text))
 
         # 逐个播放数字
@@ -150,7 +150,7 @@ async def check_bind_device(conn):
             try:
                 digit = conn.bind_code[i]
                 num_path = f"config/assets/bind_code/{digit}.wav"
-                num_packets = audio_to_data(num_path)
+                num_packets = await audio_to_data(num_path)
                 conn.tts.tts_audio_queue.put((SentenceType.MIDDLE, num_packets, None))
             except Exception as e:
                 conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
@@ -162,5 +162,5 @@ async def check_bind_device(conn):
         text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
         await send_stt_message(conn, text)
         music_path = "config/assets/bind_not_found.wav"
-        opus_packets = audio_to_data(music_path)
+        opus_packets = await audio_to_data(music_path)
         conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))

main/xiaozhi-server/core/handle/reportHandle.py

@@ -10,7 +10,6 @@ TTS上报功能已集成到ConnectionHandler类中。
 """
 
 import time
-
 import opuslib_next
 
 from config.manage_api_client import report as manage_report
@@ -18,7 +17,7 @@ from config.manage_api_client import report as manage_report
 TAG = __name__
 
 
-def report(conn, type, text, opus_data, report_time):
+async def report(conn, type, text, opus_data, report_time):
     """执行聊天记录上报操作
 
     Args:
@@ -33,8 +32,8 @@ def report(conn, type, text, opus_data, report_time):
             audio_data = opus_to_wav(conn, opus_data)
         else:
             audio_data = None
-        # 执行上报
-        manage_report(
+        # 执行异步上报
+        await manage_report(
             mac_address=conn.device_id,
             session_id=conn.session_id,
             chat_type=type,
@@ -56,41 +55,49 @@ def opus_to_wav(conn, opus_data):
     Returns:
         bytes: WAV格式的音频数据
     """
-    decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-    pcm_data = []
+    decoder = None
+    try:
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
 
-    for opus_packet in opus_data:
-        try:
-            pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-            pcm_data.append(pcm_frame)
-        except opuslib_next.OpusError as e:
-            conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
 
-    if not pcm_data:
-        raise ValueError("没有有效的PCM数据")
+        if not pcm_data:
+            raise ValueError("没有有效的PCM数据")
 
-    # 创建WAV文件头
-    pcm_data_bytes = b"".join(pcm_data)
-    num_samples = len(pcm_data_bytes) // 2  # 16-bit samples
+        # 创建WAV文件头
+        pcm_data_bytes = b"".join(pcm_data)
+        num_samples = len(pcm_data_bytes) // 2  # 16-bit samples
 
-    # WAV文件头
-    wav_header = bytearray()
-    wav_header.extend(b"RIFF")  # ChunkID
-    wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little"))  # ChunkSize
-    wav_header.extend(b"WAVE")  # Format
-    wav_header.extend(b"fmt ")  # Subchunk1ID
-    wav_header.extend((16).to_bytes(4, "little"))  # Subchunk1Size
-    wav_header.extend((1).to_bytes(2, "little"))  # AudioFormat (PCM)
-    wav_header.extend((1).to_bytes(2, "little"))  # NumChannels
-    wav_header.extend((16000).to_bytes(4, "little"))  # SampleRate
-    wav_header.extend((32000).to_bytes(4, "little"))  # ByteRate
-    wav_header.extend((2).to_bytes(2, "little"))  # BlockAlign
-    wav_header.extend((16).to_bytes(2, "little"))  # BitsPerSample
-    wav_header.extend(b"data")  # Subchunk2ID
-    wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little"))  # Subchunk2Size
+        # WAV文件头
+        wav_header = bytearray()
+        wav_header.extend(b"RIFF")  # ChunkID
+        wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little"))  # ChunkSize
+        wav_header.extend(b"WAVE")  # Format
+        wav_header.extend(b"fmt ")  # Subchunk1ID
+        wav_header.extend((16).to_bytes(4, "little"))  # Subchunk1Size
+        wav_header.extend((1).to_bytes(2, "little"))  # AudioFormat (PCM)
+        wav_header.extend((1).to_bytes(2, "little"))  # NumChannels
+        wav_header.extend((16000).to_bytes(4, "little"))  # SampleRate
+        wav_header.extend((32000).to_bytes(4, "little"))  # ByteRate
+        wav_header.extend((2).to_bytes(2, "little"))  # BlockAlign
+        wav_header.extend((16).to_bytes(2, "little"))  # BitsPerSample
+        wav_header.extend(b"data")  # Subchunk2ID
+        wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little"))  # Subchunk2Size
 
-    # 返回完整的WAV数据
-    return bytes(wav_header) + pcm_data_bytes
+        # 返回完整的WAV数据
+        return bytes(wav_header) + pcm_data_bytes
+    finally:
+        if decoder is not None:
+            try:
+                del decoder
+            except Exception as e:
+                conn.logger.bind(tag=TAG).debug(f"释放decoder资源时出错: {e}")
 
 
 def enqueue_tts_report(conn, text, opus_data):

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -4,8 +4,13 @@ import asyncio
 from core.utils import textUtils
 from core.utils.util import audio_to_data
 from core.providers.tts.dto.dto import SentenceType
+from core.utils.audioRateController import AudioRateController
 
 TAG = __name__
+# 音频帧时长（毫秒）
+AUDIO_FRAME_DURATION = 60
+# 预缓冲包数量，直接发送以减少延迟
+PRE_BUFFER_COUNT = 5
 
 
 async def sendAudioMessage(conn, sentenceType, audios, text):
@@ -15,7 +20,19 @@ async def sendAudioMessage(conn, sentenceType, audios, text):
         await send_tts_message(conn, "start", None)
 
     if sentenceType == SentenceType.FIRST:
-        await send_tts_message(conn, "sentence_start", text)
+        # 同一句子的后续消息加入流控队列，其他情况立即发送
+        if (
+            hasattr(conn, "audio_rate_controller")
+            and conn.audio_rate_controller
+            and getattr(conn, "audio_flow_control", {}).get("sentence_id")
+            == conn.sentence_id
+        ):
+            conn.audio_rate_controller.add_message(
+                lambda: send_tts_message(conn, "sentence_start", text)
+            )
+        else:
+            # 新句子或流控器未初始化，立即发送
+            await send_tts_message(conn, "sentence_start", text)
 
     await sendAudio(conn, audios)
     # 发送句子开始消息
@@ -30,29 +47,27 @@ async def sendAudioMessage(conn, sentenceType, audios, text):
             await conn.close()
 
 
-def calculate_timestamp_and_sequence(conn, start_time, packet_index, frame_duration=60):
+async def _wait_for_audio_completion(conn):
     """
-    计算音频数据包的时间戳和序列号
+    等待音频队列清空并等待预缓冲包播放完成
+
     Args:
         conn: 连接对象
-        start_time: 起始时间（性能计数器值）
-        packet_index: 数据包索引
-        frame_duration: 帧时长（毫秒），匹配 Opus 编码
-    Returns:
-        tuple: (timestamp, sequence)
     """
-    # 计算时间戳（使用播放位置计算）
-    timestamp = int((start_time + packet_index * frame_duration / 1000) * 1000) % (
-        2**32
-    )
+    if hasattr(conn, "audio_rate_controller") and conn.audio_rate_controller:
+        rate_controller = conn.audio_rate_controller
+        conn.logger.bind(tag=TAG).debug(
+            f"等待音频发送完成，队列中还有 {len(rate_controller.queue)} 个包"
+        )
+        await rate_controller.queue_empty_event.wait()
 
-    # 计算序列号
-    if hasattr(conn, "audio_flow_control"):
-        sequence = conn.audio_flow_control["sequence"]
-    else:
-        sequence = packet_index  # 如果没有流控状态，直接使用索引
+        # 等待预缓冲包播放完成
+        # 前N个包直接发送，增加2个网络抖动包，需要额外等待它们在客户端播放完成
+        frame_duration_ms = rate_controller.frame_duration
+        pre_buffer_playback_time = (PRE_BUFFER_COUNT + 2) * frame_duration_ms / 1000.0
+        await asyncio.sleep(pre_buffer_playback_time)
 
-    return timestamp, sequence
+        conn.logger.bind(tag=TAG).debug("音频发送完成")
 
 
 async def _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence):
@@ -77,135 +92,151 @@ async def _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence):
     await conn.websocket.send(complete_packet)
 
 
-# 播放音频
-async def sendAudio(conn, audios, frame_duration=60):
+async def sendAudio(conn, audios, frame_duration=AUDIO_FRAME_DURATION):
     """
-    发送单个opus包，支持流控
+    发送音频包，使用 AudioRateController 进行精确的流量控制
+
     Args:
         conn: 连接对象
-        opus_packet: 单个opus数据包
-        pre_buffer: 快速发送音频
-        frame_duration: 帧时长（毫秒），匹配 Opus 编码
+        audios: 单个opus包(bytes) 或 opus包列表
+        frame_duration: 帧时长（毫秒），默认使用全局常量AUDIO_FRAME_DURATION
     """
     if audios is None or len(audios) == 0:
         return
 
-    # 获取发送延迟配置
     send_delay = conn.config.get("tts_audio_send_delay", -1) / 1000.0
+    is_single_packet = isinstance(audios, bytes)
 
-    if isinstance(audios, bytes):
-        # 重置流控状态,第一次读取和会话发生转变时
-        if not hasattr(conn, "audio_flow_control") or conn.audio_flow_control.get("sentence_id") != conn.sentence_id:
-            conn.audio_flow_control = {
-                "last_send_time": 0,
-                "packet_count": 0,
-                "start_time": time.perf_counter(),
-                "sequence": 0,  # 添加序列号
-                "sentence_id": conn.sentence_id,
-            }
+    # 初始化或获取 RateController
+    rate_controller, flow_control = _get_or_create_rate_controller(
+        conn, frame_duration, is_single_packet
+    )
 
+    # 统一转换为列表处理
+    audio_list = [audios] if is_single_packet else audios
+
+    # 发送音频包
+    await _send_audio_with_rate_control(
+        conn, audio_list, rate_controller, flow_control, send_delay
+    )
+
+
+def _get_or_create_rate_controller(conn, frame_duration, is_single_packet):
+    """
+    获取或创建 RateController 和 flow_control
+
+    Args:
+        conn: 连接对象
+        frame_duration: 帧时长
+        is_single_packet: 是否单包模式（True: TTS流式单包, False: 批量包）
+
+    Returns:
+        (rate_controller, flow_control)
+    """
+    # 判断是否需要重置：单包模式且 sentence_id 变化，或者控制器不存在
+    need_reset = (
+        is_single_packet
+        and getattr(conn, "audio_flow_control", {}).get("sentence_id")
+        != conn.sentence_id
+    ) or not hasattr(conn, "audio_rate_controller")
+
+    if need_reset:
+        # 创建或获取 rate_controller
+        if not hasattr(conn, "audio_rate_controller"):
+            conn.audio_rate_controller = AudioRateController(frame_duration)
+        else:
+            conn.audio_rate_controller.reset()
+
+        # 初始化 flow_control
+        conn.audio_flow_control = {
+            "packet_count": 0,
+            "sequence": 0,
+            "sentence_id": conn.sentence_id,
+        }
+
+        # 启动后台发送循环
+        _start_background_sender(
+            conn, conn.audio_rate_controller, conn.audio_flow_control
+        )
+
+    return conn.audio_rate_controller, conn.audio_flow_control
+
+
+def _start_background_sender(conn, rate_controller, flow_control):
+    """
+    启动后台发送循环任务
+
+    Args:
+        conn: 连接对象
+        rate_controller: 速率控制器
+        flow_control: 流控状态
+    """
+
+    async def send_callback(packet):
+        # 检查是否应该中止
+        if conn.client_abort:
+            raise asyncio.CancelledError("客户端已中止")
+
+        conn.last_activity_time = time.time() * 1000
+        await _do_send_audio(conn, packet, flow_control)
+        conn.client_is_speaking = True
+
+    # 使用 start_sending 启动后台循环
+    rate_controller.start_sending(send_callback)
+
+
+async def _send_audio_with_rate_control(
+    conn, audio_list, rate_controller, flow_control, send_delay
+):
+    """
+    使用 rate_controller 发送音频包
+
+    Args:
+        conn: 连接对象
+        audio_list: 音频包列表
+        rate_controller: 速率控制器
+        flow_control: 流控状态
+        send_delay: 固定延迟（秒），-1表示使用动态流控
+    """
+    for packet in audio_list:
         if conn.client_abort:
             return
 
         conn.last_activity_time = time.time() * 1000
 
-        # 预缓冲：前5个包直接发送，不做延迟
-        pre_buffer_count = 5
-        flow_control = conn.audio_flow_control
-        current_time = time.perf_counter()
-
-        if flow_control["packet_count"] < pre_buffer_count:
-            # 预缓冲阶段，直接发送不延迟
-            pass
+        # 预缓冲：前N个包直接发送
+        if flow_control["packet_count"] < PRE_BUFFER_COUNT:
+            await _do_send_audio(conn, packet, flow_control)
+            conn.client_is_speaking = True
         elif send_delay > 0:
-            # 使用固定延迟
+            # 固定延迟模式
             await asyncio.sleep(send_delay)
+            await _do_send_audio(conn, packet, flow_control)
+            conn.client_is_speaking = True
         else:
-            effective_packet = flow_control["packet_count"] - pre_buffer_count
-            expected_time = flow_control["start_time"] + (
-                effective_packet * frame_duration / 1000
-            )
-            delay = expected_time - current_time
-            if delay > 0:
-                await asyncio.sleep(delay)
-            else:
-                # 纠正误差
-                flow_control["start_time"] += abs(delay)
+            # 动态流控模式：仅添加到队列，由后台循环负责发送
+            rate_controller.add_audio(packet)
 
-        if conn.conn_from_mqtt_gateway:
-            # 计算时间戳和序列号
-            timestamp, sequence = calculate_timestamp_and_sequence(
-                conn,
-                flow_control["start_time"],
-                flow_control["packet_count"],
-                frame_duration,
-            )
-            # 调用通用函数发送带头部的数据包
-            await _send_to_mqtt_gateway(conn, audios, timestamp, sequence)
-        else:
-            # 直接发送opus数据包，不添加头部
-            await conn.websocket.send(audios)
-        conn.client_is_speaking = True
 
-        # 更新流控状态
-        flow_control["packet_count"] += 1
-        flow_control["sequence"] += 1
-        flow_control["last_send_time"] = time.perf_counter()
+async def _do_send_audio(conn, opus_packet, flow_control):
+    """
+    执行实际的音频发送
+    """
+    packet_index = flow_control.get("packet_count", 0)
+    sequence = flow_control.get("sequence", 0)
+
+    if conn.conn_from_mqtt_gateway:
+        # 计算时间戳（基于播放位置）
+        start_time = time.time()
+        timestamp = int(start_time * 1000) % (2**32)
+        await _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence)
     else:
-        # 文件型音频走普通播放
-        start_time = time.perf_counter()
-        play_position = 0
+        # 直接发送opus数据包
+        await conn.websocket.send(opus_packet)
 
-        # 执行预缓冲
-        pre_buffer_frames = min(5, len(audios))
-        for i in range(pre_buffer_frames):
-            if conn.conn_from_mqtt_gateway:
-                # 计算时间戳和序列号
-                timestamp, sequence = calculate_timestamp_and_sequence(
-                    conn, start_time, i, frame_duration
-                )
-                # 调用通用函数发送带头部的数据包
-                await _send_to_mqtt_gateway(conn, audios[i], timestamp, sequence)
-            else:
-                # 直接发送预缓冲包，不添加头部
-                await conn.websocket.send(audios[i])
-            conn.client_is_speaking = True
-        remaining_audios = audios[pre_buffer_frames:]
-
-        # 播放剩余音频帧
-        for i, opus_packet in enumerate(remaining_audios):
-            if conn.client_abort:
-                break
-
-            # 重置没有声音的状态
-            conn.last_activity_time = time.time() * 1000
-
-            if send_delay > 0:
-                # 固定延迟模式
-                await asyncio.sleep(send_delay)
-            else:
-                 # 计算预期发送时间
-                expected_time = start_time + (play_position / 1000)
-                current_time = time.perf_counter()
-                delay = expected_time - current_time
-                if delay > 0:
-                    await asyncio.sleep(delay)
-
-            if conn.conn_from_mqtt_gateway:
-                # 计算时间戳和序列号（使用当前的数据包索引确保连续性）
-                packet_index = pre_buffer_frames + i
-                timestamp, sequence = calculate_timestamp_and_sequence(
-                    conn, start_time, packet_index, frame_duration
-                )
-                # 调用通用函数发送带头部的数据包
-                await _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence)
-            else:
-                # 直接发送opus数据包，不添加头部
-                await conn.websocket.send(opus_packet)
-
-            conn.client_is_speaking = True
-
-            play_position += frame_duration
+    # 更新流控状态
+    flow_control["packet_count"] = packet_index + 1
+    flow_control["sequence"] = sequence + 1
 
 
 async def send_tts_message(conn, state, text=None):
@@ -224,8 +255,10 @@ async def send_tts_message(conn, state, text=None):
             stop_tts_notify_voice = conn.config.get(
                 "stop_tts_notify_voice", "config/assets/tts_notify.mp3"
             )
-            audios = audio_to_data(stop_tts_notify_voice, is_opus=True)
+            audios = await audio_to_data(stop_tts_notify_voice, is_opus=True)
             await sendAudio(conn, audios)
+        # 等待所有音频包发送完成
+        await _wait_for_audio_completion(conn)
         # 清除服务端讲话状态
         conn.clearSpeakStatus()
 

main/xiaozhi-server/core/handle/textHandler/listenMessageHandler.py

@@ -1,12 +1,14 @@
 import time
+import asyncio
 from typing import Dict, Any
 
-from core.handle.receiveAudioHandle import handleAudioMessage, startToChat
+from core.handle.receiveAudioHandle import startToChat
 from core.handle.reportHandle import enqueue_asr_report
 from core.handle.sendAudioHandle import send_stt_message, send_tts_message
 from core.handle.textMessageHandler import TextMessageHandler
 from core.handle.textMessageType import TextMessageType
 from core.utils.util import remove_punctuation_and_length
+from core.providers.asr.dto.dto import InterfaceType
 
 TAG = __name__
 
@@ -29,8 +31,18 @@ class ListenTextMessageHandler(TextMessageHandler):
         elif msg_json["state"] == "stop":
             conn.client_have_voice = True
             conn.client_voice_stop = True
-            if len(conn.asr_audio) > 0:
-                await handleAudioMessage(conn, b"")
+            if conn.asr.interface_type == InterfaceType.STREAM:
+                # 流式模式下，发送结束请求
+                asyncio.create_task(conn.asr._send_stop_request())
+            else:
+                # 非流式模式：直接触发ASR识别
+                if len(conn.asr_audio) > 0:
+                    asr_audio_task = conn.asr_audio.copy()
+                    conn.asr_audio.clear()
+                    conn.reset_vad_states()
+
+                    if len(asr_audio_task) > 0:
+                        await conn.asr.handle_voice_stop(conn, asr_audio_task)
         elif msg_json["state"] == "detect":
             conn.client_have_voice = False
             conn.asr_audio.clear()
@@ -57,6 +69,7 @@ class ListenTextMessageHandler(TextMessageHandler):
                     enqueue_asr_report(conn, "嘿，你好呀", [])
                     await startToChat(conn, "嘿，你好呀")
                 else:
+                    conn.just_woken_up = True
                     # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
                     enqueue_asr_report(conn, original_text, [])
                     # 否则需要LLM对文字内容进行答复

main/xiaozhi-server/core/handle/textHandler/pingMessageHandler.py

@@ -0,0 +1,45 @@
+import json
+import time
+from typing import Dict, Any
+
+from core.handle.textMessageHandler import TextMessageHandler
+from core.handle.textMessageType import TextMessageType
+
+TAG = __name__
+
+
+class PingMessageHandler(TextMessageHandler):
+    """Ping消息处理器，用于保持WebSocket连接"""
+
+    @property
+    def message_type(self) -> TextMessageType:
+        return TextMessageType.PING
+
+    async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
+        """
+        处理PING消息，发送PONG响应
+        消息格式：{"type": "ping"}
+        Args:
+            conn: WebSocket连接对象
+            msg_json: PING消息的JSON数据
+        """
+        # 检查是否启用了WebSocket心跳功能
+        enable_websocket_ping = conn.config.get("enable_websocket_ping", False)
+        if not enable_websocket_ping:
+            conn.logger.debug(f"WebSocket心跳功能未启用，忽略PING消息")
+            return
+
+        try:
+            conn.logger.debug(f"收到PING消息，发送PONG响应")
+            conn.last_activity_time = time.time() * 1000
+            # 构造PONG响应消息
+            pong_message = {
+                "type": "pong",
+                "timestamp": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
+            }
+
+            # 发送PONG响应
+            await conn.websocket.send(json.dumps(pong_message))
+
+        except Exception as e:
+            conn.logger.error(f"处理PING消息时发生错误: {e}")

main/xiaozhi-server/core/handle/textMessageHandlerRegistry.py

@@ -7,6 +7,7 @@ from core.handle.textHandler.listenMessageHandler import ListenTextMessageHandle
 from core.handle.textHandler.mcpMessageHandler import McpTextMessageHandler
 from core.handle.textMessageHandler import TextMessageHandler
 from core.handle.textHandler.serverMessageHandler import ServerTextMessageHandler
+from core.handle.textHandler.pingMessageHandler import PingMessageHandler
 
 TAG = __name__
 
@@ -27,6 +28,7 @@ class TextMessageHandlerRegistry:
             IotTextMessageHandler(),
             McpTextMessageHandler(),
             ServerTextMessageHandler(),
+            PingMessageHandler(),
         ]
 
         for handler in handlers:

main/xiaozhi-server/core/handle/textMessageType.py

@@ -9,3 +9,4 @@ class TextMessageType(Enum):
     IOT = "iot"
     MCP = "mcp"
     SERVER = "server"
+    PING = "ping"

main/xiaozhi-server/core/http_server.py

@@ -33,38 +33,60 @@ class SimpleHttpServer:
             return f"ws://{local_ip}:{port}/xiaozhi/v1/"
 
     async def start(self):
-        server_config = self.config["server"]
-        read_config_from_api = self.config.get("read_config_from_api", False)
-        host = server_config.get("ip", "0.0.0.0")
-        port = int(server_config.get("http_port", 8003))
+        try:
+            server_config = self.config["server"]
+            read_config_from_api = self.config.get("read_config_from_api", False)
+            host = server_config.get("ip", "0.0.0.0")
+            port = int(server_config.get("http_port", 8003))
 
-        if port:
-            app = web.Application()
+            if port:
+                app = web.Application()
 
-            if not read_config_from_api:
-                # 如果没有开启智控台，只是单模块运行，就需要再添加简单OTA接口，用于下发websocket接口
+                if not read_config_from_api:
+                    # 如果没有开启智控台，只是单模块运行，就需要再添加简单OTA接口，用于下发websocket接口
+                    app.add_routes(
+                        [
+                            web.get("/xiaozhi/ota/", self.ota_handler.handle_get),
+                            web.post("/xiaozhi/ota/", self.ota_handler.handle_post),
+                            web.options(
+                                "/xiaozhi/ota/", self.ota_handler.handle_options
+                            ),
+                            # 下载接口，仅提供 data/bin/*.bin 下载
+                            web.get(
+                                "/xiaozhi/ota/download/{filename}",
+                                self.ota_handler.handle_download,
+                            ),
+                            web.options(
+                                "/xiaozhi/ota/download/{filename}",
+                                self.ota_handler.handle_options,
+                            ),
+                        ]
+                    )
+                # 添加路由
                 app.add_routes(
                     [
-                        web.get("/xiaozhi/ota/", self.ota_handler.handle_get),
-                        web.post("/xiaozhi/ota/", self.ota_handler.handle_post),
-                        web.options("/xiaozhi/ota/", self.ota_handler.handle_post),
+                        web.get("/mcp/vision/explain", self.vision_handler.handle_get),
+                        web.post(
+                            "/mcp/vision/explain", self.vision_handler.handle_post
+                        ),
+                        web.options(
+                            "/mcp/vision/explain", self.vision_handler.handle_options
+                        ),
                     ]
                 )
-            # 添加路由
-            app.add_routes(
-                [
-                    web.get("/mcp/vision/explain", self.vision_handler.handle_get),
-                    web.post("/mcp/vision/explain", self.vision_handler.handle_post),
-                    web.options("/mcp/vision/explain", self.vision_handler.handle_post),
-                ]
-            )
 
-            # 运行服务
-            runner = web.AppRunner(app)
-            await runner.setup()
-            site = web.TCPSite(runner, host, port)
-            await site.start()
+                # 运行服务
+                runner = web.AppRunner(app)
+                await runner.setup()
+                site = web.TCPSite(runner, host, port)
+                await site.start()
 
-            # 保持服务运行
-            while True:
-                await asyncio.sleep(3600)  # 每隔 1 小时检查一次
+                # 保持服务运行
+                while True:
+                    await asyncio.sleep(3600)  # 每隔 1 小时检查一次
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"HTTP服务器启动失败: {e}")
+            import traceback
+
+            self.logger.bind(tag=TAG).error(f"错误堆栈: {traceback.format_exc()}")
+            raise

main/xiaozhi-server/core/providers/asr/aliyun_stream.py

@@ -8,8 +8,6 @@ import asyncio
 import requests
 import websockets
 import opuslib_next
-import random
-from typing import Optional, Tuple, List
 from urllib import parse
 from datetime import datetime
 from config.logger import setup_logging
@@ -139,13 +137,13 @@ class ASRProvider(ASRProviderBase):
         conn.asr_audio.append(audio)
         conn.asr_audio = conn.asr_audio[-10:]
 
-        # 只在有声音且没有连接时建立连接
-        if audio_have_voice and not self.is_processing:
+        # 只在有声音且没有连接时建立连接（排除正在停止的情况）
+        if audio_have_voice and not self.is_processing and not self.asr_ws:
             try:
                 await self._start_recognition(conn)
             except Exception as e:
                 logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}")
-                await self._cleanup(conn)
+                await self._cleanup()
                 return
 
         if self.asr_ws and self.is_processing and self.server_ready:
@@ -185,10 +183,8 @@ class ASRProvider(ASRProviderBase):
             "header": {
                 "namespace": "SpeechTranscriber",
                 "name": "StartTranscription",
-                "status": 20000000,
                 "message_id": uuid.uuid4().hex,
                 "task_id": self.task_id,
-                "status_text": "Gateway:SUCCESS:Success.",
                 "appkey": self.appkey
             },
             "payload": {
@@ -207,18 +203,21 @@ class ASRProvider(ASRProviderBase):
     async def _forward_results(self, conn):
         """转发识别结果"""
         try:
-            while self.asr_ws and not conn.stop_event.is_set():
+            while not conn.stop_event.is_set():
                 try:
                     response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
                     result = json.loads(response)
-                    
+
                     header = result.get("header", {})
                     payload = result.get("payload", {})
                     message_name = header.get("name", "")
                     status = header.get("status", 0)
-                    
+
                     if status != 20000000:
-                        if status in [40000004, 40010004]:  # 连接超时或客户端断开
+                        if status == 40010004:
+                            logger.bind(tag=TAG).warning(f"请在服务端响应完成后再关闭链接，状态码: {status}")
+                            break
+                        if status in [40000004, 40010003]:  # 连接超时或客户端断开
                             logger.bind(tag=TAG).warning(f"连接问题，状态码: {status}")
                             break
                         elif status in [40270002, 40270003]:  # 音频问题
@@ -227,12 +226,12 @@ class ASRProvider(ASRProviderBase):
                         else:
                             logger.bind(tag=TAG).error(f"识别错误，状态码: {status}, 消息: {header.get('status_text', '')}")
                             continue
-                    
+
                     # 收到TranscriptionStarted表示服务器准备好接收音频数据
                     if message_name == "TranscriptionStarted":
                         self.server_ready = True
                         logger.bind(tag=TAG).debug("服务器已准备，开始发送缓存音频...")
-                        
+
                         # 发送缓存音频
                         if conn.asr_audio:
                             for cached_audio in conn.asr_audio[-10:]:
@@ -243,89 +242,89 @@ class ASRProvider(ASRProviderBase):
                                     logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}")
                                     break
                         continue
-                    
-                    if message_name == "TranscriptionResultChanged":
-                        # 中间结果
-                        text = payload.get("result", "")
-                        if text:
-                            self.text = text
                     elif message_name == "SentenceEnd":
-                        # 最终结果
+                        # 句子结束（每个句子都会触发）
                         text = payload.get("result", "")
                         if text:
-                            self.text = text
-                            conn.reset_vad_states()
-                            # 传递缓存的音频数据
-                            audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
-                            await self.handle_voice_stop(conn, audio_data)
-                            # 清空缓存
-                            conn.asr_audio_for_voiceprint = []
-                            break
-                    elif message_name == "TranscriptionCompleted":
-                        # 识别完成
-                        self.is_processing = False
-                        break
-                        
+                            logger.bind(tag=TAG).info(f"识别到文本: {text}")
+
+                            # 手动模式下累积识别结果
+                            if conn.client_listen_mode == "manual":
+                                if self.text:
+                                    self.text += text
+                                else:
+                                    self.text = text
+
+                                # 手动模式下，只有在收到stop信号后才触发处理（仅处理一次）
+                                if conn.client_voice_stop:
+                                    audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                                    if len(audio_data) > 0:
+                                        logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
+                                        await self.handle_voice_stop(conn, audio_data)
+                                        # 清理音频缓存
+                                        conn.asr_audio.clear()
+                                        conn.reset_vad_states()
+                                    break
+                            else:
+                                # 自动模式下直接覆盖
+                                self.text = text
+                                conn.reset_vad_states()
+                                audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                                await self.handle_voice_stop(conn, audio_data)
+                                break
+
                 except asyncio.TimeoutError:
-                    continue
-                except websockets.exceptions.ConnectionClosed:
+                    logger.bind(tag=TAG).error("接收结果超时")
+                    break
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).info("ASR服务连接已关闭")
+                    self.is_processing = False
                     break
                 except Exception as e:
                     logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}")
                     break
-                    
+
         except Exception as e:
             logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}")
         finally:
-            await self._cleanup(conn)
+            # 清理连接的音频缓存
+            await self._cleanup()
+            if conn:
+                if hasattr(conn, 'asr_audio_for_voiceprint'):
+                    conn.asr_audio_for_voiceprint = []
+                if hasattr(conn, 'asr_audio'):
+                    conn.asr_audio = []
 
-    async def _cleanup(self, conn):
-        """清理资源"""
-        logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
-        
-        # 清理连接的音频缓存
-        if conn and hasattr(conn, 'asr_audio_for_voiceprint'):
-            conn.asr_audio_for_voiceprint = []
-        
-        # 判断是否需要发送终止请求
-        should_stop = self.is_processing or self.server_ready
-        
-        # 发送停止识别请求
-        if self.asr_ws and should_stop:
+    async def _send_stop_request(self):
+        """发送停止识别请求（不关闭连接）"""
+        if self.asr_ws:
             try:
+                # 先停止音频发送
+                self.is_processing = False
+
                 stop_msg = {
                     "header": {
                         "namespace": "SpeechTranscriber",
                         "name": "StopTranscription",
-                        "status": 20000000,
                         "message_id": uuid.uuid4().hex,
                         "task_id": self.task_id,
-                        "status_text": "Client:Stop",
                         "appkey": self.appkey
                     }
                 }
-                logger.bind(tag=TAG).debug("正在发送ASR终止请求")
+                logger.bind(tag=TAG).debug("停止识别请求已发送")
                 await self.asr_ws.send(json.dumps(stop_msg, ensure_ascii=False))
-                await asyncio.sleep(0.1)
-                logger.bind(tag=TAG).debug("ASR终止请求已发送")
             except Exception as e:
-                logger.bind(tag=TAG).error(f"ASR终止请求发送失败: {e}")
-        
-        # 状态重置（在终止请求发送后）
+                logger.bind(tag=TAG).error(f"发送停止识别请求失败: {e}")
+
+    async def _cleanup(self):
+        """清理资源（关闭连接）"""
+        logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
+
+        # 状态重置
         self.is_processing = False
         self.server_ready = False
         logger.bind(tag=TAG).debug("ASR状态已重置")
 
-        # 清理任务
-        if self.forward_task and not self.forward_task.done():
-            self.forward_task.cancel()
-            try:
-                await asyncio.wait_for(self.forward_task, timeout=1.0)
-            except Exception as e:
-                logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}")
-            finally:
-                self.forward_task = None
-        
         # 关闭连接
         if self.asr_ws:
             try:
@@ -336,7 +335,10 @@ class ASRProvider(ASRProviderBase):
                 logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
             finally:
                 self.asr_ws = None
-        
+
+        # 清理任务引用
+        self.forward_task = None
+
         logger.bind(tag=TAG).debug("ASR会话清理完成")
 
     async def speech_to_text(self, opus_data, session_id, audio_format):
@@ -347,4 +349,11 @@ class ASRProvider(ASRProviderBase):
 
     async def close(self):
         """关闭资源"""
-        await self._cleanup()
+        await self._cleanup(None)
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Aliyun decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Aliyun decoder资源时出错: {e}")

main/xiaozhi-server/core/providers/asr/base.py

@@ -9,7 +9,6 @@ import asyncio
 import traceback
 import threading
 import opuslib_next
-import concurrent.futures
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
@@ -53,121 +52,89 @@ class ASRProviderBase(ABC):
 
     # 接收音频
     async def receive_audio(self, conn, audio, audio_have_voice):
-        if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime":
-            have_voice = audio_have_voice
+        if conn.client_listen_mode == "manual":
+            # 手动模式：缓存音频用于ASR识别
+            conn.asr_audio.append(audio)
         else:
-            have_voice = conn.client_have_voice
-        
-        conn.asr_audio.append(audio)
-        if not have_voice and not conn.client_have_voice:
-            conn.asr_audio = conn.asr_audio[-10:]
-            return
+            # 自动/实时模式：使用VAD检测
+            have_voice = audio_have_voice
 
-        if conn.client_voice_stop:
-            asr_audio_task = conn.asr_audio.copy()
-            conn.asr_audio.clear()
-            conn.reset_vad_states()
+            conn.asr_audio.append(audio)
+            if not have_voice and not conn.client_have_voice:
+                conn.asr_audio = conn.asr_audio[-10:]
+                return
 
-            if len(asr_audio_task) > 15:
-                await self.handle_voice_stop(conn, asr_audio_task)
+            # 自动模式下通过VAD检测到语音停止时触发识别
+            if conn.client_voice_stop:
+                asr_audio_task = conn.asr_audio.copy()
+                conn.asr_audio.clear()
+                conn.reset_vad_states()
+
+                if len(asr_audio_task) > 15:
+                    await self.handle_voice_stop(conn, asr_audio_task)
 
     # 处理语音停止
     async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
         """并行处理ASR和声纹识别"""
         try:
             total_start_time = time.monotonic()
-            
+
             # 准备音频数据
             if conn.audio_format == "pcm":
                 pcm_data = asr_audio_task
             else:
                 pcm_data = self.decode_opus(asr_audio_task)
-            
+
             combined_pcm_data = b"".join(pcm_data)
-            
+
             # 预先准备WAV数据
             wav_data = None
             if conn.voiceprint_provider and combined_pcm_data:
                 wav_data = self._pcm_to_wav(combined_pcm_data)
-            
+
             # 定义ASR任务
-            def run_asr():
-                start_time = time.monotonic()
-                try:
-                    loop = asyncio.new_event_loop()
-                    asyncio.set_event_loop(loop)
-                    try:
-                        result = loop.run_until_complete(
-                            self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
-                        )
-                        end_time = time.monotonic()
-                        logger.bind(tag=TAG).debug(f"ASR耗时: {end_time - start_time:.3f}s")
-                        return result
-                    finally:
-                        loop.close()
-                except Exception as e:
-                    end_time = time.monotonic()
-                    logger.bind(tag=TAG).error(f"ASR失败: {e}")
-                    return ("", None)
-            
-            # 定义声纹识别任务
-            def run_voiceprint():
-                if not wav_data:
-                    return None
-                try:
-                    loop = asyncio.new_event_loop()
-                    asyncio.set_event_loop(loop)
-                    try:
-                        # 使用连接的声纹识别提供者
-                        result = loop.run_until_complete(
-                            conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
-                        )
-                        return result
-                    finally:
-                        loop.close()
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"声纹识别失败: {e}")
-                    return None
-            
-            # 使用线程池执行器并行运行
-            with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor:
-                asr_future = thread_executor.submit(run_asr)
-                
-                if conn.voiceprint_provider and wav_data:
-                    voiceprint_future = thread_executor.submit(run_voiceprint)
-                    
-                    # 等待两个线程都完成
-                    asr_result = asr_future.result(timeout=15)
-                    voiceprint_result = voiceprint_future.result(timeout=15)
-                    
-                    results = {"asr": asr_result, "voiceprint": voiceprint_result}
-                else:
-                    asr_result = asr_future.result(timeout=15)
-                    results = {"asr": asr_result, "voiceprint": None}
-            
-            
-            # 处理结果
-            raw_text, _ = results.get("asr", ("", None))
-            speaker_name = results.get("voiceprint", None)
-            
-            # 记录识别结果
+            asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
+
+            if conn.voiceprint_provider and wav_data:
+                voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
+                # 并发等待两个结果
+                asr_result, voiceprint_result = await asyncio.gather(
+                    asr_task, voiceprint_task, return_exceptions=True
+                )
+            else:
+                asr_result = await asr_task
+                voiceprint_result = None
+
+            # 记录识别结果 - 检查是否为异常
+            if isinstance(asr_result, Exception):
+                logger.bind(tag=TAG).error(f"ASR识别失败: {asr_result}")
+                raw_text = ""
+            else:
+                raw_text, _ = asr_result
+
+            if isinstance(voiceprint_result, Exception):
+                logger.bind(tag=TAG).error(f"声纹识别失败: {voiceprint_result}")
+                speaker_name = ""
+            else:
+                speaker_name = voiceprint_result
+
             if raw_text:
                 logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
             if speaker_name:
                 logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
-            
+
             # 性能监控
             total_time = time.monotonic() - total_start_time
             logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s")
-            
+
             # 检查文本长度
             text_len, _ = remove_punctuation_and_length(raw_text)
             self.stop_ws_connection()
-            
+
             if text_len > 0:
                 # 构建包含说话人信息的JSON字符串
                 enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
-                
+
                 # 使用自定义模块进行上报
                 await startToChat(conn, enhanced_text)
                 enqueue_asr_report(conn, enhanced_text, asr_audio_task)
@@ -241,6 +208,7 @@ class ASRProviderBase(ABC):
     @staticmethod
     def decode_opus(opus_data: List[bytes]) -> List[bytes]:
         """将Opus音频数据解码为PCM数据"""
+        decoder = None
         try:
             decoder = opuslib_next.Decoder(16000, 1)
             pcm_data = []
@@ -265,3 +233,9 @@ class ASRProviderBase(ABC):
         except Exception as e:
             logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
             return []
+        finally:
+            if decoder is not None:
+                try:
+                    del decoder
+                except Exception as e:
+                    logger.bind(tag=TAG).debug(f"释放decoder资源时出错: {e}")

main/xiaozhi-server/core/providers/asr/doubao_stream.py

@@ -18,8 +18,6 @@ class ASRProvider(ASRProviderBase):
         self.interface_type = InterfaceType.STREAM
         self.config = config
         self.text = ""
-        self.max_retries = 3
-        self.retry_delay = 2
         self.decoder = opuslib_next.Decoder(16000, 1)
         self.asr_ws = None
         self.forward_task = None
@@ -49,6 +47,8 @@ class ASRProvider(ASRProviderBase):
         self.channel = config.get("channel", 1)
         self.auth_method = config.get("auth_method", "token")
         self.secret = config.get("secret", "access_secret")
+        end_window_size = config.get("end_window_size")
+        self.end_window_size = int(end_window_size) if end_window_size else 200
 
     async def open_audio_channels(self, conn):
         await super().open_audio_channels(conn)
@@ -56,14 +56,13 @@ class ASRProvider(ASRProviderBase):
     async def receive_audio(self, conn, audio, audio_have_voice):
         conn.asr_audio.append(audio)
         conn.asr_audio = conn.asr_audio[-10:]
-        
         # 存储音频数据
         if not hasattr(conn, 'asr_audio_for_voiceprint'):
             conn.asr_audio_for_voiceprint = []
         conn.asr_audio_for_voiceprint.append(audio)
-        
+
         # 当没有音频数据时处理完整语音片段
-        if not audio and len(conn.asr_audio_for_voiceprint) > 0:
+        if conn.client_listen_mode != "manual" and not audio and len(conn.asr_audio_for_voiceprint) > 0:
             await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint)
             conn.asr_audio_for_voiceprint = []
 
@@ -179,6 +178,7 @@ class ASRProvider(ASRProviderBase):
                                 payload.get("audio_info", {}).get("duration", 0) > 2000
                                 and not utterances
                                 and not payload["result"].get("text")
+                                and conn.client_listen_mode != "manual"
                             ):
                                 logger.bind(tag=TAG).error(f"识别文本：空")
                                 self.text = ""
@@ -187,15 +187,44 @@ class ASRProvider(ASRProviderBase):
                                     await self.handle_voice_stop(conn, audio_data)
                                 break
 
+                            # 专门处理没有文本的识别结果（手动模式下可能已经识别完成但是没松按键）
+                            elif not payload["result"].get("text") and not utterances:
+                                if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0:
+                                    logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理")
+                                    await self.handle_voice_stop(conn, audio_data)
+                                    # 清理音频缓存
+                                    conn.asr_audio.clear()
+                                    conn.reset_vad_states()
+                                    break
+
                             for utterance in utterances:
                                 if utterance.get("definite", False):
-                                    self.text = utterance["text"]
+                                    current_text = utterance["text"]
                                     logger.bind(tag=TAG).info(
-                                        f"识别到文本: {self.text}"
+                                        f"识别到文本: {current_text}"
                                     )
-                                    conn.reset_vad_states()
-                                    if len(audio_data) > 15:  # 确保有足够音频数据
-                                        await self.handle_voice_stop(conn, audio_data)
+
+                                    # 手动模式下累积识别结果
+                                    if conn.client_listen_mode == "manual":
+                                        if self.text:
+                                            self.text += current_text
+                                        else:
+                                            self.text = current_text
+
+                                        # 在接收消息中途时收到停止信号
+                                        if conn.client_voice_stop and len(audio_data) > 0:
+                                            logger.bind(tag=TAG).debug("消息中途收到停止信号，触发处理")
+                                            await self.handle_voice_stop(conn, audio_data)
+                                            # 清理音频缓存
+                                            conn.asr_audio.clear()
+                                            conn.reset_vad_states()
+                                        break
+                                    else:
+                                        # 自动模式下直接覆盖
+                                        self.text = current_text
+                                        conn.reset_vad_states()
+                                        if len(audio_data) > 15:  # 确保有足够音频数据
+                                            await self.handle_voice_stop(conn, audio_data)
                                     break
                         elif "error" in payload:
                             error_msg = payload.get("error", "未知错误")
@@ -227,8 +256,6 @@ class ASRProvider(ASRProviderBase):
                     conn.asr_audio_for_voiceprint = []
                 if hasattr(conn, 'asr_audio'):
                     conn.asr_audio = []
-                if hasattr(conn, 'has_valid_voice'):
-                    conn.has_valid_voice = False
 
     def stop_ws_connection(self):
         if self.asr_ws:
@@ -236,6 +263,20 @@ class ASRProvider(ASRProviderBase):
             self.asr_ws = None
         self.is_processing = False
 
+    async def _send_stop_request(self):
+        """发送最后一个音频帧以通知服务器结束"""
+        if self.asr_ws:
+            try:
+                # 发送结束标记的音频帧（gzip压缩的空数据）
+                empty_payload = gzip.compress(b"")
+                last_audio_request = bytearray(self.generate_last_audio_default_header())
+                last_audio_request.extend(len(empty_payload).to_bytes(4, "big"))
+                last_audio_request.extend(empty_payload)
+                await self.asr_ws.send(last_audio_request)
+                logger.bind(tag=TAG).debug("已发送结束音频帧")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"发送结束音频帧时出错: {e}")
+
     def construct_request(self, reqid):
         req = {
             "app": {
@@ -252,7 +293,7 @@ class ASRProvider(ASRProviderBase):
                 "sequence": 1,
                 "boosting_table_name": self.boosting_table_name,
                 "correct_table_name": self.correct_table_name,
-                "end_window_size": 200,
+                "end_window_size": self.end_window_size,
             },
             "audio": {
                 "format": self.format,
@@ -370,6 +411,16 @@ class ASRProvider(ASRProviderBase):
                 pass
             self.forward_task = None
         self.is_processing = False
+        
+        # 显式释放decoder资源
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Doubao decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")
+
         # 清理所有连接的音频缓冲区
         if hasattr(self, '_connections'):
             for conn in self._connections.values():
@@ -377,5 +428,3 @@ class ASRProvider(ASRProviderBase):
                     conn.asr_audio_for_voiceprint = []
                 if hasattr(conn, 'asr_audio'):
                     conn.asr_audio = []
-                if hasattr(conn, 'has_valid_voice'):
-                    conn.has_valid_voice = False

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -1,14 +1,16 @@
-import time
 import os
-import sys
 import io
+import sys
+import time
+import shutil
 import psutil
+import asyncio
+
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
-from core.providers.asr.base import ASRProviderBase
 from funasr import AutoModel
 from funasr.utils.postprocess_utils import rich_transcription_postprocess
-import shutil
+from core.providers.asr.base import ASRProviderBase
 from core.providers.asr.dto.dto import InterfaceType
 
 TAG = __name__
@@ -90,16 +92,17 @@ class ASRProvider(ASRProviderBase):
                 else:
                     file_path = self.save_audio_to_file(pcm_data, session_id)
 
-                # 语音识别
+                # 语音识别 - 使用线程池避免阻塞事件循环
                 start_time = time.time()
-                result = self.model.generate(
+                result = await asyncio.to_thread(
+                    self.model.generate,
                     input=combined_pcm_data,
                     cache={},
                     language="auto",
                     use_itn=True,
                     batch_size_s=60,
                 )
-                text = rich_transcription_postprocess(result[0]["text"])
+                text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"])
                 logger.bind(tag=TAG).debug(
                     f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
                 )

main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py

@@ -1,8 +1,5 @@
 import os
-import json
-import asyncio
 import tempfile
-import difflib
 from typing import Optional, Tuple, List
 import dashscope
 from config.logger import setup_logging
@@ -16,7 +13,8 @@ logger = setup_logging()
 class ASRProvider(ASRProviderBase):
     def __init__(self, config: dict, delete_audio_file: bool):
         super().__init__()
-        self.interface_type = InterfaceType.STREAM
+        # 音频文件上传类型，流式文本识别输出
+        self.interface_type = InterfaceType.NON_STREAM
         """Qwen3-ASR-Flash ASR初始化"""
         
         # 配置参数
@@ -130,27 +128,11 @@ class ASRProvider(ASRProviderBase):
             
             # 处理流式响应
             full_text = ""
-            last_text = ""  # 用于存储上一个文本片段
             for chunk in response:
                 try:
                     text = chunk["output"]["choices"][0]["message"].content[0]["text"]
-                    # 标准化文本片段（去除首尾空格）
-                    normalized_text = text.strip()
-                    # 只有当新文本片段与上一个不同时才处理
-                    if normalized_text != last_text:
-                        # 提取新增的文本部分
-                        # 通过比较当前文本和上一个文本，找到新增的部分
-                        if normalized_text.startswith(last_text):
-                            # 如果当前文本以最后一个文本开头，则新增部分是两者的差集
-                            new_part = normalized_text[len(last_text):]
-                        else:
-                            # 如果不以最后一个文本开头，说明识别结果发生了较大变化，直接使用当前文本
-                            new_part = normalized_text
-                        
-                        # 将新增部分添加到完整文本中
-                        full_text += new_part
-                        last_text = normalized_text
-                    # 这里可以实时处理文本片段，例如通过回调函数
+                    # 更新为最新的完整文本
+                    full_text = text.strip()
                 except:
                     pass
             

main/xiaozhi-server/core/providers/asr/xunfei_stream.py

@@ -5,6 +5,7 @@ import hashlib
 import asyncio
 import websockets
 import opuslib_next
+import gc
 from time import mktime
 from datetime import datetime
 from urllib.parse import urlencode
@@ -34,9 +35,6 @@ class ASRProvider(ASRProviderBase):
         self.forward_task = None
         self.is_processing = False
         self.server_ready = False
-        self.last_frame_sent = False  # 标记是否已发送最终帧
-        self.best_text = ""  # 保存最佳识别结果
-        self.has_final_result = False  # 标记是否收到最终识别结果
 
         # 讯飞配置
         self.app_id = config.get("app_id")
@@ -51,7 +49,6 @@ class ASRProvider(ASRProviderBase):
             "domain": config.get("domain", "slm"),
             "language": config.get("language", "zh_cn"),
             "accent": config.get("accent", "mandarin"),
-            "dwa": config.get("dwa", "wpgs"),
             "result": {"encoding": "utf8", "compress": "raw", "format": "plain"},
         }
 
@@ -115,7 +112,7 @@ class ASRProvider(ASRProviderBase):
                 await self._start_recognition(conn)
             except Exception as e:
                 logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
-                await self._cleanup(conn)
+                await self._cleanup()
                 return
 
         # 发送当前音频数据
@@ -125,7 +122,7 @@ class ASRProvider(ASRProviderBase):
                 await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
             except Exception as e:
                 logger.bind(tag=TAG).warning(f"发送音频数据时发生错误: {e}")
-                await self._cleanup(conn)
+                await self._cleanup()
 
     async def _start_recognition(self, conn):
         """开始识别会话"""
@@ -135,6 +132,10 @@ class ASRProvider(ASRProviderBase):
             ws_url = self.create_url()
             logger.bind(tag=TAG).info(f"正在连接ASR服务: {ws_url[:50]}...")
 
+            # 如果为手动模式,设置超时时长为一分钟
+            if conn.client_listen_mode == "manual":
+                self.iat_params["eos"] = 60000
+
             self.asr_ws = await websockets.connect(
                 ws_url,
                 max_size=1000000000,
@@ -145,8 +146,6 @@ class ASRProvider(ASRProviderBase):
 
             logger.bind(tag=TAG).info("ASR WebSocket连接已建立")
             self.server_ready = False
-            self.last_frame_sent = False
-            self.best_text = ""
             self.forward_task = asyncio.create_task(self._forward_results(conn))
 
             # 发送首帧音频
@@ -195,23 +194,12 @@ class ASRProvider(ASRProviderBase):
 
         await self.asr_ws.send(json.dumps(frame_data, ensure_ascii=False))
 
-        # 标记是否发送了最终帧
-        if status == STATUS_LAST_FRAME:
-            self.last_frame_sent = True
-            logger.bind(tag=TAG).info("标记最终帧已发送")
-
     async def _forward_results(self, conn):
         """转发识别结果"""
         try:
-            while self.asr_ws and not conn.stop_event.is_set():
-                # 获取当前连接的音频数据
-                audio_data = getattr(conn, "asr_audio_for_voiceprint", [])
+            while not conn.stop_event.is_set():
                 try:
-                    # 如果已发送最终帧，增加超时时间等待完整结果
-                    timeout = 3.0 if self.last_frame_sent else 30.0
-                    response = await asyncio.wait_for(
-                        self.asr_ws.recv(), timeout=timeout
-                    )
+                    response = await asyncio.wait_for(self.asr_ws.recv(), timeout=60)
                     result = json.loads(response)
                     logger.bind(tag=TAG).debug(f"收到ASR结果: {result}")
 
@@ -235,144 +223,27 @@ class ASRProvider(ASRProviderBase):
                             # 解码base64文本
                             decoded_text = base64.b64decode(text_data).decode("utf-8")
                             text_json = json.loads(decoded_text)
-
                             # 提取文本内容
                             text_ws = text_json.get("ws", [])
-                            result_text = ""
                             for i in text_ws:
                                 for j in i.get("cw", []):
                                     w = j.get("w", "")
-                                    result_text += w
+                                    self.text += w
 
-                            # 更新识别文本 - 实时更新策略
-                            # 只检查是否为空字符串，不再过滤任何标点符号
-                            # 这样可以确保所有识别到的内容，包括标点符号都能被实时更新
-                            if result_text and result_text.strip():
-                                # 实时更新：正常情况下都更新，提高响应速度
-                                should_update = True
-
-                                # 保存最佳文本
-                                # 1. 如果是识别完成状态或最终帧后收到的结果，优先保存
-                                # 2. 否则保存最长的有意义文本
-                                # 取消对标点符号的过滤，只检查是否为空
-                                # 这样可以保留所有识别到的内容，包括各种标点符号
-                                is_valid_text = len(result_text.strip()) > 0
-
-                                if (
-                                    self.last_frame_sent or status == 2
-                                ) and is_valid_text:
-                                    self.best_text = result_text
-                                    self.has_final_result = True  # 标记已收到最终结果
-                                    logger.bind(tag=TAG).debug(
-                                        f"保存最终识别结果: {self.best_text}"
-                                    )
-                                elif (
-                                    len(result_text) > len(self.best_text)
-                                    and is_valid_text
-                                    and not self.has_final_result
-                                ):
-                                    self.best_text = result_text
-                                    logger.bind(tag=TAG).debug(
-                                        f"保存中间最佳文本: {self.best_text}"
-                                    )
-
-                                # 如果已发送最终帧，只过滤空文本
-                                if self.last_frame_sent:
-                                    # 只拒绝完全空的结果
-                                    if not result_text.strip():
-                                        should_update = False
-                                        logger.bind(tag=TAG).warning(
-                                            f"最终帧后拒绝空文本"
-                                        )
-
-                                if should_update:
-                                    # 处理流式识别结果，避免简单替换导致内容丢失
-                                    # 1. 如果是中间状态（非最终帧后），可能需要替换为更完整的识别
-                                    # 2. 如果是最终帧后收到的结果，可能是对前面文本的补充
-                                    if self.last_frame_sent:
-                                        # 最终帧后收到的结果可能是标点符号等补充内容
-                                        # 检查是否需要合并文本而不是替换
-                                        # 如果当前文本是纯标点而前面已有内容，应该追加而不是替换
-                                        if len(
-                                            self.text
-                                        ) > 0 and result_text.strip() in [
-                                            "。",
-                                            ".",
-                                            "?",
-                                            "？",
-                                            "!",
-                                            "！",
-                                            ",",
-                                            "，",
-                                            ";",
-                                            "；",
-                                        ]:
-                                            # 对于标点符号，追加到现有文本后
-                                            self.text = (
-                                                self.text.rstrip().rstrip("。.")
-                                                + result_text
-                                            )
-                                        else:
-                                            # 其他情况保持替换逻辑
-                                            self.text = result_text
-                                    else:
-                                        # 中间状态替换为新的识别结果
-                                        self.text = result_text
-
-                                    logger.bind(tag=TAG).info(
-                                        f"实时更新识别文本: {self.text} (最终帧已发送: {self.last_frame_sent})"
-                                    )
-
-                    # 识别完成，但如果还没发送最终帧，继续等待
                     if status == 2:
-                        logger.bind(tag=TAG).info(
-                            f"识别完成状态已到达，当前识别文本: {self.text}"
-                        )
-
-                        # 如果还没发送最终帧，继续等待
-                        if not self.last_frame_sent:
-                            logger.bind(tag=TAG).info(
-                                "识别完成但最终帧未发送，继续等待..."
-                            )
-                            continue
-
-                        # 已发送最终帧且收到完成状态，使用最佳策略选择最终结果
-                        # 优先使用识别完成状态下的最新结果，而不是仅仅基于长度
-                        if self.best_text:
-                            # 如果当前文本是在最终帧发送后或识别完成状态下收到的，优先使用
-                            if (
-                                self.last_frame_sent or status == 2
-                            ) and self.text.strip():
-                                logger.bind(tag=TAG).info(
-                                    f"使用完成状态下的最新识别结果: {self.text}"
-                                )
-                            elif len(self.best_text) > len(self.text):
-                                logger.bind(tag=TAG).info(
-                                    f"使用更长的最佳文本作为最终结果: {self.text} -> {self.best_text}"
-                                )
-                                self.text = self.best_text
-
-                        logger.bind(tag=TAG).info(f"获取到最终完整文本: {self.text}")
+                        if conn.client_listen_mode == "manual":
+                            audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                            if len(audio_data) > 0:
+                                logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
+                                await self.handle_voice_stop(conn, audio_data)
+                                # 清理音频缓存
+                                conn.asr_audio.clear()
                         conn.reset_vad_states()
-                        if len(audio_data) > 15:  # 确保有足够音频数据
-                            # 准备处理结果
-                            pass
                         break
 
                 except asyncio.TimeoutError:
-                    if self.last_frame_sent:
-                        # 超时时也使用最佳文本
-                        if self.best_text and len(self.best_text) > len(self.text):
-                            logger.bind(tag=TAG).info(
-                                f"超时，使用最佳文本: {self.text} -> {self.best_text}"
-                            )
-                            self.text = self.best_text
-                        logger.bind(tag=TAG).info(
-                            f"最终帧后超时，使用结果: {self.text}"
-                        )
-                        break
-                    # 如果还没发送最终帧，继续等待
-                    continue
+                    logger.bind(tag=TAG).error("接收结果超时")
+                    break
                 except websockets.ConnectionClosed:
                     logger.bind(tag=TAG).info("ASR服务连接已关闭")
                     self.is_processing = False
@@ -389,17 +260,15 @@ class ASRProvider(ASRProviderBase):
             if hasattr(e, "__cause__") and e.__cause__:
                 logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
         finally:
-            if self.asr_ws:
-                await self.asr_ws.close()
-                self.asr_ws = None
-            self.is_processing = False
+            # 清理连接资源
+            await self._cleanup()
+
+            # 清理连接的音频缓存
             if conn:
                 if hasattr(conn, "asr_audio_for_voiceprint"):
                     conn.asr_audio_for_voiceprint = []
                 if hasattr(conn, "asr_audio"):
                     conn.asr_audio = []
-                if hasattr(conn, "has_valid_voice"):
-                    conn.has_valid_voice = False
 
     async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
         """处理语音停止，发送最后一帧并处理识别结果"""
@@ -407,22 +276,13 @@ class ASRProvider(ASRProviderBase):
             # 先发送最后一帧表示音频结束
             if self.asr_ws and self.is_processing:
                 try:
-                    # 取最后一个有效的音频帧作为最后一帧数据
-                    last_frame = b""
-                    if asr_audio_task:
-                        last_audio = asr_audio_task[-1]
-                        last_frame = self.decoder.decode(last_audio, 960)
-                    await self._send_audio_frame(last_frame, STATUS_LAST_FRAME)
-                    logger.bind(tag=TAG).info("已发送最后一帧")
+                    await self._send_audio_frame(b"", STATUS_LAST_FRAME)
+                    logger.bind(tag=TAG).debug(f"已发送停止请求")
 
-                    # 发送最终帧后，给_forward_results适当时间处理最终结果
                     await asyncio.sleep(0.25)
-
-                    logger.bind(tag=TAG).info(f"准备处理最终识别结果: {self.text}")
                 except Exception as e:
-                    logger.bind(tag=TAG).error(f"发送最后一帧失败: {e}")
+                    logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
 
-            # 调用父类的handle_voice_stop方法处理识别结果
             await super().handle_voice_stop(conn, asr_audio_task)
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
@@ -436,40 +296,27 @@ class ASRProvider(ASRProviderBase):
             self.asr_ws = None
         self.is_processing = False
 
-    async def _cleanup(self, conn):
-        """清理资源"""
-        logger.bind(tag=TAG).info(
+    async def _send_stop_request(self):
+        """发送停止识别请求（不关闭连接）"""
+        if self.asr_ws:
+            try:
+                # 先停止音频发送
+                self.is_processing = False
+                await self._send_audio_frame(b"", STATUS_LAST_FRAME)
+                logger.bind(tag=TAG).debug("已发送停止请求")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
+
+    async def _cleanup(self):
+        """清理资源（关闭连接）"""
+        logger.bind(tag=TAG).debug(
             f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}"
         )
 
-        # 发送最后一帧
-        if self.asr_ws and self.is_processing:
-            try:
-                await self._send_audio_frame(b"", STATUS_LAST_FRAME)
-                await asyncio.sleep(0.1)
-                logger.bind(tag=TAG).info("已发送最后一帧")
-            except Exception as e:
-                logger.bind(tag=TAG).error(f"发送最后一帧失败: {e}")
-
         # 状态重置
         self.is_processing = False
         self.server_ready = False
-        self.last_frame_sent = False
-        self.best_text = ""
-        self.has_final_result = False
-        logger.bind(tag=TAG).info("ASR状态已重置")
-
-        # 清理任务
-        if self.forward_task and not self.forward_task.done():
-            self.forward_task.cancel()
-            try:
-                await asyncio.wait_for(self.forward_task, timeout=1.0)
-            except asyncio.CancelledError:
-                pass
-            except Exception as e:
-                logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}")
-            finally:
-                self.forward_task = None
+        logger.bind(tag=TAG).debug("ASR状态已重置")
 
         # 关闭连接
         if self.asr_ws:
@@ -482,16 +329,10 @@ class ASRProvider(ASRProviderBase):
             finally:
                 self.asr_ws = None
 
-        # 清理连接的音频缓存
-        if conn:
-            if hasattr(conn, "asr_audio_for_voiceprint"):
-                conn.asr_audio_for_voiceprint = []
-            if hasattr(conn, "asr_audio"):
-                conn.asr_audio = []
-            if hasattr(conn, "has_valid_voice"):
-                conn.has_valid_voice = False
+        # 清理任务引用
+        self.forward_task = None
 
-        logger.bind(tag=TAG).info("ASR会话清理完成")
+        logger.bind(tag=TAG).debug("ASR会话清理完成")
 
     async def speech_to_text(self, opus_data, session_id, audio_format):
         """获取识别结果"""
@@ -512,6 +353,16 @@ class ASRProvider(ASRProviderBase):
                 pass
             self.forward_task = None
         self.is_processing = False
+
+        # 显式释放decoder资源
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Xunfei decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
+
         # 清理所有连接的音频缓冲区
         if hasattr(self, "_connections"):
             for conn in self._connections.values():
@@ -519,5 +370,3 @@ class ASRProvider(ASRProviderBase):
                     conn.asr_audio_for_voiceprint = []
                 if hasattr(conn, "asr_audio"):
                     conn.asr_audio = []
-                if hasattr(conn, "has_valid_voice"):
-                    conn.has_valid_voice = False

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -24,7 +24,6 @@ class LLMProvider(LLMProviderBase):
             "max_tokens": int,
             "temperature": lambda x: round(float(x), 1),
             "top_p": lambda x: round(float(x), 1),
-            "top_k": int,
             "frequency_penalty": lambda x: round(float(x), 1),
         }
 
@@ -40,7 +39,7 @@ class LLMProvider(LLMProviderBase):
                 setattr(self, param, None)
 
         logger.debug(
-            f"意图识别参数初始化: {self.temperature}, {self.max_tokens}, {self.top_p}, {self.top_k}, {self.frequency_penalty}"
+            f"意图识别参数初始化: {self.temperature}, {self.max_tokens}, {self.top_p}, {self.frequency_penalty}"
         )
 
         model_key_msg = check_model_key("LLM", self.api_key)
@@ -71,7 +70,6 @@ class LLMProvider(LLMProviderBase):
                 "max_tokens": kwargs.get("max_tokens", self.max_tokens),
                 "temperature": kwargs.get("temperature", self.temperature),
                 "top_p": kwargs.get("top_p", self.top_p),
-                "top_k": kwargs.get("top_k", self.top_k),
                 "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
             }
 
@@ -116,7 +114,6 @@ class LLMProvider(LLMProviderBase):
                 "max_tokens": kwargs.get("max_tokens", self.max_tokens),
                 "temperature": kwargs.get("temperature", self.temperature),
                 "top_p": kwargs.get("top_p", self.top_p),
-                "top_k": kwargs.get("top_k", self.top_k),
                 "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
             }
 

main/xiaozhi-server/core/providers/memory/base.py

@@ -14,7 +14,7 @@ class MemoryProviderBase(ABC):
         self.llm = llm
 
     @abstractmethod
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         """Save a new memory for specific role and return memory ID"""
         print("this is base func", msgs)
 

main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py

@@ -28,7 +28,7 @@ class MemoryProvider(MemoryProviderBase):
             logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
             self.use_mem0 = False
 
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         if not self.use_mem0:
             return None
         if len(msgs) < 2:
@@ -41,9 +41,7 @@ class MemoryProvider(MemoryProviderBase):
                 for message in msgs
                 if message.role != "system"
             ]
-            result = self.client.add(
-                messages, user_id=self.role_id
-            )
+            result = self.client.add(messages, user_id=self.role_id)
             logger.bind(tag=TAG).debug(f"Save memory result: {result}")
         except Exception as e:
             logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")

main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -4,7 +4,8 @@ import json
 import os
 import yaml
 from config.config_loader import get_project_dir
-from config.manage_api_client import save_mem_local_short
+from config.manage_api_client import generate_and_save_chat_summary
+import asyncio
 from core.utils.util import check_model_key
 
 
@@ -74,18 +75,6 @@ short_term_memory_prompt = """
 ```
 """
 
-short_term_memory_prompt_only_content = """
-你是一个经验丰富的记忆总结者，擅长将对话内容进行总结摘要，遵循以下规则：
-1、总结user的重要信息，以便在未来的对话中提供更个性化的服务
-2、不要重复总结，不要遗忘之前记忆，除非原来的记忆超过了1800字内，否则不要遗忘、不要压缩用户的历史记忆
-3、用户操控的设备音量、播放音乐、天气、退出、不想对话等和用户本身无关的内容，这些信息不需要加入到总结中
-4、聊天内容中的今天的日期时间、今天的天气情况与用户事件无关的数据，这些信息如果当成记忆存储会影响后序对话，这些信息不需要加入到总结中
-5、不要把设备操控的成果结果和失败结果加入到总结中，也不要把用户的一些废话加入到总结中
-6、不要为了总结而总结，如果用户的聊天没有意义，请返回原来的历史记录也是可以的
-7、只需要返回总结摘要，严格控制在1800字内
-8、不要包含代码、xml，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
-"""
-
 
 def extract_json_data(json_code):
     start = json_code.find("```json")
@@ -143,7 +132,7 @@ class MemoryProvider(MemoryProviderBase):
         with open(self.memory_path, "w", encoding="utf-8") as f:
             yaml.dump(all_memory, f, allow_unicode=True)
 
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         # 打印使用的模型信息
         model_info = getattr(self.llm, "model_name", str(self.llm.__class__.__name__))
         logger.bind(tag=TAG).debug(f"使用记忆保存模型: {model_info}")
@@ -187,14 +176,12 @@ class MemoryProvider(MemoryProviderBase):
             except Exception as e:
                 print("Error:", e)
         else:
-            result = self.llm.response_no_stream(
-                short_term_memory_prompt_only_content,
-                msgStr,
-                max_tokens=2000,
-                temperature=0.2,
-            )
-            save_mem_local_short(self.role_id, result)
-        logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
+            # 当save_to_file为False时，调用Java端的聊天记录总结接口
+            summary_id = session_id if session_id else self.role_id
+            await generate_and_save_chat_summary(summary_id)
+        logger.bind(tag=TAG).info(
+            f"Save memory successful - Role: {self.role_id}, Session: {session_id}"
+        )
 
         return self.short_memory
 

main/xiaozhi-server/core/providers/memory/nomem/nomem.py

@@ -11,7 +11,7 @@ class MemoryProvider(MemoryProviderBase):
     def __init__(self, config, summary_memory=None):
         super().__init__(config)
 
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
         return None
 

main/xiaozhi-server/core/providers/tools/server_mcp/mcp_manager.py

@@ -3,12 +3,8 @@
 import asyncio
 import os
 import json
-from datetime import timedelta
 from typing import Dict, Any, List
 
-from mcp import Implementation
-from mcp.client.session import SamplingFnT, ElicitationFnT, ListRootsFnT, LoggingFnT, MessageHandlerFnT
-from mcp.shared.session import ProgressFnT
 from mcp.types import LoggingMessageNotificationParams
 
 from config.config_loader import get_project_dir
@@ -33,6 +29,7 @@ class ServerMCPManager:
             )
         self.clients: Dict[str, ServerMCPClient] = {}
         self.tools = []
+        self._init_lock = asyncio.Lock()
 
     def load_config(self) -> Dict[str, Any]:
         """加载MCP服务配置"""
@@ -49,29 +46,50 @@ class ServerMCPManager:
             )
             return {}
 
+    async def _init_server(self, name: str, srv_config: Dict[str, Any]):
+        """初始化单个MCP服务"""
+        client = None
+        try:
+            # 初始化服务端MCP客户端
+            logger.bind(tag=TAG).info(f"初始化服务端MCP客户端: {name}")
+            client = ServerMCPClient(srv_config)
+            # 设置超时时间10秒
+            await asyncio.wait_for(client.initialize(logging_callback=self.logging_callback), timeout=10)
+
+            # 使用锁保护共享状态的修改
+            async with self._init_lock:
+                self.clients[name] = client
+                client_tools = client.get_available_tools()
+                self.tools.extend(client_tools)
+
+        except asyncio.TimeoutError:
+            logger.bind(tag=TAG).error(
+                f"Failed to initialize MCP server {name}: Timeout"
+            )
+            if client:
+                await client.cleanup()
+        except Exception as e:
+            logger.bind(tag=TAG).error(
+                f"Failed to initialize MCP server {name}: {e}"
+            )
+            if client:
+                await client.cleanup()
+
     async def initialize_servers(self) -> None:
         """初始化所有MCP服务"""
         config = self.load_config()
+        tasks = []
         for name, srv_config in config.items():
             if not srv_config.get("command") and not srv_config.get("url"):
                 logger.bind(tag=TAG).warning(
                     f"Skipping server {name}: neither command nor url specified"
                 )
                 continue
-
-            try:
-                # 初始化服务端MCP客户端
-                logger.bind(tag=TAG).info(f"初始化服务端MCP客户端: {name}")
-                client = ServerMCPClient(srv_config)
-                await client.initialize(logging_callback=self.logging_callback)
-                self.clients[name] = client
-                client_tools = client.get_available_tools()
-                self.tools.extend(client_tools)
-
-            except Exception as e:
-                logger.bind(tag=TAG).error(
-                    f"Failed to initialize MCP server {name}: {e}"
-                )
+            
+            tasks.append(self._init_server(name, srv_config))
+        
+        if tasks:
+            await asyncio.gather(*tasks)
 
         # 输出当前支持的服务端MCP工具列表
         if hasattr(self.conn, "func_handler") and self.conn.func_handler:

main/xiaozhi-server/core/providers/vad/silero.py

@@ -36,7 +36,18 @@ class VADProvider(VADProviderBase):
         # 至少要多少帧才算有语音
         self.frame_window_threshold = 3
 
+    def __del__(self):
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+            except Exception:
+                pass
+
     def is_vad(self, conn, opus_packet):
+        # 手动模式：直接返回True，不进行实时VAD检测，所有音频都缓存
+        if conn.client_listen_mode == "manual":
+            return True
+            
         try:
             pcm_frame = self.decoder.decode(opus_packet, 960)
             conn.client_audio_buffer.extend(pcm_frame)  # 将新数据加入缓冲区

main/xiaozhi-server/core/utils/audioRateController.py

@@ -0,0 +1,160 @@
+import time
+import asyncio
+from collections import deque
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class AudioRateController:
+    """
+    音频速率控制器 - 按照60ms帧时长精确控制音频发送
+    解决高并发下的时间累积误差问题
+    """
+
+    def __init__(self, frame_duration=60):
+        """
+        Args:
+            frame_duration: 单个音频帧时长（毫秒），默认60ms
+        """
+        self.frame_duration = frame_duration
+        self.queue = deque()
+        self.play_position = 0  # 虚拟播放位置（毫秒）
+        self.start_timestamp = None  # 开始时间戳（只读，不修改）
+        self.pending_send_task = None
+        self.logger = logger
+        self.queue_empty_event = asyncio.Event()  # 队列清空事件
+        self.queue_empty_event.set()  # 初始为空状态
+        self.queue_has_data_event = asyncio.Event()  # 队列数据事件
+
+    def reset(self):
+        """重置控制器状态"""
+        if self.pending_send_task and not self.pending_send_task.done():
+            self.pending_send_task.cancel()
+            # 取消任务后，任务会在下次事件循环时清理，无需阻塞等待
+
+        self.queue.clear()
+        self.play_position = 0
+        self.start_timestamp = None  # 由首个音频包设置
+        # 相关事件处理
+        self.queue_empty_event.set()
+        self.queue_has_data_event.clear()
+
+    def add_audio(self, opus_packet):
+        """添加音频包到队列"""
+        self.queue.append(("audio", opus_packet))
+        # 相关事件处理
+        self.queue_empty_event.clear()
+        self.queue_has_data_event.set()
+
+    def add_message(self, message_callback):
+        """
+        添加消息到队列（立即发送，不占用播放时间）
+
+        Args:
+            message_callback: 消息发送回调函数 async def()
+        """
+        self.queue.append(("message", message_callback))
+        # 相关事件处理
+        self.queue_empty_event.clear()
+        self.queue_has_data_event.set()
+
+    def _get_elapsed_ms(self):
+        """获取已经过的时间（毫秒）"""
+        if self.start_timestamp is None:
+            return 0
+        return (time.monotonic() - self.start_timestamp) * 1000
+
+    async def check_queue(self, send_audio_callback):
+        """
+        检查队列并按时发送音频/消息
+
+        Args:
+            send_audio_callback: 发送音频的回调函数 async def(opus_packet)
+        """
+        while self.queue:
+            item = self.queue[0]
+            item_type = item[0]
+
+            if item_type == "message":
+                # 消息类型：立即发送，不占用播放时间
+                _, message_callback = item
+                self.queue.popleft()
+                try:
+                    await message_callback()
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"发送消息失败: {e}")
+                    raise
+
+            elif item_type == "audio":
+                if self.start_timestamp is None:
+                    self.start_timestamp = time.monotonic()
+
+                _, opus_packet = item
+
+                # 循环等待直到时间到达
+                while True:
+                    # 计算时间差
+                    elapsed_ms = self._get_elapsed_ms()
+                    output_ms = self.play_position
+
+                    if elapsed_ms < output_ms:
+                        # 还不到发送时间，计算等待时长
+                        wait_ms = output_ms - elapsed_ms
+
+                        # 等待后继续检查（允许被中断）
+                        try:
+                            await asyncio.sleep(wait_ms / 1000)
+                        except asyncio.CancelledError:
+                            self.logger.bind(tag=TAG).debug("音频发送任务被取消")
+                            raise
+                        # 等待结束后重新检查时间（循环回到 while True）
+                    else:
+                        # 时间已到，跳出等待循环
+                        break
+
+                # 时间已到，从队列移除并发送
+                self.queue.popleft()
+                self.play_position += self.frame_duration
+                try:
+                    await send_audio_callback(opus_packet)
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"发送音频失败: {e}")
+                    raise
+
+        # 队列处理完后清除事件
+        self.queue_empty_event.set()
+        self.queue_has_data_event.clear()
+
+    def start_sending(self, send_audio_callback):
+        """
+        启动异步发送任务
+
+        Args:
+            send_audio_callback: 发送音频的回调函数
+
+        Returns:
+            asyncio.Task: 发送任务
+        """
+
+        async def _send_loop():
+            try:
+                while True:
+                    # 等待队列数据事件，不轮询等待占用CPU
+                    await self.queue_has_data_event.wait()
+
+                    await self.check_queue(send_audio_callback)
+            except asyncio.CancelledError:
+                self.logger.bind(tag=TAG).debug("音频发送循环已停止")
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"音频发送循环异常: {e}")
+
+        self.pending_send_task = asyncio.create_task(_send_loop())
+        return self.pending_send_task
+
+    def stop_sending(self):
+        """停止发送任务"""
+        if self.pending_send_task and not self.pending_send_task.done():
+            self.pending_send_task.cancel()
+            self.logger.bind(tag=TAG).debug("已取消音频发送任务")

main/xiaozhi-server/core/utils/cache/config.py

@@ -19,6 +19,7 @@ class CacheType(Enum):
     CONFIG = "config"
     DEVICE_PROMPT = "device_prompt"
     VOICEPRINT_HEALTH = "voiceprint_health"  # 声纹识别健康检查
+    AUDIO_DATA = "audio_data"  # 音频数据缓存
 
 
 @dataclass
@@ -58,5 +59,8 @@ class CacheConfig:
             CacheType.VOICEPRINT_HEALTH: cls(
                 strategy=CacheStrategy.TTL, ttl=600, max_size=100  # 10分钟过期
             ),
+            CacheType.AUDIO_DATA: cls(
+                strategy=CacheStrategy.TTL, ttl=600, max_size=100  # 10分钟过期
+            ),
         }
         return configs.get(cache_type, cls())

main/xiaozhi-server/core/utils/context_provider.py

@@ -0,0 +1,64 @@
+import httpx
+from typing import Dict, Any, List
+from config.logger import setup_logging
+
+TAG = __name__
+
+class ContextDataProvider:
+    """数据上下文填充，负责从配置的API获取数据"""
+    
+    def __init__(self, config: Dict[str, Any], logger=None):
+        self.config = config
+        self.logger = logger or setup_logging()
+        self.context_data = ""
+
+    def fetch_all(self, device_id: str) -> str:
+        """获取所有配置的上下文数据"""
+        context_providers = self.config.get("context_providers", [])
+        if not context_providers:
+            return ""
+
+        formatted_lines = []
+        for provider in context_providers:
+            url = provider.get("url")
+            headers = provider.get("headers", {})
+
+            if not url:
+                continue
+
+            try:
+                headers = headers.copy() if isinstance(headers, dict) else {}
+                # 将 device_id 添加到请求头
+                headers["device-id"] = device_id
+                
+                # 发送请求
+                response = httpx.get(url, headers=headers, timeout=3)
+                
+                if response.status_code == 200:
+                    result = response.json()
+                    if isinstance(result, dict):
+                        if result.get("code") == 0:
+                            data = result.get("data")
+                            # 格式化数据
+                            if isinstance(data, dict):
+                                for k, v in data.items():
+                                    formatted_lines.append(f"- **{k}：** {v}")
+                            elif isinstance(data, list):
+                                for item in data:
+                                    formatted_lines.append(f"- {item}")
+                            else:
+                                formatted_lines.append(f"- {data}")
+                        else:
+                            self.logger.bind(tag=TAG).warning(f"API {url} 返回错误码: {result.get('msg')}")
+                    else:
+                        self.logger.bind(tag=TAG).warning(f"API {url} 返回的不是JSON字典")
+                else:
+                    self.logger.bind(tag=TAG).warning(f"API {url} 请求失败: {response.status_code}")
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"获取上下文数据 {url} 失败: {e}")
+        
+        # 将所有格式化后的行拼接成一个字符串
+        self.context_data = "\n".join(formatted_lines)
+        if self.context_data:
+            self.logger.bind(tag=TAG).debug(f"已注入动态上下文数据:\n{self.context_data}")
+        return self.context_data

main/xiaozhi-server/core/utils/gc_manager.py

@@ -0,0 +1,122 @@
+"""
+全局GC管理模块
+定期执行垃圾回收，避免频繁触发GC导致的GIL锁问题
+"""
+
+import gc
+import asyncio
+import threading
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class GlobalGCManager:
+    """全局垃圾回收管理器"""
+
+    def __init__(self, interval_seconds=300):
+        """
+        初始化GC管理器
+
+        Args:
+            interval_seconds: GC执行间隔（秒），默认300秒（5分钟）
+        """
+        self.interval_seconds = interval_seconds
+        self._task = None
+        self._stop_event = asyncio.Event()
+        self._lock = threading.Lock()
+
+    async def start(self):
+        """启动定时GC任务"""
+        if self._task is not None:
+            logger.bind(tag=TAG).warning("GC管理器已经在运行")
+            return
+
+        logger.bind(tag=TAG).info(f"启动全局GC管理器，间隔{self.interval_seconds}秒")
+        self._stop_event.clear()
+        self._task = asyncio.create_task(self._gc_loop())
+
+    async def stop(self):
+        """停止定时GC任务"""
+        if self._task is None:
+            return
+
+        logger.bind(tag=TAG).info("停止全局GC管理器")
+        self._stop_event.set()
+
+        if self._task and not self._task.done():
+            self._task.cancel()
+            try:
+                await self._task
+            except asyncio.CancelledError:
+                pass
+
+        self._task = None
+
+    async def _gc_loop(self):
+        """GC循环任务"""
+        try:
+            while not self._stop_event.is_set():
+                # 等待指定间隔
+                try:
+                    await asyncio.wait_for(
+                        self._stop_event.wait(), timeout=self.interval_seconds
+                    )
+                    # 如果stop_event被设置，退出循环
+                    break
+                except asyncio.TimeoutError:
+                    # 超时表示到了执行GC的时间
+                    pass
+
+                # 执行GC
+                await self._run_gc()
+
+        except asyncio.CancelledError:
+            logger.bind(tag=TAG).info("GC循环任务被取消")
+            raise
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"GC循环任务异常: {e}")
+        finally:
+            logger.bind(tag=TAG).info("GC循环任务已退出")
+
+    async def _run_gc(self):
+        """执行垃圾回收"""
+        try:
+            # 在线程池中执行GC，避免阻塞事件循环
+            loop = asyncio.get_running_loop()
+
+            def do_gc():
+                with self._lock:
+                    before = len(gc.get_objects())
+                    collected = gc.collect()
+                    after = len(gc.get_objects())
+                    return before, collected, after
+
+            before, collected, after = await loop.run_in_executor(None, do_gc)
+            logger.bind(tag=TAG).debug(
+                f"全局GC执行完成 - 回收对象: {collected}, "
+                f"对象数量: {before} -> {after}"
+            )
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"执行GC时出错: {e}")
+
+
+# 全局单例
+_gc_manager_instance = None
+
+
+def get_gc_manager(interval_seconds=300):
+    """
+    获取全局GC管理器实例（单例模式）
+
+    Args:
+        interval_seconds: GC执行间隔（秒），默认300秒（5分钟）
+
+    Returns:
+        GlobalGCManager实例
+    """
+    global _gc_manager_instance
+    if _gc_manager_instance is None:
+        _gc_manager_instance = GlobalGCManager(interval_seconds)
+    return _gc_manager_instance

main/xiaozhi-server/core/utils/opus_encoder_utils.py

@@ -102,6 +102,9 @@ class OpusEncoderUtils:
     def _encode(self, frame: np.ndarray) -> Optional[bytes]:
         """编码一帧音频数据"""
         try:
+            # 编码器已释放，跳过编码
+            if not hasattr(self, 'encoder') or self.encoder is None:
+                return None
             # 将numpy数组转换为bytes
             frame_bytes = frame.tobytes()
             # opuslib要求输入字节数必须是channels*2的倍数
@@ -128,5 +131,9 @@ class OpusEncoderUtils:
 
     def close(self):
         """关闭编码器并释放资源"""
-        # opuslib没有明确的关闭方法，Python的垃圾回收会处理
-        pass
+        if hasattr(self, 'encoder') and self.encoder:
+            try:
+                del self.encoder
+                self.encoder = None
+            except Exception as e:
+                logging.error(f"Error releasing Opus encoder: {e}")

main/xiaozhi-server/core/utils/prompt_manager.py

@@ -4,7 +4,6 @@
 """
 
 import os
-import cnlunar
 from typing import Dict, Any
 from config.logger import setup_logging
 from jinja2 import Template
@@ -60,6 +59,11 @@ class PromptManager:
 
         self.cache_manager = cache_manager
         self.CacheType = CacheType
+        
+        # 初始化上下文源
+        from core.utils.context_provider import ContextDataProvider
+        self.context_provider = ContextDataProvider(config, self.logger)
+        self.context_data = {}
 
         self._load_base_template()
 
@@ -180,10 +184,33 @@ class PromptManager:
     def update_context_info(self, conn, client_ip: str):
         """同步更新上下文信息"""
         try:
-            # 获取位置信息（使用全局缓存）
-            local_address = self._get_location_info(client_ip)
-            # 获取天气信息（使用全局缓存）
-            self._get_weather_info(conn, local_address)
+            local_address = ""
+            if (
+                client_ip
+                and self.base_prompt_template
+                and (
+                    "local_address" in self.base_prompt_template
+                    or "weather_info" in self.base_prompt_template
+                )
+            ):
+                # 获取位置信息（使用全局缓存）
+                local_address = self._get_location_info(client_ip)
+
+            if (
+                self.base_prompt_template
+                and "weather_info" in self.base_prompt_template
+                and local_address
+            ):
+                # 获取天气信息（使用全局缓存）
+                self._get_weather_info(conn, local_address)
+            
+            # 获取配置的上下文数据
+            if hasattr(conn, "device_id") and conn.device_id:
+                if self.base_prompt_template and "dynamic_context" in self.base_prompt_template:
+                    self.context_data = self.context_provider.fetch_all(conn.device_id)
+                else:
+                    self.context_data = ""
+                
             self.logger.bind(tag=TAG).debug(f"上下文信息更新完成")
 
         except Exception as e:
@@ -230,6 +257,7 @@ class PromptManager:
                 emojiList=EMOJI_List,
                 device_id=device_id,
                 client_ip=client_ip,
+                dynamic_context=self.context_data,
                 *args,
                 **kwargs,
             )

main/xiaozhi-server/core/utils/util.py

@@ -4,6 +4,7 @@ import json
 import copy
 import wave
 import socket
+import asyncio
 import requests
 import subprocess
 import numpy as np
@@ -268,56 +269,82 @@ def audio_to_data_stream(
     pcm_to_data_stream(raw_data, is_opus, callback)
 
 
-def audio_to_data(audio_file_path: str, is_opus: bool = True) -> list[bytes]:
+async def audio_to_data(
+    audio_file_path: str, is_opus: bool = True, use_cache: bool = True
+) -> list[bytes]:
     """
     将音频文件转换为Opus/PCM编码的帧列表
     Args:
         audio_file_path: 音频文件路径
         is_opus: 是否进行Opus编码
+        use_cache: 是否使用缓存
     """
-    # 获取文件后缀名
-    file_type = os.path.splitext(audio_file_path)[1]
-    if file_type:
-        file_type = file_type.lstrip(".")
-    # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
-    audio = AudioSegment.from_file(
-        audio_file_path, format=file_type, parameters=["-nostdin"]
-    )
+    from core.utils.cache.manager import cache_manager
+    from core.utils.cache.config import CacheType
 
-    # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
-    audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
+    # 生成缓存键，包含文件路径和编码类型
+    cache_key = f"{audio_file_path}:{is_opus}"
 
-    # 获取原始PCM数据（16位小端）
-    raw_data = audio.raw_data
+    # 尝试从缓存获取结果
+    if use_cache:
+        cached_result = cache_manager.get(CacheType.AUDIO_DATA, cache_key)
+        if cached_result is not None:
+            return cached_result
 
-    # 初始化Opus编码器
-    encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+    def _sync_audio_to_data():
+        # 获取文件后缀名
+        file_type = os.path.splitext(audio_file_path)[1]
+        if file_type:
+            file_type = file_type.lstrip(".")
+        # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
+        audio = AudioSegment.from_file(
+            audio_file_path, format=file_type, parameters=["-nostdin"]
+        )
 
-    # 编码参数
-    frame_duration = 60  # 60ms per frame
-    frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+        # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
+        audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
 
-    datas = []
-    # 按帧处理所有音频数据（包括最后一帧可能补零）
-    for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
-        # 获取当前帧的二进制数据
-        chunk = raw_data[i : i + frame_size * 2]
+        # 获取原始PCM数据（16位小端）
+        raw_data = audio.raw_data
 
-        # 如果最后一帧不足，补零
-        if len(chunk) < frame_size * 2:
-            chunk += b"\x00" * (frame_size * 2 - len(chunk))
+        # 初始化Opus编码器
+        encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
 
-        if is_opus:
-            # 转换为numpy数组处理
-            np_frame = np.frombuffer(chunk, dtype=np.int16)
-            # 编码Opus数据
-            frame_data = encoder.encode(np_frame.tobytes(), frame_size)
-        else:
-            frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
+        # 编码参数
+        frame_duration = 60  # 60ms per frame
+        frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
 
-        datas.append(frame_data)
+        datas = []
+        # 按帧处理所有音频数据（包括最后一帧可能补零）
+        for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
+            # 获取当前帧的二进制数据
+            chunk = raw_data[i : i + frame_size * 2]
 
-    return datas
+            # 如果最后一帧不足，补零
+            if len(chunk) < frame_size * 2:
+                chunk += b"\x00" * (frame_size * 2 - len(chunk))
+
+            if is_opus:
+                # 转换为numpy数组处理
+                np_frame = np.frombuffer(chunk, dtype=np.int16)
+                # 编码Opus数据
+                frame_data = encoder.encode(np_frame.tobytes(), frame_size)
+            else:
+                frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
+
+            datas.append(frame_data)
+
+        return datas
+
+    loop = asyncio.get_running_loop()
+    # 在单独的线程中执行同步的音频处理操作
+    result = await loop.run_in_executor(None, _sync_audio_to_data)
+
+    # 将结果存入缓存，使用配置中定义的TTL（10分钟）
+    if use_cache:
+        cache_manager.set(CacheType.AUDIO_DATA, cache_key, result)
+
+    return result
 
 
 def audio_bytes_to_data_stream(
@@ -372,26 +399,33 @@ def opus_datas_to_wav_bytes(opus_datas, sample_rate=16000, channels=1):
     将opus帧列表解码为wav字节流
     """
     decoder = opuslib_next.Decoder(sample_rate, channels)
-    pcm_datas = []
+    try:
+        pcm_datas = []
 
-    frame_duration = 60  # ms
-    frame_size = int(sample_rate * frame_duration / 1000)  # 960
+        frame_duration = 60  # ms
+        frame_size = int(sample_rate * frame_duration / 1000)  # 960
 
-    for opus_frame in opus_datas:
-        # 解码为PCM（返回bytes，2字节/采样点）
-        pcm = decoder.decode(opus_frame, frame_size)
-        pcm_datas.append(pcm)
+        for opus_frame in opus_datas:
+            # 解码为PCM（返回bytes，2字节/采样点）
+            pcm = decoder.decode(opus_frame, frame_size)
+            pcm_datas.append(pcm)
 
-    pcm_bytes = b"".join(pcm_datas)
+        pcm_bytes = b"".join(pcm_datas)
 
-    # 写入wav字节流
-    wav_buffer = BytesIO()
-    with wave.open(wav_buffer, "wb") as wf:
-        wf.setnchannels(channels)
-        wf.setsampwidth(2)  # 16bit
-        wf.setframerate(sample_rate)
-        wf.writeframes(pcm_bytes)
-    return wav_buffer.getvalue()
+        # 写入wav字节流
+        wav_buffer = BytesIO()
+        with wave.open(wav_buffer, "wb") as wf:
+            wf.setnchannels(channels)
+            wf.setsampwidth(2)  # 16bit
+            wf.setframerate(sample_rate)
+            wf.writeframes(pcm_bytes)
+        return wav_buffer.getvalue()
+    finally:
+        if decoder is not None:
+            try:
+                del decoder
+            except Exception:
+                pass
 
 
 def check_vad_update(before_config, new_config):

main/xiaozhi-server/core/websocket_server.py

@@ -1,10 +1,37 @@
 import asyncio
-import json
+import logging
 
 import websockets
 from config.logger import setup_logging
+
+
+class SuppressInvalidHandshakeFilter(logging.Filter):
+    """过滤掉无效握手错误日志（如HTTPS访问WS端口）"""
+
+    def filter(self, record):
+        msg = record.getMessage()
+        suppress_keywords = [
+            "opening handshake failed",
+            "did not receive a valid HTTP request",
+            "connection closed while reading HTTP request",
+            "line without CRLF",
+        ]
+        return not any(keyword in msg for keyword in suppress_keywords)
+
+
+def _setup_websockets_logger():
+    """配置 websockets 相关的所有 logger，过滤无效握手错误"""
+    filter_instance = SuppressInvalidHandshakeFilter()
+    for logger_name in ["websockets", "websockets.server", "websockets.client"]:
+        logger = logging.getLogger(logger_name)
+        logger.addFilter(filter_instance)
+
+
+_setup_websockets_logger()
+
+
 from core.connection import ConnectionHandler
-from config.config_loader import get_config_from_api
+from config.config_loader import get_config_from_api_async
 from core.auth import AuthManager, AuthenticationError
 from core.utils.modules_initialize import initialize_modules
 from core.utils.util import check_vad_update, check_asr_update
@@ -133,8 +160,8 @@ class WebSocketServer:
         """
         try:
             async with self.config_lock:
-                # 重新获取配置
-                new_config = get_config_from_api(self.config)
+                # 重新获取配置（使用异步版本）
+                new_config = await get_config_from_api_async(self.config)
                 if new_config is None:
                     self.logger.bind(tag=TAG).error("获取新配置失败")
                     return False