resolve merge conflict

main/xiaozhi-server/core/api/base_handler.py

@@ -10,7 +10,15 @@ class BaseHandler:
     def _add_cors_headers(self, response):
         """添加CORS头信息"""
         response.headers["Access-Control-Allow-Headers"] = (
-            "client-id, content-type, device-id"
+            "client-id, content-type, device-id, authorization"
         )
         response.headers["Access-Control-Allow-Credentials"] = "true"
         response.headers["Access-Control-Allow-Origin"] = "*"
+
+    async def handle_options(self, request):
+        """处理OPTIONS请求，添加CORS头信息"""
+        response = web.Response(body=b"", content_type="text/plain")
+        self._add_cors_headers(response)
+        # 添加允许的方法
+        response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
+        return response

main/xiaozhi-server/core/api/ota_handler.py

@@ -1,15 +1,126 @@
 import json
 import time
+import base64
+import hashlib
+import hmac
+import os
+import re
+import glob
+from typing import Dict, List, Tuple
 from aiohttp import web
-from core.utils.util import get_local_ip
+
+from core.auth import AuthManager
+from core.utils.util import get_local_ip, get_vision_url
 from core.api.base_handler import BaseHandler
 
 TAG = __name__
 
 
+def _safe_basename(filename: str) -> str:
+    # Prevent directory traversal
+    return os.path.basename(filename)
+
+
+def _parse_version(ver: str) -> Tuple[int, ...]:
+    # conservative parser: split by non-digit, keep numeric parts
+    parts = re.findall(r"\d+", ver)
+    return tuple(int(p) for p in parts) if parts else (0,)
+
+
+def _is_higher_version(a: str, b: str) -> bool:
+    """Return True if version string a > b (semver-like numeric compare)."""
+    ta = _parse_version(a)
+    tb = _parse_version(b)
+    # compare tuple lexicographically, but allow different lengths
+    maxlen = max(len(ta), len(tb))
+    for i in range(maxlen):
+        ai = ta[i] if i < len(ta) else 0
+        bi = tb[i] if i < len(tb) else 0
+        if ai > bi:
+            return True
+        if ai < bi:
+            return False
+    return False
+
+
 class OTAHandler(BaseHandler):
     def __init__(self, config: dict):
         super().__init__(config)
+        auth_config = config["server"].get("auth", {})
+        self.auth_enable = auth_config.get("enabled", False)
+        # 设备白名单
+        self.allowed_devices = set(auth_config.get("allowed_devices", []))
+        secret_key = config["server"]["auth_key"]
+        expire_seconds = auth_config.get("expire_seconds")
+        self.auth = AuthManager(secret_key=secret_key, expire_seconds=expire_seconds)
+
+        # firmware storage
+        self.bin_dir = os.path.join(os.getcwd(), "data", "bin")
+        # cache structure: { 'updated_at': timestamp, 'ttl': seconds, 'files_by_model': { model: [(version, filename), ...] } }
+        self._bin_cache: Dict = {
+            "updated_at": 0,
+            "ttl": config.get("firmware_cache_ttl", 30),
+            "files_by_model": {},
+        }
+
+    def _refresh_bin_cache_if_needed(self):
+        now = int(time.time())
+        ttl = int(self._bin_cache.get("ttl", 30))
+        if now - int(
+            self._bin_cache.get("updated_at", 0)
+        ) < ttl and self._bin_cache.get("files_by_model"):
+            return
+
+        files_by_model: Dict[str, List[Tuple[str, str]]] = {}
+        try:
+            if not os.path.isdir(self.bin_dir):
+                os.makedirs(self.bin_dir, exist_ok=True)
+
+            # match files like model_1.2.3.bin (allow dots, dashes, underscores in model and version)
+            pattern = os.path.join(self.bin_dir, "*.bin")
+            for path in glob.glob(pattern):
+                fname = os.path.basename(path)
+                # filename format: {model}_{version}.bin
+                m = re.match(r"^(.+?)_([0-9][A-Za-z0-9\.\-_]*)\.bin$", fname)
+                if not m:
+                    # skip files not conforming to naming rule
+                    continue
+                model = m.group(1)
+                version = m.group(2)
+                files_by_model.setdefault(model, []).append((version, fname))
+
+            # sort versions for each model descending
+            for model, items in files_by_model.items():
+                items.sort(key=lambda it: _parse_version(it[0]), reverse=True)
+
+            self._bin_cache["files_by_model"] = files_by_model
+            self._bin_cache["updated_at"] = now
+            self.logger.bind(tag=TAG).info(
+                f"Firmware cache refreshed: {len(files_by_model)} models"
+            )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"刷新固件缓存失败: {e}")
+            # keep previous cache if any
+
+    def generate_password_signature(self, content: str, secret_key: str) -> str:
+        """生成MQTT密码签名
+
+        Args:
+            content: 签名内容 (clientId + '|' + username)
+            secret_key: 密钥
+
+        Returns:
+            str: Base64编码的HMAC-SHA256签名
+        """
+        try:
+            hmac_obj = hmac.new(
+                secret_key.encode("utf-8"), content.encode("utf-8"), hashlib.sha256
+            )
+            signature = hmac_obj.digest()
+            return base64.b64encode(signature).decode("utf-8")
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"生成MQTT密码签名失败: {e}")
+            return ""
 
     def _get_websocket_url(self, local_ip: str, port: int) -> str:
         """获取websocket地址
@@ -30,7 +141,14 @@ class OTAHandler(BaseHandler):
             return f"ws://{local_ip}:{port}/xiaozhi/v1/"
 
     async def handle_post(self, request):
-        """处理 OTA POST 请求"""
+        """处理 OTA POST 请求
+
+        This handler will:
+        - read device id/client id (as before)
+        - attempt to determine device model and current firmware version (prefer headers, fallback to body)
+        - check data/bin for newer firmware for that model
+        - if found a newer firmware, set firmware.url to the download endpoint
+        """
         try:
             data = await request.text()
             self.logger.bind(tag=TAG).debug(f"OTA请求方法: {request.method}")
@@ -43,30 +161,188 @@ class OTAHandler(BaseHandler):
             else:
                 raise Exception("OTA请求设备ID为空")
 
-            data_json = json.loads(data)
+            client_id = request.headers.get("client-id", "")
+            if client_id:
+                self.logger.bind(tag=TAG).info(f"OTA请求ClientID: {client_id}")
+            else:
+                raise Exception("OTA请求ClientID为空")
+
+            data_json = {}
+            try:
+                data_json = json.loads(data) if data else {}
+            except Exception:
+                data_json = {}
 
             server_config = self.config["server"]
-            port = int(server_config.get("port", 8000))
+            # Distinguish ports:
+            # - websocket_port is used to construct websocket URL (server["port"])
+            # - http_port is used to construct OTA download URLs (server["http_port"])
+            websocket_port = int(server_config.get("port", 8000))
+            http_port = int(server_config.get("http_port", 8003))
             local_ip = get_local_ip()
 
+            # Determine device model (prefer headers)
+            device_model = ""
+            # header candidates
+            for h in ("device-model", "device_model", "model"):
+                if h in request.headers:
+                    device_model = request.headers.get(h, "").strip()
+                    break
+            # body fallback
+            if not device_model:
+                try:
+                    if "board" in data_json and isinstance(data_json["board"], dict):
+                        device_model = data_json["board"].get("type", "")
+                    elif "model" in data_json:
+                        device_model = data_json.get("model", "")
+                except Exception:
+                    device_model = ""
+            if not device_model:
+                device_model = "default"
+
+            # Determine device current version (prefer headers)
+            device_version = ""
+            for h in (
+                "device-version",
+                "device_version",
+                "firmware-version",
+                "app-version",
+                "application-version",
+            ):
+                if h in request.headers:
+                    device_version = request.headers.get(h, "").strip()
+                    break
+            if not device_version:
+                try:
+                    device_version = data_json.get("application", {}).get("version", "")
+                except Exception:
+                    device_version = ""
+            if not device_version:
+                device_version = "0.0.0"
+
             return_json = {
                 "server_time": {
                     "timestamp": int(round(time.time() * 1000)),
                     "timezone_offset": server_config.get("timezone_offset", 8) * 60,
                 },
                 "firmware": {
-                    "version": data_json["application"].get("version", "1.0.0"),
+                    "version": device_version,
                     "url": "",
                 },
-                "websocket": {
-                    "url": self._get_websocket_url(local_ip, port),
-                },
             }
+
+            # existing mqtt/websocket logic (unchanged)
+            mqtt_gateway_endpoint = server_config.get("mqtt_gateway")
+
+            if mqtt_gateway_endpoint:  # 如果配置了非空字符串
+                # 尝试从请求数据中获取设备型号（已解析 above）
+                try:
+                    group_id = f"GID_{device_model}".replace(":", "_").replace(" ", "_")
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"获取设备型号失败: {e}")
+                    group_id = "GID_default"
+
+                mac_address_safe = device_id.replace(":", "_")
+                mqtt_client_id = f"{group_id}@@@{mac_address_safe}@@@{mac_address_safe}"
+
+                # 构建用户数据
+                user_data = {"ip": "unknown"}
+                try:
+                    user_data_json = json.dumps(user_data)
+                    username = base64.b64encode(user_data_json.encode("utf-8")).decode(
+                        "utf-8"
+                    )
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"生成用户名失败: {e}")
+                    username = ""
+
+                # 生成密码
+                password = ""
+                signature_key = server_config.get("mqtt_signature_key", "")
+                if signature_key:
+                    password = self.generate_password_signature(
+                        mqtt_client_id + "|" + username, signature_key
+                    )
+                    if not password:
+                        password = ""  # 签名失败则留空，由设备决定是否允许无密码
+                else:
+                    self.logger.bind(tag=TAG).warning("缺少MQTT签名密钥，密码留空")
+
+                # 构建MQTT配置（直接使用 mqtt_gateway 字符串）
+                return_json["mqtt"] = {
+                    "endpoint": mqtt_gateway_endpoint,
+                    "client_id": mqtt_client_id,
+                    "username": username,
+                    "password": password,
+                    "publish_topic": "device-server",
+                    "subscribe_topic": f"devices/p2p/{mac_address_safe}",
+                }
+                self.logger.bind(tag=TAG).info(f"为设备 {device_id} 下发MQTT网关配置")
+
+            else:  # 未配置 mqtt_gateway，下发 WebSocket
+                # 如果开启了认证，则进行认证校验
+                token = ""
+                if self.auth_enable:
+                    if self.allowed_devices:
+                        if device_id not in self.allowed_devices:
+                            token = self.auth.generate_token(client_id, device_id)
+                    else:
+                        token = self.auth.generate_token(client_id, device_id)
+                # NOTE: use websocket_port here
+                return_json["websocket"] = {
+                    "url": self._get_websocket_url(local_ip, websocket_port),
+                    "token": token,
+                }
+                self.logger.bind(tag=TAG).info(
+                    f"未配置MQTT网关，为设备 {device_id} 下发WebSocket配置"
+                )
+
+            # Now check firmware files for updates
+            try:
+                self._refresh_bin_cache_if_needed()
+                files_by_model = self._bin_cache.get("files_by_model", {})
+                candidates = files_by_model.get(device_model, [])
+
+                self.logger.bind(tag=TAG).info(
+                    f"查找型号 {device_model} 的固件，找到 {len(candidates)} 个候选"
+                )
+
+                chosen_url = ""
+                chosen_version = device_version
+
+                # candidates are sorted descending by version
+                for ver, fname in candidates:
+                    if _is_higher_version(ver, device_version):
+                        # build download url (only allow download via our download endpoint)
+                        chosen_version = ver
+                        # Use get_vision_url to get the base URL and replace the path
+                        vision_url = get_vision_url(self.config)
+                        # Replace the path from "/mcp/vision/explain" to "/xiaozhi/ota/download/{fname}"
+                        chosen_url = vision_url.replace(
+                            "/mcp/vision/explain", f"/xiaozhi/ota/download/{fname}"
+                        )
+                        break
+
+                if chosen_url:
+                    return_json["firmware"]["version"] = chosen_version
+                    return_json["firmware"]["url"] = chosen_url
+                    self.logger.bind(tag=TAG).info(
+                        f"为设备 {device_id} 下发固件 {chosen_version} [如果地址前缀有误，请检查配置文件中的server.vision_explain]-> {chosen_url} "
+                    )
+                else:
+                    self.logger.bind(tag=TAG).info(
+                        f"设备 {device_id} 固件已是最新: {device_version}"
+                    )
+
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"检查固件版本时出错: {e}")
+
             response = web.Response(
                 text=json.dumps(return_json, separators=(",", ":")),
                 content_type="application/json",
             )
         except Exception as e:
+            self.logger.bind(tag=TAG).error(f"OTA POST处理异常: {e}")
             return_json = {"success": False, "message": "request error."}
             response = web.Response(
                 text=json.dumps(return_json, separators=(",", ":")),
@@ -81,8 +357,9 @@ class OTAHandler(BaseHandler):
         try:
             server_config = self.config["server"]
             local_ip = get_local_ip()
-            port = int(server_config.get("port", 8000))
-            websocket_url = self._get_websocket_url(local_ip, port)
+            # use websocket port for websocket URL
+            websocket_port = int(server_config.get("port", 8000))
+            websocket_url = self._get_websocket_url(local_ip, websocket_port)
             message = f"OTA接口运行正常，向设备发送的websocket地址是：{websocket_url}"
             response = web.Response(text=message, content_type="text/plain")
         except Exception as e:
@@ -91,3 +368,48 @@ class OTAHandler(BaseHandler):
         finally:
             self._add_cors_headers(response)
             return response
+
+    async def handle_download(self, request):
+        """
+        下载固件接口
+        URL: /xiaozhi/ota/download/{filename}
+        - 只允许下载 data/bin 目录下的 .bin 文件
+        - filename 必须是 basename 且匹配安全的模式
+        """
+        try:
+            fname = request.match_info.get("filename", "")
+            if not fname:
+                raise web.HTTPBadRequest(text="filename required")
+
+            # sanitize
+            fname = _safe_basename(fname)
+            # pattern: allow letters, numbers, dot, underscore, dash
+            if not re.match(r"^[A-Za-z0-9\.\-_]+\.bin$", fname):
+                raise web.HTTPBadRequest(text="invalid filename")
+
+            file_path = os.path.join(self.bin_dir, fname)
+            # ensure realpath is under bin_dir
+            file_real = os.path.realpath(file_path)
+            bin_dir_real = os.path.realpath(self.bin_dir)
+            if (
+                not file_real.startswith(bin_dir_real + os.sep)
+                and file_real != bin_dir_real
+            ):
+                raise web.HTTPForbidden(text="forbidden")
+
+            if not os.path.isfile(file_real):
+                raise web.HTTPNotFound(text="file not found")
+
+            # use FileResponse to stream file
+            resp = web.FileResponse(path=file_real)
+        except web.HTTPError as e:
+            resp = e
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"固件下载异常: {e}")
+            resp = web.Response(text="download error", status=500)
+        finally:
+            try:
+                self._add_cors_headers(resp)
+            except Exception:
+                pass
+            return resp

main/xiaozhi-server/core/api/vision_handler.py

@@ -2,6 +2,7 @@ import json
 import copy
 from aiohttp import web
 from config.logger import setup_logging
+from core.api.base_handler import BaseHandler
 from core.utils.util import get_vision_url, is_valid_image_file
 from core.utils.vllm import create_instance
 from config.config_loader import get_private_config_from_api
@@ -16,10 +17,9 @@ TAG = __name__
 MAX_FILE_SIZE = 5 * 1024 * 1024
 
 
-class VisionHandler:
+class VisionHandler(BaseHandler):
     def __init__(self, config: dict):
-        self.config = config
-        self.logger = setup_logging()
+        super().__init__(config)
         # 初始化认证工具
         self.auth = AuthToken(config["server"]["auth_key"])
 
@@ -96,7 +96,7 @@ class VisionHandler:
             current_config = copy.deepcopy(self.config)
             read_config_from_api = current_config.get("read_config_from_api", False)
             if read_config_from_api:
-                current_config = get_private_config_from_api(
+                current_config = await get_private_config_from_api(
                     current_config,
                     device_id,
                     client_id,
@@ -172,11 +172,3 @@ class VisionHandler:
         finally:
             self._add_cors_headers(response)
             return response
-
-    def _add_cors_headers(self, response):
-        """添加CORS头信息"""
-        response.headers["Access-Control-Allow-Headers"] = (
-            "client-id, content-type, device-id"
-        )
-        response.headers["Access-Control-Allow-Credentials"] = "true"
-        response.headers["Access-Control-Allow-Origin"] = "*"

main/xiaozhi-server/core/auth.py

@@ -1,54 +1,118 @@
-from config.logger import setup_logging
-
-TAG = __name__
-logger = setup_logging()
+import hmac
+import base64
+import hashlib
+import time
 
 
 class AuthenticationError(Exception):
     """认证异常"""
+
     pass
 
 
 class AuthMiddleware:
-    def __init__(self, config):
+    """
+    认证中间件（兼容旧版命名）
+    用于 WebSocket/MQTT 连接认证
+    """
+
+    def __init__(self, config: dict):
+        """
+        初始化认证中间件
+        
+        Args:
+            config: 配置字典，包含认证相关配置
+        """
         self.config = config
-        self.auth_config = config["server"].get("auth", {})
-        # 构建token查找表
-        self.tokens = {
-            item["token"]: item["name"]
-            for item in self.auth_config.get("tokens", [])
-        }
-        # 设备白名单
-        self.allowed_devices = set(
-            self.auth_config.get("allowed_devices", [])
-        )
+        auth_config = config.get("server", {}).get("auth", {})
+        self.enabled = auth_config.get("enabled", False)
+        self.tokens = auth_config.get("tokens", [])
+        self.allowed_devices = auth_config.get("allowed_devices", [])
 
-    async def authenticate(self, headers):
-        """验证连接请求"""
-        # 检查是否启用认证
-        if not self.auth_config.get("enabled", False):
+    def authenticate(self, device_id: str, token: str = None) -> bool:
+        """
+        验证设备认证
+        
+        Args:
+            device_id: 设备 ID
+            token: 认证令牌
+            
+        Returns:
+            bool: 认证是否通过
+        """
+        if not self.enabled:
             return True
-
-        # 检查设备是否在白名单中
-        device_id = headers.get("device-id", "")
-
-        if self.allowed_devices and device_id in self.allowed_devices:
+        
+        # 检查白名单
+        if device_id in self.allowed_devices:
             return True
+        
+        # 检查 token
+        if token:
+            for token_config in self.tokens:
+                if token_config.get("token") == token:
+                    return True
+        
+        return False
 
-        # 验证Authorization header
-        auth_header = headers.get("authorization", "")
-        if not auth_header.startswith("Bearer "):
-            logger.bind(tag=TAG).error("Missing or invalid Authorization header")
-            raise AuthenticationError("Missing or invalid Authorization header")
 
-        token = auth_header.split(" ")[1]
-        if token not in self.tokens:
-            logger.bind(tag=TAG).error(f"Invalid token: {token}")
-            raise AuthenticationError("Invalid token")
+class AuthManager:
+    """
+    统一授权认证管理器
+    生成与验证 client_id device_id token（HMAC-SHA256）认证三元组
+    token 中不含明文 client_id/device_id，只携带签名 + 时间戳; client_id/device_id在连接时传递
+    在 MQTT 中 client_id: client_id, username: device_id, password: token
+    在 Websocket 中，header:{Device-ID: device_id, Client-ID: client_id, Authorization: Bearer token, ......}
+    """
 
-        logger.bind(tag=TAG).info(f"Authentication successful - Device: {device_id}, Token: {self.tokens[token]}")
-        return True
+    def __init__(self, secret_key: str, expire_seconds: int = 60 * 60 * 24 * 30):
+        if not expire_seconds or expire_seconds < 0:
+            self.expire_seconds = 60 * 60 * 24 * 30
+        else:
+            self.expire_seconds = expire_seconds
+        self.secret_key = secret_key
 
-    def get_token_name(self, token):
-        """获取token对应的设备名称"""
-        return self.tokens.get(token)
+    def _sign(self, content: str) -> str:
+        """HMAC-SHA256签名并Base64编码"""
+        sig = hmac.new(
+            self.secret_key.encode("utf-8"), content.encode("utf-8"), hashlib.sha256
+        ).digest()
+        return base64.urlsafe_b64encode(sig).decode("utf-8").rstrip("=")
+
+    def generate_token(self, client_id: str, username: str) -> str:
+        """
+        生成 token
+        Args:
+            client_id: 设备连接ID
+            username: 设备用户名（通常为deviceId）
+        Returns:
+            str: token字符串
+        """
+        ts = int(time.time())
+        content = f"{client_id}|{username}|{ts}"
+        signature = self._sign(content)
+        # token仅包含签名与时间戳，不包含明文信息
+        token = f"{signature}.{ts}"
+        return token
+
+    def verify_token(self, token: str, client_id: str, username: str) -> bool:
+        """
+        验证token有效性
+        Args:
+            token: 客户端传入的token
+            client_id: 连接使用的client_id
+            username: 连接使用的username
+        """
+        try:
+            sig_part, ts_str = token.split(".")
+            ts = int(ts_str)
+            if int(time.time()) - ts > self.expire_seconds:
+                return False  # 过期
+
+            expected_sig = self._sign(f"{client_id}|{username}|{ts}")
+            if not hmac.compare_digest(sig_part, expected_sig):
+                return False
+
+            return True
+        except Exception:
+            return False

main/xiaozhi-server/core/connection.py

@@ -10,6 +10,7 @@ import threading
 import traceback
 import subprocess
 import websockets
+
 from core.utils.util import (
     extract_json_from_string,
     check_vad_update,
@@ -69,9 +70,12 @@ class ConnectionHandler:
         self.logger = setup_logging()
         self.server = server  # 保存server实例的引用
 
-        self.auth = AuthMiddleware(config)
-        self.need_bind = False
-        self.bind_code = None
+        self.need_bind = False  # 是否需要绑定设备
+        self.bind_completed_event = asyncio.Event()
+        self.bind_code = None  # 绑定设备的验证码
+        self.last_bind_prompt_time = 0  # 上次播放绑定提示的时间戳(秒)
+        self.bind_prompt_interval = 60  # 绑定提示播放间隔(秒)
+
         self.read_config_from_api = self.config.get("read_config_from_api", False)
 
         self.websocket = None
@@ -90,7 +94,7 @@ class ConnectionHandler:
         self.client_listen_mode = "auto"
 
         # 线程任务相关
-        self.loop = asyncio.get_event_loop()
+        self.loop = None  # 在 handle_connection 中获取运行中的事件循环
         self.stop_event = threading.Event()
         self.executor = ThreadPoolExecutor(max_workers=5)
 
@@ -117,9 +121,10 @@ class ConnectionHandler:
         # vad相关变量
         self.client_audio_buffer = bytearray()
         self.client_have_voice = False
+        self.client_voice_window = deque(maxlen=5)
+        self.first_activity_time = 0.0  # 记录首次活动的时间（毫秒）
         self.last_activity_time = 0.0  # 统一的活动时间戳（毫秒）
         self.client_voice_stop = False
-        self.client_voice_window = deque(maxlen=5)
         self.last_is_voice = False
 
         # asr相关变量
@@ -156,8 +161,11 @@ class ConnectionHandler:
         # {"mcp":true} 表示启用MCP功能
         self.features = None
 
+        # 标记连接是否来自MQTT
+        self.conn_from_mqtt_gateway = False
+
         # 初始化提示词管理器
-        self.prompt_manager = PromptManager(config, self.logger)
+        self.prompt_manager = PromptManager(self.config, self.logger)
         
         # 新增：会话上下文与组件管理器（会话级清理）
         self.session_context: SessionContext = SessionContext()
@@ -166,27 +174,11 @@ class ConnectionHandler:
 
     async def handle_connection(self, ws):
         try:
+            # 获取运行中的事件循环（必须在异步上下文中）
+            self.loop = asyncio.get_running_loop()
+
             # 获取并验证headers
             self.headers = dict(ws.request.headers)
-
-            if self.headers.get("device-id", None) is None:
-                # 尝试从 URL 的查询参数中获取 device-id
-                from urllib.parse import parse_qs, urlparse
-
-                # 从 WebSocket 请求中获取路径
-                request_path = ws.request.path
-                if not request_path:
-                    self.logger.bind(tag=TAG).error("无法获取请求路径")
-                    return
-                parsed_url = urlparse(request_path)
-                query_params = parse_qs(parsed_url.query)
-                if "device-id" in query_params:
-                    self.headers["device-id"] = query_params["device-id"][0]
-                    self.headers["client-id"] = query_params["client-id"][0]
-                else:
-                    await ws.send("端口正常，如需测试连接，请使用test_page.html")
-                    await self.close(ws)
-                    return
             real_ip = self.headers.get("x-real-ip") or self.headers.get(
                 "x-forwarded-for"
             )
@@ -198,18 +190,24 @@ class ConnectionHandler:
                 f"{self.client_ip} conn - Headers: {self.headers}"
             )
 
-            # 进行认证
-            await self.auth.authenticate(self.headers)
+            self.device_id = self.headers.get("device-id", None)
 
             # 认证通过,继续处理
             self.websocket = ws
-            self.device_id = self.headers.get("device-id", None)
+            
             # 更新会话上下文关键信息
             self.session_context.headers = self.headers
             self.session_context.device_id = self.device_id
             self.session_context.client_ip = self.client_ip
 
+            # 检查是否来自MQTT连接
+            request_path = ws.request.path
+            self.conn_from_mqtt_gateway = request_path.endswith("?from=mqtt_gateway")
+            if self.conn_from_mqtt_gateway:
+                self.logger.bind(tag=TAG).info("连接来自:MQTT网关")
+
             # 初始化活动时间戳
+            self.first_activity_time = time.time() * 1000
             self.last_activity_time = time.time() * 1000
 
             # 启动超时检查任务
@@ -219,10 +217,8 @@ class ConnectionHandler:
             self.welcome_msg = self.config.xiaozhi
             self.welcome_msg["session_id"] = self.session_id
 
-            # 获取差异化配置
-            self._initialize_private_config()
-            # 异步初始化
-            self.executor.submit(self._initialize_components)
+            # 在后台初始化配置和组件（完全不阻塞主循环）
+            asyncio.create_task(self._background_initialize())
 
             try:
                 async for message in self.websocket:
@@ -273,7 +269,9 @@ class ConnectionHandler:
                         loop = asyncio.new_event_loop()
                         asyncio.set_event_loop(loop)
                         loop.run_until_complete(
-                            self.memory.save_memory(self.dialogue.dialogue)
+                            self.memory.save_memory(
+                                self.dialogue.dialogue, self.session_id
+                            )
                         )
                     except Exception as e:
                         self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
@@ -296,17 +294,116 @@ class ConnectionHandler:
                     f"保存记忆后关闭连接失败: {close_error}"
                 )
 
+    async def _discard_message_with_bind_prompt(self):
+        """丢弃消息并检查是否需要播放绑定提示"""
+        current_time = time.time()
+        # 检查是否需要播放绑定提示
+        if current_time - self.last_bind_prompt_time >= self.bind_prompt_interval:
+            self.last_bind_prompt_time = current_time
+            # 复用现有的绑定提示逻辑
+            from core.handle.receiveAudioHandle import check_bind_device
+
+            asyncio.create_task(check_bind_device(self))
+
     async def _route_message(self, message):
         """消息路由"""
+        # 检查是否已经获取到真实的绑定状态
+        if not self.bind_completed_event.is_set():
+            # 还没有获取到真实状态，等待直到获取到真实状态或超时
+            try:
+                await asyncio.wait_for(self.bind_completed_event.wait(), timeout=1)
+            except asyncio.TimeoutError:
+                # 超时仍未获取到真实状态，丢弃消息
+                await self._discard_message_with_bind_prompt()
+                return
+
+        # 已经获取到真实状态，检查是否需要绑定
+        if self.need_bind:
+            # 需要绑定，丢弃消息
+            await self._discard_message_with_bind_prompt()
+            return
+
+        # 不需要绑定，继续处理消息
+
         if isinstance(message, str):
             await handleTextMessage(self, message)
         elif isinstance(message, bytes):
-            if self.vad is None:
-                return
-            if self.asr is None:
+            if self.vad is None or self.asr is None:
                 return
+
+            # 处理来自MQTT网关的音频包
+            if self.conn_from_mqtt_gateway and len(message) >= 16:
+                handled = await self._process_mqtt_audio_message(message)
+                if handled:
+                    return
+
+            # 不需要头部处理或没有头部时，直接处理原始消息
             self.asr_audio_queue.put(message)
 
+    async def _process_mqtt_audio_message(self, message):
+        """
+        处理来自MQTT网关的音频消息，解析16字节头部并提取音频数据
+
+        Args:
+            message: 包含头部的音频消息
+
+        Returns:
+            bool: 是否成功处理了消息
+        """
+        try:
+            # 提取头部信息
+            timestamp = int.from_bytes(message[8:12], "big")
+            audio_length = int.from_bytes(message[12:16], "big")
+
+            # 提取音频数据
+            if audio_length > 0 and len(message) >= 16 + audio_length:
+                # 有指定长度，提取精确的音频数据
+                audio_data = message[16 : 16 + audio_length]
+                # 基于时间戳进行排序处理
+                self._process_websocket_audio(audio_data, timestamp)
+                return True
+            elif len(message) > 16:
+                # 没有指定长度或长度无效，去掉头部后处理剩余数据
+                audio_data = message[16:]
+                self.asr_audio_queue.put(audio_data)
+                return True
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"解析WebSocket音频包失败: {e}")
+
+        # 处理失败，返回False表示需要继续处理
+        return False
+
+    def _process_websocket_audio(self, audio_data, timestamp):
+        """处理WebSocket格式的音频包"""
+        # 初始化时间戳序列管理
+        if not hasattr(self, "audio_timestamp_buffer"):
+            self.audio_timestamp_buffer = {}
+            self.last_processed_timestamp = 0
+            self.max_timestamp_buffer_size = 20
+
+        # 如果时间戳是递增的，直接处理
+        if timestamp >= self.last_processed_timestamp:
+            self.asr_audio_queue.put(audio_data)
+            self.last_processed_timestamp = timestamp
+
+            # 处理缓冲区中的后续包
+            processed_any = True
+            while processed_any:
+                processed_any = False
+                for ts in sorted(self.audio_timestamp_buffer.keys()):
+                    if ts > self.last_processed_timestamp:
+                        buffered_audio = self.audio_timestamp_buffer.pop(ts)
+                        self.asr_audio_queue.put(buffered_audio)
+                        self.last_processed_timestamp = ts
+                        processed_any = True
+                        break
+        else:
+            # 乱序包，暂存
+            if len(self.audio_timestamp_buffer) < self.max_timestamp_buffer_size:
+                self.audio_timestamp_buffer[timestamp] = audio_data
+            else:
+                self.asr_audio_queue.put(audio_data)
+
     async def handle_restart(self, message):
         """处理服务器重启请求"""
         try:
@@ -357,6 +454,15 @@ class ConnectionHandler:
 
     def _initialize_components(self):
         try:
+            if self.tts is None:
+                self.tts = self._initialize_tts()
+            # 打开语音合成通道
+            asyncio.run_coroutine_threadsafe(
+                self.tts.open_audio_channels(self), self.loop
+            )
+            if self.need_bind:
+                self.bind_completed_event.set()
+                return
             self.selected_module_str = build_module_string(
                 self.config.get("selected_module", {})
             )
@@ -380,17 +486,10 @@ class ConnectionHandler:
 
             # 初始化声纹识别
             self._initialize_voiceprint()
-
             # 打开语音识别通道
             asyncio.run_coroutine_threadsafe(
                 self.asr.open_audio_channels(self), self.loop
             )
-            if self.tts is None:
-                self.tts = self._initialize_tts()
-            # 打开语音合成通道
-            asyncio.run_coroutine_threadsafe(
-                self.tts.open_audio_channels(self), self.loop
-            )
 
             """加载记忆"""
             self._initialize_memory()
@@ -405,6 +504,7 @@ class ConnectionHandler:
             self.logger.bind(tag=TAG).error(f"实例化组件失败: {e}")
 
     def _init_prompt_enhancement(self):
+
         # 更新上下文信息
         self.prompt_manager.update_context_info(self, self.client_ip)
         enhanced_prompt = self.prompt_manager.build_enhanced_prompt(
@@ -412,7 +512,7 @@ class ConnectionHandler:
         )
         if enhanced_prompt:
             self.change_system_prompt(enhanced_prompt)
-            self.logger.bind(tag=TAG).info("系统提示词已增强更新")
+            self.logger.bind(tag=TAG).debug("系统提示词已增强更新")
 
     def _init_report_threads(self):
         """初始化ASR和TTS上报线程"""
@@ -440,7 +540,11 @@ class ConnectionHandler:
 
     def _initialize_asr(self):
         """初始化ASR"""
-        if self._asr.interface_type == InterfaceType.LOCAL:
+        if (
+            self._asr is not None
+            and hasattr(self._asr, "interface_type")
+            and self._asr.interface_type == InterfaceType.LOCAL
+        ):
             # 如果公共ASR是本地服务，则直接返回
             # 因为本地一个实例ASR，可以被多个连接共享
             asr = self._asr
@@ -456,29 +560,46 @@ class ConnectionHandler:
         try:
             voiceprint_config = self.config.get("voiceprint", {})
             if voiceprint_config:
-                self.voiceprint_provider = VoiceprintProvider(voiceprint_config)
-                self.logger.bind(tag=TAG).info("声纹识别功能已在连接时动态启用")
+                voiceprint_provider = VoiceprintProvider(voiceprint_config)
+                if voiceprint_provider is not None and voiceprint_provider.enabled:
+                    self.voiceprint_provider = voiceprint_provider
+                    self.logger.bind(tag=TAG).info("声纹识别功能已在连接时动态启用")
+                else:
+                    self.logger.bind(tag=TAG).warning("声纹识别功能启用但配置不完整")
             else:
-                self.logger.bind(tag=TAG).info("声纹识别功能未启用或配置不完整")
+                self.logger.bind(tag=TAG).info("声纹识别功能未启用")
         except Exception as e:
             self.logger.bind(tag=TAG).warning(f"声纹识别初始化失败: {str(e)}")
 
-    def _initialize_private_config(self):
-        """如果是从配置文件获取，则进行二次实例化"""
+    async def _background_initialize(self):
+        """在后台初始化配置和组件（完全不阻塞主循环）"""
+        try:
+            # 异步获取差异化配置
+            await self._initialize_private_config_async()
+            # 在线程池中初始化组件
+            self.executor.submit(self._initialize_components)
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"后台初始化失败: {e}")
+
+    async def _initialize_private_config_async(self):
+        """从接口异步获取差异化配置（异步版本，不阻塞主循环）"""
         if not self.read_config_from_api:
+            self.need_bind = False
+            self.bind_completed_event.set()
             return
-        """从接口获取差异化的配置进行二次实例化，非全量重新实例化"""
         try:
             begin_time = time.time()
-            private_config = get_private_config_from_api(
+            private_config = await get_private_config_from_api(
                 self.config,
                 self.headers.get("device-id"),
                 self.headers.get("client-id", self.headers.get("device-id")),
             )
             private_config["delete_audio"] = bool(self.config.get("delete_audio", True))
             self.logger.bind(tag=TAG).info(
-                f"{time.time() - begin_time} 秒，获取差异化配置成功: {json.dumps(filter_sensitive_info(private_config), ensure_ascii=False)}"
+                f"{time.time() - begin_time} 秒，异步获取差异化配置成功: {json.dumps(filter_sensitive_info(private_config), ensure_ascii=False)}"
             )
+            self.need_bind = False
+            self.bind_completed_event.set()
         except DeviceNotFoundException as e:
             self.need_bind = True
             private_config = {}
@@ -488,7 +609,7 @@ class ConnectionHandler:
             private_config = {}
         except Exception as e:
             self.need_bind = True
-            self.logger.bind(tag=TAG).error(f"获取差异化配置失败: {e}")
+            self.logger.bind(tag=TAG).error(f"异步获取差异化配置失败: {e}")
             private_config = {}
 
         init_llm, init_tts, init_memory, init_intent = (
@@ -562,8 +683,14 @@ class ConnectionHandler:
             self.chat_history_conf = int(private_config["chat_history_conf"])
         if private_config.get("mcp_endpoint", None) is not None:
             self.config["mcp_endpoint"] = private_config["mcp_endpoint"]
+        if private_config.get("context_providers", None) is not None:
+            self.config["context_providers"] = private_config["context_providers"]
+
+        # 使用 run_in_executor 在线程池中执行 initialize_modules，避免阻塞主循环
         try:
-            modules = initialize_modules(
+            modules = await self.loop.run_in_executor(
+                None,  # 使用默认线程池
+                initialize_modules,
                 self.logger,
                 private_config,
                 init_vad,
@@ -685,11 +812,12 @@ class ConnectionHandler:
         self.dialogue.update_system_message(self.prompt)
 
     def chat(self, query, depth=0):
-        self.logger.bind(tag=TAG).info(f"大模型收到用户消息: {query}")
-        self.llm_finish_task = False
+        if query is not None:
+            self.logger.bind(tag=TAG).info(f"大模型收到用户消息: {query}")
 
         # 为最顶层时新建会话ID和发送FIRST请求
         if depth == 0:
+            self.llm_finish_task = False
             self.sentence_id = str(uuid.uuid4().hex)
             self.dialogue.put(Message(role="user", content=query))
             self.tts.tts_text_queue.put(
@@ -700,9 +828,31 @@ class ConnectionHandler:
                 )
             )
 
+        # 设置最大递归深度，避免无限循环，可根据实际需求调整
+        MAX_DEPTH = 5
+        force_final_answer = False  # 标记是否强制最终回答
+
+        if depth >= MAX_DEPTH:
+            self.logger.bind(tag=TAG).debug(
+                f"已达到最大工具调用深度 {MAX_DEPTH}，将强制基于现有信息回答"
+            )
+            force_final_answer = True
+            # 添加系统指令，要求 LLM 基于现有信息回答
+            self.dialogue.put(
+                Message(
+                    role="user",
+                    content="[系统提示] 已达到最大工具调用次数限制，请你基于目前已经获取的所有信息，直接给出最终答案。不要再尝试调用任何工具。",
+                )
+            )
+
         # Define intent functions
         functions = None
-        if self.intent_type == "function_call" and hasattr(self, "func_handler"):
+        # 达到最大深度时，禁用工具调用，强制 LLM 直接回答
+        if (
+            self.intent_type == "function_call"
+            and hasattr(self, "func_handler")
+            and not force_final_answer
+        ):
             functions = self.func_handler.get_functions()
         response_message = []
 
@@ -737,9 +887,8 @@ class ConnectionHandler:
 
         # 处理流式响应
         tool_call_flag = False
-        function_name = None
-        function_id = None
-        function_arguments = ""
+        # 支持多个并行工具调用 - 使用列表存储
+        tool_calls_list = []  # 格式: [{"id": "", "name": "", "arguments": ""}]
         content_arguments = ""
         self.client_abort = False
         emotion_flag = True
@@ -760,12 +909,7 @@ class ConnectionHandler:
 
                 if tools_call is not None and len(tools_call) > 0:
                     tool_call_flag = True
-                    if tools_call[0].id is not None:
-                        function_id = tools_call[0].id
-                    if tools_call[0].function.name is not None:
-                        function_name = tools_call[0].function.name
-                    if tools_call[0].function.arguments is not None:
-                        function_arguments += tools_call[0].function.arguments
+                    self._merge_tool_calls(tool_calls_list, tools_call)
             else:
                 content = response
 
@@ -791,16 +935,22 @@ class ConnectionHandler:
         # 处理function call
         if tool_call_flag:
             bHasError = False
-            if function_id is None:
+            # 处理基于文本的工具调用格式
+            if len(tool_calls_list) == 0 and content_arguments:
                 a = extract_json_from_string(content_arguments)
                 if a is not None:
                     try:
                         content_arguments_json = json.loads(a)
-                        function_name = content_arguments_json["name"]
-                        function_arguments = json.dumps(
-                            content_arguments_json["arguments"], ensure_ascii=False
+                        tool_calls_list.append(
+                            {
+                                "id": str(uuid.uuid4().hex),
+                                "name": content_arguments_json["name"],
+                                "arguments": json.dumps(
+                                    content_arguments_json["arguments"],
+                                    ensure_ascii=False,
+                                ),
+                            }
                         )
-                        function_id = str(uuid.uuid4().hex)
                     except Exception as e:
                         bHasError = True
                         response_message.append(a)
@@ -811,30 +961,43 @@ class ConnectionHandler:
                     self.logger.bind(tag=TAG).error(
                         f"function call error: {content_arguments}"
                     )
-            if not bHasError:
+
+            if not bHasError and len(tool_calls_list) > 0:
                 # 如需要大模型先处理一轮，添加相关处理后的日志情况
                 if len(response_message) > 0:
                     text_buff = "".join(response_message)
                     self.tts_MessageText = text_buff
                     self.dialogue.put(Message(role="assistant", content=text_buff))
                 response_message.clear()
-                self.logger.bind(tag=TAG).debug(
-                    f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}"
-                )
-                function_call_data = {
-                    "name": function_name,
-                    "id": function_id,
-                    "arguments": function_arguments,
-                }
 
-                # 使用统一工具处理器处理所有工具调用
-                result = asyncio.run_coroutine_threadsafe(
-                    self.func_handler.handle_llm_function_call(
-                        self, function_call_data
-                    ),
-                    self.loop,
-                ).result()
-                self._handle_function_result(result, function_call_data, depth=depth)
+                self.logger.bind(tag=TAG).debug(
+                    f"检测到 {len(tool_calls_list)} 个工具调用"
+                )
+
+                # 收集所有工具调用的 Future
+                futures_with_data = []
+                for tool_call_data in tool_calls_list:
+                    self.logger.bind(tag=TAG).debug(
+                        f"function_name={tool_call_data['name']}, function_id={tool_call_data['id']}, function_arguments={tool_call_data['arguments']}"
+                    )
+
+                    future = asyncio.run_coroutine_threadsafe(
+                        self.func_handler.handle_llm_function_call(
+                            self, tool_call_data
+                        ),
+                        self.loop,
+                    )
+                    futures_with_data.append((future, tool_call_data))
+
+                # 等待协程结束（实际等待时长为最慢的那个）
+                tool_results = []
+                for future, tool_call_data in futures_with_data:
+                    result = future.result()
+                    tool_results.append((result, tool_call_data))
+
+                # 统一处理所有工具调用结果
+                if tool_results:
+                    self._handle_function_result(tool_results, depth=depth)
 
         # 存储对话内容
         if len(response_message) > 0:
@@ -849,60 +1012,69 @@ class ConnectionHandler:
                     content_type=ContentType.ACTION,
                 )
             )
-        self.llm_finish_task = True
-        # 使用lambda延迟计算，只有在DEBUG级别时才执行get_llm_dialogue()
-        self.logger.bind(tag=TAG).debug(
-            lambda: json.dumps(
-                self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False
+            self.llm_finish_task = True
+            # 使用lambda延迟计算，只有在DEBUG级别时才执行get_llm_dialogue()
+            self.logger.bind(tag=TAG).debug(
+                lambda: json.dumps(
+                    self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False
+                )
             )
-        )
 
         return True
 
-    def _handle_function_result(self, result, function_call_data, depth):
-        if result.action == Action.RESPONSE:  # 直接回复前端
-            text = result.response
-            self.tts.tts_one_sentence(self, ContentType.TEXT, content_detail=text)
-            self.dialogue.put(Message(role="assistant", content=text))
-        elif result.action == Action.REQLLM:  # 调用函数后再请求llm生成回复
-            text = result.result
-            if text is not None and len(text) > 0:
-                function_id = function_call_data["id"]
-                function_name = function_call_data["name"]
-                function_arguments = function_call_data["arguments"]
-                self.dialogue.put(
-                    Message(
-                        role="assistant",
-                        tool_calls=[
-                            {
-                                "id": function_id,
-                                "function": {
-                                    "arguments": "{}" if function_arguments == "" else function_arguments,
-                                    "name": function_name,
-                                },
-                                "type": "function",
-                                "index": 0,
-                            }
-                        ],
-                    )
-                )
+    def _handle_function_result(self, tool_results, depth):
+        need_llm_tools = []
 
-                self.dialogue.put(
-                    Message(
-                        role="tool",
-                        tool_call_id=(
-                            str(uuid.uuid4()) if function_id is None else function_id
+        for result, tool_call_data in tool_results:
+            if result.action in [
+                Action.RESPONSE,
+                Action.NOTFOUND,
+                Action.ERROR,
+            ]:  # 直接回复前端
+                text = result.response if result.response else result.result
+                self.tts.tts_one_sentence(self, ContentType.TEXT, content_detail=text)
+                self.dialogue.put(Message(role="assistant", content=text))
+            elif result.action == Action.REQLLM:
+                # 收集需要 LLM 处理的工具
+                need_llm_tools.append((result, tool_call_data))
+            else:
+                pass
+
+        if need_llm_tools:
+            all_tool_calls = [
+                {
+                    "id": tool_call_data["id"],
+                    "function": {
+                        "arguments": (
+                            "{}"
+                            if tool_call_data["arguments"] == ""
+                            else tool_call_data["arguments"]
                         ),
-                        content=text,
+                        "name": tool_call_data["name"],
+                    },
+                    "type": "function",
+                    "index": idx,
+                }
+                for idx, (_, tool_call_data) in enumerate(need_llm_tools)
+            ]
+            self.dialogue.put(Message(role="assistant", tool_calls=all_tool_calls))
+
+            for result, tool_call_data in need_llm_tools:
+                text = result.result
+                if text is not None and len(text) > 0:
+                    self.dialogue.put(
+                        Message(
+                            role="tool",
+                            tool_call_id=(
+                                str(uuid.uuid4())
+                                if tool_call_data["id"] is None
+                                else tool_call_data["id"]
+                            ),
+                            content=text,
+                        )
                     )
-                )
-                self.chat(text, depth=depth + 1)
-        elif result.action == Action.NOTFOUND or result.action == Action.ERROR:
-            text = result.response if result.response else result.result
-            self.tts.tts_one_sentence(self, ContentType.TEXT, content_detail=text)
-            self.dialogue.put(Message(role="assistant", content=text))
-        else:
-            pass
+
+            self.chat(None, depth=depth + 1)
 
     def _report_worker(self):
         """聊天记录上报工作线程"""
@@ -930,9 +1102,9 @@ class ConnectionHandler:
     def _process_report(self, type, text, audio_data, report_time):
         """处理上报任务"""
         try:
-            # 执行上报（传入二进制数据）
-            # report(self, type, text, audio_data, report_time)  # 旧架构，已被新架构替代
-            pass  # 新架构中由ReportProcessor处理
+            # 执行异步上报（在事件循环中运行）
+            from core.handle.reportHandle import report
+            asyncio.run(report(self, type, text, audio_data, report_time))
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"上报处理异常: {e}")
         finally:
@@ -946,6 +1118,10 @@ class ConnectionHandler:
     async def close(self, ws=None):
         """资源清理方法"""
         try:
+            # 清理音频缓冲区
+            if hasattr(self, "audio_buffer"):
+                self.audio_buffer.clear()
+
             # 取消超时任务
             if self.timeout_task and not self.timeout_task.done():
                 self.timeout_task.cancel()
@@ -1019,7 +1195,6 @@ class ConnectionHandler:
                         f"关闭线程池时出错: {executor_error}"
                     )
                 self.executor = None
-
             self.logger.bind(tag=TAG).info("连接资源已释放")
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"关闭连接时出错: {e}")
@@ -1049,6 +1224,11 @@ class ConnectionHandler:
                     except queue.Empty:
                         break
 
+            # 重置音频流控器（取消后台任务并清空队列）
+            if hasattr(self, "audio_rate_controller") and self.audio_rate_controller:
+                self.audio_rate_controller.reset()
+                self.logger.bind(tag=TAG).debug("已重置音频流控器")
+
             self.logger.bind(tag=TAG).debug(
                 f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
             )
@@ -1074,13 +1254,14 @@ class ConnectionHandler:
         """检查连接超时"""
         try:
             while not self.stop_event.is_set():
+                last_activity_time = self.last_activity_time
+                if self.need_bind:
+                    last_activity_time = self.first_activity_time
+
                 # 检查是否超时（只有在时间戳已初始化的情况下）
-                if self.last_activity_time > 0.0:
+                if last_activity_time > 0.0:
                     current_time = time.time() * 1000
-                    if (
-                        current_time - self.last_activity_time
-                        > self.timeout_seconds * 1000
-                    ):
+                    if current_time - last_activity_time > self.timeout_seconds * 1000:
                         if not self.stop_event.is_set():
                             self.logger.bind(tag=TAG).info("连接超时，准备关闭")
                             # 设置停止事件，防止重复处理
@@ -1099,3 +1280,31 @@ class ConnectionHandler:
             self.logger.bind(tag=TAG).error(f"超时检查任务出错: {e}")
         finally:
             self.logger.bind(tag=TAG).info("超时检查任务已退出")
+
+    def _merge_tool_calls(self, tool_calls_list, tools_call):
+        """合并工具调用列表
+
+        Args:
+            tool_calls_list: 已收集的工具调用列表
+            tools_call: 新的工具调用
+        """
+        for tool_call in tools_call:
+            tool_index = getattr(tool_call, "index", None)
+            if tool_index is None:
+                if tool_call.function.name:
+                    # 有 function_name，说明是新的工具调用
+                    tool_index = len(tool_calls_list)
+                else:
+                    tool_index = len(tool_calls_list) - 1 if tool_calls_list else 0
+
+            # 确保列表有足够的位置
+            if tool_index >= len(tool_calls_list):
+                tool_calls_list.append({"id": "", "name": "", "arguments": ""})
+
+            # 更新工具调用信息
+            if tool_call.id:
+                tool_calls_list[tool_index]["id"] = tool_call.id
+            if tool_call.function.name:
+                tool_calls_list[tool_index]["name"] = tool_call.function.name
+            if tool_call.function.arguments:
+                tool_calls_list[tool_index]["arguments"] += tool_call.function.arguments

main/xiaozhi-server/core/handle/helloHandle.py

@@ -6,7 +6,7 @@ from core.utils.dialogue import Message
 from core.utils.util import audio_to_data
 from core.providers.tts.dto.dto import SentenceType
 from core.utils.wakeup_word import WakeupWordsConfig
-from core.handle.sendAudioHandle import sendAudioMessage, send_stt_message
+from core.handle.sendAudioHandle import sendAudioMessage, send_tts_message
 from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes
 from core.providers.tools.device_mcp import (
     MCPClient,
@@ -17,8 +17,18 @@ from core.providers.tools.device_mcp import (
 TAG = __name__
 
 WAKEUP_CONFIG = {
-    "refresh_time": 5,
-    "words": ["你好", "你好啊", "嘿，你好", "嗨"],
+    "refresh_time": 10,
+    "responses": [
+        "我一直都在呢，您请说。",
+        "在的呢，请随时吩咐我。",
+        "来啦来啦，请告诉我吧。",
+        "您请说，我正听着。",
+        "请您讲话，我准备好了。",
+        "请您说出指令吧。",
+        "我认真听着呢，请讲。",
+        "请问您需要什么帮助？",
+        "我在这里，等候您的指令。",
+    ],
 }
 
 # 创建全局的唤醒词配置管理器
@@ -33,15 +43,15 @@ async def handleHelloMessage(conn, msg_json):
     audio_params = msg_json.get("audio_params")
     if audio_params:
         format = audio_params.get("format")
-        conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}")
+        conn.logger.bind(tag=TAG).debug(f"客户端音频格式: {format}")
         conn.audio_format = format
         conn.welcome_msg["audio_params"] = audio_params
     features = msg_json.get("features")
     if features:
-        conn.logger.bind(tag=TAG).info(f"客户端特性: {features}")
+        conn.logger.bind(tag=TAG).debug(f"客户端特性: {features}")
         conn.features = features
         if features.get("mcp"):
-            conn.logger.bind(tag=TAG).info("客户端支持MCP")
+            conn.logger.bind(tag=TAG).debug("客户端支持MCP")
             conn.mcp_client = MCPClient()
             # 发送初始化
             asyncio.create_task(send_mcp_initialize_message(conn))
@@ -73,7 +83,7 @@ async def checkWakeupWords(conn, text):
         return False
 
     conn.just_woken_up = True
-    await send_stt_message(conn, text)
+    await send_tts_message(conn, "start")
 
     # 获取当前音色
     voice = getattr(conn.tts, "voice", "default")
@@ -85,13 +95,13 @@ async def checkWakeupWords(conn, text):
     if not response or not response.get("file_path"):
         response = {
             "voice": "default",
-            "file_path": "config/assets/wakeup_words.wav",
+            "file_path": "config/assets/wakeup_words_short.wav",
             "time": 0,
-            "text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",
+            "text": "我在这里哦！",
         }
 
     # 获取音频数据
-    opus_packets = audio_to_data(response.get("file_path"))
+    opus_packets = await audio_to_data(response.get("file_path"), use_cache=False)
     # 播放唤醒词回复
     conn.client_abort = False
 
@@ -110,7 +120,7 @@ async def checkWakeupWords(conn, text):
 
 
 async def wakeupWordsResponse(conn):
-    if not conn.tts or not conn.llm or not conn.llm.response_no_stream:
+    if not conn.tts:
         return
 
     try:
@@ -118,16 +128,8 @@ async def wakeupWordsResponse(conn):
         if not await _wakeup_response_lock.acquire():
             return
 
-        # 生成唤醒词回复
-        wakeup_word = random.choice(WAKEUP_CONFIG["words"])
-        question = (
-            "此刻用户正在和你说```"
-            + wakeup_word
-            + "```。\n请你根据以上用户的内容进行20-30字回复。要符合系统设置的角色情感和态度，不要像机器人一样说话。\n"
-            + "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。"
-        )
-
-        result = conn.llm.response_no_stream(conn.config["prompt"], question)
+        # 从预定义回复列表中随机选择一个回复
+        result = random.choice(WAKEUP_CONFIG["responses"])
         if not result or len(result) == 0:
             return
 
@@ -148,4 +150,4 @@ async def wakeupWordsResponse(conn):
     finally:
         # 确保在任何情况下都释放锁
         if _wakeup_response_lock.locked():
-            _wakeup_response_lock.release()
+            _wakeup_response_lock.release()

main/xiaozhi-server/core/handle/intentHandler.py

@@ -91,6 +91,30 @@ async def process_intent_result(conn, intent_result, original_text):
             if function_name == "continue_chat":
                 return False
 
+            if function_name == "result_for_context":
+                await send_stt_message(conn, original_text)
+                conn.client_abort = False
+                
+                def process_context_result():
+                    conn.dialogue.put(Message(role="user", content=original_text))
+                    
+                    from core.utils.current_time import get_current_time_info
+
+                    current_time, today_date, today_weekday, lunar_date = get_current_time_info()
+                    
+                    # 构建带上下文的基础提示
+                    context_prompt = f"""当前时间：{current_time}
+                                        今天日期：{today_date} ({today_weekday})
+                                        今天农历：{lunar_date}
+
+                                        请根据以上信息回答用户的问题：{original_text}"""
+                    
+                    response = conn.intent.replyResult(context_prompt, original_text)
+                    speak_txt(conn, response)
+                
+                conn.executor.submit(process_context_result)
+                return True
+
             function_args = {}
             if "arguments" in intent_data["function_call"]:
                 function_args = intent_data["function_call"]["arguments"]

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -14,26 +14,29 @@ async def handleAudioMessage(conn, audio):
     # 当前片段是否有人说话
     have_voice = conn.vad.is_vad(conn, audio)
     # 如果设备刚刚被唤醒，短暂忽略VAD检测
-    if have_voice and hasattr(conn, "just_woken_up") and conn.just_woken_up:
+    if hasattr(conn, "just_woken_up") and conn.just_woken_up:
         have_voice = False
         # 设置一个短暂延迟后恢复VAD检测
         conn.asr_audio.clear()
         if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
             conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
         return
+    # manual 模式下不打断正在播放的内容
     if have_voice:
-        if conn.client_is_speaking:
+        if conn.client_is_speaking and conn.client_listen_mode != "manual":
             await handleAbortMessage(conn)
     # 设备长时间空闲检测，用于say goodbye
     await no_voice_close_connect(conn, have_voice)
     # 接收音频
     await conn.asr.receive_audio(conn, audio, have_voice)
 
+
 async def resume_vad_detection(conn):
     # 等待2秒后恢复VAD检测
-    await asyncio.sleep(1)
+    await asyncio.sleep(2)
     conn.just_woken_up = False
 
+
 async def startToChat(conn, text):
     # 检查输入是否是JSON格式（包含说话人信息）
     speaker_name = None
@@ -41,11 +44,11 @@ async def startToChat(conn, text):
 
     try:
         # 尝试解析JSON格式的输入
-        if text.strip().startswith('{') and text.strip().endswith('}'):
+        if text.strip().startswith("{") and text.strip().endswith("}"):
             data = json.loads(text)
-            if 'speaker' in data and 'content' in data:
-                speaker_name = data['speaker']
-                actual_text = data['content']
+            if "speaker" in data and "content" in data:
+                speaker_name = data["speaker"]
+                actual_text = data["content"]
                 conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}")
 
                 # 直接使用JSON格式的文本，不解析
@@ -71,7 +74,8 @@ async def startToChat(conn, text):
         ):
             await max_out_size(conn)
             return
-    if conn.client_is_speaking:
+    # manual 模式下不打断正在播放的内容
+    if conn.client_is_speaking and conn.client_listen_mode != "manual":
         await handleAbortMessage(conn)
 
     # 首先进行意图分析，使用实际文本内容
@@ -119,7 +123,7 @@ async def max_out_size(conn):
     text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！"
     await send_stt_message(conn, text)
     file_path = "config/assets/max_output_size.wav"
-    opus_packets = audio_to_data(file_path)
+    opus_packets = await audio_to_data(file_path)
     conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))
     conn.close_after_chat = True
 
@@ -138,7 +142,7 @@ async def check_bind_device(conn):
 
         # 播放提示音
         music_path = "config/assets/bind_code.wav"
-        opus_packets = audio_to_data(music_path)
+        opus_packets = await audio_to_data(music_path)
         conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, text))
 
         # 逐个播放数字
@@ -146,7 +150,7 @@ async def check_bind_device(conn):
             try:
                 digit = conn.bind_code[i]
                 num_path = f"config/assets/bind_code/{digit}.wav"
-                num_packets = audio_to_data(num_path)
+                num_packets = await audio_to_data(num_path)
                 conn.tts.tts_audio_queue.put((SentenceType.MIDDLE, num_packets, None))
             except Exception as e:
                 conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
@@ -158,5 +162,5 @@ async def check_bind_device(conn):
         text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
         await send_stt_message(conn, text)
         music_path = "config/assets/bind_not_found.wav"
-        opus_packets = audio_to_data(music_path)
+        opus_packets = await audio_to_data(music_path)
         conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))

main/xiaozhi-server/core/handle/reportHandle.py

@@ -10,7 +10,6 @@ TTS上报功能已集成到ConnectionHandler类中。
 """
 
 import time
-
 import opuslib_next
 
 from config.manage_api_client import report as manage_report
@@ -18,7 +17,7 @@ from config.manage_api_client import report as manage_report
 TAG = __name__
 
 
-def report(conn, type, text, opus_data, report_time):
+async def report(conn, type, text, opus_data, report_time):
     """执行聊天记录上报操作
 
     Args:
@@ -33,8 +32,8 @@ def report(conn, type, text, opus_data, report_time):
             audio_data = opus_to_wav(conn, opus_data)
         else:
             audio_data = None
-        # 执行上报
-        manage_report(
+        # 执行异步上报
+        await manage_report(
             mac_address=conn.device_id,
             session_id=conn.session_id,
             chat_type=type,
@@ -56,41 +55,49 @@ def opus_to_wav(conn, opus_data):
     Returns:
         bytes: WAV格式的音频数据
     """
-    decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-    pcm_data = []
+    decoder = None
+    try:
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
 
-    for opus_packet in opus_data:
-        try:
-            pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-            pcm_data.append(pcm_frame)
-        except opuslib_next.OpusError as e:
-            conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
 
-    if not pcm_data:
-        raise ValueError("没有有效的PCM数据")
+        if not pcm_data:
+            raise ValueError("没有有效的PCM数据")
 
-    # 创建WAV文件头
-    pcm_data_bytes = b"".join(pcm_data)
-    num_samples = len(pcm_data_bytes) // 2  # 16-bit samples
+        # 创建WAV文件头
+        pcm_data_bytes = b"".join(pcm_data)
+        num_samples = len(pcm_data_bytes) // 2  # 16-bit samples
 
-    # WAV文件头
-    wav_header = bytearray()
-    wav_header.extend(b"RIFF")  # ChunkID
-    wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little"))  # ChunkSize
-    wav_header.extend(b"WAVE")  # Format
-    wav_header.extend(b"fmt ")  # Subchunk1ID
-    wav_header.extend((16).to_bytes(4, "little"))  # Subchunk1Size
-    wav_header.extend((1).to_bytes(2, "little"))  # AudioFormat (PCM)
-    wav_header.extend((1).to_bytes(2, "little"))  # NumChannels
-    wav_header.extend((16000).to_bytes(4, "little"))  # SampleRate
-    wav_header.extend((32000).to_bytes(4, "little"))  # ByteRate
-    wav_header.extend((2).to_bytes(2, "little"))  # BlockAlign
-    wav_header.extend((16).to_bytes(2, "little"))  # BitsPerSample
-    wav_header.extend(b"data")  # Subchunk2ID
-    wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little"))  # Subchunk2Size
+        # WAV文件头
+        wav_header = bytearray()
+        wav_header.extend(b"RIFF")  # ChunkID
+        wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little"))  # ChunkSize
+        wav_header.extend(b"WAVE")  # Format
+        wav_header.extend(b"fmt ")  # Subchunk1ID
+        wav_header.extend((16).to_bytes(4, "little"))  # Subchunk1Size
+        wav_header.extend((1).to_bytes(2, "little"))  # AudioFormat (PCM)
+        wav_header.extend((1).to_bytes(2, "little"))  # NumChannels
+        wav_header.extend((16000).to_bytes(4, "little"))  # SampleRate
+        wav_header.extend((32000).to_bytes(4, "little"))  # ByteRate
+        wav_header.extend((2).to_bytes(2, "little"))  # BlockAlign
+        wav_header.extend((16).to_bytes(2, "little"))  # BitsPerSample
+        wav_header.extend(b"data")  # Subchunk2ID
+        wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little"))  # Subchunk2Size
 
-    # 返回完整的WAV数据
-    return bytes(wav_header) + pcm_data_bytes
+        # 返回完整的WAV数据
+        return bytes(wav_header) + pcm_data_bytes
+    finally:
+        if decoder is not None:
+            try:
+                del decoder
+            except Exception as e:
+                conn.logger.bind(tag=TAG).debug(f"释放decoder资源时出错: {e}")
 
 
 def enqueue_tts_report(conn, text, opus_data):

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -4,8 +4,13 @@ import asyncio
 from core.utils import textUtils
 from core.utils.util import audio_to_data
 from core.providers.tts.dto.dto import SentenceType
+from core.utils.audioRateController import AudioRateController
 
 TAG = __name__
+# 音频帧时长（毫秒）
+AUDIO_FRAME_DURATION = 60
+# 预缓冲包数量，直接发送以减少延迟
+PRE_BUFFER_COUNT = 5
 
 
 async def sendAudioMessage(conn, sentenceType, audios, text):
@@ -15,7 +20,19 @@ async def sendAudioMessage(conn, sentenceType, audios, text):
         await send_tts_message(conn, "start", None)
 
     if sentenceType == SentenceType.FIRST:
-        await send_tts_message(conn, "sentence_start", text)
+        # 同一句子的后续消息加入流控队列，其他情况立即发送
+        if (
+            hasattr(conn, "audio_rate_controller")
+            and conn.audio_rate_controller
+            and getattr(conn, "audio_flow_control", {}).get("sentence_id")
+            == conn.sentence_id
+        ):
+            conn.audio_rate_controller.add_message(
+                lambda: send_tts_message(conn, "sentence_start", text)
+            )
+        else:
+            # 新句子或流控器未初始化，立即发送
+            await send_tts_message(conn, "sentence_start", text)
 
     await sendAudio(conn, audios)
     # 发送句子开始消息
@@ -23,85 +40,203 @@ async def sendAudioMessage(conn, sentenceType, audios, text):
         conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}")
 
     # 发送结束消息（如果是最后一个文本）
-    if conn.llm_finish_task and sentenceType == SentenceType.LAST:
+    if sentenceType == SentenceType.LAST:
         await send_tts_message(conn, "stop", None)
         conn.client_is_speaking = False
         if conn.close_after_chat:
             await conn.close()
 
 
-# 播放音频
-async def sendAudio(conn, audios, frame_duration=60):
+async def _wait_for_audio_completion(conn):
     """
-    发送单个opus包，支持流控
+    等待音频队列清空并等待预缓冲包播放完成
+
     Args:
         conn: 连接对象
-        opus_packet: 单个opus数据包
-        pre_buffer: 快速发送音频
-        frame_duration: 帧时长（毫秒），匹配 Opus 编码
+    """
+    if hasattr(conn, "audio_rate_controller") and conn.audio_rate_controller:
+        rate_controller = conn.audio_rate_controller
+        conn.logger.bind(tag=TAG).debug(
+            f"等待音频发送完成，队列中还有 {len(rate_controller.queue)} 个包"
+        )
+        await rate_controller.queue_empty_event.wait()
+
+        # 等待预缓冲包播放完成
+        # 前N个包直接发送，增加2个网络抖动包，需要额外等待它们在客户端播放完成
+        frame_duration_ms = rate_controller.frame_duration
+        pre_buffer_playback_time = (PRE_BUFFER_COUNT + 2) * frame_duration_ms / 1000.0
+        await asyncio.sleep(pre_buffer_playback_time)
+
+        conn.logger.bind(tag=TAG).debug("音频发送完成")
+
+
+async def _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence):
+    """
+    发送带16字节头部的opus数据包给mqtt_gateway
+    Args:
+        conn: 连接对象
+        opus_packet: opus数据包
+        timestamp: 时间戳
+        sequence: 序列号
+    """
+    # 为opus数据包添加16字节头部
+    header = bytearray(16)
+    header[0] = 1  # type
+    header[2:4] = len(opus_packet).to_bytes(2, "big")  # payload length
+    header[4:8] = sequence.to_bytes(4, "big")  # sequence
+    header[8:12] = timestamp.to_bytes(4, "big")  # 时间戳
+    header[12:16] = len(opus_packet).to_bytes(4, "big")  # opus长度
+
+    # 发送包含头部的完整数据包
+    complete_packet = bytes(header) + opus_packet
+    await conn.websocket.send(complete_packet)
+
+
+async def sendAudio(conn, audios, frame_duration=AUDIO_FRAME_DURATION):
+    """
+    发送音频包，使用 AudioRateController 进行精确的流量控制
+
+    Args:
+        conn: 连接对象
+        audios: 单个opus包(bytes) 或 opus包列表
+        frame_duration: 帧时长（毫秒），默认使用全局常量AUDIO_FRAME_DURATION
     """
     if audios is None or len(audios) == 0:
         return
 
-    if isinstance(audios, bytes):
+    send_delay = conn.config.get("tts_audio_send_delay", -1) / 1000.0
+    is_single_packet = isinstance(audios, bytes)
+
+    # 初始化或获取 RateController
+    rate_controller, flow_control = _get_or_create_rate_controller(
+        conn, frame_duration, is_single_packet
+    )
+
+    # 统一转换为列表处理
+    audio_list = [audios] if is_single_packet else audios
+
+    # 发送音频包
+    await _send_audio_with_rate_control(
+        conn, audio_list, rate_controller, flow_control, send_delay
+    )
+
+
+def _get_or_create_rate_controller(conn, frame_duration, is_single_packet):
+    """
+    获取或创建 RateController 和 flow_control
+
+    Args:
+        conn: 连接对象
+        frame_duration: 帧时长
+        is_single_packet: 是否单包模式（True: TTS流式单包, False: 批量包）
+
+    Returns:
+        (rate_controller, flow_control)
+    """
+    # 判断是否需要重置：单包模式且 sentence_id 变化，或者控制器不存在
+    need_reset = (
+        is_single_packet
+        and getattr(conn, "audio_flow_control", {}).get("sentence_id")
+        != conn.sentence_id
+    ) or not hasattr(conn, "audio_rate_controller")
+
+    if need_reset:
+        # 创建或获取 rate_controller
+        if not hasattr(conn, "audio_rate_controller"):
+            conn.audio_rate_controller = AudioRateController(frame_duration)
+        else:
+            conn.audio_rate_controller.reset()
+
+        # 初始化 flow_control
+        conn.audio_flow_control = {
+            "packet_count": 0,
+            "sequence": 0,
+            "sentence_id": conn.sentence_id,
+        }
+
+        # 启动后台发送循环
+        _start_background_sender(
+            conn, conn.audio_rate_controller, conn.audio_flow_control
+        )
+
+    return conn.audio_rate_controller, conn.audio_flow_control
+
+
+def _start_background_sender(conn, rate_controller, flow_control):
+    """
+    启动后台发送循环任务
+
+    Args:
+        conn: 连接对象
+        rate_controller: 速率控制器
+        flow_control: 流控状态
+    """
+
+    async def send_callback(packet):
+        # 检查是否应该中止
+        if conn.client_abort:
+            raise asyncio.CancelledError("客户端已中止")
+
+        conn.last_activity_time = time.time() * 1000
+        await _do_send_audio(conn, packet, flow_control)
+        conn.client_is_speaking = True
+
+    # 使用 start_sending 启动后台循环
+    rate_controller.start_sending(send_callback)
+
+
+async def _send_audio_with_rate_control(
+    conn, audio_list, rate_controller, flow_control, send_delay
+):
+    """
+    使用 rate_controller 发送音频包
+
+    Args:
+        conn: 连接对象
+        audio_list: 音频包列表
+        rate_controller: 速率控制器
+        flow_control: 流控状态
+        send_delay: 固定延迟（秒），-1表示使用动态流控
+    """
+    for packet in audio_list:
         if conn.client_abort:
             return
 
         conn.last_activity_time = time.time() * 1000
 
-        # 获取或初始化流控状态
-        if not hasattr(conn, "audio_flow_control"):
-            conn.audio_flow_control = {
-                "last_send_time": 0,
-                "packet_count": 0,
-                "start_time": time.perf_counter(),
-            }
+        # 预缓冲：前N个包直接发送
+        if flow_control["packet_count"] < PRE_BUFFER_COUNT:
+            await _do_send_audio(conn, packet, flow_control)
+            conn.client_is_speaking = True
+        elif send_delay > 0:
+            # 固定延迟模式
+            await asyncio.sleep(send_delay)
+            await _do_send_audio(conn, packet, flow_control)
+            conn.client_is_speaking = True
+        else:
+            # 动态流控模式：仅添加到队列，由后台循环负责发送
+            rate_controller.add_audio(packet)
 
-        flow_control = conn.audio_flow_control
-        current_time = time.perf_counter()
-        # 计算预期发送时间
-        expected_time = flow_control["start_time"] + (
-            flow_control["packet_count"] * frame_duration / 1000
-        )
-        delay = expected_time - current_time
-        if delay > 0:
-            await asyncio.sleep(delay)
 
-        # 发送数据包
-        await conn.websocket.send(audios)
+async def _do_send_audio(conn, opus_packet, flow_control):
+    """
+    执行实际的音频发送
+    """
+    packet_index = flow_control.get("packet_count", 0)
+    sequence = flow_control.get("sequence", 0)
 
-        # 更新流控状态
-        flow_control["packet_count"] += 1
-        flow_control["last_send_time"] = time.perf_counter()
+    if conn.conn_from_mqtt_gateway:
+        # 计算时间戳（基于播放位置）
+        start_time = time.time()
+        timestamp = int(start_time * 1000) % (2**32)
+        await _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence)
     else:
-        # 文件型音频走普通播放
-        start_time = time.perf_counter()
-        play_position = 0
+        # 直接发送opus数据包
+        await conn.websocket.send(opus_packet)
 
-        # 执行预缓冲
-        pre_buffer_frames = min(3, len(audios))
-        for i in range(pre_buffer_frames):
-            await conn.websocket.send(audios[i])
-        remaining_audios = audios[pre_buffer_frames:]
-
-        # 播放剩余音频帧
-        for opus_packet in remaining_audios:
-            if conn.client_abort:
-                break
-
-            # 重置没有声音的状态
-            conn.last_activity_time = time.time() * 1000
-
-            # 计算预期发送时间
-            expected_time = start_time + (play_position / 1000)
-            current_time = time.perf_counter()
-            delay = expected_time - current_time
-            if delay > 0:
-                await asyncio.sleep(delay)
-
-            await conn.websocket.send(opus_packet)
-
-            play_position += frame_duration
+    # 更新流控状态
+    flow_control["packet_count"] = packet_index + 1
+    flow_control["sequence"] = sequence + 1
 
 
 async def send_tts_message(conn, state, text=None):
@@ -120,8 +255,10 @@ async def send_tts_message(conn, state, text=None):
             stop_tts_notify_voice = conn.config.get(
                 "stop_tts_notify_voice", "config/assets/tts_notify.mp3"
             )
-            audios = audio_to_data(stop_tts_notify_voice, is_opus=True)
+            audios = await audio_to_data(stop_tts_notify_voice, is_opus=True)
             await sendAudio(conn, audios)
+        # 等待所有音频包发送完成
+        await _wait_for_audio_completion(conn)
         # 清除服务端讲话状态
         conn.clearSpeakStatus()
 
@@ -155,5 +292,4 @@ async def send_stt_message(conn, text):
     await conn.websocket.send(
         json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id})
     )
-    conn.client_is_speaking = True
     await send_tts_message(conn, "start")

main/xiaozhi-server/core/handle/textHandler/listenMessageHandler.py

@@ -1,12 +1,14 @@
 import time
+import asyncio
 from typing import Dict, Any
 
-from core.handle.receiveAudioHandle import handleAudioMessage, startToChat
+from core.handle.receiveAudioHandle import startToChat
 from core.handle.reportHandle import enqueue_asr_report
 from core.handle.sendAudioHandle import send_stt_message, send_tts_message
 from core.handle.textMessageHandler import TextMessageHandler
 from core.handle.textMessageType import TextMessageType
 from core.utils.util import remove_punctuation_and_length
+from core.providers.asr.dto.dto import InterfaceType
 
 TAG = __name__
 
@@ -29,8 +31,18 @@ class ListenTextMessageHandler(TextMessageHandler):
         elif msg_json["state"] == "stop":
             conn.client_have_voice = True
             conn.client_voice_stop = True
-            if len(conn.asr_audio) > 0:
-                await handleAudioMessage(conn, b"")
+            if conn.asr.interface_type == InterfaceType.STREAM:
+                # 流式模式下，发送结束请求
+                asyncio.create_task(conn.asr._send_stop_request())
+            else:
+                # 非流式模式：直接触发ASR识别
+                if len(conn.asr_audio) > 0:
+                    asr_audio_task = conn.asr_audio.copy()
+                    conn.asr_audio.clear()
+                    conn.reset_vad_states()
+
+                    if len(asr_audio_task) > 0:
+                        await conn.asr.handle_voice_stop(conn, asr_audio_task)
         elif msg_json["state"] == "detect":
             conn.client_have_voice = False
             conn.asr_audio.clear()
@@ -57,6 +69,7 @@ class ListenTextMessageHandler(TextMessageHandler):
                     enqueue_asr_report(conn, "嘿，你好呀", [])
                     await startToChat(conn, "嘿，你好呀")
                 else:
+                    conn.just_woken_up = True
                     # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
                     enqueue_asr_report(conn, original_text, [])
                     # 否则需要LLM对文字内容进行答复

main/xiaozhi-server/core/handle/textHandler/pingMessageHandler.py

@@ -0,0 +1,45 @@
+import json
+import time
+from typing import Dict, Any
+
+from core.handle.textMessageHandler import TextMessageHandler
+from core.handle.textMessageType import TextMessageType
+
+TAG = __name__
+
+
+class PingMessageHandler(TextMessageHandler):
+    """Ping消息处理器，用于保持WebSocket连接"""
+
+    @property
+    def message_type(self) -> TextMessageType:
+        return TextMessageType.PING
+
+    async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
+        """
+        处理PING消息，发送PONG响应
+        消息格式：{"type": "ping"}
+        Args:
+            conn: WebSocket连接对象
+            msg_json: PING消息的JSON数据
+        """
+        # 检查是否启用了WebSocket心跳功能
+        enable_websocket_ping = conn.config.get("enable_websocket_ping", False)
+        if not enable_websocket_ping:
+            conn.logger.debug(f"WebSocket心跳功能未启用，忽略PING消息")
+            return
+
+        try:
+            conn.logger.debug(f"收到PING消息，发送PONG响应")
+            conn.last_activity_time = time.time() * 1000
+            # 构造PONG响应消息
+            pong_message = {
+                "type": "pong",
+                "timestamp": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
+            }
+
+            # 发送PONG响应
+            await conn.websocket.send(json.dumps(pong_message))
+
+        except Exception as e:
+            conn.logger.error(f"处理PING消息时发生错误: {e}")

main/xiaozhi-server/core/handle/textMessageHandlerRegistry.py

@@ -7,6 +7,7 @@ from core.handle.textHandler.listenMessageHandler import ListenTextMessageHandle
 from core.handle.textHandler.mcpMessageHandler import McpTextMessageHandler
 from core.handle.textMessageHandler import TextMessageHandler
 from core.handle.textHandler.serverMessageHandler import ServerTextMessageHandler
+from core.handle.textHandler.pingMessageHandler import PingMessageHandler
 
 TAG = __name__
 
@@ -27,6 +28,7 @@ class TextMessageHandlerRegistry:
             IotTextMessageHandler(),
             McpTextMessageHandler(),
             ServerTextMessageHandler(),
+            PingMessageHandler(),
         ]
 
         for handler in handlers:

main/xiaozhi-server/core/handle/textMessageType.py

@@ -9,3 +9,4 @@ class TextMessageType(Enum):
     IOT = "iot"
     MCP = "mcp"
     SERVER = "server"
+    PING = "ping"

main/xiaozhi-server/core/http_server.py

@@ -33,38 +33,60 @@ class SimpleHttpServer:
             return f"ws://{local_ip}:{port}/xiaozhi/v1/"
 
     async def start(self):
-        server_config = self.config["server"]
-        read_config_from_api = self.config.get("read_config_from_api", False)
-        host = server_config.get("ip", "0.0.0.0")
-        port = int(server_config.get("http_port", 8003))
+        try:
+            server_config = self.config["server"]
+            read_config_from_api = self.config.get("read_config_from_api", False)
+            host = server_config.get("ip", "0.0.0.0")
+            port = int(server_config.get("http_port", 8003))
 
-        if port:
-            app = web.Application()
+            if port:
+                app = web.Application()
 
-            if not read_config_from_api:
-                # 如果没有开启智控台，只是单模块运行，就需要再添加简单OTA接口，用于下发websocket接口
+                if not read_config_from_api:
+                    # 如果没有开启智控台，只是单模块运行，就需要再添加简单OTA接口，用于下发websocket接口
+                    app.add_routes(
+                        [
+                            web.get("/xiaozhi/ota/", self.ota_handler.handle_get),
+                            web.post("/xiaozhi/ota/", self.ota_handler.handle_post),
+                            web.options(
+                                "/xiaozhi/ota/", self.ota_handler.handle_options
+                            ),
+                            # 下载接口，仅提供 data/bin/*.bin 下载
+                            web.get(
+                                "/xiaozhi/ota/download/{filename}",
+                                self.ota_handler.handle_download,
+                            ),
+                            web.options(
+                                "/xiaozhi/ota/download/{filename}",
+                                self.ota_handler.handle_options,
+                            ),
+                        ]
+                    )
+                # 添加路由
                 app.add_routes(
                     [
-                        web.get("/xiaozhi/ota/", self.ota_handler.handle_get),
-                        web.post("/xiaozhi/ota/", self.ota_handler.handle_post),
-                        web.options("/xiaozhi/ota/", self.ota_handler.handle_post),
+                        web.get("/mcp/vision/explain", self.vision_handler.handle_get),
+                        web.post(
+                            "/mcp/vision/explain", self.vision_handler.handle_post
+                        ),
+                        web.options(
+                            "/mcp/vision/explain", self.vision_handler.handle_options
+                        ),
                     ]
                 )
-            # 添加路由
-            app.add_routes(
-                [
-                    web.get("/mcp/vision/explain", self.vision_handler.handle_get),
-                    web.post("/mcp/vision/explain", self.vision_handler.handle_post),
-                    web.options("/mcp/vision/explain", self.vision_handler.handle_post),
-                ]
-            )
 
-            # 运行服务
-            runner = web.AppRunner(app)
-            await runner.setup()
-            site = web.TCPSite(runner, host, port)
-            await site.start()
+                # 运行服务
+                runner = web.AppRunner(app)
+                await runner.setup()
+                site = web.TCPSite(runner, host, port)
+                await site.start()
 
-            # 保持服务运行
-            while True:
-                await asyncio.sleep(3600)  # 每隔 1 小时检查一次
+                # 保持服务运行
+                while True:
+                    await asyncio.sleep(3600)  # 每隔 1 小时检查一次
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"HTTP服务器启动失败: {e}")
+            import traceback
+
+            self.logger.bind(tag=TAG).error(f"错误堆栈: {traceback.format_exc()}")
+            raise

main/xiaozhi-server/core/providers/asr/aliyun_stream.py

@@ -8,8 +8,6 @@ import asyncio
 import requests
 import websockets
 import opuslib_next
-import random
-from typing import Optional, Tuple, List
 from urllib import parse
 from datetime import datetime
 from config.logger import setup_logging
@@ -96,6 +94,8 @@ class ASRProvider(ASRProviderBase):
         self.delete_audio_file = delete_audio_file
         self.expire_time = None
 
+        self.task_id = uuid.uuid4().hex
+
         # Token管理
         if self.access_key_id and self.access_key_secret:
             self._refresh_token()
@@ -137,13 +137,13 @@ class ASRProvider(ASRProviderBase):
         conn.asr_audio.append(audio)
         conn.asr_audio = conn.asr_audio[-10:]
 
-        # 只在有声音且没有连接时建立连接
-        if audio_have_voice and not self.is_processing:
+        # 只在有声音且没有连接时建立连接（排除正在停止的情况）
+        if audio_have_voice and not self.is_processing and not self.asr_ws:
             try:
                 await self._start_recognition(conn)
             except Exception as e:
                 logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}")
-                await self._cleanup(conn)
+                await self._cleanup()
                 return
 
         if self.asr_ws and self.is_processing and self.server_ready:
@@ -169,20 +169,22 @@ class ASRProvider(ASRProviderBase):
             ping_timeout=None,
             close_timeout=5,
         )
-        
+
+        self.task_id = uuid.uuid4().hex
+
+        logger.bind(tag=TAG).debug(f"WebSocket连接建立成功, task_id: {self.task_id}")
+
         self.is_processing = True
         self.server_ready = False  # 重置服务器准备状态
         self.forward_task = asyncio.create_task(self._forward_results(conn))
-        
+
         # 发送开始请求
         start_request = {
             "header": {
                 "namespace": "SpeechTranscriber",
                 "name": "StartTranscription",
-                "status": 20000000,
-                "message_id": ''.join(random.choices('0123456789abcdef', k=32)),
-                "task_id": ''.join(random.choices('0123456789abcdef', k=32)),
-                "status_text": "Gateway:SUCCESS:Success.",
+                "message_id": uuid.uuid4().hex,
+                "task_id": self.task_id,
                 "appkey": self.appkey
             },
             "payload": {
@@ -196,23 +198,26 @@ class ASRProvider(ASRProviderBase):
             }
         }
         await self.asr_ws.send(json.dumps(start_request, ensure_ascii=False))
-        logger.bind(tag=TAG).info("已发送开始请求，等待服务器准备...")
+        logger.bind(tag=TAG).debug("已发送开始请求，等待服务器准备...")
 
     async def _forward_results(self, conn):
         """转发识别结果"""
         try:
-            while self.asr_ws and not conn.stop_event.is_set():
+            while not conn.stop_event.is_set():
                 try:
                     response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
                     result = json.loads(response)
-                    
+
                     header = result.get("header", {})
                     payload = result.get("payload", {})
                     message_name = header.get("name", "")
                     status = header.get("status", 0)
-                    
+
                     if status != 20000000:
-                        if status in [40000004, 40010004]:  # 连接超时或客户端断开
+                        if status == 40010004:
+                            logger.bind(tag=TAG).warning(f"请在服务端响应完成后再关闭链接，状态码: {status}")
+                            break
+                        if status in [40000004, 40010003]:  # 连接超时或客户端断开
                             logger.bind(tag=TAG).warning(f"连接问题，状态码: {status}")
                             break
                         elif status in [40270002, 40270003]:  # 音频问题
@@ -221,12 +226,12 @@ class ASRProvider(ASRProviderBase):
                         else:
                             logger.bind(tag=TAG).error(f"识别错误，状态码: {status}, 消息: {header.get('status_text', '')}")
                             continue
-                    
+
                     # 收到TranscriptionStarted表示服务器准备好接收音频数据
                     if message_name == "TranscriptionStarted":
                         self.server_ready = True
-                        logger.bind(tag=TAG).info("服务器已准备，开始发送缓存音频...")
-                        
+                        logger.bind(tag=TAG).debug("服务器已准备，开始发送缓存音频...")
+
                         # 发送缓存音频
                         if conn.asr_audio:
                             for cached_audio in conn.asr_audio[-10:]:
@@ -237,88 +242,89 @@ class ASRProvider(ASRProviderBase):
                                     logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}")
                                     break
                         continue
-                    
-                    if message_name == "TranscriptionResultChanged":
-                        # 中间结果
-                        text = payload.get("result", "")
-                        if text:
-                            self.text = text
                     elif message_name == "SentenceEnd":
-                        # 最终结果
+                        # 句子结束（每个句子都会触发）
                         text = payload.get("result", "")
                         if text:
-                            self.text = text
-                            conn.reset_vad_states()
-                            # 传递缓存的音频数据
-                            audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
-                            await self.handle_voice_stop(conn, audio_data)
-                            # 清空缓存
-                            conn.asr_audio_for_voiceprint = []
-                            break
-                    elif message_name == "TranscriptionCompleted":
-                        # 识别完成
-                        self.is_processing = False
-                        break
-                        
+                            logger.bind(tag=TAG).info(f"识别到文本: {text}")
+
+                            # 手动模式下累积识别结果
+                            if conn.client_listen_mode == "manual":
+                                if self.text:
+                                    self.text += text
+                                else:
+                                    self.text = text
+
+                                # 手动模式下，只有在收到stop信号后才触发处理（仅处理一次）
+                                if conn.client_voice_stop:
+                                    audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                                    if len(audio_data) > 0:
+                                        logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
+                                        await self.handle_voice_stop(conn, audio_data)
+                                        # 清理音频缓存
+                                        conn.asr_audio.clear()
+                                        conn.reset_vad_states()
+                                    break
+                            else:
+                                # 自动模式下直接覆盖
+                                self.text = text
+                                conn.reset_vad_states()
+                                audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                                await self.handle_voice_stop(conn, audio_data)
+                                break
+
                 except asyncio.TimeoutError:
-                    continue
-                except websockets.exceptions.ConnectionClosed:
+                    logger.bind(tag=TAG).error("接收结果超时")
+                    break
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).info("ASR服务连接已关闭")
+                    self.is_processing = False
                     break
                 except Exception as e:
                     logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}")
                     break
-                    
+
         except Exception as e:
             logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}")
         finally:
-            await self._cleanup(conn)
+            # 清理连接的音频缓存
+            await self._cleanup()
+            if conn:
+                if hasattr(conn, 'asr_audio_for_voiceprint'):
+                    conn.asr_audio_for_voiceprint = []
+                if hasattr(conn, 'asr_audio'):
+                    conn.asr_audio = []
 
-    async def _cleanup(self, conn):
-        """清理资源"""
-        logger.bind(tag=TAG).info(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
-        
-        # 清理连接的音频缓存
-        if conn and hasattr(conn, 'asr_audio_for_voiceprint'):
-            conn.asr_audio_for_voiceprint = []
-        
-        # 判断是否需要发送终止请求
-        should_stop = self.is_processing or self.server_ready
-        
-        # 发送停止识别请求
-        if self.asr_ws and should_stop:
+    async def _send_stop_request(self):
+        """发送停止识别请求（不关闭连接）"""
+        if self.asr_ws:
             try:
+                # 先停止音频发送
+                self.is_processing = False
+
                 stop_msg = {
                     "header": {
                         "namespace": "SpeechTranscriber",
                         "name": "StopTranscription",
-                        "status": 20000000,
-                        "message_id": ''.join(random.choices('0123456789abcdef', k=32)),
-                        "status_text": "Client:Stop",
+                        "message_id": uuid.uuid4().hex,
+                        "task_id": self.task_id,
                         "appkey": self.appkey
                     }
                 }
-                logger.bind(tag=TAG).info("正在发送ASR终止请求")
+                logger.bind(tag=TAG).debug("停止识别请求已发送")
                 await self.asr_ws.send(json.dumps(stop_msg, ensure_ascii=False))
-                await asyncio.sleep(0.1)
-                logger.bind(tag=TAG).info("ASR终止请求已发送")
             except Exception as e:
-                logger.bind(tag=TAG).error(f"ASR终止请求发送失败: {e}")
-        
-        # 状态重置（在终止请求发送后）
+                logger.bind(tag=TAG).error(f"发送停止识别请求失败: {e}")
+
+    async def _cleanup(self):
+        """清理资源（关闭连接）"""
+        logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
+
+        # 状态重置
         self.is_processing = False
         self.server_ready = False
-        logger.bind(tag=TAG).info("ASR状态已重置")
+        logger.bind(tag=TAG).debug("ASR状态已重置")
 
-        # 清理任务
-        if self.forward_task and not self.forward_task.done():
-            self.forward_task.cancel()
-            try:
-                await asyncio.wait_for(self.forward_task, timeout=1.0)
-            except Exception as e:
-                logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}")
-            finally:
-                self.forward_task = None
-        
         # 关闭连接
         if self.asr_ws:
             try:
@@ -329,8 +335,11 @@ class ASRProvider(ASRProviderBase):
                 logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
             finally:
                 self.asr_ws = None
-        
-        logger.bind(tag=TAG).info("ASR会话清理完成")
+
+        # 清理任务引用
+        self.forward_task = None
+
+        logger.bind(tag=TAG).debug("ASR会话清理完成")
 
     async def speech_to_text(self, opus_data, session_id, audio_format):
         """获取识别结果"""
@@ -340,4 +349,11 @@ class ASRProvider(ASRProviderBase):
 
     async def close(self):
         """关闭资源"""
-        await self._cleanup()
+        await self._cleanup(None)
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Aliyun decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Aliyun decoder资源时出错: {e}")

main/xiaozhi-server/core/providers/asr/base.py

@@ -9,7 +9,6 @@ import asyncio
 import traceback
 import threading
 import opuslib_next
-import concurrent.futures
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
@@ -118,121 +117,89 @@ class ASRProviderBase(ABC):
 
     # 接收音频
     async def receive_audio(self, conn, audio, audio_have_voice):
-        if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime":
-            have_voice = audio_have_voice
+        if conn.client_listen_mode == "manual":
+            # 手动模式：缓存音频用于ASR识别
+            conn.asr_audio.append(audio)
         else:
-            have_voice = conn.client_have_voice
-        
-        conn.asr_audio.append(audio)
-        if not have_voice and not conn.client_have_voice:
-            conn.asr_audio = conn.asr_audio[-10:]
-            return
+            # 自动/实时模式：使用VAD检测
+            have_voice = audio_have_voice
 
-        if conn.client_voice_stop:
-            asr_audio_task = conn.asr_audio.copy()
-            conn.asr_audio.clear()
-            conn.reset_vad_states()
+            conn.asr_audio.append(audio)
+            if not have_voice and not conn.client_have_voice:
+                conn.asr_audio = conn.asr_audio[-10:]
+                return
 
-            if len(asr_audio_task) > 15:
-                await self.handle_voice_stop(conn, asr_audio_task)
+            # 自动模式下通过VAD检测到语音停止时触发识别
+            if conn.client_voice_stop:
+                asr_audio_task = conn.asr_audio.copy()
+                conn.asr_audio.clear()
+                conn.reset_vad_states()
+
+                if len(asr_audio_task) > 15:
+                    await self.handle_voice_stop(conn, asr_audio_task)
 
     # 处理语音停止
     async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
         """并行处理ASR和声纹识别"""
         try:
             total_start_time = time.monotonic()
-            
+
             # 准备音频数据
             if conn.audio_format == "pcm":
                 pcm_data = asr_audio_task
             else:
                 pcm_data = self.decode_opus(asr_audio_task)
-            
+
             combined_pcm_data = b"".join(pcm_data)
-            
+
             # 预先准备WAV数据
             wav_data = None
             if conn.voiceprint_provider and combined_pcm_data:
                 wav_data = self._pcm_to_wav(combined_pcm_data)
-            
+
             # 定义ASR任务
-            def run_asr():
-                start_time = time.monotonic()
-                try:
-                    loop = asyncio.new_event_loop()
-                    asyncio.set_event_loop(loop)
-                    try:
-                        result = loop.run_until_complete(
-                            self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
-                        )
-                        end_time = time.monotonic()
-                        logger.bind(tag=TAG).info(f"ASR耗时: {end_time - start_time:.3f}s")
-                        return result
-                    finally:
-                        loop.close()
-                except Exception as e:
-                    end_time = time.monotonic()
-                    logger.bind(tag=TAG).error(f"ASR失败: {e}")
-                    return ("", None)
-            
-            # 定义声纹识别任务
-            def run_voiceprint():
-                if not wav_data:
-                    return None
-                try:
-                    loop = asyncio.new_event_loop()
-                    asyncio.set_event_loop(loop)
-                    try:
-                        # 使用连接的声纹识别提供者
-                        result = loop.run_until_complete(
-                            conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
-                        )
-                        return result
-                    finally:
-                        loop.close()
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"声纹识别失败: {e}")
-                    return None
-            
-            # 使用线程池执行器并行运行
-            with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor:
-                asr_future = thread_executor.submit(run_asr)
-                
-                if conn.voiceprint_provider and wav_data:
-                    voiceprint_future = thread_executor.submit(run_voiceprint)
-                    
-                    # 等待两个线程都完成
-                    asr_result = asr_future.result(timeout=15)
-                    voiceprint_result = voiceprint_future.result(timeout=15)
-                    
-                    results = {"asr": asr_result, "voiceprint": voiceprint_result}
-                else:
-                    asr_result = asr_future.result(timeout=15)
-                    results = {"asr": asr_result, "voiceprint": None}
-            
-            
-            # 处理结果
-            raw_text, _ = results.get("asr", ("", None))
-            speaker_name = results.get("voiceprint", None)
-            
-            # 记录识别结果
+            asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
+
+            if conn.voiceprint_provider and wav_data:
+                voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
+                # 并发等待两个结果
+                asr_result, voiceprint_result = await asyncio.gather(
+                    asr_task, voiceprint_task, return_exceptions=True
+                )
+            else:
+                asr_result = await asr_task
+                voiceprint_result = None
+
+            # 记录识别结果 - 检查是否为异常
+            if isinstance(asr_result, Exception):
+                logger.bind(tag=TAG).error(f"ASR识别失败: {asr_result}")
+                raw_text = ""
+            else:
+                raw_text, _ = asr_result
+
+            if isinstance(voiceprint_result, Exception):
+                logger.bind(tag=TAG).error(f"声纹识别失败: {voiceprint_result}")
+                speaker_name = ""
+            else:
+                speaker_name = voiceprint_result
+
             if raw_text:
                 logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
             if speaker_name:
                 logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
-            
+
             # 性能监控
             total_time = time.monotonic() - total_start_time
-            logger.bind(tag=TAG).info(f"总处理耗时: {total_time:.3f}s")
-            
+            logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s")
+
             # 检查文本长度
             text_len, _ = remove_punctuation_and_length(raw_text)
             self.stop_ws_connection()
-            
+
             if text_len > 0:
                 # 构建包含说话人信息的JSON字符串
                 enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
-                
+
                 # 使用自定义模块进行上报
                 await startToChat(conn, enhanced_text)
                 enqueue_asr_report(conn, enhanced_text, asr_audio_task)
@@ -306,6 +273,7 @@ class ASRProviderBase(ABC):
     @staticmethod
     def decode_opus(opus_data: List[bytes]) -> List[bytes]:
         """将Opus音频数据解码为PCM数据"""
+        decoder = None
         try:
             decoder = opuslib_next.Decoder(16000, 1)
             pcm_data = []
@@ -330,3 +298,9 @@ class ASRProviderBase(ABC):
         except Exception as e:
             logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
             return []
+        finally:
+            if decoder is not None:
+                try:
+                    del decoder
+                except Exception as e:
+                    logger.bind(tag=TAG).debug(f"释放decoder资源时出错: {e}")

main/xiaozhi-server/core/providers/asr/doubao_stream.py

@@ -18,8 +18,6 @@ class ASRProvider(ASRProviderBase):
         self.interface_type = InterfaceType.STREAM
         self.config = config
         self.text = ""
-        self.max_retries = 3
-        self.retry_delay = 2
         self.decoder = opuslib_next.Decoder(16000, 1)
         self.asr_ws = None
         self.forward_task = None
@@ -49,6 +47,8 @@ class ASRProvider(ASRProviderBase):
         self.channel = config.get("channel", 1)
         self.auth_method = config.get("auth_method", "token")
         self.secret = config.get("secret", "access_secret")
+        end_window_size = config.get("end_window_size")
+        self.end_window_size = int(end_window_size) if end_window_size else 200
 
     async def open_audio_channels(self, conn):
         await super().open_audio_channels(conn)
@@ -56,14 +56,13 @@ class ASRProvider(ASRProviderBase):
     async def receive_audio(self, conn, audio, audio_have_voice):
         conn.asr_audio.append(audio)
         conn.asr_audio = conn.asr_audio[-10:]
-        
         # 存储音频数据
         if not hasattr(conn, 'asr_audio_for_voiceprint'):
             conn.asr_audio_for_voiceprint = []
         conn.asr_audio_for_voiceprint.append(audio)
-        
+
         # 当没有音频数据时处理完整语音片段
-        if not audio and len(conn.asr_audio_for_voiceprint) > 0:
+        if conn.client_listen_mode != "manual" and not audio and len(conn.asr_audio_for_voiceprint) > 0:
             await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint)
             conn.asr_audio_for_voiceprint = []
 
@@ -179,6 +178,7 @@ class ASRProvider(ASRProviderBase):
                                 payload.get("audio_info", {}).get("duration", 0) > 2000
                                 and not utterances
                                 and not payload["result"].get("text")
+                                and conn.client_listen_mode != "manual"
                             ):
                                 logger.bind(tag=TAG).error(f"识别文本：空")
                                 self.text = ""
@@ -187,15 +187,44 @@ class ASRProvider(ASRProviderBase):
                                     await self.handle_voice_stop(conn, audio_data)
                                 break
 
+                            # 专门处理没有文本的识别结果（手动模式下可能已经识别完成但是没松按键）
+                            elif not payload["result"].get("text") and not utterances:
+                                if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0:
+                                    logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理")
+                                    await self.handle_voice_stop(conn, audio_data)
+                                    # 清理音频缓存
+                                    conn.asr_audio.clear()
+                                    conn.reset_vad_states()
+                                    break
+
                             for utterance in utterances:
                                 if utterance.get("definite", False):
-                                    self.text = utterance["text"]
+                                    current_text = utterance["text"]
                                     logger.bind(tag=TAG).info(
-                                        f"识别到文本: {self.text}"
+                                        f"识别到文本: {current_text}"
                                     )
-                                    conn.reset_vad_states()
-                                    if len(audio_data) > 15:  # 确保有足够音频数据
-                                        await self.handle_voice_stop(conn, audio_data)
+
+                                    # 手动模式下累积识别结果
+                                    if conn.client_listen_mode == "manual":
+                                        if self.text:
+                                            self.text += current_text
+                                        else:
+                                            self.text = current_text
+
+                                        # 在接收消息中途时收到停止信号
+                                        if conn.client_voice_stop and len(audio_data) > 0:
+                                            logger.bind(tag=TAG).debug("消息中途收到停止信号，触发处理")
+                                            await self.handle_voice_stop(conn, audio_data)
+                                            # 清理音频缓存
+                                            conn.asr_audio.clear()
+                                            conn.reset_vad_states()
+                                        break
+                                    else:
+                                        # 自动模式下直接覆盖
+                                        self.text = current_text
+                                        conn.reset_vad_states()
+                                        if len(audio_data) > 15:  # 确保有足够音频数据
+                                            await self.handle_voice_stop(conn, audio_data)
                                     break
                         elif "error" in payload:
                             error_msg = payload.get("error", "未知错误")
@@ -227,8 +256,6 @@ class ASRProvider(ASRProviderBase):
                     conn.asr_audio_for_voiceprint = []
                 if hasattr(conn, 'asr_audio'):
                     conn.asr_audio = []
-                if hasattr(conn, 'has_valid_voice'):
-                    conn.has_valid_voice = False
 
     def stop_ws_connection(self):
         if self.asr_ws:
@@ -236,6 +263,20 @@ class ASRProvider(ASRProviderBase):
             self.asr_ws = None
         self.is_processing = False
 
+    async def _send_stop_request(self):
+        """发送最后一个音频帧以通知服务器结束"""
+        if self.asr_ws:
+            try:
+                # 发送结束标记的音频帧（gzip压缩的空数据）
+                empty_payload = gzip.compress(b"")
+                last_audio_request = bytearray(self.generate_last_audio_default_header())
+                last_audio_request.extend(len(empty_payload).to_bytes(4, "big"))
+                last_audio_request.extend(empty_payload)
+                await self.asr_ws.send(last_audio_request)
+                logger.bind(tag=TAG).debug("已发送结束音频帧")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"发送结束音频帧时出错: {e}")
+
     def construct_request(self, reqid):
         req = {
             "app": {
@@ -252,7 +293,7 @@ class ASRProvider(ASRProviderBase):
                 "sequence": 1,
                 "boosting_table_name": self.boosting_table_name,
                 "correct_table_name": self.correct_table_name,
-                "end_window_size": 200,
+                "end_window_size": self.end_window_size,
             },
             "audio": {
                 "format": self.format,
@@ -370,6 +411,16 @@ class ASRProvider(ASRProviderBase):
                 pass
             self.forward_task = None
         self.is_processing = False
+        
+        # 显式释放decoder资源
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Doubao decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")
+
         # 清理所有连接的音频缓冲区
         if hasattr(self, '_connections'):
             for conn in self._connections.values():
@@ -377,5 +428,3 @@ class ASRProvider(ASRProviderBase):
                     conn.asr_audio_for_voiceprint = []
                 if hasattr(conn, 'asr_audio'):
                     conn.asr_audio = []
-                if hasattr(conn, 'has_valid_voice'):
-                    conn.has_valid_voice = False

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -1,14 +1,16 @@
-import time
 import os
-import sys
 import io
+import sys
+import time
+import shutil
 import psutil
+import asyncio
+
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
-from core.providers.asr.base import ASRProviderBase
 from funasr import AutoModel
 from funasr.utils.postprocess_utils import rich_transcription_postprocess
-import shutil
+from core.providers.asr.base import ASRProviderBase
 from core.providers.asr.dto.dto import InterfaceType
 
 TAG = __name__
@@ -90,16 +92,17 @@ class ASRProvider(ASRProviderBase):
                 else:
                     file_path = self.save_audio_to_file(pcm_data, session_id)
 
-                # 语音识别
+                # 语音识别 - 使用线程池避免阻塞事件循环
                 start_time = time.time()
-                result = self.model.generate(
+                result = await asyncio.to_thread(
+                    self.model.generate,
                     input=combined_pcm_data,
                     cache={},
                     language="auto",
                     use_itn=True,
                     batch_size_s=60,
                 )
-                text = rich_transcription_postprocess(result[0]["text"])
+                text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"])
                 logger.bind(tag=TAG).debug(
                     f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
                 )

main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py

@@ -0,0 +1,151 @@
+import os
+import tempfile
+from typing import Optional, Tuple, List
+import dashscope
+from config.logger import setup_logging
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+
+tag = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        super().__init__()
+        # 音频文件上传类型，流式文本识别输出
+        self.interface_type = InterfaceType.NON_STREAM
+        """Qwen3-ASR-Flash ASR初始化"""
+        
+        # 配置参数
+        self.api_key = config.get("api_key")
+        if not self.api_key:
+            raise ValueError("Qwen3-ASR-Flash 需要配置 api_key")
+            
+        self.model_name = config.get("model_name", "qwen3-asr-flash")
+        self.output_dir = config.get("output_dir", "./audio_output")
+        self.delete_audio_file = delete_audio_file
+        
+        # ASR选项配置
+        self.enable_lid = config.get("enable_lid", True)  # 自动语种检测
+        self.enable_itn = config.get("enable_itn", True)  # 逆文本归一化
+        self.language = config.get("language", None)  # 指定语种，默认自动检测
+        self.context = config.get("context", "")  # 上下文信息，用于提高识别准确率
+        
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def _prepare_audio_file(self, pcm_data: bytes) -> str:
+        """将PCM数据转换为WAV文件并返回文件路径"""
+        try:
+            import wave
+            
+            # 创建临时WAV文件
+            with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file:
+                temp_path = temp_file.name
+                
+            # 写入WAV格式
+            with wave.open(temp_path, 'wb') as wav_file:
+                wav_file.setnchannels(1)      # 单声道
+                wav_file.setsampwidth(2)      # 16位
+                wav_file.setframerate(16000)  # 16kHz采样率
+                wav_file.writeframes(pcm_data)
+                
+            return temp_path
+            
+        except Exception as e:
+            logger.bind(tag=tag).error(f"音频文件准备失败: {e}")
+            return None
+
+    async def speech_to_text(
+        self, opus_data: List[bytes], session_id: str, audio_format="opus"
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        temp_file_path = None
+        file_path = None
+        
+        try:
+            # 解码音频数据
+            if audio_format == "pcm":
+                pcm_data = opus_data
+            else:
+                pcm_data = self.decode_opus(opus_data)
+            
+            combined_pcm_data = b"".join(pcm_data)
+            if len(combined_pcm_data) == 0:
+                logger.bind(tag=tag).warning("音频数据为空")
+                return "", None
+            
+            # 准备音频文件
+            temp_file_path = self._prepare_audio_file(combined_pcm_data)
+            if not temp_file_path:
+                return "", None
+            
+            # 保存音频文件（如果需要）
+            if not self.delete_audio_file:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
+            
+            # 构造请求消息
+            messages = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"audio": temp_file_path}
+                    ]
+                }
+            ]
+            
+            # 如果有上下文信息，添加system消息
+            if self.context:
+                messages.insert(0, {
+                    "role": "system", 
+                    "content": [
+                        {"text": self.context}
+                    ]
+                })
+            
+            # 准备ASR选项
+            asr_options = {
+                "enable_lid": self.enable_lid,
+                "enable_itn": self.enable_itn
+            }
+            
+            # 如果指定了语种，添加到选项中
+            if self.language:
+                asr_options["language"] = self.language
+            
+            # 设置API密钥
+            dashscope.api_key = self.api_key
+            
+            # 发送流式请求
+            response = dashscope.MultiModalConversation.call(
+                model=self.model_name,
+                messages=messages,
+                result_format="message",
+                asr_options=asr_options,
+                stream=True
+            )
+            
+            # 处理流式响应
+            full_text = ""
+            for chunk in response:
+                try:
+                    text = chunk["output"]["choices"][0]["message"].content[0]["text"]
+                    # 更新为最新的完整文本
+                    full_text = text.strip()
+                except:
+                    pass
+            
+            return full_text, file_path
+                
+        except Exception as e:
+            logger.bind(tag=tag).error(f"语音识别失败: {e}")
+            return "", file_path
+            
+        finally:
+            # 清理临时文件
+            if temp_file_path and os.path.exists(temp_file_path):
+                try:
+                    os.unlink(temp_file_path)
+                except Exception as e:
+                    logger.bind(tag=tag).warning(f"清理临时文件失败: {e}")

main/xiaozhi-server/core/providers/asr/vosk.py

@@ -0,0 +1,114 @@
+import os
+import json
+import time
+from typing import Optional, Tuple, List
+from .base import ASRProviderBase
+from config.logger import setup_logging
+from core.providers.asr.dto.dto import InterfaceType
+import vosk
+
+TAG = __name__
+logger = setup_logging()
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool = True):
+        super().__init__()
+        self.interface_type = InterfaceType.LOCAL
+        self.model_path = config.get("model_path")
+        self.output_dir = config.get("output_dir", "tmp/")
+        self.delete_audio_file = delete_audio_file
+        
+        # 初始化VOSK模型
+        self.model = None
+        self.recognizer = None
+        self._load_model()
+        
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def _load_model(self):
+        """加载VOSK模型"""
+        try:
+            if not os.path.exists(self.model_path):
+                raise FileNotFoundError(f"VOSK模型路径不存在: {self.model_path}")
+                
+            logger.bind(tag=TAG).info(f"正在加载VOSK模型: {self.model_path}")
+            self.model = vosk.Model(self.model_path)
+
+            # 初始化VOSK识别器（采样率必须为16kHz）
+            self.recognizer = vosk.KaldiRecognizer(self.model, 16000)
+
+            logger.bind(tag=TAG).info("VOSK模型加载成功")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"加载VOSK模型失败: {e}")
+            raise
+
+    async def speech_to_text(
+        self, audio_data: List[bytes], session_id: str, audio_format: str = "opus"
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        file_path = None
+        try:
+            # 检查模型是否加载成功
+            if not self.model:
+                logger.bind(tag=TAG).error("VOSK模型未加载，无法进行识别")
+                return "", None
+
+            # 解码音频（如果原始格式是Opus）
+            if audio_format == "pcm":
+                pcm_data = audio_data
+            else:
+                pcm_data = self.decode_opus(audio_data)
+                
+            if not pcm_data:
+                logger.bind(tag=TAG).warning("解码后的PCM数据为空，无法进行识别")
+                return "", None
+                
+            # 合并PCM数据
+            combined_pcm_data = b"".join(pcm_data)
+            if len(combined_pcm_data) == 0:
+                logger.bind(tag=TAG).warning("合并后的PCM数据为空")
+                return "", None
+
+            # 判断是否保存为WAV文件
+            if not self.delete_audio_file:
+                file_path = self.save_audio_to_file(pcm_data, session_id)
+
+            start_time = time.time()
+            
+            
+            # 进行识别（VOSK推荐每次送入2000字节的数据）
+            chunk_size = 2000
+            text_result = ""
+            
+            for i in range(0, len(combined_pcm_data), chunk_size):
+                chunk = combined_pcm_data[i:i+chunk_size]
+                if self.recognizer.AcceptWaveform(chunk):
+                    result = json.loads(self.recognizer.Result())
+                    text = result.get('text', '')
+                    if text:
+                        text_result += text + " "
+            
+            # 获取最终结果
+            final_result = json.loads(self.recognizer.FinalResult())
+            final_text = final_result.get('text', '')
+            if final_text:
+                text_result += final_text
+            
+            logger.bind(tag=TAG).debug(
+                f"VOSK语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text_result.strip()}"
+            )
+            
+            return text_result.strip(), file_path
+            
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"VOSK语音识别失败: {e}")
+            return "", None
+        finally:
+            # 文件清理逻辑
+            if self.delete_audio_file and file_path and os.path.exists(file_path):
+                try:
+                    os.remove(file_path)
+                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/asr/xunfei_stream.py

@@ -0,0 +1,372 @@
+import json
+import hmac
+import base64
+import hashlib
+import asyncio
+import websockets
+import opuslib_next
+import gc
+from time import mktime
+from datetime import datetime
+from urllib.parse import urlencode
+from typing import List
+from config.logger import setup_logging
+from wsgiref.handlers import format_date_time
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+# 帧状态常量
+STATUS_FIRST_FRAME = 0  # 第一帧的标识
+STATUS_CONTINUE_FRAME = 1  # 中间帧标识
+STATUS_LAST_FRAME = 2  # 最后一帧的标识
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__()
+        self.interface_type = InterfaceType.STREAM
+        self.config = config
+        self.text = ""
+        self.decoder = opuslib_next.Decoder(16000, 1)
+        self.asr_ws = None
+        self.forward_task = None
+        self.is_processing = False
+        self.server_ready = False
+
+        # 讯飞配置
+        self.app_id = config.get("app_id")
+        self.api_key = config.get("api_key")
+        self.api_secret = config.get("api_secret")
+
+        if not all([self.app_id, self.api_key, self.api_secret]):
+            raise ValueError("必须提供app_id、api_key和api_secret")
+
+        # 识别参数
+        self.iat_params = {
+            "domain": config.get("domain", "slm"),
+            "language": config.get("language", "zh_cn"),
+            "accent": config.get("accent", "mandarin"),
+            "result": {"encoding": "utf8", "compress": "raw", "format": "plain"},
+        }
+
+        self.output_dir = config.get("output_dir", "tmp/")
+        self.delete_audio_file = delete_audio_file
+
+    def create_url(self) -> str:
+        """生成认证URL"""
+        url = "ws://iat.cn-huabei-1.xf-yun.com/v1"
+        # 生成RFC1123格式的时间戳
+        now = datetime.now()
+        date = format_date_time(mktime(now.timetuple()))
+
+        # 拼接字符串
+        signature_origin = "host: " + "iat.cn-huabei-1.xf-yun.com" + "\n"
+        signature_origin += "date: " + date + "\n"
+        signature_origin += "GET " + "/v1 " + "HTTP/1.1"
+
+        # 进行hmac-sha256进行加密
+        signature_sha = hmac.new(
+            self.api_secret.encode("utf-8"),
+            signature_origin.encode("utf-8"),
+            digestmod=hashlib.sha256,
+        ).digest()
+        signature_sha = base64.b64encode(signature_sha).decode(encoding="utf-8")
+
+        authorization_origin = (
+            'api_key="%s", algorithm="%s", headers="%s", signature="%s"'
+            % (self.api_key, "hmac-sha256", "host date request-line", signature_sha)
+        )
+        authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(
+            encoding="utf-8"
+        )
+
+        # 将请求的鉴权参数组合为字典
+        v = {
+            "authorization": authorization,
+            "date": date,
+            "host": "iat.cn-huabei-1.xf-yun.com",
+        }
+
+        # 拼接鉴权参数，生成url
+        url = url + "?" + urlencode(v)
+        return url
+
+    async def open_audio_channels(self, conn):
+        await super().open_audio_channels(conn)
+
+    async def receive_audio(self, conn, audio, audio_have_voice):
+        # 先调用父类方法处理基础逻辑
+        await super().receive_audio(conn, audio, audio_have_voice)
+
+        # 存储音频数据用于声纹识别
+        if not hasattr(conn, "asr_audio_for_voiceprint"):
+            conn.asr_audio_for_voiceprint = []
+        conn.asr_audio_for_voiceprint.append(audio)
+
+        # 如果本次有声音，且之前没有建立连接
+        if audio_have_voice and self.asr_ws is None and not self.is_processing:
+            try:
+                await self._start_recognition(conn)
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
+                await self._cleanup()
+                return
+
+        # 发送当前音频数据
+        if self.asr_ws and self.is_processing and self.server_ready:
+            try:
+                pcm_frame = self.decoder.decode(audio, 960)
+                await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"发送音频数据时发生错误: {e}")
+                await self._cleanup()
+
+    async def _start_recognition(self, conn):
+        """开始识别会话"""
+        try:
+            self.is_processing = True
+            # 建立WebSocket连接
+            ws_url = self.create_url()
+            logger.bind(tag=TAG).info(f"正在连接ASR服务: {ws_url[:50]}...")
+
+            # 如果为手动模式,设置超时时长为一分钟
+            if conn.client_listen_mode == "manual":
+                self.iat_params["eos"] = 60000
+
+            self.asr_ws = await websockets.connect(
+                ws_url,
+                max_size=1000000000,
+                ping_interval=None,
+                ping_timeout=None,
+                close_timeout=10,
+            )
+
+            logger.bind(tag=TAG).info("ASR WebSocket连接已建立")
+            self.server_ready = False
+            self.forward_task = asyncio.create_task(self._forward_results(conn))
+
+            # 发送首帧音频
+            if conn.asr_audio and len(conn.asr_audio) > 0:
+                first_audio = conn.asr_audio[-1] if conn.asr_audio else b""
+                pcm_frame = (
+                    self.decoder.decode(first_audio, 960) if first_audio else b""
+                )
+                await self._send_audio_frame(pcm_frame, STATUS_FIRST_FRAME)
+                self.server_ready = True
+                logger.bind(tag=TAG).info("已发送首帧，开始识别")
+
+                # 发送缓存的音频数据
+                for cached_audio in conn.asr_audio[-10:]:
+                    try:
+                        pcm_frame = self.decoder.decode(cached_audio, 960)
+                        await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
+                    except Exception as e:
+                        logger.bind(tag=TAG).info(f"发送缓存音频数据时发生错误: {e}")
+                        break
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
+            if hasattr(e, "__cause__") and e.__cause__:
+                logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+            if self.asr_ws:
+                await self.asr_ws.close()
+                self.asr_ws = None
+            self.is_processing = False
+            raise
+
+    async def _send_audio_frame(self, audio_data: bytes, status: int):
+        """发送音频帧"""
+        if not self.asr_ws:
+            return
+
+        audio_b64 = base64.b64encode(audio_data).decode("utf-8")
+
+        frame_data = {
+            "header": {"status": status, "app_id": self.app_id},
+            "parameter": {"iat": self.iat_params},
+            "payload": {
+                "audio": {"audio": audio_b64, "sample_rate": 16000, "encoding": "raw"}
+            },
+        }
+
+        await self.asr_ws.send(json.dumps(frame_data, ensure_ascii=False))
+
+    async def _forward_results(self, conn):
+        """转发识别结果"""
+        try:
+            while not conn.stop_event.is_set():
+                try:
+                    response = await asyncio.wait_for(self.asr_ws.recv(), timeout=60)
+                    result = json.loads(response)
+                    logger.bind(tag=TAG).debug(f"收到ASR结果: {result}")
+
+                    header = result.get("header", {})
+                    payload = result.get("payload", {})
+                    code = header.get("code", 0)
+                    status = header.get("status", 0)
+
+                    if code != 0:
+                        logger.bind(tag=TAG).error(
+                            f"识别错误，错误码: {code}, 消息: {header.get('message', '')}"
+                        )
+                        if code in [10114, 10160]:  # 连接问题
+                            break
+                        continue
+
+                    # 处理识别结果
+                    if payload and "result" in payload:
+                        text_data = payload["result"]["text"]
+                        if text_data:
+                            # 解码base64文本
+                            decoded_text = base64.b64decode(text_data).decode("utf-8")
+                            text_json = json.loads(decoded_text)
+                            # 提取文本内容
+                            text_ws = text_json.get("ws", [])
+                            for i in text_ws:
+                                for j in i.get("cw", []):
+                                    w = j.get("w", "")
+                                    self.text += w
+
+                    if status == 2:
+                        if conn.client_listen_mode == "manual":
+                            audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                            if len(audio_data) > 0:
+                                logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
+                                await self.handle_voice_stop(conn, audio_data)
+                                # 清理音频缓存
+                                conn.asr_audio.clear()
+                        conn.reset_vad_states()
+                        break
+
+                except asyncio.TimeoutError:
+                    logger.bind(tag=TAG).error("接收结果超时")
+                    break
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).info("ASR服务连接已关闭")
+                    self.is_processing = False
+                    break
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"处理ASR结果时发生错误: {str(e)}")
+                    if hasattr(e, "__cause__") and e.__cause__:
+                        logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+                    self.is_processing = False
+                    break
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR结果转发任务发生错误: {str(e)}")
+            if hasattr(e, "__cause__") and e.__cause__:
+                logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
+        finally:
+            # 清理连接资源
+            await self._cleanup()
+
+            # 清理连接的音频缓存
+            if conn:
+                if hasattr(conn, "asr_audio_for_voiceprint"):
+                    conn.asr_audio_for_voiceprint = []
+                if hasattr(conn, "asr_audio"):
+                    conn.asr_audio = []
+
+    async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
+        """处理语音停止，发送最后一帧并处理识别结果"""
+        try:
+            # 先发送最后一帧表示音频结束
+            if self.asr_ws and self.is_processing:
+                try:
+                    await self._send_audio_frame(b"", STATUS_LAST_FRAME)
+                    logger.bind(tag=TAG).debug(f"已发送停止请求")
+
+                    await asyncio.sleep(0.25)
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
+
+            await super().handle_voice_stop(conn, asr_audio_task)
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
+            import traceback
+
+            logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
+
+    def stop_ws_connection(self):
+        if self.asr_ws:
+            asyncio.create_task(self.asr_ws.close())
+            self.asr_ws = None
+        self.is_processing = False
+
+    async def _send_stop_request(self):
+        """发送停止识别请求（不关闭连接）"""
+        if self.asr_ws:
+            try:
+                # 先停止音频发送
+                self.is_processing = False
+                await self._send_audio_frame(b"", STATUS_LAST_FRAME)
+                logger.bind(tag=TAG).debug("已发送停止请求")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
+
+    async def _cleanup(self):
+        """清理资源（关闭连接）"""
+        logger.bind(tag=TAG).debug(
+            f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}"
+        )
+
+        # 状态重置
+        self.is_processing = False
+        self.server_ready = False
+        logger.bind(tag=TAG).debug("ASR状态已重置")
+
+        # 关闭连接
+        if self.asr_ws:
+            try:
+                logger.bind(tag=TAG).debug("正在关闭WebSocket连接")
+                await asyncio.wait_for(self.asr_ws.close(), timeout=2.0)
+                logger.bind(tag=TAG).debug("WebSocket连接已关闭")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
+            finally:
+                self.asr_ws = None
+
+        # 清理任务引用
+        self.forward_task = None
+
+        logger.bind(tag=TAG).debug("ASR会话清理完成")
+
+    async def speech_to_text(self, opus_data, session_id, audio_format):
+        """获取识别结果"""
+        result = self.text
+        self.text = ""
+        return result, None
+
+    async def close(self):
+        """资源清理方法"""
+        if self.asr_ws:
+            await self.asr_ws.close()
+            self.asr_ws = None
+        if self.forward_task:
+            self.forward_task.cancel()
+            try:
+                await self.forward_task
+            except asyncio.CancelledError:
+                pass
+            self.forward_task = None
+        self.is_processing = False
+
+        # 显式释放decoder资源
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Xunfei decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
+
+        # 清理所有连接的音频缓冲区
+        if hasattr(self, "_connections"):
+            for conn in self._connections.values():
+                if hasattr(conn, "asr_audio_for_voiceprint"):
+                    conn.asr_audio_for_voiceprint = []
+                if hasattr(conn, "asr_audio"):
+                    conn.asr_audio = []

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -53,24 +53,32 @@ class IntentProvider(IntentProviderBase):
             functions_desc += "---\n"
 
         prompt = (
+            "【严格格式要求】你必须只能返回JSON格式，绝对不能返回任何自然语言！\n\n"
             "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
+            "【重要规则】以下类型的查询请直接返回result_for_context，无需调用函数：\n"
+            "- 询问当前时间（如：现在几点、当前时间、查询时间等）\n"
+            "- 询问今天日期（如：今天几号、今天星期几、今天是什么日期等）\n"
+            "- 询问今天农历（如：今天农历几号、今天什么节气等）\n"
+            "- 询问所在城市（如：我现在在哪里、你知道我在哪个城市吗等）"
+            "系统会根据上下文信息直接构建回答。\n\n"
             "- 如果用户使用疑问词（如'怎么'、'为什么'、'如何'）询问退出相关的问题（例如'怎么退出了？'），注意这不是让你退出，请返回 {'function_call': {'name': 'continue_chat'}\n"
             "- 仅当用户明确使用'退出系统'、'结束对话'、'我不想和你说话了'等指令时，才触发 handle_exit_intent\n\n"
             f"{functions_desc}\n"
             "处理步骤:\n"
             "1. 分析用户输入，确定用户意图\n"
-            "2. 从可用函数列表中选择最匹配的函数\n"
-            "3. 如果找到匹配的函数，生成对应的function_call 格式\n"
-            '4. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
+            "2. 检查是否为上述基础信息查询（时间、日期等），如是则返回result_for_context\n"
+            "3. 从可用函数列表中选择最匹配的函数\n"
+            "4. 如果找到匹配的函数，生成对应的function_call 格式\n"
+            '5. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
             "返回格式要求：\n"
-            "1. 必须返回纯JSON格式\n"
+            "1. 必须返回纯JSON格式，不要包含任何其他文字\n"
             "2. 必须包含function_call字段\n"
             "3. function_call必须包含name字段\n"
             "4. 如果函数需要参数，必须包含arguments字段\n\n"
             "示例：\n"
             "```\n"
             "用户: 现在几点了？\n"
-            '返回: {"function_call": {"name": "get_time"}}\n'
+            '返回: {"function_call": {"name": "result_for_context"}}\n'
             "```\n"
             "```\n"
             "用户: 当前电池电量是多少？\n"
@@ -94,12 +102,15 @@ class IntentProvider(IntentProviderBase):
             "```\n\n"
             "注意：\n"
             "1. 只返回JSON格式，不要包含任何其他文字\n"
-            '2. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
-            "3. 确保返回的JSON格式正确，包含所有必要的字段\n"
+            '2. 优先检查用户查询是否为基础信息（时间、日期等），如是则返回{"function_call": {"name": "result_for_context"}}，不需要arguments参数\n'
+            '3. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
+            "4. 确保返回的JSON格式正确，包含所有必要的字段\n"
+            "5. result_for_context不需要任何参数，系统会自动从上下文获取信息\n"
             "特殊说明：\n"
             "- 当用户单次输入包含多个指令时（如'打开灯并且调高音量'）\n"
             "- 请返回多个function_call组成的JSON数组\n"
-            "- 示例：{'function_calls': [{name:'light_on'}, {name:'volume_up'}]}"
+            "- 示例：{'function_calls': [{name:'light_on'}, {name:'volume_up'}]}\n\n"
+            "【最终警告】绝对禁止输出任何自然语言、表情符号或解释文字！只能输出有效JSON格式！违反此规则将导致系统错误！"
         )
         return prompt
 
@@ -190,7 +201,7 @@ class IntentProvider(IntentProviderBase):
         # 记录LLM调用完成时间
         llm_time = time.time() - llm_start_time
         logger.bind(tag=TAG).debug(
-            f"LLM意图识别完成, 模型: {model_info}, 调用耗时: {llm_time:.4f}秒"
+            f"外挂的大模型意图识别完成, 模型: {model_info}, 调用耗时: {llm_time:.4f}秒"
         )
 
         # 记录后处理开始时间
@@ -223,8 +234,15 @@ class IntentProvider(IntentProviderBase):
                     f"llm 识别到意图: {function_name}, 参数: {function_args}"
                 )
 
-                # 如果是继续聊天，清理工具调用相关的历史消息
-                if function_name == "continue_chat":
+                # 处理不同类型的意图
+                if function_name == "result_for_context":
+                    # 处理基础信息查询，直接从context构建结果
+                    logger.bind(tag=TAG).info(
+                        "检测到result_for_context意图，将使用上下文信息直接回答"
+                    )
+
+                elif function_name == "continue_chat":
+                    # 处理普通对话
                     # 保留非工具相关的消息
                     clean_history = [
                         msg
@@ -233,25 +251,15 @@ class IntentProvider(IntentProviderBase):
                     ]
                     conn.dialogue.dialogue = clean_history
 
-                # 添加到缓存
-                self.cache_manager.set(self.CacheType.INTENT, cache_key, intent)
+                else:
+                    # 处理函数调用
+                    logger.bind(tag=TAG).info(f"检测到函数调用意图: {function_name}")
 
-                # 后处理时间
-                postprocess_time = time.time() - postprocess_start_time
-                logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
-
-                # 确保返回完全序列化的JSON字符串
-                return intent
-            else:
-                # 添加到缓存
-                self.cache_manager.set(self.CacheType.INTENT, cache_key, intent)
-
-                # 后处理时间
-                postprocess_time = time.time() - postprocess_start_time
-                logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
-
-                # 返回普通意图
-                return intent
+            # 统一缓存处理和返回
+            self.cache_manager.set(self.CacheType.INTENT, cache_key, intent)
+            postprocess_time = time.time() - postprocess_start_time
+            logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
+            return intent
         except json.JSONDecodeError:
             # 后处理时间
             postprocess_time = time.time() - postprocess_start_time
@@ -259,4 +267,4 @@ class IntentProvider(IntentProviderBase):
                 f"无法解析意图JSON: {intent}, 后处理耗时: {postprocess_time:.4f}秒"
             )
             # 如果解析失败，默认返回继续聊天意图
-            return '{"intent": "继续聊天"}'
+            return '{"function_call": {"name": "continue_chat"}}'

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -17,27 +17,26 @@ class LLMProvider(LLMProviderBase):
             self.base_url = config.get("base_url")
         else:
             self.base_url = config.get("url")
-        # 增加timeout的配置项，单位为秒
         timeout = config.get("timeout", 300)
         self.timeout = int(timeout) if timeout else 300
 
         param_defaults = {
-            "max_tokens": (500, int),
-            "temperature": (0.7, lambda x: round(float(x), 1)),
-            "top_p": (1.0, lambda x: round(float(x), 1)),
-            "frequency_penalty": (0, lambda x: round(float(x), 1)),
+            "max_tokens": int,
+            "temperature": lambda x: round(float(x), 1),
+            "top_p": lambda x: round(float(x), 1),
+            "frequency_penalty": lambda x: round(float(x), 1),
         }
 
-        for param, (default, converter) in param_defaults.items():
+        for param, converter in param_defaults.items():
             value = config.get(param)
             try:
                 setattr(
                     self,
                     param,
-                    converter(value) if value not in (None, "") else default,
+                    converter(value) if value not in (None, "") else None,
                 )
             except (ValueError, TypeError):
-                setattr(self, param, default)
+                setattr(self, param, None)
 
         logger.debug(
             f"意图识别参数初始化: {self.temperature}, {self.max_tokens}, {self.top_p}, {self.frequency_penalty}"
@@ -48,34 +47,46 @@ class LLMProvider(LLMProviderBase):
             logger.bind(tag=TAG).error(model_key_msg)
         self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url, timeout=httpx.Timeout(self.timeout))
 
+    @staticmethod
+    def normalize_dialogue(dialogue):
+        """自动修复 dialogue 中缺失 content 的消息"""
+        for msg in dialogue:
+            if "role" in msg and "content" not in msg:
+                msg["content"] = ""
+        return dialogue
+
     def response(self, session_id, dialogue, **kwargs):
         try:
-            responses = self.client.chat.completions.create(
-                model=self.model_name,
-                messages=dialogue,
-                stream=True,
-                max_tokens=kwargs.get("max_tokens", self.max_tokens),
-                temperature=kwargs.get("temperature", self.temperature),
-                top_p=kwargs.get("top_p", self.top_p),
-                frequency_penalty=kwargs.get(
-                    "frequency_penalty", self.frequency_penalty
-                ),
-            )
+            dialogue = self.normalize_dialogue(dialogue)
+
+            request_params = {
+                "model": self.model_name,
+                "messages": dialogue,
+                "stream": True,
+            }
+
+            # 添加可选参数,只有当参数不为None时才添加
+            optional_params = {
+                "max_tokens": kwargs.get("max_tokens", self.max_tokens),
+                "temperature": kwargs.get("temperature", self.temperature),
+                "top_p": kwargs.get("top_p", self.top_p),
+                "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
+            }
+
+            for key, value in optional_params.items():
+                if value is not None:
+                    request_params[key] = value
+
+            responses = self.client.chat.completions.create(**request_params)
 
             is_active = True
             for chunk in responses:
                 try:
-                    # 检查是否存在有效的choice且content不为空
-                    delta = (
-                        chunk.choices[0].delta
-                        if getattr(chunk, "choices", None)
-                        else None
-                    )
-                    content = delta.content if hasattr(delta, "content") else ""
+                    delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
+                    content = getattr(delta, "content", "") if delta else ""
                 except IndexError:
                     content = ""
                 if content:
-                    # 处理标签跨多个chunk的情况
                     if "<think>" in content:
                         is_active = False
                         content = content.split("<think>")[0]
@@ -88,19 +99,36 @@ class LLMProvider(LLMProviderBase):
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")
 
-    def response_with_functions(self, session_id, dialogue, functions=None):
+    def response_with_functions(self, session_id, dialogue, functions=None, **kwargs):
         try:
-            stream = self.client.chat.completions.create(
-                model=self.model_name, messages=dialogue, stream=True, tools=functions
-            )
+            dialogue = self.normalize_dialogue(dialogue)
+
+            request_params = {
+                "model": self.model_name,
+                "messages": dialogue,
+                "stream": True,
+                "tools": functions,
+            }
+
+            optional_params = {
+                "max_tokens": kwargs.get("max_tokens", self.max_tokens),
+                "temperature": kwargs.get("temperature", self.temperature),
+                "top_p": kwargs.get("top_p", self.top_p),
+                "frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
+            }
+
+            for key, value in optional_params.items():
+                if value is not None:
+                    request_params[key] = value
+
+            stream = self.client.chat.completions.create(**request_params)
 
             for chunk in stream:
-                # 检查是否存在有效的choice且content不为空
                 if getattr(chunk, "choices", None):
-                    yield chunk.choices[0].delta.content, chunk.choices[
-                        0
-                    ].delta.tool_calls
-                # 存在 CompletionUsage 消息时，生成 Token 消耗 log
+                    delta = chunk.choices[0].delta
+                    content = getattr(delta, "content", "")
+                    tool_calls = getattr(delta, "tool_calls", None)
+                    yield content, tool_calls
                 elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
                     usage_info = getattr(chunk, "usage", None)
                     logger.bind(tag=TAG).info(

main/xiaozhi-server/core/providers/memory/base.py

@@ -14,7 +14,7 @@ class MemoryProviderBase(ABC):
         self.llm = llm
 
     @abstractmethod
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         """Save a new memory for specific role and return memory ID"""
         print("this is base func", msgs)
 

main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py

@@ -28,7 +28,7 @@ class MemoryProvider(MemoryProviderBase):
             logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
             self.use_mem0 = False
 
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         if not self.use_mem0:
             return None
         if len(msgs) < 2:
@@ -41,9 +41,7 @@ class MemoryProvider(MemoryProviderBase):
                 for message in msgs
                 if message.role != "system"
             ]
-            result = self.client.add(
-                messages, user_id=self.role_id, output_format=self.api_version
-            )
+            result = self.client.add(messages, user_id=self.role_id)
             logger.bind(tag=TAG).debug(f"Save memory result: {result}")
         except Exception as e:
             logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
@@ -53,9 +51,12 @@ class MemoryProvider(MemoryProviderBase):
         if not self.use_mem0:
             return ""
         try:
-            results = self.client.search(
-                query, user_id=self.role_id, output_format=self.api_version
-            )
+            if not getattr(self, "role_id", None):
+                return ""
+
+            filters = {"user_id": self.role_id}
+
+            results = self.client.search(query, filters=filters)
             if not results or "results" not in results:
                 return ""
 

main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -4,7 +4,8 @@ import json
 import os
 import yaml
 from config.config_loader import get_project_dir
-from config.manage_api_client import save_mem_local_short
+from config.manage_api_client import generate_and_save_chat_summary
+import asyncio
 from core.utils.util import check_model_key
 
 
@@ -74,18 +75,6 @@ short_term_memory_prompt = """
 ```
 """
 
-short_term_memory_prompt_only_content = """
-你是一个经验丰富的记忆总结者，擅长将对话内容进行总结摘要，遵循以下规则：
-1、总结user的重要信息，以便在未来的对话中提供更个性化的服务
-2、不要重复总结，不要遗忘之前记忆，除非原来的记忆超过了1800字内，否则不要遗忘、不要压缩用户的历史记忆
-3、用户操控的设备音量、播放音乐、天气、退出、不想对话等和用户本身无关的内容，这些信息不需要加入到总结中
-4、聊天内容中的今天的日期时间、今天的天气情况与用户事件无关的数据，这些信息如果当成记忆存储会影响后序对话，这些信息不需要加入到总结中
-5、不要把设备操控的成果结果和失败结果加入到总结中，也不要把用户的一些废话加入到总结中
-6、不要为了总结而总结，如果用户的聊天没有意义，请返回原来的历史记录也是可以的
-7、只需要返回总结摘要，严格控制在1800字内
-8、不要包含代码、xml，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
-"""
-
 
 def extract_json_data(json_code):
     start = json_code.find("```json")
@@ -143,7 +132,7 @@ class MemoryProvider(MemoryProviderBase):
         with open(self.memory_path, "w", encoding="utf-8") as f:
             yaml.dump(all_memory, f, allow_unicode=True)
 
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         # 打印使用的模型信息
         model_info = getattr(self.llm, "model_name", str(self.llm.__class__.__name__))
         logger.bind(tag=TAG).debug(f"使用记忆保存模型: {model_info}")
@@ -187,14 +176,12 @@ class MemoryProvider(MemoryProviderBase):
             except Exception as e:
                 print("Error:", e)
         else:
-            result = self.llm.response_no_stream(
-                short_term_memory_prompt_only_content,
-                msgStr,
-                max_tokens=2000,
-                temperature=0.2,
-            )
-            save_mem_local_short(self.role_id, result)
-        logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
+            # 当save_to_file为False时，调用Java端的聊天记录总结接口
+            summary_id = session_id if session_id else self.role_id
+            await generate_and_save_chat_summary(summary_id)
+        logger.bind(tag=TAG).info(
+            f"Save memory successful - Role: {self.role_id}, Session: {session_id}"
+        )
 
         return self.short_memory
 

main/xiaozhi-server/core/providers/memory/nomem/nomem.py

@@ -11,7 +11,7 @@ class MemoryProvider(MemoryProviderBase):
     def __init__(self, config, summary_memory=None):
         super().__init__(config)
 
-    async def save_memory(self, msgs):
+    async def save_memory(self, msgs, session_id=None):
         logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
         return None
 

main/xiaozhi-server/core/providers/tools/device_mcp/mcp_handler.py

@@ -116,14 +116,14 @@ async def send_mcp_message(conn, payload: dict, transport=None):
             await conn.transport.send(message)
         else:
             raise AttributeError("无法找到可用的传输层接口")
-        logger.bind(tag=TAG).info(f"成功发送MCP消息: {message}")
+        logger.bind(tag=TAG).debug(f"成功发送MCP消息: {message}")
     except Exception as e:
         logger.bind(tag=TAG).error(f"发送MCP消息失败: {e}")
 
 
 async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict, transport=None):
     """处理MCP消息,包括初始化、工具列表和工具调用响应等"""
-    logger.bind(tag=TAG).info(f"处理MCP消息: {str(payload)[:100]}")
+    logger.bind(tag=TAG).debug(f"处理MCP消息: {str(payload)[:100]}")
 
     if not isinstance(payload, dict):
         logger.bind(tag=TAG).error("MCP消息缺少payload字段或格式错误")
@@ -148,7 +148,7 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict, transpo
             if isinstance(server_info, dict):
                 name = server_info.get("name")
                 version = server_info.get("version")
-                logger.bind(tag=TAG).info(
+                logger.bind(tag=TAG).debug(
                     f"客户端MCP服务器信息: name={name}, version={version}"
                 )
             return
@@ -205,11 +205,11 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict, transpo
 
                 next_cursor = result.get("nextCursor", "")
                 if next_cursor:
-                    logger.bind(tag=TAG).info(f"有更多工具，nextCursor: {next_cursor}")
+                    logger.bind(tag=TAG).debug(f"有更多工具，nextCursor: {next_cursor}")
                     await send_mcp_tools_list_continue_request(conn, next_cursor, transport)
                 else:
                     await mcp_client.set_ready(True)
-                    logger.bind(tag=TAG).info("所有工具已获取，MCP客户端准备就绪")
+                    logger.bind(tag=TAG).debug("所有工具已获取，MCP客户端准备就绪")
 
                     # 刷新工具缓存，确保MCP工具被包含在函数列表中
                     if hasattr(conn, "func_handler") and conn.func_handler:
@@ -265,7 +265,7 @@ async def send_mcp_initialize_message(conn, transport=None):
             },
         },
     }
-    logger.bind(tag=TAG).info("发送MCP初始化消息")
+    logger.bind(tag=TAG).debug("发送MCP初始化消息")
     await send_mcp_message(conn, payload, transport)
 
 

main/xiaozhi-server/core/providers/tools/mcp_endpoint/mcp_endpoint_handler.py

@@ -358,7 +358,9 @@ async def call_mcp_endpoint_tool(
     }
 
     message = json.dumps(payload)
-    logger.bind(tag=TAG).info(f"发送MCP接入点工具调用请求: {actual_name}，参数: {args}")
+    logger.bind(tag=TAG).info(
+        f"发送MCP接入点工具调用请求: {actual_name}，参数: {json.dumps(arguments, ensure_ascii=False)}"
+    )
     await mcp_client.send_message(message)
 
     try:

main/xiaozhi-server/core/providers/tools/server_mcp/mcp_client.py

@@ -10,9 +10,13 @@ import concurrent.futures
 from contextlib import AsyncExitStack
 from typing import Optional, List, Dict, Any
 
-from mcp import ClientSession, StdioServerParameters
+from mcp import ClientSession, StdioServerParameters, Implementation
+from mcp.client.session import SamplingFnT, ElicitationFnT, ListRootsFnT, LoggingFnT, MessageHandlerFnT
 from mcp.client.stdio import stdio_client
 from mcp.client.sse import sse_client
+from mcp.client.streamable_http import streamablehttp_client
+from mcp.shared.session import ProgressFnT
+
 from config.logger import setup_logging
 from core.utils.util import sanitize_tool_name
 
@@ -40,13 +44,25 @@ class ServerMCPClient:
         self.tools_dict: Dict[str, Any] = {}
         self.name_mapping: Dict[str, str] = {}
 
-    async def initialize(self):
+    async def initialize(self, read_timeout_seconds: timedelta | None = None,
+             sampling_callback: SamplingFnT | None = None,
+             elicitation_callback: ElicitationFnT | None = None,
+             list_roots_callback: ListRootsFnT | None = None,
+             logging_callback: LoggingFnT | None = None,
+             message_handler: MessageHandlerFnT | None = None,
+             client_info: Implementation | None = None):
         """初始化MCP客户端连接"""
         if self._worker_task:
             return
 
         self._worker_task = asyncio.create_task(
-            self._worker(), name="ServerMCPClientWorker"
+            self._worker(read_timeout_seconds=read_timeout_seconds,
+                        sampling_callback=sampling_callback,
+                        elicitation_callback=elicitation_callback,
+                        list_roots_callback=list_roots_callback,
+                        logging_callback=logging_callback,
+                        message_handler=message_handler,
+                        client_info=client_info), name="ServerMCPClientWorker"
         )
         await self._ready_evt.wait()
 
@@ -96,12 +112,15 @@ class ServerMCPClient:
             for name, tool in self.tools_dict.items()
         ]
 
-    async def call_tool(self, name: str, args: dict) -> Any:
+    async def call_tool(self, name: str, arguments: dict, read_timeout_seconds: timedelta | None = None, progress_callback: ProgressFnT | None = None, *, meta: dict[str, Any] | None = None) -> Any:
         """调用指定工具
 
         Args:
             name: 工具名称
-            args: 工具参数
+            arguments: 工具参数
+            read_timeout_seconds:
+            progress_callback: 进度回调函数
+            meta:
 
         Returns:
             Any: 工具执行结果
@@ -114,7 +133,7 @@ class ServerMCPClient:
 
         real_name = self.name_mapping.get(name, name)
         loop = self._worker_task.get_loop()
-        coro = self.session.call_tool(real_name, args)
+        coro = self.session.call_tool(real_name, arguments=arguments, read_timeout_seconds=read_timeout_seconds, progress_callback=progress_callback, meta=meta)
 
         if loop is asyncio.get_running_loop():
             return await coro
@@ -143,7 +162,13 @@ class ServerMCPClient:
         # 所有检查都通过，连接正常
         return True
 
-    async def _worker(self):
+    async def _worker(self, read_timeout_seconds: timedelta | None = None,
+             sampling_callback: SamplingFnT | None = None,
+             elicitation_callback: ElicitationFnT | None = None,
+             list_roots_callback: ListRootsFnT | None = None,
+             logging_callback: LoggingFnT | None = None,
+             message_handler: MessageHandlerFnT | None = None,
+             client_info: Implementation | None = None):
         """MCP客户端工作协程"""
         async with AsyncExitStack() as stack:
             try:
@@ -167,16 +192,38 @@ class ServerMCPClient:
 
                 # 建立SSEClient
                 elif "url" in self.config:
+                    headers = dict(self.config.get("headers", {}))
+                    # TODO 兼容旧版本
                     if "API_ACCESS_TOKEN" in self.config:
-                        headers = {
-                            "Authorization": f"Bearer {self.config['API_ACCESS_TOKEN']}"
-                        }
+                        headers["Authorization"] = f"Bearer {self.config['API_ACCESS_TOKEN']}"
+                        self.logger.bind(tag=TAG).warning(f"你正在使用旧过时的配置 API_ACCESS_TOKEN ，请在.mcp_server_settings.json中将API_ACCESS_TOKEN直接设置在headers中，例如 'Authorization': 'Bearer API_ACCESS_TOKEN'")
+                   
+                    # 根据transport类型选择不同的客户端，默认为SSE
+                    transport_type = self.config.get("transport", "sse")
+
+                    if transport_type == "streamable-http" or transport_type == "http":
+                        # 使用 Streamable HTTP 传输
+                        http_r, http_w, get_session_id = await stack.enter_async_context(
+                            streamablehttp_client(
+                                url=self.config["url"],
+                                headers=headers,
+                                timeout=self.config.get("timeout", 30),
+                                sse_read_timeout=self.config.get("sse_read_timeout", 60 * 5),
+                                terminate_on_close=self.config.get("terminate_on_close", True)
+                            )
+                        )
+                        read_stream, write_stream = http_r, http_w
                     else:
-                        headers = {}
-                    sse_r, sse_w = await stack.enter_async_context(
-                        sse_client(self.config["url"], headers=headers)
-                    )
-                    read_stream, write_stream = sse_r, sse_w
+                        # 使用传统的 SSE 传输
+                        sse_r, sse_w = await stack.enter_async_context(
+                            sse_client(
+                                url=self.config["url"],
+                                headers=headers,
+                                timeout=self.config.get("timeout", 5),
+                                sse_read_timeout=self.config.get("sse_read_timeout", 60 * 5)
+                            )
+                        )
+                        read_stream, write_stream = sse_r, sse_w
 
                 else:
                     raise ValueError("MCP客户端配置必须包含'command'或'url'")
@@ -185,7 +232,13 @@ class ServerMCPClient:
                     ClientSession(
                         read_stream=read_stream,
                         write_stream=write_stream,
-                        read_timeout_seconds=timedelta(seconds=15),
+                        read_timeout_seconds=read_timeout_seconds,
+                        sampling_callback=sampling_callback,
+                        elicitation_callback=elicitation_callback,
+                        list_roots_callback=list_roots_callback,
+                        logging_callback=logging_callback,
+                        message_handler=message_handler,
+                        client_info=client_info
                     )
                 )
                 await self.session.initialize()

main/xiaozhi-server/core/providers/tools/server_mcp/mcp_manager.py

@@ -4,6 +4,9 @@ import asyncio
 import os
 import json
 from typing import Dict, Any, List
+
+from mcp.types import LoggingMessageNotificationParams
+
 from config.config_loader import get_project_dir
 from config.logger import setup_logging
 from .mcp_client import ServerMCPClient
@@ -26,6 +29,7 @@ class ServerMCPManager:
             )
         self.clients: Dict[str, ServerMCPClient] = {}
         self.tools = []
+        self._init_lock = asyncio.Lock()
 
     def load_config(self) -> Dict[str, Any]:
         """加载MCP服务配置"""
@@ -42,29 +46,50 @@ class ServerMCPManager:
             )
             return {}
 
+    async def _init_server(self, name: str, srv_config: Dict[str, Any]):
+        """初始化单个MCP服务"""
+        client = None
+        try:
+            # 初始化服务端MCP客户端
+            logger.bind(tag=TAG).info(f"初始化服务端MCP客户端: {name}")
+            client = ServerMCPClient(srv_config)
+            # 设置超时时间10秒
+            await asyncio.wait_for(client.initialize(logging_callback=self.logging_callback), timeout=10)
+
+            # 使用锁保护共享状态的修改
+            async with self._init_lock:
+                self.clients[name] = client
+                client_tools = client.get_available_tools()
+                self.tools.extend(client_tools)
+
+        except asyncio.TimeoutError:
+            logger.bind(tag=TAG).error(
+                f"Failed to initialize MCP server {name}: Timeout"
+            )
+            if client:
+                await client.cleanup()
+        except Exception as e:
+            logger.bind(tag=TAG).error(
+                f"Failed to initialize MCP server {name}: {e}"
+            )
+            if client:
+                await client.cleanup()
+
     async def initialize_servers(self) -> None:
         """初始化所有MCP服务"""
         config = self.load_config()
+        tasks = []
         for name, srv_config in config.items():
             if not srv_config.get("command") and not srv_config.get("url"):
                 logger.bind(tag=TAG).warning(
                     f"Skipping server {name}: neither command nor url specified"
                 )
                 continue
-
-            try:
-                # 初始化服务端MCP客户端
-                logger.bind(tag=TAG).info(f"初始化服务端MCP客户端: {name}")
-                client = ServerMCPClient(srv_config)
-                await client.initialize()
-                self.clients[name] = client
-                client_tools = client.get_available_tools()
-                self.tools.extend(client_tools)
-
-            except Exception as e:
-                logger.bind(tag=TAG).error(
-                    f"Failed to initialize MCP server {name}: {e}"
-                )
+            
+            tasks.append(self._init_server(name, srv_config))
+        
+        if tasks:
+            await asyncio.gather(*tasks)
 
         # 输出当前支持的服务端MCP工具列表
         if hasattr(self.conn, "func_handler") and self.conn.func_handler:
@@ -109,7 +134,7 @@ class ServerMCPManager:
         # 带重试机制的工具调用
         for attempt in range(max_retries):
             try:
-                return await target_client.call_tool(tool_name, arguments)
+                return await target_client.call_tool(tool_name, arguments, progress_callback=self.progress_callback)
             except Exception as e:
                 # 最后一次尝试失败时直接抛出异常
                 if attempt == max_retries - 1:
@@ -131,7 +156,7 @@ class ServerMCPManager:
                     config = self.load_config()
                     if client_name in config:
                         client = ServerMCPClient(config[client_name])
-                        await client.initialize()
+                        await client.initialize(logging_callback=self.logging_callback)
                         self.clients[client_name] = client
                         target_client = client
                         logger.bind(tag=TAG).info(
@@ -159,3 +184,11 @@ class ServerMCPManager:
             except (asyncio.TimeoutError, Exception) as e:
                 logger.bind(tag=TAG).error(f"关闭服务端MCP客户端 {name} 时出错: {e}")
         self.clients.clear()
+
+    # 可选回调方法
+
+    async def logging_callback(self, params: LoggingMessageNotificationParams):
+        logger.bind(tag=TAG).info(f"[Server Log - {params.level.upper()}] {params.data}")
+
+    async def progress_callback(self, progress: float, total: float | None, message: str | None) -> None:
+        logger.bind(tag=TAG).info(f"[Progress {progress}/{total}]: {message}")

main/xiaozhi-server/core/providers/tools/server_plugins/plugin_executor.py

@@ -71,6 +71,19 @@ class ServerPluginExecutor(ToolExecutor):
         for func_name in all_required_functions:
             func_item = all_function_registry.get(func_name)
             if func_item:
+                # 从函数注册中获取描述
+                fun_description = (
+                    self.config.get("plugins", {})
+                    .get(func_name, {})
+                    .get("description", "")
+                )
+                if fun_description is not None and len(fun_description) > 0:
+                    if "function" in func_item.description and isinstance(
+                        func_item.description["function"], dict
+                    ):
+                        func_item.description["function"][
+                            "description"
+                        ] = fun_description
                 tools[func_name] = ToolDefinition(
                     name=func_name,
                     description=func_item.description,

main/xiaozhi-server/core/providers/tools/unified_tool_handler.py

@@ -69,7 +69,7 @@ class UnifiedToolHandler:
             self._initialize_home_assistant()
 
             self.finish_init = True
-            self.logger.info("统一工具处理器初始化完成")
+            self.logger.debug("统一工具处理器初始化完成")
 
             # 输出当前支持的所有工具列表
             self.current_support_functions()

main/xiaozhi-server/core/providers/tools/unified_tool_manager.py

@@ -20,7 +20,7 @@ class ToolManager:
         """注册工具执行器"""
         self.executors[tool_type] = executor
         self._invalidate_cache()
-        self.logger.info(f"注册工具执行器: {tool_type.value}")
+        self.logger.debug(f"注册工具执行器: {tool_type.value}")
 
     def _invalidate_cache(self):
         """使缓存失效"""
@@ -109,7 +109,7 @@ class ToolManager:
     def refresh_tools(self):
         """刷新工具缓存"""
         self._invalidate_cache()
-        self.logger.info("工具缓存已刷新")
+        self.logger.debug("工具缓存已刷新")
 
     def get_tool_statistics(self) -> Dict[str, int]:
         """获取工具统计信息"""

main/xiaozhi-server/core/providers/tts/alibl_stream.py

@@ -0,0 +1,522 @@
+import os
+import uuid
+import json
+import time
+import queue
+import asyncio
+import traceback
+import websockets
+from asyncio import Task
+from config.logger import setup_logging
+from core.utils import opus_encoder_utils
+from core.utils.tts import MarkdownCleaner
+from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+
+        self.interface_type = InterfaceType.DUAL_STREAM
+        # 基础配置
+        self.api_key = config.get("api_key")
+        if not self.api_key:
+            raise ValueError("api_key is required for CosyVoice TTS")
+
+        # WebSocket配置
+        self.ws_url = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/"
+        self.ws = None
+        self._monitor_task = None
+        self.last_active_time = None
+
+        # 模型和音色配置
+        self.model = config.get("model", "cosyvoice-v2")
+        self.voice = config.get("voice", "longxiaochun_v2")  # 默认音色
+        if config.get("private_voice"):
+            self.voice = config.get("private_voice")
+
+        # 音频参数配置
+        self.format = config.get("format", "pcm")
+        sample_rate = config.get("sample_rate", "24000")
+        self.sample_rate = int(sample_rate) if sample_rate else 24000
+
+        volume = config.get("volume", "50")
+        self.volume = int(volume) if volume else 50
+
+        rate = config.get("rate", "1.0")
+        self.rate = float(rate) if rate else 1.0
+
+        pitch = config.get("pitch", "1.0")
+        self.pitch = float(pitch) if pitch else 1.0
+
+        self.header = {
+            "Authorization": f"Bearer {self.api_key}",
+            # "user-agent": "your_platform_info", // 可选
+            # "X-DashScope-WorkSpace": workspace, // 可选，阿里云百炼业务空间ID
+            "X-DashScope-DataInspection": "enable",
+        }
+
+        # 创建Opus编码器
+        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
+            sample_rate=self.sample_rate, channels=1, frame_size_ms=60
+        )
+
+    async def _ensure_connection(self):
+        """确保WebSocket连接可用，支持60秒内连接复用"""
+        try:
+            current_time = time.time()
+            if self.ws and current_time - self.last_active_time < 60:
+                # 一分钟内才可以复用链接进行连续对话
+                logger.bind(tag=TAG).info(f"使用已有链接...")
+                return self.ws
+            logger.bind(tag=TAG).info("开始建立新连接...")
+
+            self.ws = await websockets.connect(
+                self.ws_url,
+                additional_headers=self.header,
+                ping_interval=30,
+                ping_timeout=10,
+                close_timeout=10,
+            )
+
+            logger.bind(tag=TAG).info("WebSocket连接建立成功")
+            self.last_active_time = current_time
+            return self.ws
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}")
+            self.ws = None
+            self.last_active_time = None
+            raise
+
+    def tts_text_priority_thread(self):
+        """流式TTS文本处理线程"""
+        while not self.conn.stop_event.is_set():
+            try:
+                message = self.tts_text_queue.get(timeout=1)
+                logger.bind(tag=TAG).debug(
+                    f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}"
+                )
+
+                if message.sentence_type == SentenceType.FIRST:
+                    self.conn.client_abort = False
+
+                if self.conn.client_abort:
+                    try:
+                        logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
+                        continue
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"取消TTS会话失败: {str(e)}")
+                        continue
+
+                if message.sentence_type == SentenceType.FIRST:
+                    # 初始化会话
+                    try:
+                        if not getattr(self.conn, "sentence_id", None): 
+                            self.conn.sentence_id = uuid.uuid4().hex
+                            logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}")
+
+                        logger.bind(tag=TAG).info("开始启动TTS会话...")
+                        future = asyncio.run_coroutine_threadsafe(
+                            self.start_session(self.conn.sentence_id),
+                            loop=self.conn.loop,
+                        )
+                        future.result()
+                        self.before_stop_play_files.clear()
+                        logger.bind(tag=TAG).info("TTS会话启动成功")
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
+                        continue
+
+                elif ContentType.TEXT == message.content_type:
+                    if message.content_detail:
+                        try:
+                            logger.bind(tag=TAG).debug(
+                                f"开始发送TTS文本: {message.content_detail}"
+                            )
+                            future = asyncio.run_coroutine_threadsafe(
+                                self.text_to_speak(message.content_detail, None),
+                                loop=self.conn.loop,
+                            )
+                            future.result()
+                            logger.bind(tag=TAG).debug("TTS文本发送成功")
+                        except Exception as e:
+                            logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
+                            continue
+
+                elif ContentType.FILE == message.content_type:
+                    logger.bind(tag=TAG).info(
+                        f"添加音频文件到待播放列表: {message.content_file}"
+                    )
+                    if message.content_file and os.path.exists(message.content_file):
+                        # 先处理文件音频数据
+                        self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
+
+                if message.sentence_type == SentenceType.LAST:
+                    try:
+                        logger.bind(tag=TAG).info("开始结束TTS会话...")
+                        future = asyncio.run_coroutine_threadsafe(
+                            self.finish_session(self.conn.sentence_id),
+                            loop=self.conn.loop,
+                        )
+                        future.result()
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"结束TTS会话失败: {str(e)}")
+                        continue
+
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
+                )
+                continue
+
+    async def text_to_speak(self, text, _):
+        """发送文本到TTS服务进行合成"""
+        try:
+            if self.ws is None:
+                logger.bind(tag=TAG).warning("WebSocket连接不存在，终止发送文本")
+                return
+
+            # 过滤Markdown
+            filtered_text = MarkdownCleaner.clean_markdown(text)
+
+            # 发送continue-task消息
+            continue_task_message = {
+                "header": {
+                    "action": "continue-task",
+                    "task_id": self.conn.sentence_id,
+                    "streaming": "duplex",
+                },
+                "payload": {"input": {"text": filtered_text}},
+            }
+
+            await self.ws.send(json.dumps(continue_task_message))
+            self.last_active_time = time.time()
+            logger.bind(tag=TAG).debug(f"已发送文本: {filtered_text}")
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
+            if self.ws:
+                try:
+                    await self.ws.close()
+                except:
+                    pass
+                self.ws = None
+            raise
+
+    async def start_session(self, session_id):
+        """启动TTS会话"""
+        logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
+        try:
+            # 检查并清理上一个会话的监听任务
+            if (
+                self._monitor_task is not None
+                and isinstance(self._monitor_task, Task)
+                and not self._monitor_task.done()
+            ):
+                logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务...")
+                await self.close()
+
+            # 确保连接可用
+            await self._ensure_connection()
+
+            # 启动监听任务
+            self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
+
+            # 发送run-task消息启动会话
+            run_task_message = {
+                "header": {
+                    "action": "run-task",
+                    "task_id": session_id,
+                    "streaming": "duplex",
+                },
+                "payload": {
+                    "task_group": "audio",
+                    "task": "tts",
+                    "function": "SpeechSynthesizer",
+                    "model": self.model,
+                    "parameters": {
+                        "text_type": "PlainText",
+                        "voice": self.voice,
+                        "format": self.format,
+                        "sample_rate": self.sample_rate,
+                        "volume": self.volume,
+                        "rate": self.rate,
+                        "pitch": self.pitch,
+                    },
+                    "input": {}
+                },
+            }
+
+            await self.ws.send(json.dumps(run_task_message))
+            self.last_active_time = time.time()
+            logger.bind(tag=TAG).info("会话启动请求已发送")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
+            await self.close()
+            raise
+
+    async def finish_session(self, session_id):
+        """结束TTS会话"""
+        logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
+        try:
+            if self.ws and session_id:
+                # 发送finish-task消息
+                finish_task_message = {
+                    "header": {
+                        "action": "finish-task",
+                        "task_id": session_id,
+                        "streaming": "duplex",
+                    },
+                    "payload": {
+                        "input": {}
+                    }
+                }
+
+                await self.ws.send(json.dumps(finish_task_message))
+                self.last_active_time = time.time()
+                logger.bind(tag=TAG).info("会话结束请求已发送")
+                # 等待监听任务完成
+                if self._monitor_task:
+                    try:
+                        await self._monitor_task
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(
+                            f"等待监听任务完成时发生错误: {str(e)}"
+                        )
+                    finally:
+                        self._monitor_task = None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
+            await self.close()
+            raise
+
+    async def close(self):
+        """清理资源"""
+        # 取消监听任务
+        if self._monitor_task:
+            try:
+                self._monitor_task.cancel()
+                await self._monitor_task
+            except asyncio.CancelledError:
+                pass
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}")
+            self._monitor_task = None
+
+        # 关闭WebSocket连接
+        if self.ws:
+            try:
+                await self.ws.close()
+            except:
+                pass
+            self.ws = None
+            self.last_active_time = None
+
+    async def _start_monitor_tts_response(self):
+        """监听TTS响应"""
+        try:
+            session_finished = False
+            while not self.conn.stop_event.is_set():
+                try:
+                    msg = await self.ws.recv()
+                    self.last_active_time = time.time()
+
+                    # 检查客户端是否中止
+                    if self.conn.client_abort:
+                        logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应")
+                        break
+
+                    if isinstance(msg, str):  # JSON控制消息
+                        try:
+                            data = json.loads(msg)
+                            event = data["header"].get("event")
+
+                            if event == "task-started":
+                                logger.bind(tag=TAG).debug("TTS任务启动成功~")
+                                self.tts_audio_queue.put((SentenceType.FIRST, [], None))
+                            elif event == "result-generated":
+                                # 发送缓存的数据
+                                if self.conn.tts_MessageText:
+                                    logger.bind(tag=TAG).info(
+                                        f"句子语音生成成功： {self.conn.tts_MessageText}"
+                                    )
+                                    self.tts_audio_queue.put(
+                                        (SentenceType.FIRST, [], self.conn.tts_MessageText)
+                                    )
+                                    self.conn.tts_MessageText = None
+                            elif event == "task-finished":
+                                logger.bind(tag=TAG).debug("TTS任务完成~")
+                                self._process_before_stop_play_files()
+                                session_finished = True
+                                break
+                            elif event == "task-failed":
+                                error_code = data["header"].get("error_code", "unknown")
+                                error_message = data["header"].get("error_message", "未知错误")
+                                logger.bind(tag=TAG).error(
+                                    f"TTS任务失败: {error_code} - {error_message}"
+                                )
+                                break
+                        except json.JSONDecodeError:
+                            logger.bind(tag=TAG).warning("收到无效的JSON消息")
+                    elif isinstance(msg, (bytes, bytearray)):
+                        self.opus_encoder.encode_pcm_to_opus_stream(
+                            msg, False, callback=self.handle_opus
+                        )
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).warning("WebSocket连接已关闭")
+                    break
+                except Exception as e:
+                    logger.bind(tag=TAG).error(
+                        f"处理TTS响应时出错: {e}\n{traceback.format_exc()}"
+                    )
+                    break
+
+            # 仅在连接异常且非正常结束时才关闭连接
+            if not session_finished and self.ws:
+                try:
+                    await self.ws.close()
+                except:
+                    pass
+                self.ws = None
+        # 监听任务退出时清理引用
+        finally:
+            self._monitor_task = None
+
+    def to_tts(self, text: str) -> list:
+        """非流式生成音频数据，用于生成音频及测试场景"""
+        try:
+            # 创建事件循环
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+            # 生成会话ID
+            session_id = uuid.uuid4().hex
+            # 存储音频数据
+            audio_data = []
+
+            async def _generate_audio():
+                ws = await websockets.connect(
+                    self.ws_url,
+                    additional_headers=self.header,
+                    ping_interval=30,
+                    ping_timeout=10,
+                    close_timeout=10,
+                    max_size=10 * 1024 * 1024,
+                )
+
+                try:
+                    # 发送run-task消息启动会话
+                    run_task_message = {
+                        "header": {
+                            "action": "run-task",
+                            "task_id": session_id,
+                            "streaming": "duplex",
+                        },
+                        "payload": {
+                            "task_group": "audio",
+                            "task": "tts",
+                            "function": "SpeechSynthesizer",
+                            "model": self.model,
+                            "parameters": {
+                                "text_type": "PlainText",
+                                "voice": self.voice,
+                                "format": self.format,
+                                "sample_rate": self.sample_rate,
+                                "volume": self.volume,
+                                "rate": self.rate,
+                                "pitch": self.pitch,
+                            },
+                            "input": {}
+                        },
+                    }
+                    await ws.send(json.dumps(run_task_message))
+
+                    # 等待任务启动
+                    task_started = False
+                    while not task_started:
+                        msg = await ws.recv()
+                        if isinstance(msg, str):
+                            data = json.loads(msg)
+                            header = data.get("header", {})
+                            if header.get("event") == "task-started":
+                                task_started = True
+                                logger.bind(tag=TAG).debug("TTS任务已启动")
+                            elif header.get("event") == "task-failed":
+                                error_code = header.get("error_code", "unknown")
+                                error_message = header.get("error_message", "未知错误")
+                                raise Exception(
+                                    f"启动任务失败: {error_code} - {error_message}"
+                                )
+
+                    # 发送文本
+                    filtered_text = MarkdownCleaner.clean_markdown(text)
+                    # 发送continue-task消息
+                    continue_task_message = {
+                        "header": {
+                            "action": "continue-task",
+                            "task_id": session_id,
+                            "streaming": "duplex",
+                        },
+                        "payload": {"input": {"text": filtered_text}},
+                    }
+                    await ws.send(json.dumps(continue_task_message))
+
+                    # 发送finish-task消息
+                    finish_task_message = {
+                        "header": {
+                            "action": "finish-task",
+                            "task_id": session_id,
+                            "streaming": "duplex",
+                        },
+                        "payload": {
+                            "input": {}
+                        }
+                    }
+                    await ws.send(json.dumps(finish_task_message))
+
+                    # 接收音频数据
+                    task_finished = False
+                    while not task_finished:
+                        msg = await ws.recv()
+                        if isinstance(msg, (bytes, bytearray)):
+                            self.opus_encoder.encode_pcm_to_opus_stream(
+                                msg,
+                                end_of_stream=False,
+                                callback=lambda opus: audio_data.append(opus)
+                            )
+                        elif isinstance(msg, str):
+                            data = json.loads(msg)
+                            header = data.get("header", {})
+                            if header.get("event") == "task-finished":
+                                task_finished = True
+                                logger.bind(tag=TAG).debug("TTS任务完成")
+                            elif header.get("event") == "task-failed":
+                                error_code = header.get("error_code", "unknown")
+                                error_message = header.get("error_message", "未知错误")
+                                raise Exception(
+                                    f"合成失败: {error_code} - {error_message}"
+                                )
+
+                finally:
+                    # 清理资源
+                    try:
+                        await ws.close()
+                    except:
+                        pass
+
+            # 运行异步任务
+            loop.run_until_complete(_generate_audio())
+            loop.close()
+
+            return audio_data
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}")
+            return []

main/xiaozhi-server/core/providers/tts/aliyun_stream.py

@@ -1,3 +1,4 @@
+import random
 import uuid
 import json
 import hmac
@@ -131,7 +132,7 @@ class TTSProvider(TTSProviderBase):
         self.last_active_time = None
 
         # 专属tts设置
-        self.message_id = ""
+        self.task_id = uuid.uuid4().hex
 
         # 创建Opus编码器
         self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
@@ -185,9 +186,10 @@ class TTSProvider(TTSProviderBase):
             current_time = time.time()
             if self.ws and current_time - self.last_active_time < 10:
                 # 10秒内才可以复用链接进行连续对话
-                logger.bind(tag=TAG).info(f"使用已有链接...")
+                self.task_id = uuid.uuid4().hex
+                logger.bind(tag=TAG).info(f"使用已有链接..., task_id: {self.task_id}")
                 return self.ws
-            logger.bind(tag=TAG).info("开始建立新连接...")
+            logger.bind(tag=TAG).debug("开始建立新连接...")
 
             self.ws = await websockets.connect(
                 self.ws_url,
@@ -196,7 +198,8 @@ class TTSProvider(TTSProviderBase):
                 ping_timeout=10,
                 close_timeout=10,
             )
-            logger.bind(tag=TAG).info("WebSocket连接建立成功")
+            self.task_id = uuid.uuid4().hex
+            logger.bind(tag=TAG).debug(f"WebSocket连接建立成功, task_id: {self.task_id}")
             self.last_active_time = time.time()
             return self.ws
         except Exception as e:
@@ -224,23 +227,14 @@ class TTSProvider(TTSProviderBase):
                 if message.sentence_type == SentenceType.FIRST:
                     # 初始化参数
                     try:
-                        if not getattr(self.conn, "sentence_id", None):
-                            self.conn.sentence_id = uuid.uuid4().hex
-                            logger.bind(tag=TAG).info(
-                                f"自动生成新的 会话ID: {self.conn.sentence_id}"
-                            )
-
-                        # aliyunStream独有的参数生成
-                        self.message_id = str(uuid.uuid4().hex)
-
-                        logger.bind(tag=TAG).info("开始启动TTS会话...")
+                        logger.bind(tag=TAG).debug("开始启动TTS会话...")
                         future = asyncio.run_coroutine_threadsafe(
-                            self.start_session(self.conn.sentence_id),
+                            self.start_session(self.task_id),
                             loop=self.conn.loop,
                         )
                         future.result()
                         self.before_stop_play_files.clear()
-                        logger.bind(tag=TAG).info("TTS会话启动成功")
+                        logger.bind(tag=TAG).debug("TTS会话启动成功")
 
                     except Exception as e:
                         logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
@@ -271,9 +265,9 @@ class TTSProvider(TTSProviderBase):
                         self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
                 if message.sentence_type == SentenceType.LAST:
                     try:
-                        logger.bind(tag=TAG).info("开始结束TTS会话...")
+                        logger.bind(tag=TAG).debug("开始结束TTS会话...")
                         future = asyncio.run_coroutine_threadsafe(
-                            self.finish_session(self.conn.sentence_id),
+                            self.finish_session(self.task_id),
                             loop=self.conn.loop,
                         )
                         future.result()
@@ -296,8 +290,8 @@ class TTSProvider(TTSProviderBase):
             filtered_text = MarkdownCleaner.clean_markdown(text)
             run_request = {
                 "header": {
-                    "message_id": self.message_id,
-                    "task_id": self.conn.sentence_id,
+                    "message_id": uuid.uuid4().hex,
+                    "task_id": self.task_id,
                     "namespace": "FlowingSpeechSynthesizer",
                     "name": "RunSynthesis",
                     "appkey": self.appkey,
@@ -318,8 +312,8 @@ class TTSProvider(TTSProviderBase):
                 self.ws = None
             raise
 
-    async def start_session(self, session_id):
-        logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
+    async def start_session(self, task_id):
+        logger.bind(tag=TAG).debug("开始会话～～")
         try:
             # 会话开始时检测上个会话的监听状态
             if (
@@ -340,8 +334,8 @@ class TTSProvider(TTSProviderBase):
 
             start_request = {
                 "header": {
-                    "message_id": self.message_id,
-                    "task_id": self.conn.sentence_id,
+                    "message_id": uuid.uuid4().hex,
+                    "task_id": self.task_id,
                     "namespace": "FlowingSpeechSynthesizer",
                     "name": "StartSynthesis",
                     "appkey": self.appkey,
@@ -358,28 +352,28 @@ class TTSProvider(TTSProviderBase):
             }
             await self.ws.send(json.dumps(start_request))
             self.last_active_time = time.time()
-            logger.bind(tag=TAG).info("会话启动请求已发送")
+            logger.bind(tag=TAG).debug("会话启动请求已发送")
         except Exception as e:
             logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
             # 确保清理资源
             await self.close()
             raise
 
-    async def finish_session(self, session_id):
-        logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
+    async def finish_session(self, task_id):
+        logger.bind(tag=TAG).debug(f"关闭会话～～{task_id}")
         try:
             if self.ws:
                 stop_request = {
                     "header": {
-                        "message_id": self.message_id,
-                        "task_id": self.conn.sentence_id,
+                        "message_id": uuid.uuid4().hex,
+                        "task_id": self.task_id,
                         "namespace": "FlowingSpeechSynthesizer",
                         "name": "StopSynthesis",
                         "appkey": self.appkey,
                     }
                 }
                 await self.ws.send(json.dumps(stop_request))
-                logger.bind(tag=TAG).info("会话结束请求已发送")
+                logger.bind(tag=TAG).debug("会话结束请求已发送")
                 self.last_active_time = time.time()
                 if self._monitor_task:
                     try:
@@ -457,7 +451,6 @@ class TTSProvider(TTSProviderBase):
                             logger.bind(tag=TAG).warning("收到无效的JSON消息")
                     # 二进制消息（音频数据）
                     elif isinstance(msg, (bytes, bytearray)):
-                        logger.bind(tag=TAG).debug(f"推送数据到队列里面～～")
                         self.opus_encoder.encode_pcm_to_opus_stream(msg, False, self.handle_opus)
                 except websockets.ConnectionClosed:
                     logger.bind(tag=TAG).warning("WebSocket连接已关闭")
@@ -485,8 +478,6 @@ class TTSProvider(TTSProviderBase):
             loop = asyncio.new_event_loop()
             asyncio.set_event_loop(loop)
 
-            # 生成会话ID
-            session_id = uuid.uuid4().hex
             # 存储音频数据
             audio_data = []
 
@@ -505,11 +496,10 @@ class TTSProvider(TTSProviderBase):
                 )
                 try:
                     # 发送StartSynthesis请求
-                    start_message_id = str(uuid.uuid4().hex)
                     start_request = {
                         "header": {
-                            "message_id": start_message_id,
-                            "task_id": session_id,
+                            "message_id": uuid.uuid4().hex,
+                            "task_id": self.task_id,
                             "namespace": "FlowingSpeechSynthesizer",
                             "name": "StartSynthesis",
                             "appkey": self.appkey,
@@ -550,11 +540,10 @@ class TTSProvider(TTSProviderBase):
 
                     # 发送文本合成请求
                     filtered_text = MarkdownCleaner.clean_markdown(text)
-                    run_message_id = str(uuid.uuid4().hex)
                     run_request = {
                         "header": {
-                            "message_id": run_message_id,
-                            "task_id": session_id,
+                            "message_id": uuid.uuid4().hex,
+                            "task_id": self.task_id,
                             "namespace": "FlowingSpeechSynthesizer",
                             "name": "RunSynthesis",
                             "appkey": self.appkey,
@@ -564,11 +553,10 @@ class TTSProvider(TTSProviderBase):
                     await ws.send(json.dumps(run_request))
 
                     # 发送停止合成请求
-                    stop_message_id = str(uuid.uuid4().hex)
                     stop_request = {
                         "header": {
-                            "message_id": stop_message_id,
-                            "task_id": session_id,
+                            "message_id": uuid.uuid4().hex,
+                            "task_id": self.task_id,
                             "namespace": "FlowingSpeechSynthesizer",
                             "name": "StopSynthesis",
                             "appkey": self.appkey,

main/xiaozhi-server/core/providers/tts/base.py

@@ -395,14 +395,6 @@ class TTSProviderBase(ABC):
 
                 # 收到下一个文本开始或会话结束时进行上报
                 if sentence_type is not SentenceType.MIDDLE:
-                    # 重置音频流控状态（新句子开始或者结束）
-                    if hasattr(self.conn, 'audio_flow_control'):
-                        self.conn.audio_flow_control = {
-                            'last_send_time': 0,
-                            'packet_count': 0,
-                            'start_time': time.perf_counter()
-                        }
-                    
                     # 上报TTS数据
                     if enqueue_text is not None and enqueue_audio is not None:
                         enqueue_tts_report(self.conn, enqueue_text, enqueue_audio)

main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py

@@ -149,6 +149,7 @@ class TTSProvider(TTSProviderBase):
         self.access_token = config.get("access_token")
         self.cluster = config.get("cluster")
         self.resource_id = config.get("resource_id")
+        self.activate_session = False
         if config.get("private_voice"):
             self.voice = config.get("private_voice")
         else:
@@ -162,7 +163,8 @@ class TTSProvider(TTSProviderBase):
         self.ws_url = config.get("ws_url")
         self.authorization = config.get("authorization")
         self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
-        self.enable_two_way = True
+        enable_ws_reuse_value = config.get("enable_ws_reuse", True)
+        self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() in ('false', 'False') else True
         self.tts_text = ""
         self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
             sample_rate=16000, channels=1, frame_size_ms=60
@@ -180,12 +182,18 @@ class TTSProvider(TTSProviderBase):
             raise
 
     async def _ensure_connection(self):
-        """建立新的WebSocket连接"""
+        """建立新的WebSocket连接，并启动监听任务（仅第一次）"""
         try:
             if self.ws:
-                logger.bind(tag=TAG).info(f"使用已有链接...")
-                return self.ws
-            logger.bind(tag=TAG).info("开始建立新连接...")
+                if self.enable_ws_reuse:
+                    logger.bind(tag=TAG).info(f"使用已有链接...")
+                    return self.ws
+                else:
+                    try:
+                        await self.finish_connection()
+                    except:
+                        pass
+            logger.bind(tag=TAG).debug("开始建立新连接...")
             ws_header = {
                 "X-Api-App-Key": self.appId,
                 "X-Api-Access-Key": self.access_token,
@@ -195,12 +203,34 @@ class TTSProvider(TTSProviderBase):
             self.ws = await websockets.connect(
                 self.ws_url, additional_headers=ws_header, max_size=1000000000
             )
-            logger.bind(tag=TAG).info("WebSocket连接建立成功")
+            logger.bind(tag=TAG).debug("WebSocket连接建立成功")
+            
+            # 连接建立成功后，启动监听任务
+            if self._monitor_task is None or self._monitor_task.done():
+                logger.bind(tag=TAG).debug("启动监听任务...")
+                self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
+            
             return self.ws
         except Exception as e:
             logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}")
             self.ws = None
             raise
+    
+    async def finish_connection(self):
+        """发送 FinishConnection 事件，等待服务端返回 EVENT_ConnectionFinished"""
+        try:
+            if self.ws:
+                logger.bind(tag=TAG).debug("开始关闭连接...")
+                header = Header(
+                    message_type=FULL_CLIENT_REQUEST,
+                    message_type_specific_flags=MsgTypeFlagWithEvent,
+                    serial_method=JSON,
+                ).as_bytes()
+                optional = Optional(event=EVENT_FinishConnection).as_bytes()
+                payload = str.encode("{}")
+                await self.send_event(self.ws, header, optional, payload)
+        except:
+            pass
 
     def tts_text_priority_thread(self):
         """火山引擎双流式TTS的文本处理线程"""
@@ -217,10 +247,16 @@ class TTSProvider(TTSProviderBase):
                 if self.conn.client_abort:
                     try:
                         logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
-                        asyncio.run_coroutine_threadsafe(
-                            self.cancel_session(self.conn.sentence_id),
-                            loop=self.conn.loop,
-                        )
+                        if self.enable_ws_reuse:
+                            asyncio.run_coroutine_threadsafe(
+                                self.cancel_session(self.conn.sentence_id),
+                                loop=self.conn.loop,
+                            )
+                        else:
+                            asyncio.run_coroutine_threadsafe(
+                                self.finish_connection(),
+                                loop=self.conn.loop,
+                            )
                         continue
                     except Exception as e:
                         logger.bind(tag=TAG).error(f"取消TTS会话失败: {str(e)}")
@@ -231,16 +267,16 @@ class TTSProvider(TTSProviderBase):
                     try:
                         if not getattr(self.conn, "sentence_id", None): 
                             self.conn.sentence_id = uuid.uuid4().hex
-                            logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}")
+                            logger.bind(tag=TAG).debug(f"自动生成新的 会话ID: {self.conn.sentence_id}")
 
-                        logger.bind(tag=TAG).info("开始启动TTS会话...")
+                        logger.bind(tag=TAG).debug("开始启动TTS会话...")
                         future = asyncio.run_coroutine_threadsafe(
                             self.start_session(self.conn.sentence_id),
                             loop=self.conn.loop,
                         )
                         future.result()
                         self.before_stop_play_files.clear()
-                        logger.bind(tag=TAG).info("TTS会话启动成功")
+                        logger.bind(tag=TAG).debug("TTS会话启动成功")
                     except Exception as e:
                         logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
                         continue
@@ -270,7 +306,7 @@ class TTSProvider(TTSProviderBase):
                         self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
                 if message.sentence_type == SentenceType.LAST:
                     try:
-                        logger.bind(tag=TAG).info("开始结束TTS会话...")
+                        logger.bind(tag=TAG).debug("开始结束TTS会话...")
                         future = asyncio.run_coroutine_threadsafe(
                             self.finish_session(self.conn.sentence_id),
                             loop=self.conn.loop,
@@ -313,23 +349,25 @@ class TTSProvider(TTSProviderBase):
             raise
 
     async def start_session(self, session_id):
-        logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
-        try:
-            # 会话开始时检测上个会话的监听状态
-            if (
-                self._monitor_task is not None
-                and isinstance(self._monitor_task, Task)
-                and not self._monitor_task.done()
-            ):
-                logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务和连接...")
+        logger.bind(tag=TAG).debug(f"开始会话～～{session_id}")
+        try:       
+            # 等待上一个会话结束，最多等待3次
+            for _ in range(3):
+                if not self.activate_session:
+                    break
+                logger.bind(tag=TAG).debug(f"等待上一个会话结束...")
+                await asyncio.sleep(0.1)
+            else:
+                # 等待超时，强制清除连接状态
+                logger.bind(tag=TAG).debug("等待上一个会话超时，清除连接状态...")
                 await self.close()
-
-            # 建立新连接
+            
+            # 设置会话激活标志
+            self.activate_session = True
+            
+            # 确保连接建立
             await self._ensure_connection()
 
-            # 启动监听任务
-            self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
-
             header = Header(
                 message_type=FULL_CLIENT_REQUEST,
                 message_type_specific_flags=MsgTypeFlagWithEvent,
@@ -342,7 +380,7 @@ class TTSProvider(TTSProviderBase):
                 event=EVENT_StartSession, speaker=self.voice
             )
             await self.send_event(self.ws, header, optional, payload)
-            logger.bind(tag=TAG).info("会话启动请求已发送")
+            logger.bind(tag=TAG).debug("会话启动请求已发送")
         except Exception as e:
             logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
             # 确保清理资源
@@ -350,7 +388,7 @@ class TTSProvider(TTSProviderBase):
             raise
 
     async def finish_session(self, session_id):
-        logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
+        logger.bind(tag=TAG).debug(f"关闭会话～～{session_id}")
         try:
             if self.ws:
                 header = Header(
@@ -363,18 +401,7 @@ class TTSProvider(TTSProviderBase):
                 ).as_bytes()
                 payload = str.encode("{}")
                 await self.send_event(self.ws, header, optional, payload)
-                logger.bind(tag=TAG).info("会话结束请求已发送")
-
-                # 等待监听任务完成
-                if self._monitor_task:
-                    try:
-                        await self._monitor_task
-                    except Exception as e:
-                        logger.bind(tag=TAG).error(
-                            f"等待监听任务完成时发生错误: {str(e)}"
-                        )
-                    finally:
-                        self._monitor_task = None
+                logger.bind(tag=TAG).debug("会话结束请求已发送")
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
@@ -383,7 +410,7 @@ class TTSProvider(TTSProviderBase):
             raise
 
     async def cancel_session(self,session_id):
-        logger.bind(tag=TAG).info(f"取消会话，释放服务端资源～～{session_id}")
+        logger.bind(tag=TAG).debug(f"取消会话，释放服务端资源～～{session_id}")
         try:
             if self.ws:
                 header = Header(
@@ -396,7 +423,7 @@ class TTSProvider(TTSProviderBase):
                 ).as_bytes()
                 payload = str.encode("{}")
                 await self.send_event(self.ws, header, optional, payload)
-                logger.bind(tag=TAG).info("会话取消请求已发送")
+                logger.bind(tag=TAG).debug("会话取消请求已发送")
         except Exception as e:
             logger.bind(tag=TAG).error(f"取消会话失败: {str(e)}")
             # 确保清理资源
@@ -405,6 +432,7 @@ class TTSProvider(TTSProviderBase):
 
     async def close(self):
         """资源清理方法"""
+        self.activate_session = False
         # 取消监听任务
         if self._monitor_task:
             try:
@@ -424,9 +452,8 @@ class TTSProvider(TTSProviderBase):
             self.ws = None
 
     async def _start_monitor_tts_response(self):
-        """监听TTS响应"""
+        """监听TTS响应 - 长期运行"""
         try:
-            session_finished = False  # 标记会话是否正常结束
             while not self.conn.stop_event.is_set():
                 try:
                     # 确保 `recv()` 运行在同一个 event loop
@@ -434,10 +461,22 @@ class TTSProvider(TTSProviderBase):
                     res = self.parser_response(msg)
                     self.print_response(res, "send_text res:")
 
+                    # 优先处理连接级别事件
+                    if res.optional.event == EVENT_ConnectionFinished:
+                        logger.bind(tag=TAG).debug(f"链接关闭成功～～")
+                        break
+
+                    # 只处理当前活跃会话的响应
+                    if res.optional.sessionId and self.conn.sentence_id != res.optional.sessionId:
+                        # 如果是会话结束相关事件，即使会话ID不匹配也要重置状态
+                        if res.optional.event in [EVENT_SessionCanceled, EVENT_SessionFailed, EVENT_SessionFinished]:
+                            logger.bind(tag=TAG).debug(f"收到残余下行结束响应重置会话状态～～")
+                            self.activate_session = False
+                        continue
+
                     if res.optional.event == EVENT_SessionCanceled:
                         logger.bind(tag=TAG).debug(f"释放服务端资源成功～～")
-                        session_finished = True
-                        break
+                        self.activate_session = False
                     elif res.optional.event == EVENT_TTSSentenceStart:
                         json_data = json.loads(res.payload.decode("utf-8"))
                         self.tts_text = json_data.get("text", "")
@@ -449,15 +488,16 @@ class TTSProvider(TTSProviderBase):
                         res.optional.event == EVENT_TTSResponse
                         and res.header.message_type == AUDIO_ONLY_RESPONSE
                     ):
-                        logger.bind(tag=TAG).debug(f"推送数据到队列里面～～")
                         self.wav_to_opus_data_audio_raw_stream(res.payload, callback=self.handle_opus)
                     elif res.optional.event == EVENT_TTSSentenceEnd:
                         logger.bind(tag=TAG).info(f"句子语音生成成功：{self.tts_text}")
                     elif res.optional.event == EVENT_SessionFinished:
                         logger.bind(tag=TAG).debug(f"会话结束～～")
+                        self.activate_session = False
                         self._process_before_stop_play_files()
-                        session_finished = True
-                        break
+                        # 非复用模式下，会话结束后发送 FinishConnection
+                        if not self.enable_ws_reuse:
+                            await self.finish_connection()
                 except websockets.ConnectionClosed:
                     logger.bind(tag=TAG).warning("WebSocket连接已关闭")
                     break
@@ -467,8 +507,8 @@ class TTSProvider(TTSProviderBase):
                     )
                     traceback.print_exc()
                     break
-            # 仅在连接异常时才关闭
-            if not session_finished and self.ws:
+            # 连接异常时关闭WebSocket
+            if self.ws:
                 try:
                     await self.ws.close()
                 except:
@@ -476,6 +516,7 @@ class TTSProvider(TTSProviderBase):
                 self.ws = None
         # 监听任务退出时清理引用
         finally:
+            self.activate_session = False
             self._monitor_task = None
 
     async def send_event(
@@ -514,7 +555,7 @@ class TTSProvider(TTSProviderBase):
     def read_res_content(self, res: bytes, offset: int):
         content_size = int.from_bytes(res[offset : offset + 4], "big", signed=True)
         offset += 4
-        content = str(res[offset : offset + content_size])
+        content = res[offset : offset + content_size].decode('utf-8')
         offset += content_size
         return content, offset
 
@@ -616,12 +657,13 @@ class TTSProvider(TTSProviderBase):
                             "speech_rate": self.speech_rate,
                             "loudness_rate": self.loudness_rate
                         },
+                        "additions": json.dumps({
+                            "post_process": {
+                                "pitch": self.pitch
+                            }
+                        })
                     },
-                    "additions": {
-                        "post_process": {
-                            "pitch": self.pitch
-                        }
-                    }
+
                 }
             )
         )

main/xiaozhi-server/core/providers/tts/index_stream.py

@@ -190,7 +190,7 @@ class TTSProvider(TTSProviderBase):
         start_time = time.time()
         text = MarkdownCleaner.clean_markdown(text)
 
-        payload = {"text": text, "character": self.character}
+        payload = {"text": text, "character": self.voice}
 
         try:
             with requests.post(self.api_url, json=payload, timeout=5) as response:

main/xiaozhi-server/core/providers/tts/minimax.py

@@ -1,95 +0,0 @@
-import os
-import uuid
-import json
-import requests
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-from core.utils.util import parse_string_to_list
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.group_id = config.get("group_id")
-        self.api_key = config.get("api_key")
-        self.model = config.get("model")
-        if config.get("private_voice"):
-            self.voice = config.get("private_voice")
-        else:
-            self.voice = config.get("voice_id")
-
-        default_voice_setting = {
-            "voice_id": "female-shaonv",
-            "speed": 1,
-            "vol": 1,
-            "pitch": 0,
-            "emotion": "happy",
-        }
-        default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}
-        defult_audio_setting = {
-            "sample_rate": 32000,
-            "bitrate": 128000,
-            "format": "mp3",
-            "channel": 1,
-        }
-        self.voice_setting = {
-            **default_voice_setting,
-            **config.get("voice_setting", {}),
-        }
-        self.pronunciation_dict = {
-            **default_pronunciation_dict,
-            **config.get("pronunciation_dict", {}),
-        }
-        self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
-        self.timber_weights = parse_string_to_list(config.get("timber_weights"))
-
-        if self.voice:
-            self.voice_setting["voice_id"] = self.voice
-
-        self.host = "api.minimax.chat"
-        self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
-        self.header = {
-            "Content-Type": "application/json",
-            "Authorization": f"Bearer {self.api_key}",
-        }
-        self.audio_file_type = defult_audio_setting.get("format", "mp3")
-
-    def generate_filename(self, extension=".mp3"):
-        return os.path.join(
-            self.output_file,
-            f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
-        )
-
-    async def text_to_speak(self, text, output_file):
-        request_json = {
-            "model": self.model,
-            "text": text,
-            "stream": False,
-            "voice_setting": self.voice_setting,
-            "pronunciation_dict": self.pronunciation_dict,
-            "audio_setting": self.audio_setting,
-        }
-
-        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
-            request_json["timber_weights"] = self.timber_weights
-            request_json["voice_setting"]["voice_id"] = ""
-
-        try:
-            resp = requests.post(
-                self.api_url, json.dumps(request_json), headers=self.header
-            )
-            # 检查返回请求数据的status_code是否为0
-            if resp.json()["base_resp"]["status_code"] == 0:
-                data = resp.json()["data"]["audio"]
-                audio_bytes = bytes.fromhex(data)
-                if output_file:
-                    with open(output_file, "wb") as file_to_save:
-                        file_to_save.write(audio_bytes)
-                else:
-                    return audio_bytes
-            else:
-                raise Exception(
-                    f"{__name__} status_code: {resp.status_code} response: {resp.content}"
-                )
-        except Exception as e:
-            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/minimax_httpstream.py

@@ -1,11 +1,20 @@
 import os
-import uuid
 import json
+import time
+import queue
+import asyncio
+import aiohttp
 import requests
-from datetime import datetime
-from typing import Iterator, Optional, Union
-from core.providers.tts.base import TTSProviderBase
+import traceback
+from config.logger import setup_logging
+from core.utils.tts import MarkdownCleaner
 from core.utils.util import parse_string_to_list
+from core.providers.tts.base import TTSProviderBase
+from core.utils import opus_encoder_utils, textUtils
+from core.providers.tts.dto.dto import SentenceType, ContentType
+
+TAG = __name__
+logger = setup_logging()
 
 
 class TTSProvider(TTSProviderBase):
@@ -28,9 +37,9 @@ class TTSProvider(TTSProviderBase):
         }
         default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}
         defult_audio_setting = {
-            "sample_rate": 32000,
+            "sample_rate": 24000,
             "bitrate": 128000,
-            "format": "mp3",
+            "format": "pcm",
             "channel": 1,
         }
         self.voice_setting = {
@@ -47,66 +56,101 @@ class TTSProvider(TTSProviderBase):
         if self.voice:
             self.voice_setting["voice_id"] = self.voice
 
-        self.host = "api.minimax.chat"
+        self.host = "api.minimaxi.com"  # 备用地址：api-bj.minimaxi.com
         self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
         self.header = {
             "Content-Type": "application/json",
             "Authorization": f"Bearer {self.api_key}",
         }
-        self.audio_file_type = defult_audio_setting.get("format", "mp3")
+        self.audio_file_type = defult_audio_setting.get("format", "pcm")
 
-    def generate_filename(self, extension=".mp3"):
-        return os.path.join(
-            self.output_file,
-            f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
+        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
+            sample_rate=24000, channels=1, frame_size_ms=60
         )
 
-    async def text_to_speak(self, text, output_file):
-        """非流式语音合成（保留原有实现）"""
-        request_json = {
-            "model": self.model,
-            "text": text,
-            "stream": False,
-            "voice_setting": self.voice_setting,
-            "pronunciation_dict": self.pronunciation_dict,
-            "audio_setting": self.audio_setting,
-        }
+        # PCM缓冲区
+        self.pcm_buffer = bytearray()
 
-        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
-            request_json["timber_weights"] = self.timber_weights
-            request_json["voice_setting"]["voice_id"] = ""
+    def tts_text_priority_thread(self):
+        """流式文本处理线程"""
+        while not self.conn.stop_event.is_set():
+            try:
+                message = self.tts_text_queue.get(timeout=1)
+                if message.sentence_type == SentenceType.FIRST:
+                    # 初始化参数
+                    self.tts_stop_request = False
+                    self.processed_chars = 0
+                    self.tts_text_buff = []
+                    self.before_stop_play_files.clear()
+                elif ContentType.TEXT == message.content_type:
+                    self.tts_text_buff.append(message.content_detail)
+                    segment_text = self._get_segment_text()
+                    if segment_text:
+                        self.to_tts_single_stream(segment_text)
 
-        try:
-            resp = requests.post(
-                self.api_url, json.dumps(request_json), headers=self.header
-            )
-            if resp.json()["base_resp"]["status_code"] == 0:
-                data = resp.json()["data"]["audio"]
-                audio_bytes = bytes.fromhex(data)
-                if output_file:
-                    with open(output_file, "wb") as file_to_save:
-                        file_to_save.write(audio_bytes)
-                else:
-                    return audio_bytes
+                elif ContentType.FILE == message.content_type:
+                    logger.bind(tag=TAG).info(
+                        f"添加音频文件到待播放列表: {message.content_file}"
+                    )
+                    if message.content_file and os.path.exists(message.content_file):
+                        # 先处理文件音频数据
+                        self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
+                if message.sentence_type == SentenceType.LAST:
+                    # 处理剩余的文本
+                    self._process_remaining_text_stream(True)
+
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
+                )
+
+    def _process_remaining_text_stream(self, is_last=False):
+        """处理剩余的文本并生成语音
+        Returns:
+            bool: 是否成功处理了文本
+        """
+        full_text = "".join(self.tts_text_buff)
+        remaining_text = full_text[self.processed_chars :]
+        if remaining_text:
+            segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text)
+            if segment_text:
+                self.to_tts_single_stream(segment_text, is_last)
+                self.processed_chars += len(full_text)
             else:
-                raise Exception(
-                    f"{__name__} status_code: {resp.status_code} response: {resp.content}"
+                self._process_before_stop_play_files()
+        else:
+            self._process_before_stop_play_files()
+
+    def to_tts_single_stream(self, text, is_last=False):
+        try:
+            max_repeat_time = 5
+            text = MarkdownCleaner.clean_markdown(text)
+            try:
+                asyncio.run(self.text_to_speak(text, is_last))
+            except Exception as e:
+                logger.bind(tag=TAG).warning(
+                    f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}"
+                )
+                max_repeat_time -= 1
+
+            if max_repeat_time > 0:
+                logger.bind(tag=TAG).info(
+                    f"语音生成成功: {text}，重试{5 - max_repeat_time}次"
+                )
+            else:
+                logger.bind(tag=TAG).error(
+                    f"语音生成失败: {text}，请检查网络或服务是否正常"
                 )
         except Exception as e:
-            raise Exception(f"{__name__} error: {e}")
+            logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
+        finally:
+            return None
 
-    def text_to_speak_stream(
-        self, 
-        text: str, 
-        chunk_callback: Optional[callable] = None
-    ) -> Iterator[bytes]:
-        """
-        流式语音合成方法
-        :param text: 要合成的文本
-        :param chunk_callback: 可选的回调函数，用于处理每个音频块
-        :return: 生成器，每次产生一个音频数据块(bytes)
-        """
-        request_json = {
+    async def text_to_speak(self, text, is_last):
+        """流式处理TTS音频，每句只推送一次音频列表"""
+        payload = {
             "model": self.model,
             "text": text,
             "stream": True,
@@ -115,116 +159,183 @@ class TTSProvider(TTSProviderBase):
             "audio_setting": self.audio_setting,
         }
 
-        if isinstance(self.timber_weights, list) and len(self.timber_weights) > 0:
-            request_json["timber_weights"] = self.timber_weights
-            request_json["voice_setting"]["voice_id"] = ""
+        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
+            payload["timber_weights"] = self.timber_weights
+            payload["voice_setting"]["voice_id"] = ""
+
+        frame_bytes = int(
+            self.opus_encoder.sample_rate
+            * self.opus_encoder.channels  # 1
+            * self.opus_encoder.frame_size_ms
+            / 1000
+            * 2
+        )  # 16-bit = 2 bytes
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(
+                    self.api_url,
+                    headers=self.header,
+                    data=json.dumps(payload),
+                    timeout=10,
+                ) as resp:
+
+                    if resp.status != 200:
+                        logger.bind(tag=TAG).error(
+                            f"TTS请求失败: {resp.status}, {await resp.text()}"
+                        )
+                        self.tts_audio_queue.put((SentenceType.LAST, [], None))
+                        return
+
+                    self.pcm_buffer.clear()
+                    self.tts_audio_queue.put((SentenceType.FIRST, [], text))
+
+                    # 处理音频流数据
+                    buffer = b""
+                    async for chunk in resp.content.iter_any():
+                        if not chunk:
+                            continue
+
+                        buffer += chunk
+                        while True:
+                            # 查找数据块分隔符
+                            header_pos = buffer.find(b"data: ")
+                            if header_pos == -1:
+                                break
+
+                            end_pos = buffer.find(b"\n\n", header_pos)
+                            if end_pos == -1:
+                                break
+
+                            # 提取单个完整JSON块
+                            json_str = buffer[header_pos + 6 : end_pos].decode("utf-8")
+                            buffer = buffer[end_pos + 2 :]
+
+                            try:
+                                data = json.loads(json_str)
+                                status = data.get("data", {}).get("status", 1)
+                                audio_hex = data.get("data", {}).get("audio")
+
+                                # 仅处理status=1的有效音频块 忽略status=2的结束汇总块
+                                if status == 1 and audio_hex:
+                                    pcm_data = bytes.fromhex(audio_hex)
+                                    self.pcm_buffer.extend(pcm_data)
+
+                            except json.JSONDecodeError as e:
+                                logger.bind(tag=TAG).error(f"JSON解析失败: {e}")
+                                continue
+
+                        while len(self.pcm_buffer) >= frame_bytes:
+                            frame = bytes(self.pcm_buffer[:frame_bytes])
+                            del self.pcm_buffer[:frame_bytes]
+
+                            self.opus_encoder.encode_pcm_to_opus_stream(
+                                frame, end_of_stream=False, callback=self.handle_opus
+                            )
+
+                    # flush 剩余不足一帧的数据
+                    if self.pcm_buffer:
+                        self.opus_encoder.encode_pcm_to_opus_stream(
+                            bytes(self.pcm_buffer),
+                            end_of_stream=True,
+                            callback=self.handle_opus,
+                        )
+                        self.pcm_buffer.clear()
+
+                    # 如果是最后一段，输出音频获取完毕
+                    if is_last:
+                        self._process_before_stop_play_files()
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
+            self.tts_audio_queue.put((SentenceType.LAST, [], None))
+
+    async def close(self):
+        """资源清理"""
+        await super().close()
+        if hasattr(self, "opus_encoder"):
+            self.opus_encoder.close()
+
+    def to_tts(self, text: str) -> list:
+        """非流式TTS处理，用于测试及保存音频文件的场景
+        Args:
+            text: 要转换的文本
+        Returns:
+            list: 返回opus编码后的音频数据列表
+        """
+        start_time = time.time()
+        text = MarkdownCleaner.clean_markdown(text)
+
+        payload = {
+            "model": self.model,
+            "text": text,
+            "stream": True,
+            "voice_setting": self.voice_setting,
+            "pronunciation_dict": self.pronunciation_dict,
+            "audio_setting": self.audio_setting,
+        }
+
+        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
+            payload["timber_weights"] = self.timber_weights
+            payload["voice_setting"]["voice_id"] = ""
+
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}",
+        }
 
         try:
             with requests.post(
-                self.api_url,
-                data=json.dumps(request_json),
-                headers=self.header,
-                stream=True
+                self.api_url, data=json.dumps(payload), headers=headers, timeout=5
             ) as response:
-                
-                # 检查HTTP状态码
                 if response.status_code != 200:
-                    raise Exception(
-                        f"HTTP error: {response.status_code}, response: {response.text}"
+                    logger.bind(tag=TAG).error(
+                        f"TTS请求失败: {response.status_code}, {response.text}"
                     )
-                
-                # 处理流式响应
-                for line in response.iter_lines():
-                    if line:  # 过滤空行
-                        # 检查是否为数据行 (SSE格式)
-                        if line.startswith(b'data:'):
-                            try:
-                                data = json.loads(line[5:].strip())  # 去掉"data:"前缀
-                                
-                                # 检查API状态码
-                                if data.get("base_resp", {}).get("status_code", -1) != 0:
-                                    raise Exception(
-                                        f"API error: {data.get('base_resp', {}).get('status_msg')}"
-                                    )
-                                
-                                # 跳过非音频数据块
-                                if "extra_info" in data:
-                                    continue
-                                    
-                                # 提取音频数据
-                                audio_hex = data.get("data", {}).get("audio")
-                                if audio_hex:
-                                    audio_chunk = bytes.fromhex(audio_hex)
-                                    if chunk_callback:
-                                        chunk_callback(audio_chunk)
-                                    yield audio_chunk
-                                    
-                            except json.JSONDecodeError:
-                                # 忽略JSON解析错误（可能是心跳包等）
-                                continue
-                            except Exception as e:
-                                raise e
-                                
-        except Exception as e:
-            raise Exception(f"{__name__} stream error: {e}")
+                    return []
 
-    def save_stream_to_file(
-        self, 
-        text: str, 
-        output_file: Optional[str] = None,
-        progress_callback: Optional[callable] = None
-    ) -> str:
-        """
-        流式合成并保存到文件
-        :param text: 要合成的文本
-        :param output_file: 输出文件路径，如果为None则自动生成
-        :param progress_callback: 可选的回调函数，接收已写入的字节数
-        :return: 保存的文件路径
-        """
-        if not output_file:
-            output_file = self.generate_filename(extension=f".{self.audio_file_type}")
-        
-        os.makedirs(os.path.dirname(output_file), exist_ok=True)
-        
-        total_bytes = 0
-        try:
-            with open(output_file, "wb") as audio_file:
-                for audio_chunk in self.text_to_speak_stream(text):
-                    audio_file.write(audio_chunk)
-                    audio_file.flush()
-                    total_bytes += len(audio_chunk)
-                    if progress_callback:
-                        progress_callback(total_bytes)
-            return output_file
-        except Exception as e:
-            # 清理可能创建的不完整文件
-            if os.path.exists(output_file):
-                os.remove(output_file)
-            raise e
+                logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒")
+
+                # 使用opus编码器处理PCM数据
+                opus_datas = []
+                full_content = response.content.decode('utf-8')
+                pcm_data = bytearray()
+                for data_block in full_content.split('\n\n'):
+                    if not data_block.startswith('data: '):
+                        continue
+
+                    try:
+                        json_str = data_block[6:]  # 去除'data: '前缀
+                        data = json.loads(json_str)
+                        if data.get('data', {}).get('status') == 1:
+                            audio_hex = data['data']['audio']
+                            pcm_data.extend(bytes.fromhex(audio_hex))
+                    except (json.JSONDecodeError, KeyError) as e:
+                        logger.bind(tag=TAG).warning(f"无效数据块: {e}")
+                        continue
+
+                # 计算每帧的字节数
+                frame_bytes = int(
+                    self.opus_encoder.sample_rate
+                    * self.opus_encoder.channels
+                    * self.opus_encoder.frame_size_ms
+                    / 1000
+                    * 2
+                )
+
+                # 分帧处理合并后的PCM数据
+                for i in range(0, len(pcm_data), frame_bytes):
+                    frame = bytes(pcm_data[i:i+frame_bytes])
+                    if len(frame) < frame_bytes:
+                        frame += b"\x00" * (frame_bytes - len(frame))
+ 
+                    self.opus_encoder.encode_pcm_to_opus_stream(
+                        frame,
+                        end_of_stream=(i + frame_bytes >= len(pcm_data)),
+                        callback=lambda opus: opus_datas.append(opus)
+                    )
+
+                return opus_datas
 
-    def stream_to_audio_player(self, text: str, player_command: list = None):
-        """
-        流式合成并直接播放音频
-        :param text: 要合成的文本
-        :param player_command: 音频播放器命令，默认使用mpv
-        """
-        if player_command is None:
-            player_command = ["mpv", "--no-cache", "--no-terminal", "--", "fd://0"]
-        
-        try:
-            import subprocess
-            player_process = subprocess.Popen(
-                player_command,
-                stdin=subprocess.PIPE,
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.DEVNULL,
-            )
-            
-            for audio_chunk in self.text_to_speak_stream(text):
-                player_process.stdin.write(audio_chunk)
-                player_process.stdin.flush()
-                
-            player_process.stdin.close()
-            player_process.wait()
         except Exception as e:
-            raise Exception(f"Audio player error: {e}")
+            logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
+            return []

main/xiaozhi-server/core/providers/tts/minimax_webSocket.py

@@ -1,180 +0,0 @@
-import os
-import uuid
-import json
-import asyncio
-import websockets
-import ssl
-from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
-from core.utils.util import parse_string_to_list
-
-
-class TTSProvider(TTSProviderBase):
-    def __init__(self, config, delete_audio_file):
-        super().__init__(config, delete_audio_file)
-        self.group_id = config.get("group_id")
-        self.api_key = config.get("api_key")
-        self.model = config.get("model")
-        
-        # 初始化语音设置
-        default_voice_setting = {
-            "voice_id": "female-shaonv",
-            "speed": 1,
-            "vol": 1,
-            "pitch": 0,
-            "emotion": "happy",
-        }
-        default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}
-        default_audio_setting = {
-            "sample_rate": 32000,
-            "bitrate": 128000,
-            "format": "mp3",
-            "channel": 1,
-        }
-        
-        # 合并配置
-        self.voice_setting = {
-            **default_voice_setting,
-            **config.get("voice_setting", {}),
-        }
-        self.pronunciation_dict = {
-            **default_pronunciation_dict,
-            **config.get("pronunciation_dict", {}),
-        }
-        self.audio_setting = {
-            **default_audio_setting, 
-            **config.get("audio_setting", {})
-        }
-        self.timber_weights = parse_string_to_list(config.get("timber_weights"))
-        
-        # 设置语音ID
-        if config.get("private_voice"):
-            self.voice_setting["voice_id"] = config.get("private_voice")
-        elif config.get("voice_id"):
-            self.voice_setting["voice_id"] = config.get("voice_id")
-        
-        # WebSocket配置
-        self.ws_url = "wss://api.minimaxi.com/ws/v1/t2a_v2"
-        self.headers = {
-            "Authorization": f"Bearer {self.api_key}",
-            "GroupId": self.group_id
-        }
-        self.audio_file_type = self.audio_setting.get("format", "mp3")
-
-    def generate_filename(self, extension=".mp3"):
-        """生成唯一的音频文件名"""
-        return os.path.join(
-            self.output_file,
-            f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
-        )
-
-    async def _establish_connection(self):
-        """建立WebSocket连接"""
-        ssl_context = ssl.create_default_context()
-        ssl_context.check_hostname = False
-        ssl_context.verify_mode = ssl.CERT_NONE
-
-        try:
-            ws = await websockets.connect(
-                self.ws_url,
-                additional_headers=self.headers,
-                ssl=ssl_context
-            )
-            connected = json.loads(await ws.recv())
-            if connected.get("event") == "connected_success":
-                print("连接成功")
-                return ws
-            return None
-        except Exception as e:
-            print(f"连接失败: {e}")
-            return None
-
-    async def _start_task(self, websocket):
-        """发送任务开始请求"""
-        start_msg = {
-            "event": "task_start",
-            "model": self.model,
-            "voice_setting": self.voice_setting,
-            "pronunciation_dict": self.pronunciation_dict,
-            "audio_setting": self.audio_setting
-        }
-        
-        if self.timber_weights and len(self.timber_weights) > 0:
-            start_msg["timber_weights"] = self.timber_weights
-            start_msg["voice_setting"]["voice_id"] = ""
-
-        await websocket.send(json.dumps(start_msg))
-        response = json.loads(await websocket.recv())
-        return response.get("event") == "task_started"
-
-    async def _continue_task(self, websocket, text):
-        """发送继续请求并收集音频数据"""
-        await websocket.send(json.dumps({
-            "event": "task_continue",
-            "text": text
-        }))
-
-        audio_chunks = []
-        while True:
-            response = json.loads(await websocket.recv())
-            if "data" in response and "audio" in response["data"]:
-                audio_chunks.append(response["data"]["audio"])
-            if response.get("is_final"):
-                break
-        return "".join(audio_chunks)
-
-    async def _close_connection(self, websocket):
-        """关闭连接"""
-        if websocket:
-            await websocket.send(json.dumps({"event": "task_finish"}))
-            await websocket.close()
-            print("连接已关闭")
-
-    async def text_to_speak(self, text, output_file=None):
-        """主方法：文本转语音"""
-        ws = await self._establish_connection()
-        if not ws:
-            raise Exception("无法建立WebSocket连接")
-
-        try:
-            if not await self._start_task(ws):
-                raise Exception("任务启动失败")
-
-            hex_audio = await self._continue_task(ws, text)
-            audio_bytes = bytes.fromhex(hex_audio)
-
-            # 保存到文件或返回二进制数据
-            if output_file:
-                with open(output_file, "wb") as f:
-                    f.write(audio_bytes)
-                print(f"音频已保存为{output_file}")
-                return output_file
-            else:
-                # 返回音频二进制数据（不播放）
-                return audio_bytes
-
-        finally:
-            await self._close_connection(ws)
-
-
-async def main():
-    """测试用主函数"""
-    # 示例配置
-    config = {
-        "group_id": "YOUR_GROUP_ID",  # 替换为实际的group_id
-        "api_key": "YOUR_API_KEY",    # 替换为实际的api_key
-        "model": "your-model",        # 替换为实际的模型名称
-        "voice_id": "male-qn-qingse",
-        "voice_setting": {
-            "speed": 1.2,
-            "emotion": "happy"
-        }
-    }
-    
-    tts = TTSProvider(config, delete_audio_file=True)
-    output_file = tts.generate_filename()
-    await tts.text_to_speak("这是一个测试文本，用于验证流式语音合成功能", output_file)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

main/xiaozhi-server/core/providers/tts/xunfei_stream.py

@@ -0,0 +1,527 @@
+import os
+import time
+import uuid
+import json
+import hmac
+import queue
+import base64
+import hashlib
+import asyncio
+import traceback
+import websockets
+from asyncio import Task
+from config.logger import setup_logging
+from core.utils import opus_encoder_utils
+from core.utils.tts import MarkdownCleaner
+from urllib.parse import urlencode, urlparse
+from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+
+class XunfeiWSAuth:
+    @staticmethod
+    def create_auth_url(api_key, api_secret, api_url):
+        """生成讯飞WebSocket认证URL"""
+        parsed_url = urlparse(api_url)
+        host = parsed_url.netloc
+        path = parsed_url.path
+
+        # 获取UTC时间，讯飞要求使用RFC1123格式
+        now = time.gmtime()
+        date = time.strftime('%a, %d %b %Y %H:%M:%S GMT', now)
+
+        # 构造签名字符串
+        signature_origin = f"host: {host}\ndate: {date}\nGET {path} HTTP/1.1"
+
+        # 计算签名
+        signature_sha = hmac.new(
+            api_secret.encode('utf-8'),
+            signature_origin.encode('utf-8'),
+            digestmod=hashlib.sha256
+        ).digest()
+        signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding='utf-8')
+
+        # 构造authorization
+        authorization_origin = f'api_key="{api_key}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha_base64}"'
+        authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8')
+
+        # 构造最终的WebSocket URL
+        v = {
+            "authorization": authorization,
+            "date": date,
+            "host": host
+        }
+        url = api_url + '?' + urlencode(v)
+        return url
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+
+        # 设置为流式接口类型
+        self.interface_type = InterfaceType.DUAL_STREAM
+
+        # 基础配置
+        self.app_id = config.get("app_id")
+        self.api_key = config.get("api_key")
+        self.api_secret = config.get("api_secret")
+
+        # 接口地址
+        self.api_url = config.get("api_url", "wss://cbm01.cn-huabei-1.xf-yun.com/v1/private/mcd9m97e6")
+
+        # 音色配置
+        self.voice = config.get("voice", "x5_lingxiaoxuan_flow")
+        if config.get("private_voice"):
+            self.voice = config.get("private_voice")
+
+        # 音频参数配置
+        speed = config.get("speed", "50")
+        self.speed = int(speed) if speed else 50
+
+        volume = config.get("volume", "50")
+        self.volume = int(volume) if volume else 50
+
+        pitch = config.get("pitch", "50")
+        self.pitch = int(pitch) if pitch else 50
+
+        # 音频编码配置
+        self.format = config.get("format", "raw")
+
+        sample_rate = config.get("sample_rate", "24000")
+        self.sample_rate = int(sample_rate) if sample_rate else 24000
+
+        # 口语化配置
+        self.oral_level = config.get("oral_level", "mid")
+
+        spark_assist = config.get("spark_assist", "1")
+        self.spark_assist = int(spark_assist) if spark_assist else 1
+
+        stop_split = config.get("stop_split", "0")
+        self.stop_split = int(stop_split) if stop_split else 0
+    
+        remain = config.get("remain", "0")
+        self.remain = int(remain) if remain else 0
+
+        # WebSocket配置
+        self.ws = None
+        self._monitor_task = None
+
+        # 序列号管理
+        self.text_seq = 0
+
+        # 创建Opus编码器
+        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
+            sample_rate=self.sample_rate, channels=1, frame_size_ms=60
+        )
+
+        # 验证必需参数
+        if not all([self.app_id, self.api_key, self.api_secret]):
+            raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
+
+    async def _ensure_connection(self):
+        """确保WebSocket连接可用"""
+        try:
+            logger.bind(tag=TAG).info("开始建立新连接...")
+
+            # 生成认证URL
+            auth_url = XunfeiWSAuth.create_auth_url(
+                self.api_key, self.api_secret, self.api_url
+            )
+
+            self.ws = await websockets.connect(
+                auth_url,
+                ping_interval=30,
+                ping_timeout=10,
+                close_timeout=10,
+            )
+            logger.bind(tag=TAG).info("WebSocket连接建立成功")
+            return self.ws
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}")
+            self.ws = None
+            raise
+
+    def tts_text_priority_thread(self):
+        """流式文本处理线程"""
+        while not self.conn.stop_event.is_set():
+            try:
+                message = self.tts_text_queue.get(timeout=1)
+                logger.bind(tag=TAG).debug(
+                    f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}"
+                )
+
+                if message.sentence_type == SentenceType.FIRST:
+                    # 重置序列号
+                    self.text_seq = 0
+                    self.conn.client_abort = False
+                # 增加序列号
+                self.text_seq += 1
+                if self.conn.client_abort:
+                    logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
+                    continue
+
+                if message.sentence_type == SentenceType.FIRST:
+                    # 初始化参数
+                    try:
+                        if not getattr(self.conn, "sentence_id", None):
+                            self.conn.sentence_id = uuid.uuid4().hex
+                            logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}")
+
+                        logger.bind(tag=TAG).info("开始启动TTS会话...")
+                        future = asyncio.run_coroutine_threadsafe(
+                            self.start_session(self.conn.sentence_id),
+                            loop=self.conn.loop,
+                        )
+                        future.result()
+                        self.before_stop_play_files.clear()
+                        logger.bind(tag=TAG).info("TTS会话启动成功")
+
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
+                        continue
+
+                # 处理文本内容
+                if ContentType.TEXT == message.content_type:
+                    if message.content_detail:
+                        try:
+                            logger.bind(tag=TAG).debug(
+                                f"开始发送TTS文本: {message.content_detail}"
+                            )
+                            future = asyncio.run_coroutine_threadsafe(
+                                self.text_to_speak(message.content_detail, None),
+                                loop=self.conn.loop,
+                            )
+                            future.result()
+                            logger.bind(tag=TAG).debug("TTS文本发送成功")
+                        except Exception as e:
+                            logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
+                            # 不使用continue，确保后续处理不被中断
+
+                # 处理文件内容
+                if ContentType.FILE == message.content_type:
+                    logger.bind(tag=TAG).info(
+                        f"添加音频文件到待播放列表: {message.content_file}"
+                    )
+                    if message.content_file and os.path.exists(message.content_file):
+                        # 先处理文件音频数据
+                        self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
+
+                # 处理会话结束
+                if message.sentence_type == SentenceType.LAST:
+                    try:
+                        logger.bind(tag=TAG).info("开始结束TTS会话...")
+                        asyncio.run_coroutine_threadsafe(
+                            self.finish_session(self.conn.sentence_id),
+                            loop=self.conn.loop,
+                        )
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"结束TTS会话失败: {str(e)}")
+                        continue
+
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
+                )
+
+    async def text_to_speak(self, text, _):
+        """发送文本到TTS服务进行合成"""
+        try:
+            if self.ws is None:
+                logger.bind(tag=TAG).warning(f"WebSocket连接不存在，终止发送文本")
+                return
+
+            filtered_text = MarkdownCleaner.clean_markdown(text)
+
+            # 发送文本合成请求
+            run_request = self._build_base_request(status=1,text=filtered_text)
+            await self.ws.send(json.dumps(run_request))
+            return
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
+            if self.ws:
+                try:
+                    await self.ws.close()
+                except:
+                    pass
+                self.ws = None
+            raise
+
+    async def start_session(self, session_id):
+        logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
+        try:
+            # 会话开始时检测上个会话的监听状态
+            if (
+                self._monitor_task is not None
+                and isinstance(self._monitor_task, Task)
+                and not self._monitor_task.done()
+            ):
+                logger.bind(tag=TAG).info(
+                    "检测到未完成的上个会话，关闭监听任务和连接..."
+                )
+                await self.close()
+
+            # 建立新连接
+            await self._ensure_connection()
+
+            # 启动监听任务
+            self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
+
+            # 发送会话启动请求
+            start_request = self._build_base_request(status=0)
+
+            await self.ws.send(json.dumps(start_request))
+            logger.bind(tag=TAG).info("会话启动请求已发送")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
+            # 确保清理资源
+            await self.close()
+            raise
+
+    async def finish_session(self, session_id):
+        logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
+        try:
+            if self.ws:
+                # 发送会话结束请求
+                stop_request = self._build_base_request(status=2)
+                await self.ws.send(json.dumps(stop_request))
+                logger.bind(tag=TAG).info("会话结束请求已发送")
+
+                if self._monitor_task:
+                    try:
+                        await self._monitor_task
+                    except Exception as e:
+                        logger.bind(tag=TAG).error(f"等待监听任务完成时发生错误: {str(e)}")
+                    finally:
+                        self._monitor_task = None
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
+            await self.close()
+            raise
+
+    async def close(self):
+        """资源清理"""
+        if self._monitor_task:
+            try:
+                self._monitor_task.cancel()
+                await self._monitor_task
+            except asyncio.CancelledError:
+                pass
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}")
+            self._monitor_task = None
+
+        if self.ws:
+            try:
+                await self.ws.close()
+            except:
+                pass
+            self.ws = None
+
+    async def _start_monitor_tts_response(self):
+        """监听TTS响应"""
+        try:
+            while not self.conn.stop_event.is_set():
+                try:
+                    msg = await self.ws.recv()
+
+                    # 检查客户端是否中止
+                    if self.conn.client_abort:
+                        logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应")
+                        break
+
+                    try:
+                        data = json.loads(msg)
+                        header = data.get("header", {})
+                        code = header.get("code")
+
+                        if code == 0:
+                            payload = data.get("payload", {})
+                            audio_payload = payload.get("audio", {})
+
+                            if audio_payload:
+                                status = audio_payload.get("status", 0)
+                                audio_data = audio_payload.get("audio", "")
+                                if status == 0:
+                                    logger.bind(tag=TAG).debug("TTS合成已启动")
+                                    self.tts_audio_queue.put(
+                                        (SentenceType.FIRST, [], None)
+                                    )
+                                elif status == 2:
+                                    logger.bind(tag=TAG).debug("收到结束状态的音频数据，TTS合成完成")
+                                    self._process_before_stop_play_files()
+                                    break
+                                else:
+                                    if self.conn.tts_MessageText:
+                                        logger.bind(tag=TAG).info(
+                                            f"句子语音生成成功： {self.conn.tts_MessageText}"
+                                        )
+                                        self.tts_audio_queue.put(
+                                            (SentenceType.FIRST, [], self.conn.tts_MessageText)
+                                        )
+                                        self.conn.tts_MessageText = None
+                                    try:
+                                        audio_bytes = base64.b64decode(audio_data)
+                                        self.opus_encoder.encode_pcm_to_opus_stream(
+                                            audio_bytes, False, self.handle_opus
+                                        )
+
+                                    except Exception as e:
+                                        logger.bind(tag=TAG).error(f"处理音频数据失败: {e}")
+
+                        else:
+                            message = header.get("message", "未知错误")
+                            logger.bind(tag=TAG).error(f"TTS合成错误: {code} - {message}")
+                            break
+
+                    except json.JSONDecodeError:
+                        logger.bind(tag=TAG).warning("收到无效的JSON消息")
+
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).warning("WebSocket连接已关闭")
+                    break
+
+                except Exception as e:
+                    logger.bind(tag=TAG).error(
+                        f"处理TTS响应时出错: {e}\n{traceback.format_exc()}"
+                    )
+                    break
+
+            # 链接不可复用
+            if self.ws:
+                try:
+                    await self.ws.close()
+                except:
+                    pass
+                self.ws = None
+        # 监听任务退出时清理引用
+        finally:
+            self._monitor_task = None
+
+    def to_tts(self, text: str) -> list:
+        """非流式TTS处理，用于测试及保存音频文件的场景"""
+        try:
+            # 创建新的事件循环
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+            # 存储音频数据
+            audio_data = []
+
+            async def _generate_audio():
+                # 生成认证URL
+                auth_url = XunfeiWSAuth.create_auth_url(
+                    self.api_key, self.api_secret, self.api_url
+                )
+
+                # 建立WebSocket连接
+                ws = await websockets.connect(
+                    auth_url,
+                    ping_interval=30,
+                    ping_timeout=10,
+                    close_timeout=10,
+                )
+
+                try:
+                    filtered_text = MarkdownCleaner.clean_markdown(text)
+
+                    text_request = self._build_base_request(status=2,text=filtered_text)
+
+                    await ws.send(json.dumps(text_request))
+
+                    task_finished = False
+                    while not task_finished:
+                        msg = await ws.recv()
+
+                        data = json.loads(msg)
+                        header = data.get("header", {})
+                        code = header.get("code")
+
+                        if code == 0:
+                            payload = data.get("payload", {})
+                            audio_payload = payload.get("audio", {})
+                            if audio_payload:
+                                status = audio_payload.get("status", 0)
+                                audio_base64 = audio_payload.get("audio", "")
+                                if status == 1:
+                                    try:
+                                        audio_bytes = base64.b64decode(audio_base64)
+                                        self.opus_encoder.encode_pcm_to_opus_stream(
+                                            audio_bytes,
+                                            end_of_stream=False,
+                                            callback=lambda opus: audio_data.append(opus)
+                                        )
+                                    except Exception as e:
+                                        logger.bind(tag=TAG).error(f"处理音频数据失败: {e}")
+                                elif status == 2:
+                                    task_finished = True
+                                    logger.bind(tag=TAG).debug("TTS任务完成")
+
+                        else:
+                            message = header.get("message", "未知错误")
+                            raise Exception(f"合成失败: {code} - {message}")
+
+                finally:
+                    # 清理资源
+                    try:
+                        await ws.close()
+                    except:
+                        pass
+
+            loop.run_until_complete(_generate_audio())
+            loop.close()
+
+            return audio_data
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}")
+            return []        
+    
+    def _build_base_request(self, status,text=" "):
+        """构建基础请求结构"""
+        return {
+            "header": {
+                "app_id": self.app_id,
+                "status": status,
+            },
+            "parameter": {
+                "oral": {
+                    "oral_level": self.oral_level,
+                    "spark_assist": self.spark_assist,
+                    "stop_split": self.stop_split,
+                    "remain": self.remain
+                },
+                "tts": {
+                    "vcn": self.voice,
+                    "speed": self.speed,
+                    "volume": self.volume,
+                    "pitch": self.pitch,
+                    "bgs": 0,
+                    "reg": 0,
+                    "rdn": 0,
+                    "rhy": 0,
+                    "audio": {
+                        "encoding": self.format,
+                        "sample_rate": self.sample_rate,
+                        "channels": 1,
+                        "bit_depth": 16,
+                        "frame_size": 0
+                    }
+                }
+            },
+            "payload": {
+                "text": {
+                    "encoding": "utf8",
+                    "compress": "raw",
+                    "format": "plain",
+                    "status": status,
+                    "seq": self.text_seq,
+                    "text": base64.b64encode(text.encode('utf-8')).decode('utf-8')
+                }
+            }
+        }

main/xiaozhi-server/core/providers/vad/silero.py

@@ -36,7 +36,18 @@ class VADProvider(VADProviderBase):
         # 至少要多少帧才算有语音
         self.frame_window_threshold = 3
 
+    def __del__(self):
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+            except Exception:
+                pass
+
     def is_vad(self, conn, opus_packet):
+        # 手动模式：直接返回True，不进行实时VAD检测，所有音频都缓存
+        if conn.client_listen_mode == "manual":
+            return True
+            
         try:
             pcm_frame = self.decoder.decode(opus_packet, 960)
             conn.client_audio_buffer.extend(pcm_frame)  # 将新数据加入缓冲区
@@ -70,7 +81,9 @@ class VADProvider(VADProviderBase):
 
                 # 更新滑动窗口
                 conn.client_voice_window.append(is_voice)
-                client_have_voice = (conn.client_voice_window.count(True) >= self.frame_window_threshold)
+                client_have_voice = (
+                    conn.client_voice_window.count(True) >= self.frame_window_threshold
+                )
 
                 # 如果之前有声音，但本次没有声音，且与上次有声音的时间差已经超过了静默阈值，则认为已经说完一句话
                 if conn.client_have_voice and not client_have_voice:

main/xiaozhi-server/core/utils/audioRateController.py

@@ -0,0 +1,160 @@
+import time
+import asyncio
+from collections import deque
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class AudioRateController:
+    """
+    音频速率控制器 - 按照60ms帧时长精确控制音频发送
+    解决高并发下的时间累积误差问题
+    """
+
+    def __init__(self, frame_duration=60):
+        """
+        Args:
+            frame_duration: 单个音频帧时长（毫秒），默认60ms
+        """
+        self.frame_duration = frame_duration
+        self.queue = deque()
+        self.play_position = 0  # 虚拟播放位置（毫秒）
+        self.start_timestamp = None  # 开始时间戳（只读，不修改）
+        self.pending_send_task = None
+        self.logger = logger
+        self.queue_empty_event = asyncio.Event()  # 队列清空事件
+        self.queue_empty_event.set()  # 初始为空状态
+        self.queue_has_data_event = asyncio.Event()  # 队列数据事件
+
+    def reset(self):
+        """重置控制器状态"""
+        if self.pending_send_task and not self.pending_send_task.done():
+            self.pending_send_task.cancel()
+            # 取消任务后，任务会在下次事件循环时清理，无需阻塞等待
+
+        self.queue.clear()
+        self.play_position = 0
+        self.start_timestamp = None  # 由首个音频包设置
+        # 相关事件处理
+        self.queue_empty_event.set()
+        self.queue_has_data_event.clear()
+
+    def add_audio(self, opus_packet):
+        """添加音频包到队列"""
+        self.queue.append(("audio", opus_packet))
+        # 相关事件处理
+        self.queue_empty_event.clear()
+        self.queue_has_data_event.set()
+
+    def add_message(self, message_callback):
+        """
+        添加消息到队列（立即发送，不占用播放时间）
+
+        Args:
+            message_callback: 消息发送回调函数 async def()
+        """
+        self.queue.append(("message", message_callback))
+        # 相关事件处理
+        self.queue_empty_event.clear()
+        self.queue_has_data_event.set()
+
+    def _get_elapsed_ms(self):
+        """获取已经过的时间（毫秒）"""
+        if self.start_timestamp is None:
+            return 0
+        return (time.monotonic() - self.start_timestamp) * 1000
+
+    async def check_queue(self, send_audio_callback):
+        """
+        检查队列并按时发送音频/消息
+
+        Args:
+            send_audio_callback: 发送音频的回调函数 async def(opus_packet)
+        """
+        while self.queue:
+            item = self.queue[0]
+            item_type = item[0]
+
+            if item_type == "message":
+                # 消息类型：立即发送，不占用播放时间
+                _, message_callback = item
+                self.queue.popleft()
+                try:
+                    await message_callback()
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"发送消息失败: {e}")
+                    raise
+
+            elif item_type == "audio":
+                if self.start_timestamp is None:
+                    self.start_timestamp = time.monotonic()
+
+                _, opus_packet = item
+
+                # 循环等待直到时间到达
+                while True:
+                    # 计算时间差
+                    elapsed_ms = self._get_elapsed_ms()
+                    output_ms = self.play_position
+
+                    if elapsed_ms < output_ms:
+                        # 还不到发送时间，计算等待时长
+                        wait_ms = output_ms - elapsed_ms
+
+                        # 等待后继续检查（允许被中断）
+                        try:
+                            await asyncio.sleep(wait_ms / 1000)
+                        except asyncio.CancelledError:
+                            self.logger.bind(tag=TAG).debug("音频发送任务被取消")
+                            raise
+                        # 等待结束后重新检查时间（循环回到 while True）
+                    else:
+                        # 时间已到，跳出等待循环
+                        break
+
+                # 时间已到，从队列移除并发送
+                self.queue.popleft()
+                self.play_position += self.frame_duration
+                try:
+                    await send_audio_callback(opus_packet)
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"发送音频失败: {e}")
+                    raise
+
+        # 队列处理完后清除事件
+        self.queue_empty_event.set()
+        self.queue_has_data_event.clear()
+
+    def start_sending(self, send_audio_callback):
+        """
+        启动异步发送任务
+
+        Args:
+            send_audio_callback: 发送音频的回调函数
+
+        Returns:
+            asyncio.Task: 发送任务
+        """
+
+        async def _send_loop():
+            try:
+                while True:
+                    # 等待队列数据事件，不轮询等待占用CPU
+                    await self.queue_has_data_event.wait()
+
+                    await self.check_queue(send_audio_callback)
+            except asyncio.CancelledError:
+                self.logger.bind(tag=TAG).debug("音频发送循环已停止")
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"音频发送循环异常: {e}")
+
+        self.pending_send_task = asyncio.create_task(_send_loop())
+        return self.pending_send_task
+
+    def stop_sending(self):
+        """停止发送任务"""
+        if self.pending_send_task and not self.pending_send_task.done():
+            self.pending_send_task.cancel()
+            self.logger.bind(tag=TAG).debug("已取消音频发送任务")

main/xiaozhi-server/core/utils/cache/config.py

@@ -19,6 +19,7 @@ class CacheType(Enum):
     CONFIG = "config"
     DEVICE_PROMPT = "device_prompt"
     VOICEPRINT_HEALTH = "voiceprint_health"  # 声纹识别健康检查
+    AUDIO_DATA = "audio_data"  # 音频数据缓存
 
 
 @dataclass
@@ -58,5 +59,8 @@ class CacheConfig:
             CacheType.VOICEPRINT_HEALTH: cls(
                 strategy=CacheStrategy.TTL, ttl=600, max_size=100  # 10分钟过期
             ),
+            CacheType.AUDIO_DATA: cls(
+                strategy=CacheStrategy.TTL, ttl=600, max_size=100  # 10分钟过期
+            ),
         }
         return configs.get(cache_type, cls())

main/xiaozhi-server/core/utils/context_provider.py

@@ -0,0 +1,64 @@
+import httpx
+from typing import Dict, Any, List
+from config.logger import setup_logging
+
+TAG = __name__
+
+class ContextDataProvider:
+    """数据上下文填充，负责从配置的API获取数据"""
+    
+    def __init__(self, config: Dict[str, Any], logger=None):
+        self.config = config
+        self.logger = logger or setup_logging()
+        self.context_data = ""
+
+    def fetch_all(self, device_id: str) -> str:
+        """获取所有配置的上下文数据"""
+        context_providers = self.config.get("context_providers", [])
+        if not context_providers:
+            return ""
+
+        formatted_lines = []
+        for provider in context_providers:
+            url = provider.get("url")
+            headers = provider.get("headers", {})
+
+            if not url:
+                continue
+
+            try:
+                headers = headers.copy() if isinstance(headers, dict) else {}
+                # 将 device_id 添加到请求头
+                headers["device-id"] = device_id
+                
+                # 发送请求
+                response = httpx.get(url, headers=headers, timeout=3)
+                
+                if response.status_code == 200:
+                    result = response.json()
+                    if isinstance(result, dict):
+                        if result.get("code") == 0:
+                            data = result.get("data")
+                            # 格式化数据
+                            if isinstance(data, dict):
+                                for k, v in data.items():
+                                    formatted_lines.append(f"- **{k}：** {v}")
+                            elif isinstance(data, list):
+                                for item in data:
+                                    formatted_lines.append(f"- {item}")
+                            else:
+                                formatted_lines.append(f"- {data}")
+                        else:
+                            self.logger.bind(tag=TAG).warning(f"API {url} 返回错误码: {result.get('msg')}")
+                    else:
+                        self.logger.bind(tag=TAG).warning(f"API {url} 返回的不是JSON字典")
+                else:
+                    self.logger.bind(tag=TAG).warning(f"API {url} 请求失败: {response.status_code}")
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"获取上下文数据 {url} 失败: {e}")
+        
+        # 将所有格式化后的行拼接成一个字符串
+        self.context_data = "\n".join(formatted_lines)
+        if self.context_data:
+            self.logger.bind(tag=TAG).debug(f"已注入动态上下文数据:\n{self.context_data}")
+        return self.context_data

main/xiaozhi-server/core/utils/current_time.py

@@ -0,0 +1,68 @@
+"""
+时间工具模块
+提供统一的时间获取功能
+"""
+
+import cnlunar
+from datetime import datetime
+
+WEEKDAY_MAP = {
+    "Monday": "星期一",
+    "Tuesday": "星期二", 
+    "Wednesday": "星期三",
+    "Thursday": "星期四",
+    "Friday": "星期五",
+    "Saturday": "星期六",
+    "Sunday": "星期日",
+}
+
+
+def get_current_time() -> str:
+    """
+    获取当前时间字符串 (格式: HH:MM)
+    """
+    return datetime.now().strftime("%H:%M")
+
+
+def get_current_date() -> str:
+    """
+    获取今天日期字符串 (格式: YYYY-MM-DD)
+    """
+    return datetime.now().strftime("%Y-%m-%d")
+
+
+def get_current_weekday() -> str:
+    """
+    获取今天星期几
+    """
+    now = datetime.now()
+    return WEEKDAY_MAP[now.strftime("%A")]
+
+
+def get_current_lunar_date() -> str:
+    """
+    获取农历日期字符串
+    """
+    try:
+        now = datetime.now()
+        today_lunar = cnlunar.Lunar(now, godType="8char")
+        return "%s年%s%s" % (
+            today_lunar.lunarYearCn,
+            today_lunar.lunarMonthCn[:-1],
+            today_lunar.lunarDayCn,
+        )
+    except Exception:
+        return "农历获取失败"
+
+
+def get_current_time_info() -> tuple:
+    """
+    获取当前时间信息
+    返回: (当前时间字符串, 今天日期, 今天星期, 农历日期)
+    """
+    current_time = get_current_time()
+    today_date = get_current_date()
+    today_weekday = get_current_weekday()
+    lunar_date = get_current_lunar_date()
+    
+    return current_time, today_date, today_weekday, lunar_date

main/xiaozhi-server/core/utils/gc_manager.py

@@ -0,0 +1,122 @@
+"""
+全局GC管理模块
+定期执行垃圾回收，避免频繁触发GC导致的GIL锁问题
+"""
+
+import gc
+import asyncio
+import threading
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class GlobalGCManager:
+    """全局垃圾回收管理器"""
+
+    def __init__(self, interval_seconds=300):
+        """
+        初始化GC管理器
+
+        Args:
+            interval_seconds: GC执行间隔（秒），默认300秒（5分钟）
+        """
+        self.interval_seconds = interval_seconds
+        self._task = None
+        self._stop_event = asyncio.Event()
+        self._lock = threading.Lock()
+
+    async def start(self):
+        """启动定时GC任务"""
+        if self._task is not None:
+            logger.bind(tag=TAG).warning("GC管理器已经在运行")
+            return
+
+        logger.bind(tag=TAG).info(f"启动全局GC管理器，间隔{self.interval_seconds}秒")
+        self._stop_event.clear()
+        self._task = asyncio.create_task(self._gc_loop())
+
+    async def stop(self):
+        """停止定时GC任务"""
+        if self._task is None:
+            return
+
+        logger.bind(tag=TAG).info("停止全局GC管理器")
+        self._stop_event.set()
+
+        if self._task and not self._task.done():
+            self._task.cancel()
+            try:
+                await self._task
+            except asyncio.CancelledError:
+                pass
+
+        self._task = None
+
+    async def _gc_loop(self):
+        """GC循环任务"""
+        try:
+            while not self._stop_event.is_set():
+                # 等待指定间隔
+                try:
+                    await asyncio.wait_for(
+                        self._stop_event.wait(), timeout=self.interval_seconds
+                    )
+                    # 如果stop_event被设置，退出循环
+                    break
+                except asyncio.TimeoutError:
+                    # 超时表示到了执行GC的时间
+                    pass
+
+                # 执行GC
+                await self._run_gc()
+
+        except asyncio.CancelledError:
+            logger.bind(tag=TAG).info("GC循环任务被取消")
+            raise
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"GC循环任务异常: {e}")
+        finally:
+            logger.bind(tag=TAG).info("GC循环任务已退出")
+
+    async def _run_gc(self):
+        """执行垃圾回收"""
+        try:
+            # 在线程池中执行GC，避免阻塞事件循环
+            loop = asyncio.get_running_loop()
+
+            def do_gc():
+                with self._lock:
+                    before = len(gc.get_objects())
+                    collected = gc.collect()
+                    after = len(gc.get_objects())
+                    return before, collected, after
+
+            before, collected, after = await loop.run_in_executor(None, do_gc)
+            logger.bind(tag=TAG).debug(
+                f"全局GC执行完成 - 回收对象: {collected}, "
+                f"对象数量: {before} -> {after}"
+            )
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"执行GC时出错: {e}")
+
+
+# 全局单例
+_gc_manager_instance = None
+
+
+def get_gc_manager(interval_seconds=300):
+    """
+    获取全局GC管理器实例（单例模式）
+
+    Args:
+        interval_seconds: GC执行间隔（秒），默认300秒（5分钟）
+
+    Returns:
+        GlobalGCManager实例
+    """
+    global _gc_manager_instance
+    if _gc_manager_instance is None:
+        _gc_manager_instance = GlobalGCManager(interval_seconds)
+    return _gc_manager_instance

main/xiaozhi-server/core/utils/opus_encoder_utils.py

@@ -102,6 +102,9 @@ class OpusEncoderUtils:
     def _encode(self, frame: np.ndarray) -> Optional[bytes]:
         """编码一帧音频数据"""
         try:
+            # 编码器已释放，跳过编码
+            if not hasattr(self, 'encoder') or self.encoder is None:
+                return None
             # 将numpy数组转换为bytes
             frame_bytes = frame.tobytes()
             # opuslib要求输入字节数必须是channels*2的倍数
@@ -128,5 +131,9 @@ class OpusEncoderUtils:
 
     def close(self):
         """关闭编码器并释放资源"""
-        # opuslib没有明确的关闭方法，Python的垃圾回收会处理
-        pass
+        if hasattr(self, 'encoder') and self.encoder:
+            try:
+                del self.encoder
+                self.encoder = None
+            except Exception as e:
+                logging.error(f"Error releasing Opus encoder: {e}")

main/xiaozhi-server/core/utils/prompt_manager.py

@@ -4,7 +4,6 @@
 """
 
 import os
-import cnlunar
 from typing import Dict, Any
 from config.logger import setup_logging
 from jinja2 import Template
@@ -60,13 +59,20 @@ class PromptManager:
 
         self.cache_manager = cache_manager
         self.CacheType = CacheType
+        
+        # 初始化上下文源
+        from core.utils.context_provider import ContextDataProvider
+        self.context_provider = ContextDataProvider(config, self.logger)
+        self.context_data = {}
 
         self._load_base_template()
 
     def _load_base_template(self):
         """加载基础提示词模板"""
         try:
-            template_path = "agent-base-prompt.txt"
+            template_path = self.config.get("prompt_template", None)
+            if not template_path:
+                template_path = "agent-base-prompt.txt"
             cache_key = f"prompt_template:{template_path}"
 
             # 先从缓存获取
@@ -88,7 +94,7 @@ class PromptManager:
                 self.base_prompt_template = template_content
                 self.logger.bind(tag=TAG).debug("成功加载基础提示词模板并缓存")
             else:
-                self.logger.bind(tag=TAG).warning("未找到agent-base-prompt.txt文件")
+                self.logger.bind(tag=TAG).warning(f"未找到{template_path}文件")
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"加载提示词模板失败: {e}")
 
@@ -117,18 +123,16 @@ class PromptManager:
 
     def _get_current_time_info(self) -> tuple:
         """获取当前时间信息"""
-        from datetime import datetime
-
-        now = datetime.now()
-        today_date = now.strftime("%Y-%m-%d")
-        today_weekday = WEEKDAY_MAP[now.strftime("%A")]
-        today_lunar = cnlunar.Lunar(now, godType="8char")
-        lunar_date = "%s年%s%s\n" % (
-            today_lunar.lunarYearCn,
-            today_lunar.lunarMonthCn[:-1],
-            today_lunar.lunarDayCn,
+        from .current_time import (
+            get_current_date,
+            get_current_weekday,
+            get_current_lunar_date,
         )
 
+        today_date = get_current_date()
+        today_weekday = get_current_weekday()
+        lunar_date = get_current_lunar_date() + "\n"
+
         return today_date, today_weekday, lunar_date
 
     def _get_location_info(self, client_ip: str) -> str:
@@ -180,17 +184,40 @@ class PromptManager:
     def update_context_info(self, conn, client_ip: str):
         """同步更新上下文信息"""
         try:
-            # 获取位置信息（使用全局缓存）
-            local_address = self._get_location_info(client_ip)
-            # 获取天气信息（使用全局缓存）
-            self._get_weather_info(conn, local_address)
-            self.logger.bind(tag=TAG).info(f"上下文信息更新完成")
+            local_address = ""
+            if (
+                client_ip
+                and self.base_prompt_template
+                and (
+                    "local_address" in self.base_prompt_template
+                    or "weather_info" in self.base_prompt_template
+                )
+            ):
+                # 获取位置信息（使用全局缓存）
+                local_address = self._get_location_info(client_ip)
+
+            if (
+                self.base_prompt_template
+                and "weather_info" in self.base_prompt_template
+                and local_address
+            ):
+                # 获取天气信息（使用全局缓存）
+                self._get_weather_info(conn, local_address)
+            
+            # 获取配置的上下文数据
+            if hasattr(conn, "device_id") and conn.device_id:
+                if self.base_prompt_template and "dynamic_context" in self.base_prompt_template:
+                    self.context_data = self.context_provider.fetch_all(conn.device_id)
+                else:
+                    self.context_data = ""
+                
+            self.logger.bind(tag=TAG).debug(f"上下文信息更新完成")
 
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"更新上下文信息失败: {e}")
 
     def build_enhanced_prompt(
-        self, user_prompt: str, device_id: str, client_ip: str = None
+        self, user_prompt: str, device_id: str, client_ip: str = None, *args, **kwargs
     ) -> str:
         """构建增强的系统提示词"""
         if not self.base_prompt_template:
@@ -198,9 +225,7 @@ class PromptManager:
 
         try:
             # 获取最新的时间信息（不缓存）
-            today_date, today_weekday, lunar_date = (
-                self._get_current_time_info()
-            )
+            today_date, today_weekday, lunar_date = self._get_current_time_info()
 
             # 获取缓存的上下文信息
             local_address = ""
@@ -230,6 +255,11 @@ class PromptManager:
                 local_address=local_address,
                 weather_info=weather_info,
                 emojiList=EMOJI_List,
+                device_id=device_id,
+                client_ip=client_ip,
+                dynamic_context=self.context_data,
+                *args,
+                **kwargs,
             )
             device_cache_key = f"device_prompt:{device_id}"
             self.cache_manager.set(

main/xiaozhi-server/core/utils/tts.py

@@ -6,6 +6,27 @@ import importlib
 
 logger = setup_logging()
 
+punctuation_set = {
+    "，",
+    ",",  # 中文逗号 + 英文逗号
+    "。",
+    ".",  # 中文句号 + 英文句号
+    "！",
+    "!",  # 中文感叹号 + 英文感叹号
+    "“",
+    "”",
+    '"',  # 中文双引号 + 英文引号
+    "：",
+    ":",  # 中文冒号 + 英文冒号
+    "-",
+    "－",  # 英文连字符 + 中文全角横线
+    "、",  # 中文顿号
+    "[",
+    "]",  # 方括号
+    "【",
+    "】",  # 中文方括号
+    "~",  # 波浪号
+}
 
 def create_instance(class_name, *args, **kwargs):
     # 创建TTS实例
@@ -107,6 +128,11 @@ class MarkdownCleaner:
         """
         主入口方法：依序执行所有正则，移除或替换 Markdown 元素
         """
+        # 检查文本是否全为英文和基本标点符号
+        if text and all((c.isascii() or c.isspace() or c in punctuation_set) for c in text):
+            # 保留原始空格，直接返回
+            return text
+
         for regex, replacement in MarkdownCleaner.REGEXES:
             text = regex.sub(replacement, text)
         return text.strip()

main/xiaozhi-server/core/utils/util.py

@@ -4,6 +4,7 @@ import json
 import copy
 import wave
 import socket
+import asyncio
 import requests
 import subprocess
 import numpy as np
@@ -176,31 +177,67 @@ def parse_string_to_list(value, separator=";"):
     return []
 
 
-def check_ffmpeg_installed():
-    ffmpeg_installed = False
+def check_ffmpeg_installed() -> bool:
+    """
+    检查当前环境中是否已正确安装并可执行 ffmpeg。
+
+    Returns:
+        bool: 如果 ffmpeg 正常可用，返回 True；否则抛出 ValueError 异常。
+
+    Raises:
+        ValueError: 当检测到 ffmpeg 未安装或依赖缺失时，抛出详细的提示信息。
+    """
     try:
-        # 执行ffmpeg -version命令，并捕获输出
+        # 尝试执行 ffmpeg 命令
         result = subprocess.run(
             ["ffmpeg", "-version"],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             text=True,
-            check=True,  # 如果返回码非零则抛出异常
+            check=True,  # 非零退出码会触发 CalledProcessError
         )
-        # 检查输出中是否包含版本信息（可选）
-        output = result.stdout + result.stderr
-        if "ffmpeg version" in output.lower():
-            ffmpeg_installed = True
-        return False
-    except (subprocess.CalledProcessError, FileNotFoundError):
-        # 命令执行失败或未找到
-        ffmpeg_installed = False
-    if not ffmpeg_installed:
-        error_msg = "您的电脑还没正确安装ffmpeg\n"
-        error_msg += "\n建议您：\n"
-        error_msg += "1、按照项目的安装文档，正确进入conda环境\n"
-        error_msg += "2、查阅安装文档，如何在conda环境中安装ffmpeg\n"
-        raise ValueError(error_msg)
+
+        output = (result.stdout + result.stderr).lower()
+        if "ffmpeg version" in output:
+            return True
+
+        # 如果未检测到版本信息，也视为异常情况
+        raise ValueError("未检测到有效的 ffmpeg 版本输出。")
+
+    except (subprocess.CalledProcessError, FileNotFoundError) as e:
+        # 提取错误输出
+        stderr_output = ""
+        if isinstance(e, subprocess.CalledProcessError):
+            stderr_output = (e.stderr or "").strip()
+        else:
+            stderr_output = str(e).strip()
+
+        # 构建基础错误提示
+        error_msg = [
+            "❌ 检测到 ffmpeg 无法正常运行。\n",
+            "建议您：",
+            "1. 确认已正确激活 conda 环境；",
+            "2. 查阅项目安装文档，了解如何在 conda 环境中安装 ffmpeg。\n",
+        ]
+
+        # 🎯 针对具体错误信息提供额外提示
+        if "libiconv.so.2" in stderr_output:
+            error_msg.append("⚠️ 发现缺少依赖库：libiconv.so.2")
+            error_msg.append("解决方法：在当前 conda 环境中执行：")
+            error_msg.append("   conda install -c conda-forge libiconv\n")
+        elif (
+            "no such file or directory" in stderr_output
+            and "ffmpeg" in stderr_output.lower()
+        ):
+            error_msg.append("⚠️ 系统未找到 ffmpeg 可执行文件。")
+            error_msg.append("解决方法：在当前 conda 环境中执行：")
+            error_msg.append("   conda install -c conda-forge ffmpeg\n")
+        else:
+            error_msg.append("错误详情：")
+            error_msg.append(stderr_output or "未知错误。")
+
+        # 抛出详细异常信息
+        raise ValueError("\n".join(error_msg)) from e
 
 
 def extract_json_from_string(input_string):
@@ -212,7 +249,9 @@ def extract_json_from_string(input_string):
     return None
 
 
-def audio_to_data_stream(audio_file_path, is_opus=True, callback: Callable[[Any], Any]=None) -> None:
+def audio_to_data_stream(
+    audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None
+) -> None:
     # 获取文件后缀名
     file_type = os.path.splitext(audio_file_path)[1]
     if file_type:
@@ -229,58 +268,88 @@ def audio_to_data_stream(audio_file_path, is_opus=True, callback: Callable[[Any]
     raw_data = audio.raw_data
     pcm_to_data_stream(raw_data, is_opus, callback)
 
-def audio_to_data(audio_file_path: str, is_opus: bool = True) -> list[bytes]:
+
+async def audio_to_data(
+    audio_file_path: str, is_opus: bool = True, use_cache: bool = True
+) -> list[bytes]:
     """
     将音频文件转换为Opus/PCM编码的帧列表
     Args:
         audio_file_path: 音频文件路径
         is_opus: 是否进行Opus编码
+        use_cache: 是否使用缓存
     """
-    # 获取文件后缀名
-    file_type = os.path.splitext(audio_file_path)[1]
-    if file_type:
-        file_type = file_type.lstrip(".")
-    # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
-    audio = AudioSegment.from_file(
-        audio_file_path, format=file_type, parameters=["-nostdin"]
-    )
+    from core.utils.cache.manager import cache_manager
+    from core.utils.cache.config import CacheType
 
-    # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
-    audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
+    # 生成缓存键，包含文件路径和编码类型
+    cache_key = f"{audio_file_path}:{is_opus}"
 
-    # 获取原始PCM数据（16位小端）
-    raw_data = audio.raw_data
+    # 尝试从缓存获取结果
+    if use_cache:
+        cached_result = cache_manager.get(CacheType.AUDIO_DATA, cache_key)
+        if cached_result is not None:
+            return cached_result
 
-    # 初始化Opus编码器
-    encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+    def _sync_audio_to_data():
+        # 获取文件后缀名
+        file_type = os.path.splitext(audio_file_path)[1]
+        if file_type:
+            file_type = file_type.lstrip(".")
+        # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
+        audio = AudioSegment.from_file(
+            audio_file_path, format=file_type, parameters=["-nostdin"]
+        )
 
-    # 编码参数
-    frame_duration = 60  # 60ms per frame
-    frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+        # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
+        audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
 
-    datas = []
-    # 按帧处理所有音频数据（包括最后一帧可能补零）
-    for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
-        # 获取当前帧的二进制数据
-        chunk = raw_data[i : i + frame_size * 2]
+        # 获取原始PCM数据（16位小端）
+        raw_data = audio.raw_data
 
-        # 如果最后一帧不足，补零
-        if len(chunk) < frame_size * 2:
-            chunk += b"\x00" * (frame_size * 2 - len(chunk))
+        # 初始化Opus编码器
+        encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
 
-        if is_opus:
-            # 转换为numpy数组处理
-            np_frame = np.frombuffer(chunk, dtype=np.int16)
-            # 编码Opus数据
-            frame_data = encoder.encode(np_frame.tobytes(), frame_size)
-        else:
-            frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
+        # 编码参数
+        frame_duration = 60  # 60ms per frame
+        frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
 
-        datas.append(frame_data)
+        datas = []
+        # 按帧处理所有音频数据（包括最后一帧可能补零）
+        for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
+            # 获取当前帧的二进制数据
+            chunk = raw_data[i : i + frame_size * 2]
 
-    return datas
+            # 如果最后一帧不足，补零
+            if len(chunk) < frame_size * 2:
+                chunk += b"\x00" * (frame_size * 2 - len(chunk))
 
-def audio_bytes_to_data_stream(audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]) -> None:
+            if is_opus:
+                # 转换为numpy数组处理
+                np_frame = np.frombuffer(chunk, dtype=np.int16)
+                # 编码Opus数据
+                frame_data = encoder.encode(np_frame.tobytes(), frame_size)
+            else:
+                frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
+
+            datas.append(frame_data)
+
+        return datas
+
+    loop = asyncio.get_running_loop()
+    # 在单独的线程中执行同步的音频处理操作
+    result = await loop.run_in_executor(None, _sync_audio_to_data)
+
+    # 将结果存入缓存，使用配置中定义的TTL（10分钟）
+    if use_cache:
+        cache_manager.set(CacheType.AUDIO_DATA, cache_key, result)
+
+    return result
+
+
+def audio_bytes_to_data_stream(
+    audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]
+) -> None:
     """
     直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3
     """
@@ -324,31 +393,40 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] =
             frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
             callback(frame_data)
 
+
 def opus_datas_to_wav_bytes(opus_datas, sample_rate=16000, channels=1):
     """
     将opus帧列表解码为wav字节流
     """
     decoder = opuslib_next.Decoder(sample_rate, channels)
-    pcm_datas = []
+    try:
+        pcm_datas = []
 
-    frame_duration = 60  # ms
-    frame_size = int(sample_rate * frame_duration / 1000)  # 960
+        frame_duration = 60  # ms
+        frame_size = int(sample_rate * frame_duration / 1000)  # 960
 
-    for opus_frame in opus_datas:
-        # 解码为PCM（返回bytes，2字节/采样点）
-        pcm = decoder.decode(opus_frame, frame_size)
-        pcm_datas.append(pcm)
+        for opus_frame in opus_datas:
+            # 解码为PCM（返回bytes，2字节/采样点）
+            pcm = decoder.decode(opus_frame, frame_size)
+            pcm_datas.append(pcm)
 
-    pcm_bytes = b"".join(pcm_datas)
+        pcm_bytes = b"".join(pcm_datas)
+
+        # 写入wav字节流
+        wav_buffer = BytesIO()
+        with wave.open(wav_buffer, "wb") as wf:
+            wf.setnchannels(channels)
+            wf.setsampwidth(2)  # 16bit
+            wf.setframerate(sample_rate)
+            wf.writeframes(pcm_bytes)
+        return wav_buffer.getvalue()
+    finally:
+        if decoder is not None:
+            try:
+                del decoder
+            except Exception:
+                pass
 
-    # 写入wav字节流
-    wav_buffer = BytesIO()
-    with wave.open(wav_buffer, "wb") as wf:
-        wf.setnchannels(channels)
-        wf.setsampwidth(2)  # 16bit
-        wf.setframerate(sample_rate)
-        wf.writeframes(pcm_bytes)
-    return wav_buffer.getvalue()
 
 def check_vad_update(before_config, new_config):
     if (
@@ -423,6 +501,17 @@ def filter_sensitive_info(config: dict) -> dict:
                 filtered[k] = _filter_dict(v)
             elif isinstance(v, list):
                 filtered[k] = [_filter_dict(i) if isinstance(i, dict) else i for i in v]
+            elif isinstance(v, str):
+                try:
+                    json_data = json.loads(v)
+                    if isinstance(json_data, dict):
+                        filtered[k] = json.dumps(
+                            _filter_dict(json_data), ensure_ascii=False
+                        )
+                    else:
+                        filtered[k] = v
+                except (json.JSONDecodeError, TypeError):
+                    filtered[k] = v
             else:
                 filtered[k] = v
         return filtered

main/xiaozhi-server/core/utils/voiceprint_provider.py

@@ -19,6 +19,8 @@ class VoiceprintProvider:
         self.original_url = config.get("url", "")
         self.speakers = config.get("speakers", [])
         self.speaker_map = self._parse_speakers()
+        # 声纹识别相似度阈值，默认0.4
+        self.similarity_threshold = float(config.get("similarity_threshold", 0.4))
         
         # 解析API地址和密钥
         self.api_url = None
@@ -62,7 +64,7 @@ class VoiceprintProvider:
                     # 进行健康检查，验证服务器是否可用
                     if self._check_server_health():
                         self.enabled = True
-                        logger.bind(tag=TAG).info(f"声纹识别已启用: API={self.api_url}, 说话人={len(self.speaker_ids)}个")
+                        logger.bind(tag=TAG).info(f"声纹识别已启用: API={self.api_url}, 说话人={len(self.speaker_ids)}个, 相似度阈值={self.similarity_threshold}")
                     else:
                         self.enabled = False
                         logger.bind(tag=TAG).warning(f"声纹识别服务器不可用，声纹识别已禁用: {self.api_url}")
@@ -169,12 +171,14 @@ class VoiceprintProvider:
                         
                         logger.bind(tag=TAG).info(f"声纹识别耗时: {total_elapsed_time:.3f}s")
                         
-                        # 置信度检查
-                        if score < 0.5:
-                            logger.bind(tag=TAG).warning(f"声纹识别置信度较低: {score:.3f}")
+                        # 相似度阈值检查
+                        if score < self.similarity_threshold:
+                            logger.bind(tag=TAG).warning(f"声纹识别相似度{score:.3f}低于阈值{self.similarity_threshold}")
+                            return "未知说话人"
                         
                         if speaker_id and speaker_id in self.speaker_map:
                             result_name = self.speaker_map[speaker_id]["name"]
+                            logger.bind(tag=TAG).info(f"声纹识别成功: {result_name} (相似度: {score:.3f})")
                             return result_name
                         else:
                             logger.bind(tag=TAG).warning(f"未识别的说话人ID: {speaker_id}")

main/xiaozhi-server/core/utils/wakeup_word.py

@@ -55,7 +55,7 @@ class WakeupWordsConfig:
             return self._config_cache
 
         try:
-            with open(self.config_file, "a+") as f:
+            with open(self.config_file, "a+", encoding="utf-8") as f:
                 with FileLock(f, timeout=self._lock_timeout):
                     f.seek(0)
                     content = f.read()
@@ -73,7 +73,7 @@ class WakeupWordsConfig:
     def _save_config(self, config: Dict):
         """保存配置到文件，使用文件锁保护"""
         try:
-            with open(self.config_file, "w") as f:
+            with open(self.config_file, "w", encoding="utf-8") as f:
                 with FileLock(f, timeout=self._lock_timeout):
                     yaml.dump(config, f, allow_unicode=True)
                     self._config_cache = config

main/xiaozhi-server/core/websocket_server.py

@@ -1,8 +1,38 @@
 import asyncio
+import logging
+
 import websockets
 from config.logger import setup_logging
+
+
+class SuppressInvalidHandshakeFilter(logging.Filter):
+    """过滤掉无效握手错误日志（如HTTPS访问WS端口）"""
+
+    def filter(self, record):
+        msg = record.getMessage()
+        suppress_keywords = [
+            "opening handshake failed",
+            "did not receive a valid HTTP request",
+            "connection closed while reading HTTP request",
+            "line without CRLF",
+        ]
+        return not any(keyword in msg for keyword in suppress_keywords)
+
+
+def _setup_websockets_logger():
+    """配置 websockets 相关的所有 logger，过滤无效握手错误"""
+    filter_instance = SuppressInvalidHandshakeFilter()
+    for logger_name in ["websockets", "websockets.server", "websockets.client"]:
+        logger = logging.getLogger(logger_name)
+        logger.addFilter(filter_instance)
+
+
+_setup_websockets_logger()
+
+
 from core.connection import ConnectionHandler
-from config.config_loader import get_config_from_api
+from config.config_loader import get_config_from_api_async
+from core.auth import AuthManager, AuthenticationError
 from core.utils.modules_initialize import initialize_modules
 from core.utils.util import check_vad_update, check_asr_update
 
@@ -30,7 +60,13 @@ class WebSocketServer:
         self._intent = modules["intent"] if "intent" in modules else None
         self._memory = modules["memory"] if "memory" in modules else None
 
-        self.active_connections = set()
+        auth_config = self.config["server"].get("auth", {})
+        self.auth_enable = auth_config.get("enabled", False)
+        # 设备白名单
+        self.allowed_devices = set(auth_config.get("allowed_devices", []))
+        secret_key = self.config["server"]["auth_key"]
+        expire_seconds = auth_config.get("expire_seconds", None)
+        self.auth = AuthManager(secret_key=secret_key, expire_seconds=expire_seconds)
 
     async def start(self):
         server_config = self.config["server"]
@@ -43,7 +79,40 @@ class WebSocketServer:
             await asyncio.Future()
 
     async def _handle_connection(self, websocket):
+        headers = dict(websocket.request.headers)
+        if headers.get("device-id", None) is None:
+            # 尝试从 URL 的查询参数中获取 device-id
+            from urllib.parse import parse_qs, urlparse
+
+            # 从 WebSocket 请求中获取路径
+            request_path = websocket.request.path
+            if not request_path:
+                self.logger.bind(tag=TAG).error("无法获取请求路径")
+                await websocket.close()
+                return
+            parsed_url = urlparse(request_path)
+            query_params = parse_qs(parsed_url.query)
+            if "device-id" not in query_params:
+                await websocket.send("端口正常，如需测试连接，请使用test_page.html")
+                await websocket.close()
+                return
+            else:
+                websocket.request.headers["device-id"] = query_params["device-id"][0]
+            if "client-id" in query_params:
+                websocket.request.headers["client-id"] = query_params["client-id"][0]
+            if "authorization" in query_params:
+                websocket.request.headers["authorization"] = query_params[
+                    "authorization"
+                ][0]
+
         """处理新连接，每次创建独立的ConnectionHandler"""
+        # 先认证，后建立连接
+        try:
+            await self._handle_auth(websocket)
+        except AuthenticationError:
+            await websocket.send("认证失败")
+            await websocket.close()
+            return
         # 创建ConnectionHandler时传入当前server实例
         handler = ConnectionHandler(
             self.config,
@@ -54,14 +123,11 @@ class WebSocketServer:
             self._intent,
             self,  # 传入server实例
         )
-        self.active_connections.add(handler)
         try:
             await handler.handle_connection(websocket)
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"处理连接时出错: {e}")
         finally:
-            # 确保从活动连接集合中移除
-            self.active_connections.discard(handler)
             # 强制关闭连接（如果还没有关闭的话）
             try:
                 # 安全地检查WebSocket状态并关闭
@@ -94,8 +160,8 @@ class WebSocketServer:
         """
         try:
             async with self.config_lock:
-                # 重新获取配置
-                new_config = get_config_from_api(self.config)
+                # 重新获取配置（使用异步版本）
+                new_config = await get_config_from_api_async(self.config)
                 if new_config is None:
                     self.logger.bind(tag=TAG).error("获取新配置失败")
                     return False
@@ -136,3 +202,26 @@ class WebSocketServer:
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"更新服务器配置失败: {str(e)}")
             return False
+
+    async def _handle_auth(self, websocket):
+        # 先认证，后建立连接
+        if self.auth_enable:
+            headers = dict(websocket.request.headers)
+            device_id = headers.get("device-id", None)
+            client_id = headers.get("client-id", None)
+            if self.allowed_devices and device_id in self.allowed_devices:
+                # 如果属于白名单内的设备，不校验token，直接放行
+                return
+            else:
+                # 否则校验token
+                token = headers.get("authorization", "")
+                if token.startswith("Bearer "):
+                    token = token[7:]  # 移除'Bearer '前缀
+                else:
+                    raise AuthenticationError("Missing or invalid Authorization header")
+                # 进行认证
+                auth_success = self.auth.verify_token(
+                    token, client_id=client_id, username=device_id
+                )
+                if not auth_success:
+                    raise AuthenticationError("Invalid token")