From 0c582ed3b6f4f153f314e7029f65426ebcd836ea Mon Sep 17 00:00:00 2001 From: caixypromise Date: Mon, 27 Jul 2026 02:05:26 +0800 Subject: [PATCH] refactor: introduce transport-neutral connection runtime --- main/xiaozhi-server/app.py | 161 ++- .../config/manage_api_client.py | 26 + main/xiaozhi-server/core/auth.py | 158 +++ .../core/components/adapters/asr_adapter.py | 116 ++ .../components/adapters/intent_adapter.py | 92 ++ .../core/components/adapters/llm_adapter.py | 64 + .../components/adapters/memory_adapter.py | 103 ++ .../core/components/adapters/tts_adapter.py | 76 ++ .../core/components/adapters/vad_adapter.py | 64 + .../core/components/component_manager.py | 268 ++++ .../core/components/component_registry.py | 73 ++ .../core/context/session_context.py | 505 ++++++++ .../core/handle/sendAudioHandle.py | 14 +- main/xiaozhi-server/core/http_server.py | 61 +- .../core/pipeline/message_pipeline.py | 26 + .../core/processors/abort_processor.py | 132 ++ .../processors/audio_receive_processor.py | 439 +++++++ .../core/processors/audio_send_processor.py | 578 +++++++++ .../core/processors/auth_processor.py | 55 + .../core/processors/chat_processor.py | 1042 +++++++++++++++ .../core/processors/goodbye_processor.py | 34 + .../core/processors/hello_processor.py | 351 +++++ .../core/processors/iot_processor.py | 124 ++ .../core/processors/listen_processor.py | 433 +++++++ .../core/processors/mcp_processor.py | 96 ++ .../core/processors/message_router.py | 124 ++ .../core/processors/ping_processor.py | 40 + .../core/processors/report_processor.py | 301 +++++ .../core/processors/server_processor.py | 177 +++ .../core/processors/text_processor.py | 54 + .../core/processors/timeout_processor.py | 48 + .../core/providers/asr/aliyun_stream.py | 39 +- .../core/providers/asr/aliyunbl_stream.py | 35 +- .../xiaozhi-server/core/providers/asr/base.py | 63 +- .../core/providers/asr/doubao_stream.py | 69 +- .../core/providers/asr/fun_local.py | 5 +- .../core/providers/asr/shared_asr_manager.py | 522 ++++++++ .../core/providers/asr/shared_asr_proxy.py | 119 ++ .../core/providers/asr/xunfei_stream.py | 51 +- .../tools/device_iot/iot_executor.py | 11 +- .../providers/tools/device_mcp/mcp_client.py | 11 +- .../providers/tools/device_mcp/mcp_handler.py | 157 +-- .../tools/mcp_endpoint/mcp_endpoint_client.py | 13 + .../mcp_endpoint/mcp_endpoint_handler.py | 10 +- .../providers/tools/server_mcp/mcp_client.py | 19 +- .../xiaozhi-server/core/providers/tts/base.py | 103 +- .../core/servers/multi_protocol_server.py | 450 +++++++ .../core/services/audio_ingress_service.py | 229 ++++ .../core/services/connection_service.py | 1140 +++++++++++++++++ .../core/transport/transport_interface.py | 90 ++ .../core/transport/websocket_transport.py | 154 +++ .../core/utils/config_validation.py | 106 ++ main/xiaozhi-server/core/utils/restart.py | 18 + main/xiaozhi-server/core/utils/textUtils.py | 30 +- .../core/websocket_server_new.py | 452 +++++++ .../core/xiaozhi_server_facade.py | 437 +++++++ 56 files changed, 9931 insertions(+), 237 deletions(-) create mode 100644 main/xiaozhi-server/core/components/adapters/asr_adapter.py create mode 100644 main/xiaozhi-server/core/components/adapters/intent_adapter.py create mode 100644 main/xiaozhi-server/core/components/adapters/llm_adapter.py create mode 100644 main/xiaozhi-server/core/components/adapters/memory_adapter.py create mode 100644 main/xiaozhi-server/core/components/adapters/tts_adapter.py create mode 100644 main/xiaozhi-server/core/components/adapters/vad_adapter.py create mode 100644 main/xiaozhi-server/core/components/component_manager.py create mode 100644 main/xiaozhi-server/core/components/component_registry.py create mode 100644 main/xiaozhi-server/core/context/session_context.py create mode 100644 main/xiaozhi-server/core/pipeline/message_pipeline.py create mode 100644 main/xiaozhi-server/core/processors/abort_processor.py create mode 100644 main/xiaozhi-server/core/processors/audio_receive_processor.py create mode 100644 main/xiaozhi-server/core/processors/audio_send_processor.py create mode 100644 main/xiaozhi-server/core/processors/auth_processor.py create mode 100644 main/xiaozhi-server/core/processors/chat_processor.py create mode 100644 main/xiaozhi-server/core/processors/goodbye_processor.py create mode 100644 main/xiaozhi-server/core/processors/hello_processor.py create mode 100644 main/xiaozhi-server/core/processors/iot_processor.py create mode 100644 main/xiaozhi-server/core/processors/listen_processor.py create mode 100644 main/xiaozhi-server/core/processors/mcp_processor.py create mode 100644 main/xiaozhi-server/core/processors/message_router.py create mode 100644 main/xiaozhi-server/core/processors/ping_processor.py create mode 100644 main/xiaozhi-server/core/processors/report_processor.py create mode 100644 main/xiaozhi-server/core/processors/server_processor.py create mode 100644 main/xiaozhi-server/core/processors/text_processor.py create mode 100644 main/xiaozhi-server/core/processors/timeout_processor.py create mode 100644 main/xiaozhi-server/core/providers/asr/shared_asr_manager.py create mode 100644 main/xiaozhi-server/core/providers/asr/shared_asr_proxy.py create mode 100644 main/xiaozhi-server/core/servers/multi_protocol_server.py create mode 100644 main/xiaozhi-server/core/services/audio_ingress_service.py create mode 100644 main/xiaozhi-server/core/services/connection_service.py create mode 100644 main/xiaozhi-server/core/transport/transport_interface.py create mode 100644 main/xiaozhi-server/core/transport/websocket_transport.py create mode 100644 main/xiaozhi-server/core/utils/config_validation.py create mode 100644 main/xiaozhi-server/core/utils/restart.py create mode 100644 main/xiaozhi-server/core/websocket_server_new.py create mode 100644 main/xiaozhi-server/core/xiaozhi_server_facade.py diff --git a/main/xiaozhi-server/app.py b/main/xiaozhi-server/app.py index 9fb3a9ea..4da8f1d4 100644 --- a/main/xiaozhi-server/app.py +++ b/main/xiaozhi-server/app.py @@ -3,13 +3,14 @@ import uuid import signal import asyncio from aioconsole import ainput -from config.settings import load_config +from config.config_loader import load_config from config.logger import setup_logging from core.utils.util import get_local_ip, validate_mcp_endpoint from core.http_server import SimpleHttpServer -from core.websocket_server import WebSocketServer +from core.xiaozhi_server_facade import XiaozhiServerFacade from core.utils.util import check_ffmpeg_installed from core.utils.gc_manager import get_gc_manager +from config.manage_api_client import manage_api_http_close TAG = __name__ logger = setup_logging() @@ -51,14 +52,14 @@ async def main(): # auth_key用于jwt认证，比如视觉分析接口的jwt认证、ota接口的token生成与websocket认证 # 获取配置文件中的auth_key auth_key = config["server"].get("auth_key", "") - + # 验证auth_key，无效则尝试使用manager-api.secret if not auth_key or len(auth_key) == 0 or "你" in auth_key: auth_key = config.get("manager-api", {}).get("secret", "") # 验证secret，无效则生成随机密钥 if not auth_key or len(auth_key) == 0 or "你" in auth_key: auth_key = str(uuid.uuid4().hex) - + config["server"]["auth_key"] = auth_key # 添加 stdin 监控任务 @@ -68,12 +69,47 @@ async def main(): gc_manager = get_gc_manager(interval_seconds=300) await gc_manager.start() - # 启动 WebSocket 服务器 - ws_server = WebSocketServer(config) - ws_task = asyncio.create_task(ws_server.start()) - # 启动 Simple http 服务器 - ota_server = SimpleHttpServer(config) - ota_task = asyncio.create_task(ota_server.start()) + # 启动小智服务器门面 + xiaozhi_server = XiaozhiServerFacade(config) + ota_server = None + ota_task = None + try: + # Facade.start() returns after every listener is ready. Await it so bind + # or configuration failures stop startup instead of leaving a partial process. + await xiaozhi_server.start() + ota_server = SimpleHttpServer(config) + ota_task = asyncio.create_task( + ota_server.start(), name="xiaozhi-http-server" + ) + await ota_server.wait_started( + ota_task, + timeout=float(config.get("server_startup_timeout", 10)), + ) + except Exception: + if ota_server: + try: + await ota_server.stop() + except Exception as cleanup_error: + logger.bind(tag=TAG).error(f"停止HTTP服务器失败: {cleanup_error}") + if ota_task: + if not ota_task.done(): + ota_task.cancel() + await asyncio.gather(ota_task, return_exceptions=True) + try: + await xiaozhi_server.stop() + except Exception as cleanup_error: + logger.bind(tag=TAG).error(f"停止协议服务器失败: {cleanup_error}") + try: + await gc_manager.stop() + except Exception as cleanup_error: + logger.bind(tag=TAG).error(f"停止GC管理器失败: {cleanup_error}") + try: + await manage_api_http_close() + except Exception as cleanup_error: + logger.bind(tag=TAG).error(f"关闭管理API客户端失败: {cleanup_error}") + stdin_task.cancel() + await asyncio.gather(stdin_task, return_exceptions=True) + raise read_config_from_api = config.get("read_config_from_api", False) port = int(config["server"].get("http_port", 8003)) @@ -100,24 +136,31 @@ async def main(): logger.bind(tag=TAG).error("mcp接入点不符合规范") config["mcp_endpoint"] = "你的接入点 websocket地址" - # 获取WebSocket配置，使用安全的默认值 - websocket_port = 8000 - server_config = config.get("server", {}) - if isinstance(server_config, dict): - websocket_port = int(server_config.get("port", 8000)) + # 显示协议连接信息 + connection_info = xiaozhi_server.get_connection_info() - logger.bind(tag=TAG).info( - "Websocket地址是\tws://{}:{}/xiaozhi/v1/", - get_local_ip(), - websocket_port, - ) + # WebSocket信息 + websocket_info = connection_info.get("websocket", {}) + if websocket_info.get("enabled", False): + websocket_port = websocket_info.get("port", 8000) + logger.bind(tag=TAG).info( + "WebSocket地址是\tws://{}:{}/xiaozhi/v1/", + get_local_ip(), + websocket_port, + ) + + # 显示启用的协议 + enabled_protocols = xiaozhi_server.config.get("enabled_protocols", []) + logger.bind(tag=TAG).info(f"启用的协议: {', '.join(enabled_protocols)}") + + if "websocket" in enabled_protocols: + logger.bind(tag=TAG).info( + "=======上面的WebSocket地址请勿用浏览器访问=======" + ) + logger.bind(tag=TAG).info( + "如想测试WebSocket请启动digital-human模块，打开浏览器交互测试" + ) - logger.bind(tag=TAG).info( - "=======上面的地址是websocket协议地址，请勿用浏览器访问=======" - ) - logger.bind(tag=TAG).info( - "如想测试websocket请启动digital-human模块，打开浏览器交互测试" - ) logger.bind(tag=TAG).info( "=============================================================\n" ) @@ -127,22 +170,66 @@ async def main(): except asyncio.CancelledError: print("任务被取消，清理资源中...") finally: + shutdown_errors = [] + if ota_server: + try: + await ota_server.stop() + except Exception as e: + shutdown_errors.append(("HTTP服务器", e)) + logger.bind(tag=TAG).error(f"停止HTTP服务器失败: {e}") + + # 停止小智服务器 + try: + await xiaozhi_server.stop() + except Exception as e: + shutdown_errors.append(("协议服务器", e)) + logger.bind(tag=TAG).error(f"停止小智服务器失败: {e}") + # 停止全局GC管理器 - await gc_manager.stop() + try: + await gc_manager.stop() + except Exception as e: + shutdown_errors.append(("GC管理器", e)) + logger.bind(tag=TAG).error(f"停止GC管理器失败: {e}") - # 取消所有任务（关键修复点） + try: + await manage_api_http_close() + except Exception as e: + shutdown_errors.append(("管理API客户端", e)) + logger.bind(tag=TAG).error(f"关闭管理API客户端失败: {e}") + + # stdin 可立即取消；HTTP task 先获得窗口完成 runner.cleanup()。 stdin_task.cancel() - ws_task.cancel() - if ota_task: - ota_task.cancel() + await asyncio.gather(stdin_task, return_exceptions=True) - # 等待任务终止（必须加超时） - await asyncio.wait( - [stdin_task, ws_task, ota_task] if ota_task else [stdin_task, ws_task], - timeout=3.0, - return_when=asyncio.ALL_COMPLETED, - ) + if ota_task: + done, pending = await asyncio.wait({ota_task}, timeout=3.0) + if pending: + ota_task.cancel() + await asyncio.gather(ota_task, return_exceptions=True) + elif done and not ota_task.cancelled(): + try: + ota_task.result() + except Exception as e: + shutdown_errors.append((ota_task.get_name(), e)) + logger.bind(tag=TAG).error( + f"后台任务退出失败: {ota_task.get_name()}: {e}" + ) + + # If task cancellation or its first cleanup attempt retained the + # runner, retry after the task has fully relinquished ownership. + if ota_server: + try: + await ota_server.stop() + except Exception as e: + shutdown_errors.append(("HTTP服务器重试清理", e)) + logger.bind(tag=TAG).error(f"重试停止HTTP服务器失败: {e}") print("服务器已关闭，程序退出。") + if shutdown_errors: + details = ", ".join( + f"{owner}: {error}" for owner, error in shutdown_errors + ) + raise RuntimeError(f"服务器清理失败: {details}") if __name__ == "__main__": diff --git a/main/xiaozhi-server/config/manage_api_client.py b/main/xiaozhi-server/config/manage_api_client.py index 8eb2da20..b3e57ad5 100644 --- a/main/xiaozhi-server/config/manage_api_client.py +++ b/main/xiaozhi-server/config/manage_api_client.py @@ -80,6 +80,28 @@ class ManageApiClient: # 如果没有运行中的事件循环，创建一个临时的 raise Exception("必须在异步上下文中调用") + @classmethod + async def close_current_loop_client(cls): + """Close the client owned by the running loop before that loop exits.""" + import asyncio + + loop_id = id(asyncio.get_running_loop()) + client = cls._async_clients.pop(loop_id, None) + if client is not None: + await client.aclose() + + @classmethod + async def close_all_clients(cls): + """Close remaining clients after session/report workers have stopped.""" + clients = list(cls._async_clients.values()) + cls._async_clients.clear() + for client in clients: + try: + await client.aclose() + except Exception: + pass + cls._instance = None + @classmethod async def _async_request(cls, method: str, endpoint: str, **kwargs) -> Dict: """发送单次异步HTTP请求并处理响应""" @@ -266,3 +288,7 @@ def init_service(config): def manage_api_http_safe_close(): ManageApiClient.safe_close() + + +async def manage_api_http_close(): + await ManageApiClient.close_all_clients() diff --git a/main/xiaozhi-server/core/auth.py b/main/xiaozhi-server/core/auth.py index be95da21..c0f2d8a7 100755 --- a/main/xiaozhi-server/core/auth.py +++ b/main/xiaozhi-server/core/auth.py @@ -10,6 +10,164 @@ class AuthenticationError(Exception): pass +class AuthMiddleware: + """ + 认证中间件 + 用于 WebSocket/MQTT 连接认证 + 集成 AuthManager 的 token 验证逻辑，支持多种认证方式 + """ + + def __init__(self, config: dict): + """ + 初始化认证中间件 + + Args: + config: 配置字典，包含认证相关配置 + """ + self.config = config + server_config = config.get("server", {}) + auth_config = server_config.get("auth", {}) + + self.enabled = auth_config.get("enabled", False) + self.tokens = auth_config.get("tokens", []) + self.allowed_devices = set(auth_config.get("allowed_devices", [])) + + # 获取 auth_key 用于 HMAC token 验证 + self.auth_key = server_config.get("auth_key", "") + expire_seconds = auth_config.get("expire_seconds", None) + + # 创建 AuthManager 实例用于 HMAC token 验证 + if self.auth_key: + self._auth_manager = AuthManager( + secret_key=self.auth_key, + expire_seconds=expire_seconds + ) + else: + self._auth_manager = None + + def authenticate(self, device_id: str, token: str = None, client_id: str = None) -> bool: + """ + 验证设备认证（同步方法） + + Args: + device_id: 设备 ID + token: 认证令牌（可以是静态 token 或 HMAC token） + client_id: 客户端 ID（用于 HMAC token 验证） + + Returns: + bool: 认证是否通过 + """ + from config.logger import setup_logging + logger = setup_logging() + + if not self.enabled: + logger.debug("[AuthMiddleware.authenticate] 认证未启用") + return True + + # 1. 检查白名单 + if device_id and device_id in self.allowed_devices: + logger.debug(f"[AuthMiddleware.authenticate] 设备 {device_id} 在白名单中，放行") + return True + + # 2. 检查静态 token + if token: + # 移除 Bearer 前缀（如果有） + if token.startswith("Bearer "): + token = token[7:] + + logger.debug(f"[AuthMiddleware.authenticate] 检查静态token, 配置了 {len(self.tokens)} 个token") + for token_config in self.tokens: + configured_token = token_config.get("token") + if configured_token == token: + logger.debug(f"[AuthMiddleware.authenticate] 静态token匹配成功") + return True + logger.debug(f"[AuthMiddleware.authenticate] 静态token不匹配") + + # 3. 检查 HMAC token（需要 AuthManager） + if token and self._auth_manager and client_id and device_id: + logger.debug(f"[AuthMiddleware.authenticate] 尝试HMAC token验证") + if self._auth_manager.verify_token(token, client_id, device_id): + logger.debug(f"[AuthMiddleware.authenticate] HMAC token验证成功") + return True + logger.debug(f"[AuthMiddleware.authenticate] HMAC token验证失败") + + logger.debug(f"[AuthMiddleware.authenticate] 所有认证方式均失败") + return False + + async def authenticate_async(self, headers: dict) -> bool: + """ + 从 headers 中提取信息并进行异步认证 + + Args: + headers: HTTP 请求头字典 + + Returns: + bool: 认证是否通过 + + Raises: + AuthenticationError: 认证失败时抛出 + """ + from config.logger import setup_logging + logger = setup_logging() + + logger.debug(f"[AuthMiddleware] 开始认证, enabled={self.enabled}") + + if not self.enabled: + logger.debug("[AuthMiddleware] 认证未启用，跳过认证") + return True + + device_id = headers.get("device-id") + client_id = headers.get("client-id") + authorization = headers.get("authorization", "") + + logger.debug(f"[AuthMiddleware] device_id={device_id}, client_id={client_id}, has_auth={bool(authorization)}") + + # 提取 token + token = None + if authorization: + if authorization.startswith("Bearer "): + token = authorization[7:] + else: + token = authorization + + # 执行认证 + auth_result = self.authenticate(device_id, token, client_id) + logger.debug(f"[AuthMiddleware] 认证结果: {auth_result}") + + if auth_result: + return True + + raise AuthenticationError(f"认证失败: device_id={device_id}") + + def authenticate_websocket(self, websocket) -> bool: + """ + WebSocket 连接认证 + + Args: + websocket: WebSocket 连接对象 + + Returns: + bool: 认证是否通过 + """ + if not self.enabled: + return True + + headers = dict(websocket.request.headers) + device_id = headers.get("device-id") + client_id = headers.get("client-id") + authorization = headers.get("authorization", "") + + # 提取 token + token = None + if authorization: + if authorization.startswith("Bearer "): + token = authorization[7:] + else: + token = authorization + + return self.authenticate(device_id, token, client_id) + + class AuthManager: """ 统一授权认证管理器 diff --git a/main/xiaozhi-server/core/components/adapters/asr_adapter.py b/main/xiaozhi-server/core/components/adapters/asr_adapter.py new file mode 100644 index 00000000..fcc44b4e --- /dev/null +++ b/main/xiaozhi-server/core/components/adapters/asr_adapter.py @@ -0,0 +1,116 @@ +from typing import Any, Dict +from core.components.component_manager import Component, ComponentType, ComponentFactory +from core.utils import asr +from core.utils.modules_initialize import initialize_asr +from config.logger import setup_logging + +logger = setup_logging() +TAG = __name__ + + +class ASRAdapter(Component): + """ + ASR组件适配器：将现有ASR组件包装为新的组件接口 + + 支持两种模式： + 1. 共享实例模式：使用 SharedASRManager 的全局共享实例 + 2. 独立实例模式：每个连接创建独立的 ASR 实例（原有逻辑） + """ + + def __init__(self, config: Dict[str, Any]): + super().__init__(ComponentType.ASR, config) + self._asr_instance = None + self._delete_audio = config.get("delete_audio", True) + self._using_shared = False # 是否使用共享实例 + self._shared_owner = None + + async def _do_initialize(self, context: Any) -> None: + """初始化ASR组件""" + try: + # 获取ASR配置 + selected_module = self.config.get("selected_module", {}).get("ASR") + if not selected_module: + raise ValueError("未配置ASR模块") + + # 检查是否有全局共享 ASR 管理器 + shared_manager = getattr(context, 'shared_asr_manager', None) + + acquired_manager = None + if shared_manager and hasattr(shared_manager, "acquire_for_config"): + acquired_manager = await shared_manager.acquire_for_config( + self.config + ) + elif ( + shared_manager + and shared_manager.is_ready() + and shared_manager.matches_config(self.config) + ): + acquired_manager = shared_manager + + if acquired_manager is not None: + # 使用共享实例模式 + logger.bind(tag=TAG).info(f"使用共享 ASR 实例: {selected_module}") + from core.providers.asr.shared_asr_proxy import SharedASRProxy + self._asr_instance = SharedASRProxy(acquired_manager) + self._using_shared = True + self._shared_owner = shared_manager + else: + # 使用独立实例模式（原有逻辑） + if shared_manager and shared_manager.is_ready(): + logger.bind(tag=TAG).info( + f"连接ASR配置与共享实例不匹配，使用独立实例: {selected_module}" + ) + logger.bind(tag=TAG).info(f"使用独立 ASR 实例: {selected_module}") + self._asr_instance = initialize_asr(self.config) + self._using_shared = False + + # Legacy providers inspect conn.asr while handling stream state. + context.asr = self._asr_instance + + # 打开音频通道（如果需要） + if hasattr(self._asr_instance, 'open_audio_channels'): + await self._asr_instance.open_audio_channels(context) + + logger.bind(tag=TAG).info( + f"ASR组件初始化完成: {selected_module}, " + f"共享模式: {self._using_shared}" + ) + + except Exception as e: + logger.bind(tag=TAG).error(f"ASR组件初始化失败: {e}") + raise + + async def _do_cleanup(self) -> None: + """清理ASR组件""" + if self._asr_instance: + instance = self._asr_instance + if self._using_shared: + await self._close_resource(instance) + if self._shared_owner and hasattr( + self._shared_owner, "release_for_config" + ): + await self._shared_owner.release_for_config(instance.manager) + logger.bind(tag=TAG).debug("共享 ASR 代理已释放") + else: + await self._close_resource(instance) + if hasattr(instance, 'cleanup_audio_files'): + instance.cleanup_audio_files() + logger.bind(tag=TAG).info("ASR组件清理完成") + self._asr_instance = None + self._using_shared = False + self._shared_owner = None + + @property + def asr_instance(self): + """获取ASR实例""" + return self._asr_instance + + +class ASRFactory(ComponentFactory): + """ASR组件工厂""" + + def create(self, config: Dict[str, Any]) -> Component: + return ASRAdapter(config) + + def get_component_type(self) -> ComponentType: + return ComponentType.ASR diff --git a/main/xiaozhi-server/core/components/adapters/intent_adapter.py b/main/xiaozhi-server/core/components/adapters/intent_adapter.py new file mode 100644 index 00000000..9dde58a9 --- /dev/null +++ b/main/xiaozhi-server/core/components/adapters/intent_adapter.py @@ -0,0 +1,92 @@ +from typing import Any, Dict +from core.components.component_manager import Component, ComponentType, ComponentFactory +from core.utils import intent, llm +from config.logger import setup_logging + +logger = setup_logging() + + +class IntentAdapter(Component): + """Intent组件适配器：将现有Intent组件包装为新的组件接口""" + + def __init__(self, config: Dict[str, Any]): + super().__init__(ComponentType.INTENT, config) + self._intent_instance = None + self._owned_llm_instance = None + + async def _do_initialize(self, context: Any) -> None: + """初始化Intent组件""" + try: + # 获取Intent配置 + selected_module = self.config.get("selected_module", {}).get("Intent") + if not selected_module: + raise ValueError("未配置Intent模块") + + # 获取Intent类型 + intent_type = ( + selected_module + if "type" not in self.config["Intent"][selected_module] + else self.config["Intent"][selected_module]["type"] + ) + + # 创建Intent实例 + self._intent_instance = intent.create_instance( + intent_type, + self.config["Intent"][selected_module], + ) + + # intent_llm 可配置独立模型；未配置时回退主 LLM。 + if intent_type == "intent_llm": + llm_component = None + if getattr(context, "component_manager", None): + llm_component = await context.component_manager.get_component(ComponentType.LLM, context) + main_llm = getattr(llm_component, "llm_instance", None) + intent_config = self.config["Intent"][selected_module] + dedicated_llm_name = intent_config.get("llm") + selected_llm = main_llm + if ( + dedicated_llm_name + and dedicated_llm_name in self.config.get("LLM", {}) + ): + dedicated_config = self.config["LLM"][dedicated_llm_name] + dedicated_type = dedicated_config.get("type", dedicated_llm_name) + self._owned_llm_instance = llm.create_instance( + dedicated_type, dedicated_config + ) + selected_llm = self._owned_llm_instance + logger.info(f"为意图识别创建专用LLM: {dedicated_llm_name}") + if selected_llm and hasattr(self._intent_instance, "set_llm"): + self._intent_instance.set_llm(selected_llm) + + logger.info(f"Intent组件初始化完成: {intent_type}") + + except Exception as e: + logger.error(f"Intent组件初始化失败: {e}") + raise + + async def _do_cleanup(self) -> None: + """清理Intent组件""" + if self._intent_instance: + instance = self._intent_instance + await self._close_resource(instance) + self._intent_instance = None + logger.info("Intent组件清理完成") + if self._owned_llm_instance: + instance = self._owned_llm_instance + await self._close_resource(instance) + self._owned_llm_instance = None + + @property + def intent_instance(self): + """获取Intent实例""" + return self._intent_instance + + +class IntentFactory(ComponentFactory): + """Intent组件工厂""" + + def create(self, config: Dict[str, Any]) -> Component: + return IntentAdapter(config) + + def get_component_type(self) -> ComponentType: + return ComponentType.INTENT diff --git a/main/xiaozhi-server/core/components/adapters/llm_adapter.py b/main/xiaozhi-server/core/components/adapters/llm_adapter.py new file mode 100644 index 00000000..71d954ea --- /dev/null +++ b/main/xiaozhi-server/core/components/adapters/llm_adapter.py @@ -0,0 +1,64 @@ +from typing import Any, Dict +from core.components.component_manager import Component, ComponentType, ComponentFactory +from core.utils import llm +from config.logger import setup_logging + +logger = setup_logging() + + +class LLMAdapter(Component): + """LLM组件适配器：将现有LLM组件包装为新的组件接口""" + + def __init__(self, config: Dict[str, Any]): + super().__init__(ComponentType.LLM, config) + self._llm_instance = None + + async def _do_initialize(self, context: Any) -> None: + """初始化LLM组件""" + try: + # 获取LLM配置 + selected_module = self.config.get("selected_module", {}).get("LLM") + if not selected_module: + raise ValueError("未配置LLM模块") + + # 获取LLM类型 + llm_type = ( + selected_module + if "type" not in self.config["LLM"][selected_module] + else self.config["LLM"][selected_module]["type"] + ) + + # 创建LLM实例 + self._llm_instance = llm.create_instance( + llm_type, + self.config["LLM"][selected_module], + ) + + logger.info(f"LLM组件初始化完成: {llm_type}") + + except Exception as e: + logger.error(f"LLM组件初始化失败: {e}") + raise + + async def _do_cleanup(self) -> None: + """清理LLM组件""" + if self._llm_instance: + instance = self._llm_instance + await self._close_resource(instance) + self._llm_instance = None + logger.info("LLM组件清理完成") + + @property + def llm_instance(self): + """获取LLM实例""" + return self._llm_instance + + +class LLMFactory(ComponentFactory): + """LLM组件工厂""" + + def create(self, config: Dict[str, Any]) -> Component: + return LLMAdapter(config) + + def get_component_type(self) -> ComponentType: + return ComponentType.LLM diff --git a/main/xiaozhi-server/core/components/adapters/memory_adapter.py b/main/xiaozhi-server/core/components/adapters/memory_adapter.py new file mode 100644 index 00000000..53f450ef --- /dev/null +++ b/main/xiaozhi-server/core/components/adapters/memory_adapter.py @@ -0,0 +1,103 @@ +from typing import Any, Dict +from core.components.component_manager import Component, ComponentType, ComponentFactory +from core.utils import llm, memory +from config.logger import setup_logging + +logger = setup_logging() + + +class MemoryAdapter(Component): + """Memory组件适配器：将现有Memory组件包装为新的组件接口""" + + def __init__(self, config: Dict[str, Any]): + super().__init__(ComponentType.MEMORY, config) + self._memory_instance = None + self._owned_llm_instance = None + + async def _do_initialize(self, context: Any) -> None: + """初始化Memory组件""" + try: + # 获取Memory配置 + selected_module = self.config.get("selected_module", {}).get("Memory") + if not selected_module: + raise ValueError("未配置Memory模块") + + # 获取Memory类型 + memory_type = ( + selected_module + if "type" not in self.config["Memory"][selected_module] + else self.config["Memory"][selected_module]["type"] + ) + + # 创建Memory实例 + self._memory_instance = memory.create_instance( + memory_type, + self.config["Memory"][selected_module], + self.config.get("summaryMemory", None), + ) + + # 初始化记忆模块 + main_llm = None + if hasattr(self._memory_instance, 'init_memory'): + # 需要LLM实例来初始化记忆 + llm_component = None + if getattr(context, "component_manager", None): + llm_component = await context.component_manager.get_component(ComponentType.LLM, context) + if llm_component and hasattr(llm_component, 'llm_instance'): + main_llm = llm_component.llm_instance + self._memory_instance.init_memory( + role_id=context.device_id, + llm=main_llm, + summary_memory=self.config.get("summaryMemory", None), + save_to_file=not self.config.get("read_config_from_api", False), + ) + + memory_config = self.config["Memory"][selected_module] + dedicated_llm_name = memory_config.get("llm") + if ( + memory_type == "mem_local_short" + and dedicated_llm_name + and dedicated_llm_name in self.config.get("LLM", {}) + ): + dedicated_config = self.config["LLM"][dedicated_llm_name] + dedicated_type = dedicated_config.get("type", dedicated_llm_name) + self._owned_llm_instance = llm.create_instance( + dedicated_type, dedicated_config + ) + self._memory_instance.set_llm(self._owned_llm_instance) + logger.info(f"为记忆总结创建专用LLM: {dedicated_llm_name}") + elif main_llm and hasattr(self._memory_instance, "set_llm"): + self._memory_instance.set_llm(main_llm) + + logger.info(f"Memory组件初始化完成: {memory_type}") + + except Exception as e: + logger.error(f"Memory组件初始化失败: {e}") + raise + + async def _do_cleanup(self) -> None: + """清理Memory组件""" + if self._memory_instance: + instance = self._memory_instance + await self._close_resource(instance) + self._memory_instance = None + logger.info("Memory组件清理完成") + if self._owned_llm_instance: + instance = self._owned_llm_instance + await self._close_resource(instance) + self._owned_llm_instance = None + + @property + def memory_instance(self): + """获取Memory实例""" + return self._memory_instance + + +class MemoryFactory(ComponentFactory): + """Memory组件工厂""" + + def create(self, config: Dict[str, Any]) -> Component: + return MemoryAdapter(config) + + def get_component_type(self) -> ComponentType: + return ComponentType.MEMORY diff --git a/main/xiaozhi-server/core/components/adapters/tts_adapter.py b/main/xiaozhi-server/core/components/adapters/tts_adapter.py new file mode 100644 index 00000000..70e8bf97 --- /dev/null +++ b/main/xiaozhi-server/core/components/adapters/tts_adapter.py @@ -0,0 +1,76 @@ +from typing import Any, Dict +from core.components.component_manager import Component, ComponentType, ComponentFactory +from core.utils import tts +from config.logger import setup_logging + +logger = setup_logging() + + +class TTSAdapter(Component): + """TTS组件适配器：将现有TTS组件包装为新的组件接口""" + + def __init__(self, config: Dict[str, Any]): + super().__init__(ComponentType.TTS, config) + self._tts_instance = None + self._delete_audio = config.get("delete_audio", True) + + async def _do_initialize(self, context: Any) -> None: + """初始化TTS组件""" + try: + # 获取TTS配置 + selected_module = self.config.get("selected_module", {}).get("TTS") + if not selected_module: + raise ValueError("未配置TTS模块") + + # 获取TTS类型 + tts_type = ( + selected_module + if "type" not in self.config["TTS"][selected_module] + else self.config["TTS"][selected_module]["type"] + ) + + # 创建TTS实例 + self._tts_instance = tts.create_instance( + tts_type, + self.config["TTS"][selected_module], + str(self._delete_audio).lower() in ("true", "1", "yes"), + ) + + # 打开音频通道 + if hasattr(self._tts_instance, 'open_audio_channels'): + await self._tts_instance.open_audio_channels(context) + + # 设置兼容属性（用于向后兼容） + if hasattr(context, 'tts'): + context.tts = self._tts_instance + + logger.info(f"TTS组件初始化完成: {tts_type}") + + except Exception as e: + logger.error(f"TTS组件初始化失败: {e}") + raise + + async def _do_cleanup(self) -> None: + """清理TTS组件""" + if self._tts_instance: + instance = self._tts_instance + await self._close_resource(instance) + if hasattr(instance, 'cleanup_audio_files'): + instance.cleanup_audio_files() + self._tts_instance = None + logger.info("TTS组件清理完成") + + @property + def tts_instance(self): + """获取TTS实例""" + return self._tts_instance + + +class TTSFactory(ComponentFactory): + """TTS组件工厂""" + + def create(self, config: Dict[str, Any]) -> Component: + return TTSAdapter(config) + + def get_component_type(self) -> ComponentType: + return ComponentType.TTS diff --git a/main/xiaozhi-server/core/components/adapters/vad_adapter.py b/main/xiaozhi-server/core/components/adapters/vad_adapter.py new file mode 100644 index 00000000..4f18a8f0 --- /dev/null +++ b/main/xiaozhi-server/core/components/adapters/vad_adapter.py @@ -0,0 +1,64 @@ +from typing import Any, Dict +from core.components.component_manager import Component, ComponentType, ComponentFactory +from core.utils import vad +from config.logger import setup_logging + +logger = setup_logging() + + +class VADAdapter(Component): + """VAD组件适配器：将现有VAD组件包装为新的组件接口""" + + def __init__(self, config: Dict[str, Any]): + super().__init__(ComponentType.VAD, config) + self._vad_instance = None + + async def _do_initialize(self, context: Any) -> None: + """初始化VAD组件""" + try: + # 获取VAD配置 + selected_module = self.config.get("selected_module", {}).get("VAD") + if not selected_module: + raise ValueError("未配置VAD模块") + + # 获取VAD类型 + vad_type = ( + selected_module + if "type" not in self.config["VAD"][selected_module] + else self.config["VAD"][selected_module]["type"] + ) + + # 创建VAD实例 + self._vad_instance = vad.create_instance( + vad_type, + self.config["VAD"][selected_module], + ) + + logger.info(f"VAD组件初始化完成: {vad_type}") + + except Exception as e: + logger.error(f"VAD组件初始化失败: {e}") + raise + + async def _do_cleanup(self) -> None: + """清理VAD组件""" + if self._vad_instance: + instance = self._vad_instance + await self._close_resource(instance) + self._vad_instance = None + logger.info("VAD组件清理完成") + + @property + def vad_instance(self): + """获取VAD实例""" + return self._vad_instance + + +class VADFactory(ComponentFactory): + """VAD组件工厂""" + + def create(self, config: Dict[str, Any]) -> Component: + return VADAdapter(config) + + def get_component_type(self) -> ComponentType: + return ComponentType.VAD diff --git a/main/xiaozhi-server/core/components/component_manager.py b/main/xiaozhi-server/core/components/component_manager.py new file mode 100644 index 00000000..bea2ce7a --- /dev/null +++ b/main/xiaozhi-server/core/components/component_manager.py @@ -0,0 +1,268 @@ +import asyncio +import inspect +import weakref +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, Optional, Type, TypeVar, Generic +from enum import Enum +from config.logger import setup_logging + +T = TypeVar('T') + +logger = setup_logging() + + +class ComponentType(Enum): + """组件类型枚举""" + TTS = "tts" + ASR = "asr" + VAD = "vad" + LLM = "llm" + MEMORY = "memory" + INTENT = "intent" + + +class ComponentState(Enum): + """组件状态枚举""" + UNINITIALIZED = "uninitialized" + INITIALIZING = "initializing" + READY = "ready" + ERROR = "error" + CLEANING = "cleaning" + CLEANED = "cleaned" + + +class Component(ABC): + """组件基类：定义统一的组件接口和生命周期管理""" + + def __init__(self, component_type: ComponentType, config: Dict[str, Any]): + self.component_type = component_type + self.config = config + self.state = ComponentState.UNINITIALIZED + self._initialization_lock = asyncio.Lock() + self._cleanup_lock = asyncio.Lock() + self._dependencies: Dict[str, 'Component'] = {} + self._dependents: weakref.WeakSet['Component'] = weakref.WeakSet() + self._resources: list = [] # 存储需要清理的资源 + + @abstractmethod + async def _do_initialize(self, context: Any) -> None: + """子类实现具体的初始化逻辑""" + pass + + @abstractmethod + async def _do_cleanup(self) -> None: + """子类实现具体的清理逻辑""" + pass + + async def initialize(self, context: Any) -> None: + """初始化组件（带锁保护）""" + async with self._initialization_lock: + if self.state != ComponentState.UNINITIALIZED: + return + + try: + self.state = ComponentState.INITIALIZING + logger.info(f"正在初始化组件: {self.component_type.value}") + + # 初始化依赖组件 + await self._initialize_dependencies(context) + + # 执行具体初始化 + await self._do_initialize(context) + + self.state = ComponentState.READY + logger.info(f"组件初始化完成: {self.component_type.value}") + + except Exception as e: + self.state = ComponentState.ERROR + logger.error(f"组件初始化失败: {self.component_type.value}, 错误: {e}") + # 初始化可能已经分配线程、连接或文件，失败路径也必须释放。 + try: + await self._do_cleanup() + await self._cleanup_resources() + except Exception as cleanup_error: + logger.error( + f"组件初始化回滚失败: {self.component_type.value}, " + f"错误: {cleanup_error}" + ) + raise + + async def cleanup(self) -> None: + """清理组件（带锁保护）""" + async with self._cleanup_lock: + if self.state in [ComponentState.CLEANING, ComponentState.CLEANED]: + return + + try: + self.state = ComponentState.CLEANING + logger.info(f"正在清理组件: {self.component_type.value}") + + # 清理依赖此组件的其他组件 + await self._cleanup_dependents() + + # 执行具体清理 + await self._do_cleanup() + + # 清理资源 + await self._cleanup_resources() + + self.state = ComponentState.CLEANED + logger.info(f"组件清理完成: {self.component_type.value}") + + except Exception as e: + logger.error(f"组件清理失败: {self.component_type.value}, 错误: {e}") + self.state = ComponentState.ERROR + raise + + def add_dependency(self, name: str, component: 'Component') -> None: + """添加依赖组件""" + self._dependencies[name] = component + component._dependents.add(self) + + def add_resource(self, resource: Any) -> None: + """添加需要清理的资源""" + self._resources.append(resource) + + async def _initialize_dependencies(self, context: Any) -> None: + """初始化依赖组件""" + for name, dep in self._dependencies.items(): + if dep.state == ComponentState.UNINITIALIZED: + await dep.initialize(context) + + async def _cleanup_dependents(self) -> None: + """清理依赖此组件的其他组件""" + for dependent in list(self._dependents): + await dependent.cleanup() + + async def _cleanup_resources(self) -> None: + """清理所有注册的资源""" + for resource in self._resources: + try: + await self._close_resource(resource) + except Exception as e: + logger.warning(f"清理资源时出错: {e}") + self._resources.clear() + + @staticmethod + async def _close_resource(resource: Any) -> None: + """Close either sync or async providers without invoking them twice.""" + cleanup = getattr(resource, "close", None) or getattr(resource, "cleanup", None) + if not cleanup: + return + result = cleanup() + if inspect.isawaitable(result): + await result + + +class ComponentFactory(ABC): + """组件工厂基类""" + + @abstractmethod + def create(self, config: Dict[str, Any]) -> Component: + """创建组件实例""" + pass + + @abstractmethod + def get_component_type(self) -> ComponentType: + """获取组件类型""" + pass + + +class ComponentManager: + """ + 组件管理器：统一管理连接期内的组件实例生命周期。 + 支持分类管理、依赖注入、按需懒加载与统一清理。 + """ + + def __init__(self, config: Dict[str, Any]): + self._config = config + self._components: Dict[str, Component] = {} + self._factories: Dict[ComponentType, ComponentFactory] = {} + self._component_locks: Dict[str, asyncio.Lock] = {} + self._initialization_order: list[ComponentType] = [] + self.lazy_enabled = True + + def register_factory(self, factory: ComponentFactory) -> None: + """注册组件工厂""" + component_type = factory.get_component_type() + self._factories[component_type] = factory + logger.debug(f"注册组件工厂: {component_type.value}") + + def set_initialization_order(self, order: list[ComponentType]) -> None: + """设置组件初始化顺序""" + self._initialization_order = order + + async def get_component(self, component_type: ComponentType, context: Any) -> Optional[Component]: + """获取组件实例（按需创建）""" + key = component_type.value + + lock = self._component_locks.setdefault(key, asyncio.Lock()) + async with lock: + if key in self._components: + return self._components[key] + if not self.lazy_enabled: + logger.warning(f"组件未初始化且已关闭懒加载: {component_type.value}") + return None + factory = self._factories.get(component_type) + if factory is None: + logger.warning(f"未找到组件工厂: {component_type.value}") + return None + + try: + instance = factory.create(self._config) + # 先登记所有权，确保初始化中途失败时资源仍然可达。 + self._components[key] = instance + await instance.initialize(context) + logger.info(f"组件创建并初始化完成: {component_type.value}") + except Exception as e: + logger.error(f"组件创建失败: {component_type.value}, 错误: {e}") + failed = self._components.pop(key, None) + if failed is not None: + await failed.cleanup() + return None + + return self._components.get(key) + + def get(self, component_name: str) -> Optional[Component]: + """获取已初始化的组件实例（兼容接口）""" + return self._components.get(component_name) + + async def initialize_all(self, context: Any) -> None: + """按顺序初始化所有组件""" + for component_type in self._initialization_order: + await self.get_component(component_type, context) + + async def cleanup_all(self) -> None: + """清理所有组件（逆序清理）""" + errors = [] + # 按逆序清理，确保依赖关系正确 + for component_type in reversed(self._initialization_order): + key = component_type.value + if key in self._components: + component = self._components.pop(key) + try: + await component.cleanup() + except Exception as e: + errors.append((key, e)) + + # 清理可能遗漏的组件 + remaining_components = list(self._components.items()) + for key, component in remaining_components: + try: + await component.cleanup() + except Exception as e: + errors.append((key, e)) + + self._components.clear() + self._component_locks.clear() + logger.info("所有组件已清理完成") + if errors: + details = ", ".join(f"{name}: {error}" for name, error in errors) + raise RuntimeError(f"部分组件清理失败: {details}") + + def get_component_status(self) -> Dict[str, str]: + """获取所有组件状态""" + return { + name: component.state.value + for name, component in self._components.items() + } diff --git a/main/xiaozhi-server/core/components/component_registry.py b/main/xiaozhi-server/core/components/component_registry.py new file mode 100644 index 00000000..5d06a001 --- /dev/null +++ b/main/xiaozhi-server/core/components/component_registry.py @@ -0,0 +1,73 @@ +from typing import Dict, Any +from core.components.component_manager import ComponentManager, ComponentType +from core.components.adapters.tts_adapter import TTSFactory +from core.components.adapters.asr_adapter import ASRFactory +from core.components.adapters.vad_adapter import VADFactory +from core.components.adapters.llm_adapter import LLMFactory +from core.components.adapters.memory_adapter import MemoryFactory +from core.components.adapters.intent_adapter import IntentFactory +from config.logger import setup_logging + +logger = setup_logging() + + +class ComponentRegistry: + """组件注册器：统一管理所有组件工厂的注册""" + + _factories_registered = False + + @classmethod + def create_component_manager(cls, config: Dict[str, Any]) -> ComponentManager: + """创建并配置组件管理器""" + manager = ComponentManager(config) + + # 只在第一次时记录注册日志 + if not cls._factories_registered: + logger.info("注册组件工厂") + cls._factories_registered = True + + # 注册所有组件工厂（每个manager都需要注册，但不重复记录日志） + manager.register_factory(TTSFactory()) + manager.register_factory(ASRFactory()) + manager.register_factory(VADFactory()) + manager.register_factory(LLMFactory()) + manager.register_factory(MemoryFactory()) + manager.register_factory(IntentFactory()) + + # 设置组件初始化顺序（考虑依赖关系） + # VAD -> ASR -> LLM -> Memory/Intent -> TTS + initialization_order = [ + ComponentType.VAD, + ComponentType.ASR, + ComponentType.LLM, + ComponentType.MEMORY, + ComponentType.INTENT, + ComponentType.TTS, + ] + manager.set_initialization_order(initialization_order) + + if not cls._factories_registered: + logger.info("组件管理器创建完成，已注册所有组件工厂") + + return manager + + @staticmethod + def get_required_components(config: Dict[str, Any]) -> list[ComponentType]: + """根据配置获取需要的组件类型""" + required = [] + selected_modules = config.get("selected_module", {}) + + if selected_modules.get("VAD"): + required.append(ComponentType.VAD) + if selected_modules.get("ASR"): + required.append(ComponentType.ASR) + if selected_modules.get("LLM"): + required.append(ComponentType.LLM) + if selected_modules.get("TTS"): + required.append(ComponentType.TTS) + if selected_modules.get("Memory"): + required.append(ComponentType.MEMORY) + if selected_modules.get("Intent"): + required.append(ComponentType.INTENT) + + return required diff --git a/main/xiaozhi-server/core/context/session_context.py b/main/xiaozhi-server/core/context/session_context.py new file mode 100644 index 00000000..efe3038a --- /dev/null +++ b/main/xiaozhi-server/core/context/session_context.py @@ -0,0 +1,505 @@ +import copy +import uuid +import time +import queue +import asyncio +import threading +from dataclasses import dataclass, field +from typing import Any, Dict, Optional, List, Callable, Awaitable, Union +from collections import deque +from concurrent.futures import ThreadPoolExecutor +from core.utils.dialogue import Dialogue +from core.auth import AuthMiddleware +from core.utils.prompt_manager import PromptManager +from core.utils.voiceprint_provider import VoiceprintProvider +from config.logger import setup_logging + + +@dataclass +class SessionContext: + """ + 会话上下文：完全替换ConnectionHandler的所有功能 + 承载单连接生命周期内的状态、组件、资源管理 + 与传输层解耦，支持WebSocket/MQTT/UDP等多协议 + """ + + # === 基础标识 === + session_id: str = field(default_factory=lambda: str(uuid.uuid4())) + device_id: Optional[str] = None + client_ip: Optional[str] = None + headers: Dict[str, str] = field(default_factory=dict) + + # === 配置管理 === + config: Dict[str, Any] = field(default_factory=dict) + common_config: Dict[str, Any] = field(default_factory=dict) + private_config: Dict[str, Any] = field(default_factory=dict) + selected_module_str: str = "" + + # === 认证与绑定 === + is_authenticated: bool = False + need_bind: bool = False + bind_code: Optional[str] = None + read_config_from_api: bool = False + max_output_size: int = 0 + chat_history_conf: int = 0 + + # === 会话状态 === + is_speaking: bool = False + listen_mode: str = "auto" + abort_requested: bool = False + close_after_chat: bool = False + conversation_active: bool = False + last_finalized_session_id: Optional[str] = None + just_woken_up: bool = False + calling: bool = False + incoming_call: Optional[Any] = None + load_function_plugin: bool = False + intent_type: str = "nointent" + + # === 音频相关 === + audio_format: str = "opus" + # sample_rate/channels/frame_duration remain the output-side compatibility + # fields consumed by existing TTS providers. + sample_rate: int = 24000 + channels: int = 1 + frame_duration: int = 60 + input_sample_rate: int = 16000 + input_channels: int = 1 + input_frame_duration: int = 60 + output_sample_rate: int = 24000 + output_channels: int = 1 + output_frame_duration: int = 60 + client_aec: bool = False + # Native MQTT control and UDP audio use different channels. Keep UDP + # closed until the first listen/start of a logical session. It stays open + # through the bounded listen/stop grace period so queued tail frames are + # included in the same ASR turn. + accepting_input_audio: bool = False + wake_audio_suppression_task: Optional[asyncio.Task] = None + listen_stop_pending: bool = False + listen_stop_task: Optional[asyncio.Task] = None + listen_stop_deadline: float = 0.0 + listen_start_task: Optional[asyncio.Task] = None + client_have_voice: bool = False + client_voice_stop: bool = False + client_audio_buffer: bytearray = field(default_factory=bytearray) + client_voice_window: deque = field(default_factory=lambda: deque(maxlen=5)) + last_is_voice: bool = False + audio_flow_control: Dict[str, Any] = field(default_factory=dict) + aec_audio_cache: Dict[int, bytes] = field(default_factory=dict) + aec_audio_cache_time: Dict[int, float] = field(default_factory=dict) + + # === ASR相关 === + asr_audio: List[bytes] = field(default_factory=list) + asr_audio_queue: queue.Queue = field(default_factory=queue.Queue) + asr_priority_thread: Optional[threading.Thread] = None + asr_result_handler: Optional[Callable[[str, List[bytes]], Awaitable[None]]] = None + uses_pipeline_runtime: bool = True + + # === LLM相关 === + llm_finish_task: bool = True + dialogue: Optional[Dialogue] = None + current_speaker: Optional[str] = None + introduced_speakers: set = field(default_factory=set) + system_introduced_speakers: set = field(default_factory=set) + sentence_id: Optional[str] = None + + # === TTS相关 === + tts_MessageText: str = "" + + # === IoT相关 === + iot_descriptors: Dict[str, Any] = field(default_factory=dict) + func_handler: Optional[Any] = None + + # === 时间管理 === + last_activity_time_ms: float = field(default_factory=lambda: time.time() * 1000) + created_at: float = field(default_factory=lambda: time.time()) + timeout_seconds: int = 180 # 默认超时时间 + timeout_task: Optional[asyncio.Task] = None + + # === 组件实例 === + # components属性通过@property方法提供，指向component_manager + + # === 其他状态 === + welcome_msg: Optional[Dict[str, Any]] = None + prompt: Optional[str] = None + features: Optional[Dict[str, Any]] = None + mcp_client: Optional[Any] = None + cmd_exit: List[str] = field(default_factory=list) + + # === 线程与并发 === + loop: Optional[asyncio.AbstractEventLoop] = None + stop_event: Optional[threading.Event] = None + executor: Optional[ThreadPoolExecutor] = None + conversation_tasks: set = field(default_factory=set) + turn_tasks: set = field(default_factory=set) + background_tasks: set = field(default_factory=set) + + # === 队列管理 === + report_queue: queue.Queue = field( + default_factory=lambda: queue.Queue(maxsize=64) + ) + report_thread: Optional[threading.Thread] = None + report_asr_enable: bool = False + report_tts_enable: bool = False + + # === 组件管理器 === + component_manager: Optional[Any] = None + + # === 兼容属性（用于向后兼容TTS处理） === + tts: Optional[Any] = None + websocket: Optional[Any] = None # 兼容旧TTS组件 + transport: Optional[Any] = None # 新的transport接口 + + # === 工具类 === + auth: Optional[AuthMiddleware] = None + prompt_manager: Optional[PromptManager] = None + voiceprint_provider: Optional[VoiceprintProvider] = None + server: Optional[Any] = None # WebSocket服务器引用 + + # === 会话级清理回调 === + _cleanup_callbacks: List[Callable[[], Union[None, Awaitable[None]]]] = field(default_factory=list) + + def __post_init__(self): + """初始化后处理""" + # 深拷贝配置避免污染 + if self.config: + self.common_config = self.config + self.config = copy.deepcopy(self.config) + + # 从配置中读取相关设置 + self.read_config_from_api = self.config.get("read_config_from_api", False) + self.max_output_size = self.config.get("max_output_size", 0) + self.chat_history_conf = self.config.get("chat_history_conf", 0) + self.cmd_exit = self.config.get("exit_commands", []) + self.timeout_seconds = int(self.config.get("close_connection_no_voice_time", 120)) + 60 + + # 初始化认证中间件 + self.auth = AuthMiddleware(self.config) + + # 初始化提示词管理器 + self.prompt_manager = PromptManager(self.config, setup_logging()) + + # 初始化对话管理 + if not self.dialogue: + self.dialogue = Dialogue() + + # 初始化线程相关 + if not self.loop: + try: + self.loop = asyncio.get_event_loop() + except RuntimeError: + self.loop = asyncio.new_event_loop() + + if not self.stop_event: + self.stop_event = threading.Event() + + if not self.executor: + self.executor = ThreadPoolExecutor(max_workers=5) + + # 初始化上报设置 + self.report_asr_enable = self.read_config_from_api + self.report_tts_enable = self.read_config_from_api + + def update_activity(self) -> None: + """刷新最后活跃时间""" + self.last_activity_time_ms = time.time() * 1000 + + def clearSpeakStatus(self) -> None: + """清除服务端讲话状态（兼容方法）""" + self.is_speaking = False + logger = setup_logging() + logger.debug("清除服务端讲话状态") + + def change_system_prompt(self, prompt: str) -> None: + """Update the active role prompt for legacy function plugins.""" + self.prompt = prompt + self.dialogue.update_system_message(prompt) + + def reset_vad_states(self) -> None: + """重置VAD状态（兼容方法）""" + self.client_audio_buffer = bytearray() + self.client_have_voice = False + self.client_voice_stop = False + self.last_is_voice = False + self.client_voice_window.clear() + logger = setup_logging() + logger.debug("VAD states reset.") + + def reset_audio_states(self) -> None: + """Reset all turn-scoped input state expected by legacy ASR providers.""" + self.asr_audio.clear() + self.listen_stop_pending = False + self.reset_vad_states() + + def is_timeout(self, timeout_seconds: int) -> bool: + """检查是否超时""" + now_ms = time.time() * 1000 + return (now_ms - self.last_activity_time_ms) > (timeout_seconds * 1000) + + def register_cleanup(self, callback: Callable[[], Union[None, Awaitable[None]]]) -> None: + """注册会话结束时需要执行的清理回调""" + self._cleanup_callbacks.append(callback) + + def unregister_cleanup( + self, callback: Callable[[], Union[None, Awaitable[None]]] + ) -> None: + """Remove a callback when its connection-scoped owner is replaced.""" + self._cleanup_callbacks = [ + registered + for registered in self._cleanup_callbacks + if registered != callback + ] + + def create_background_task( + self, + coroutine, + *, + conversation_scoped: bool = True, + turn_scoped: bool = False, + ) -> asyncio.Task: + """Create a tracked task with an explicit turn/session/connection owner.""" + task = asyncio.create_task(coroutine) + if turn_scoped: + owner = self.turn_tasks + elif conversation_scoped: + owner = self.conversation_tasks + else: + owner = self.background_tasks + owner.add(task) + task.add_done_callback(owner.discard) + return task + + async def cancel_turn_tasks(self) -> None: + """Cancel producers belonging only to the active speech/chat turn.""" + current_task = asyncio.current_task() + pending_tasks = [ + task + for task in list(self.turn_tasks) + if task is not current_task and not task.done() + ] + for task in pending_tasks: + task.cancel() + if pending_tasks: + await asyncio.gather(*pending_tasks, return_exceptions=True) + self.turn_tasks = { + task for task in self.turn_tasks if task is current_task + } + + async def cancel_conversation_tasks(self) -> None: + """Cancel all producers owned by the current logical conversation.""" + await self.cancel_turn_tasks() + current_task = asyncio.current_task() + while True: + pending_tasks = [ + task + for task in list(self.conversation_tasks) + if task is not current_task and not task.done() + ] + if not pending_tasks: + break + for task in pending_tasks: + task.cancel() + await asyncio.gather(*pending_tasks, return_exceptions=True) + self.conversation_tasks = { + task for task in self.conversation_tasks if task is current_task + } + + async def run_cleanup(self) -> None: + """执行所有注册的清理回调""" + logger = setup_logging() + logger.info(f"Session {self.session_id} 开始执行会话级清理 ({len(self._cleanup_callbacks)} 个回调)") + + # 停止所有线程 + if self.stop_event: + self.stop_event.set() + + # 取消超时任务 + if self.timeout_task and not self.timeout_task.done(): + self.timeout_task.cancel() + + await self.cancel_conversation_tasks() + + current_task = asyncio.current_task() + pending_tasks = [ + task + for task in list(self.background_tasks) + if task is not current_task and not task.done() + ] + for task in pending_tasks: + task.cancel() + if pending_tasks: + await asyncio.gather(*pending_tasks, return_exceptions=True) + self.background_tasks.clear() + + # 执行清理回调 + for callback in reversed(self._cleanup_callbacks): + try: + result = callback() + if asyncio.iscoroutine(result): + await result + except Exception as e: + logger.error(f"Session {self.session_id} 清理回调执行失败: {e}", exc_info=True) + + self._cleanup_callbacks.clear() + + # 所有异步生产者退出后再关闭线程池，避免任务继续提交工作。 + if self.executor: + self.executor.shutdown(wait=False) + self.executor = None + logger.info(f"Session {self.session_id} 会话级清理完成") + + # === 兼容旧代码的属性访问 === + @property + def client_is_speaking(self) -> bool: + """兼容旧代码的属性名""" + return self.is_speaking + + @client_is_speaking.setter + def client_is_speaking(self, value: bool): + self.is_speaking = value + + @property + def client_listen_mode(self) -> str: + """兼容旧代码的属性名""" + return self.listen_mode + + @client_listen_mode.setter + def client_listen_mode(self, value: str): + self.listen_mode = value + + @property + def client_abort(self) -> bool: + """兼容旧代码的属性名""" + return self.abort_requested + + @client_abort.setter + def client_abort(self, value: bool): + self.abort_requested = value + + @property + def components(self): + """组件访问器（兼容属性）""" + return self.component_manager + + @components.setter + def components(self, value): + """组件设置器（兼容属性）- 实际设置到component_manager""" + # 如果尝试设置components，我们忽略它或者给出警告 + # 因为components应该通过component_manager管理 + logger = setup_logging() + logger.warning("尝试直接设置components属性，请使用component_manager") + + @property + def last_activity_time(self) -> float: + """兼容旧代码：返回毫秒级时间戳""" + return self.last_activity_time_ms + + @last_activity_time.setter + def last_activity_time(self, value: float): + """兼容旧代码：接受毫秒级时间戳""" + self.last_activity_time_ms = value + + # === 日志相关 === + @property + def logger(self): + """获取日志记录器""" + return setup_logging() + + # === 工具方法 === + def get_component(self, component_name: str) -> Optional[Any]: + """获取组件实例""" + return self.components.get(component_name) + + def set_component(self, component_name: str, component_instance: Any) -> None: + """设置组件实例""" + if self.component_manager: + self.component_manager._components[component_name] = component_instance + + def has_component(self, component_name: str) -> bool: + """检查是否有指定组件""" + return component_name in self.components + + def clear_audio_buffer(self) -> None: + """清空音频缓冲区""" + self.client_audio_buffer.clear() + self.asr_audio.clear() + + # 清空队列 + try: + while not self.asr_audio_queue.empty(): + self.asr_audio_queue.get_nowait() + except queue.Empty: + pass + + def reset_voice_state(self) -> None: + """重置语音状态""" + self.client_have_voice = False + self.client_voice_stop = False + self.last_is_voice = False + self.client_voice_window.clear() + + def initialize_private_config(self) -> None: + """初始化差异化配置（从ConnectionHandler迁移）""" + from config.config_loader import get_private_config_from_api + from config.manage_api_client import DeviceNotFoundException, DeviceBindException + + if not self.read_config_from_api: + return + + try: + # 获取设备私有配置 + private_config = get_private_config_from_api( + self.config, self.device_id, self.headers.get("client-id") + ) + + if private_config: + self.private_config = private_config + # 合并私有配置到主配置 + self.config.update(private_config) + + except DeviceNotFoundException: + self.logger.error(f"设备 {self.device_id} 未找到") + self.need_bind = True + except DeviceBindException as e: + self.logger.error(f"设备绑定异常: {e}") + self.need_bind = True + self.bind_code = str(e) + except Exception as e: + self.logger.error(f"获取私有配置失败: {e}") + + async def initialize_components(self) -> None: + """异步初始化组件（从ConnectionHandler迁移）""" + if not self.component_manager: + return + + try: + # 初始化各个组件 + from core.components.component_registry import ComponentType + + # 按依赖顺序初始化组件 + component_types = [ + ComponentType.VAD, + ComponentType.ASR, + ComponentType.LLM, + ComponentType.MEMORY, + ComponentType.INTENT, + ComponentType.TTS + ] + + for component_type in component_types: + try: + component = await self.component_manager.get_component(component_type, self) + if component: + self.logger.info(f"组件 {component_type} 初始化成功") + except Exception as e: + self.logger.error(f"组件 {component_type} 初始化失败: {e}") + + except Exception as e: + self.logger.error(f"组件初始化失败: {e}") + + def __str__(self) -> str: + return f"SessionContext(session_id={self.session_id}, device_id={self.device_id})" + + def __repr__(self) -> str: + return self.__str__() diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index 421554d6..d3604dee 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -350,4 +350,16 @@ async def send_display_message(conn: "ConnectionHandler", text): "text": text, "session_id": conn.session_id } - await conn.websocket.send(json.dumps(message)) + transport = getattr(conn, "transport", None) + if transport is not None: + send_json = getattr(transport, "send_json", None) + if callable(send_json): + await send_json(message) + else: + await transport.send(json.dumps(message)) + return + + websocket = getattr(conn, "websocket", None) + if websocket is None: + raise AttributeError("无法找到可用的传输层接口") + await websocket.send(json.dumps(message)) diff --git a/main/xiaozhi-server/core/http_server.py b/main/xiaozhi-server/core/http_server.py index feb96f3b..da6251d6 100644 --- a/main/xiaozhi-server/core/http_server.py +++ b/main/xiaozhi-server/core/http_server.py @@ -13,6 +13,33 @@ class SimpleHttpServer: self.logger = setup_logging() self.ota_handler = OTAHandler(config) self.vision_handler = VisionHandler(config) + self._started_event = asyncio.Event() + self._stop_event = asyncio.Event() + self._runner = None + self._start_active = False + self._cleanup_lock = asyncio.Lock() + + async def wait_started(self, task: asyncio.Task, timeout: float = 10) -> None: + """Wait until the HTTP listener is bound or surface startup failure.""" + event_waiter = asyncio.create_task(self._started_event.wait()) + try: + done, _ = await asyncio.wait( + {task, event_waiter}, + timeout=timeout, + return_when=asyncio.FIRST_COMPLETED, + ) + if self._started_event.is_set(): + if task.done(): + await task + return + if task in done: + await task + raise RuntimeError("HTTP服务器在就绪前退出") + raise TimeoutError("等待HTTP服务器启动超时") + finally: + if not event_waiter.done(): + event_waiter.cancel() + await asyncio.gather(event_waiter, return_exceptions=True) def _get_websocket_url(self, local_ip: str, port: int) -> str: """获取websocket地址 @@ -33,7 +60,11 @@ class SimpleHttpServer: return f"ws://{local_ip}:{port}/xiaozhi/v1/" async def start(self): + runner = None + self._start_active = True try: + self._started_event.clear() + self._stop_event.clear() server_config = self.config["server"] read_config_from_api = self.config.get("read_config_from_api", False) host = server_config.get("ip", "0.0.0.0") @@ -74,19 +105,39 @@ class SimpleHttpServer: ), ] ) - # 运行服务 runner = web.AppRunner(app) + self._runner = runner await runner.setup() site = web.TCPSite(runner, host, port) await site.start() - - # 保持服务运行 - while True: - await asyncio.sleep(3600) # 每隔 1 小时检查一次 + self._started_event.set() + await self._stop_event.wait() except Exception as e: self.logger.bind(tag=TAG).error(f"HTTP服务器启动失败: {e}") import traceback self.logger.bind(tag=TAG).error(f"错误堆栈: {traceback.format_exc()}") raise + finally: + self._started_event.clear() + try: + if runner is not None: + await self._cleanup_runner() + finally: + self._start_active = False + + async def _cleanup_runner(self): + """Cleanup the owned runner and retain it when cleanup must be retried.""" + async with self._cleanup_lock: + runner = self._runner + if runner is None: + return + await runner.cleanup() + self._runner = None + + async def stop(self): + """Stop the HTTP loop and retry cleanup left by a failed start task.""" + self._stop_event.set() + if not self._start_active: + await self._cleanup_runner() diff --git a/main/xiaozhi-server/core/pipeline/message_pipeline.py b/main/xiaozhi-server/core/pipeline/message_pipeline.py new file mode 100644 index 00000000..e1ce45e8 --- /dev/null +++ b/main/xiaozhi-server/core/pipeline/message_pipeline.py @@ -0,0 +1,26 @@ +from abc import ABC, abstractmethod +from typing import Any, List + + +class MessageProcessor(ABC): + """消息处理器接口。返回 True 表示已处理并中止后续处理。""" + + @abstractmethod + async def process(self, context: Any, transport: Any, message: Any) -> bool: + raise NotImplementedError + + +class MessagePipeline: + """责任链式消息处理管道。""" + + def __init__(self) -> None: + self._processors: List[MessageProcessor] = [] + + def add_processor(self, processor: MessageProcessor) -> None: + self._processors.append(processor) + + async def process_message(self, context: Any, transport: Any, message: Any) -> None: + for processor in self._processors: + handled = await processor.process(context, transport, message) + if handled: + return diff --git a/main/xiaozhi-server/core/processors/abort_processor.py b/main/xiaozhi-server/core/processors/abort_processor.py new file mode 100644 index 00000000..521504cf --- /dev/null +++ b/main/xiaozhi-server/core/processors/abort_processor.py @@ -0,0 +1,132 @@ +import json +import uuid +from typing import Any +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from core.components.component_manager import ComponentType +from config.logger import setup_logging + +logger = setup_logging() + + +class AbortProcessor(MessageProcessor): + """中断消息处理器：完整迁移abortMessageHandler.py和abortHandle.py的所有功能""" + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """处理abort类型的消息""" + msg_json = None + if isinstance(message, str): + try: + msg_json = json.loads(message) + except json.JSONDecodeError: + msg_json = None + elif isinstance(message, dict): + msg_json = message + + if isinstance(msg_json, dict) and msg_json.get("type") == "abort": + await self.handle_abort_message(context, transport, msg_json) + return True + return False + + async def handle_abort_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict): + """处理中断消息 - 完整迁移自abortHandle.py的handleAbortMessage""" + msg_session_id = msg_json.get("session_id") + if msg_session_id and msg_session_id != context.session_id: + logger.warning( + f"忽略非当前会话的abort消息: " + f"msg={msg_session_id}, current={context.session_id}" + ) + return + logger.info("Abort message received") + + # 设置成打断状态，会自动打断llm、tts任务 - 完整迁移原逻辑 + context.abort_requested = True + context.close_after_chat = False + + # Stop the old LLM/tool producers before a subsequent listen/start can + # clear abort_requested and begin a new turn. + cancel_tasks = getattr(context, "cancel_turn_tasks", None) + if callable(cancel_tasks): + await cancel_tasks() + + # 清理队列 - 完整迁移原逻辑 + await self._clear_queues(context) + + # 打断客户端说话状态 - 完整迁移原逻辑 + await transport.send_json({ + "type": "tts", + "state": "stop", + "session_id": context.session_id + }) + + # 清理说话状态 - 完整迁移原逻辑 + self._clear_speak_status(context) + + logger.info("Abort message received-end") + + async def _clear_queues(self, context: SessionContext): + """清理所有队列 - 完整迁移原clear_queues逻辑""" + try: + # 清理TTS音频队列 + tts_component = await self._get_component(context, ComponentType.TTS) + if tts_component and hasattr(tts_component, 'tts_instance'): + tts_instance = tts_component.tts_instance + for queue_name in ('tts_text_queue', 'tts_audio_queue'): + pending_queue = getattr(tts_instance, queue_name, None) + if pending_queue is None: + continue + try: + while not pending_queue.empty(): + pending_queue.get_nowait() + except: + pass + + # Rotate the turn token so late text/audio from the aborted turn is stale. + context.sentence_id = uuid.uuid4().hex + + # 清理ASR音频队列 + context.clear_audio_buffer() + + # 清理其他可能的队列 + if hasattr(context, 'clear_queues'): + context.clear_queues() + + except Exception as e: + logger.error(f"清理队列时出错: {e}") + + def _clear_speak_status(self, context: SessionContext): + """清理说话状态 - 完整迁移原clearSpeakStatus逻辑""" + try: + # 清理说话状态 + context.is_speaking = False + + # 如果有其他说话状态相关的属性，也一并清理 + if hasattr(context, 'clearSpeakStatus'): + context.clearSpeakStatus() + + # 重置相关状态 + context.client_have_voice = False + context.client_voice_stop = True + + except Exception as e: + logger.error(f"清理说话状态时出错: {e}") + + async def _send_abort_confirmation(self, transport: TransportInterface, session_id: str): + """发送中断确认响应（可选）""" + response = { + "type": "abort", + "status": "success", + "message": "中断操作已完成", + "session_id": session_id + } + + try: + await transport.send(json.dumps(response)) + except Exception as e: + logger.error(f"发送中断确认响应失败: {e}") + + async def _get_component(self, context: SessionContext, component_type: ComponentType): + if not context.component_manager: + return None + return await context.component_manager.get_component(component_type, context) diff --git a/main/xiaozhi-server/core/processors/audio_receive_processor.py b/main/xiaozhi-server/core/processors/audio_receive_processor.py new file mode 100644 index 00000000..43aac780 --- /dev/null +++ b/main/xiaozhi-server/core/processors/audio_receive_processor.py @@ -0,0 +1,439 @@ +import time +import json +import asyncio +import uuid +from typing import Any +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from core.components.component_manager import ComponentType +from core.services.audio_ingress_service import AudioIngressService +from core.utils.util import audio_to_data +from core.utils.output_counter import check_device_output_limit +from config.logger import setup_logging + +logger = setup_logging() + + +def arm_wake_audio_suppression(context: SessionContext) -> None: + """Ignore reordered wake-word audio for a bounded interval.""" + context.just_woken_up = True + previous = getattr(context, "wake_audio_suppression_task", None) + if previous and not previous.done(): + previous.cancel() + + delay_ms = max( + 0, + int(context.config.get("wake_audio_suppression_ms", 2000)), + ) + if delay_ms == 0: + context.just_woken_up = False + context.wake_audio_suppression_task = None + return + + session_id = context.session_id + context.wake_audio_suppression_task = context.create_background_task( + _release_wake_audio_suppression( + context, + session_id, + delay_ms / 1000, + ), + conversation_scoped=True, + ) + + +async def _release_wake_audio_suppression( + context: SessionContext, + session_id: str, + delay_seconds: float, +) -> None: + current_task = asyncio.current_task() + try: + await asyncio.sleep(delay_seconds) + if context.session_id == session_id: + context.just_woken_up = False + finally: + if getattr(context, "wake_audio_suppression_task", None) is current_task: + context.wake_audio_suppression_task = None + + +class AudioReceiveProcessor(MessageProcessor): + """音频接收处理器：完整迁移receiveAudioHandle.py的所有功能""" + + def __init__(self, audio_ingress_service=None): + self.audio_ingress_service = audio_ingress_service or AudioIngressService() + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """处理音频消息""" + if isinstance(message, bytes): + await self.handle_audio_message(context, transport, message, 0) + return True + if isinstance(message, dict) and message.get("type") == "audio": + audio_data = message.get("data") + if isinstance(audio_data, (bytes, bytearray)): + await self.handle_audio_message( + context, + transport, + bytes(audio_data), + int(message.get("timestamp", 0) or 0), + ) + return True + return True + return False + + async def handle_audio_message( + self, + context: SessionContext, + transport: TransportInterface, + audio: bytes, + timestamp: int = 0, + ): + """处理音频消息 - 完整迁移自handleAudioMessage""" + if ( + getattr(transport, "requires_audio_tail_grace", False) + and not getattr(context, "accepting_input_audio", False) + ): + return + + pcm_frame = self.audio_ingress_service.process(context, audio, timestamp) + if not pcm_frame: + return + + # Do not feed encoded wake-word history into stateful VAD. Native MQTT + # gates the usual pre-start history above; this bounded suppression only + # covers frames that crossed the independent control/audio channels. + if context.just_woken_up: + if not getattr(context, "wake_audio_suppression_task", None): + arm_wake_audio_suppression(context) + context.asr_audio.clear() + context.reset_vad_states() + return + + # 获取VAD组件 + vad_component = await self._get_component(context, ComponentType.VAD) + if not vad_component or not hasattr(vad_component, 'vad_instance'): + now = time.time() + last_log = getattr(context, "_vad_missing_last_log", 0.0) + if now - last_log > 10: + logger.warning("VAD组件未初始化") + context._vad_missing_last_log = now + return + + vad_instance = vad_component.vad_instance + + # 当前片段是否有人说话 + have_voice = vad_instance.is_vad(context, pcm_frame) + + if have_voice: + if ( + getattr(context, "client_aec", False) + and context.is_speaking + and context.listen_mode != "manual" + ): + await self._handle_abort_message(context, transport) + + # 设备长时间空闲检测，用于say goodbye + await self._no_voice_close_connect(context, transport, have_voice) + + # 接收音频 + asr_component = await self._get_component(context, ComponentType.ASR) + asr_instance = asr_component.asr_instance if asr_component and hasattr(asr_component, 'asr_instance') else None + + # 自动模式兜底：没有收到 listen stop 时，基于VAD触发一次停止 + if ( + not have_voice + and context.client_have_voice + and not context.client_voice_stop + and not getattr(context, "listen_stop_pending", False) + and context.listen_mode != "manual" + and asr_instance is not None + ): + context.client_voice_stop = True + try: + from core.providers.asr.dto.dto import InterfaceType + if hasattr(asr_instance, "interface_type") and asr_instance.interface_type == InterfaceType.STREAM: + if hasattr(asr_instance, "_send_stop_request"): + context.create_background_task( + asr_instance._send_stop_request(), + turn_scoped=True, + ) + else: + if len(context.asr_audio) > 0: + asr_audio_task = context.asr_audio.copy() + context.asr_audio.clear() + context.reset_vad_states() + await asr_instance.handle_voice_stop(context, asr_audio_task) + except Exception as e: + logger.error(f"自动VAD停止处理失败: {e}") + + if asr_instance and hasattr(asr_instance, 'receive_audio'): + await asr_instance.receive_audio(context, pcm_frame, have_voice) + + async def start_to_chat( + self, + context: SessionContext, + transport: TransportInterface, + text: str, + *, + preserve_close_after_chat: bool = False, + ): + """开始聊天 - 完整迁移自startToChat""" + # 检查输入是否是JSON格式（包含说话人信息） + speaker_name = None + actual_text = text + + try: + # 尝试解析JSON格式的输入 + if text.strip().startswith('{') and text.strip().endswith('}'): + data = json.loads(text) + if 'speaker' in data and 'content' in data: + speaker_name = data['speaker'] + actual_content = data['content'] + logger.info(f"解析到说话人信息: {speaker_name}") + + if speaker_name not in context.introduced_speakers: + context.introduced_speakers.add(speaker_name) + actual_text = text + else: + actual_text = actual_content + except (json.JSONDecodeError, KeyError): + # 如果解析失败，继续使用原始文本 + pass + + # 保存说话人信息到上下文 + if speaker_name: + context.current_speaker = speaker_name + else: + context.current_speaker = None + + # 检查设备绑定 + if context.need_bind: + await self.prompt_bind_device(context, transport) + return + + # 如果当日的输出字数大于限定的字数 + if context.max_output_size > 0: + if check_device_output_limit( + context.headers.get("device-id"), context.max_output_size + ): + await self._max_out_size(context, transport) + return + + if context.is_speaking and getattr(context, "listen_mode", "auto") != "manual": + await self._handle_abort_message(context, transport) + + context.abort_requested = False + if not preserve_close_after_chat: + context.close_after_chat = False + + # 首先进行意图分析，使用实际文本内容 + from core.processors.chat_processor import ChatProcessor + chat_processor = ChatProcessor() + intent_handled = await chat_processor.handle_user_intent(context, transport, actual_text) + + if intent_handled: + # 如果意图已被处理，不再进行聊天 + return + + # 意图未被处理，继续常规聊天流程，使用实际文本内容 + await self._send_stt_message(context, transport, actual_text) + + # 与旧架构一致：将聊天处理放入线程池，避免阻塞事件循环 + from core.processors.chat_processor import ChatProcessor + chat_processor = ChatProcessor() + + if hasattr(context, "create_background_task"): + context.create_background_task( + chat_processor.handle_chat( + context, + transport, + actual_text, + skip_intent=True, + ), + turn_scoped=True, + ) + else: + await chat_processor.handle_chat( + context, transport, actual_text, skip_intent=True + ) + + async def _no_voice_close_connect(self, context: SessionContext, transport: TransportInterface, have_voice: bool): + """无声音时关闭连接检测 - 完整迁移自no_voice_close_connect""" + if have_voice: + context.update_activity() + return + + # 只有在已经初始化过时间戳的情况下才进行超时检查 + if context.last_activity_time_ms > 0.0: + no_voice_time = time.time() * 1000 - context.last_activity_time_ms + close_connection_no_voice_time = int( + context.config.get("close_connection_no_voice_time", 120) + ) + + if ( + not context.close_after_chat + and no_voice_time > 1000 * close_connection_no_voice_time + ): + context.close_after_chat = True + context.abort_requested = False + + end_prompt = context.config.get("end_prompt", {}) + if end_prompt and end_prompt.get("enable", True) is False: + logger.info("结束对话，无需发送结束提示语") + if transport.keeps_connection_between_sessions: + session_id = context.session_id + end_conversation = getattr(context, "end_conversation", None) + if callable(end_conversation): + await end_conversation(session_id) + await transport.end_session(session_id) + context.close_after_chat = False + context.reset_vad_states() + else: + await transport.close() + return + + prompt = end_prompt.get("prompt") + if not prompt: + prompt = "请你以```时间过得真快```未来头，用富有感情、依依不舍的话来结束这场对话吧。！" + await self.start_to_chat( + context, + transport, + prompt, + preserve_close_after_chat=True, + ) + + async def _max_out_size(self, context: SessionContext, transport: TransportInterface): + """超出最大输出字数处理 - 完整迁移自max_out_size""" + # 播放超出最大输出字数的提示 + context.abort_requested = False + text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！" + await self._send_stt_message(context, transport, text) + + file_path = "config/assets/max_output_size.wav" + opus_packets = await audio_to_data(file_path) + + # 获取TTS组件并添加到队列 + tts_component = await self._get_component(context, ComponentType.TTS) + if tts_component and hasattr(tts_component, 'tts_instance'): + tts_instance = tts_component.tts_instance + if hasattr(tts_instance, 'tts_audio_queue'): + from core.providers.tts.dto.dto import SentenceType + tts_instance.tts_audio_queue.put( + (SentenceType.LAST, opus_packets, text, context.sentence_id) + ) + + context.close_after_chat = True + + async def prompt_bind_device( + self, + context: SessionContext, + transport: TransportInterface, + ): + """检查设备绑定 - 完整迁移自check_bind_device""" + bind_code = context.bind_code + + if bind_code: + # 确保bind_code是6位数字 + if len(bind_code) != 6: + logger.error(f"无效的绑定码格式: {bind_code}") + text = "绑定码格式错误，请检查配置。" + await self._send_stt_message(context, transport, text) + return + + text = f"请登录控制面板，输入{bind_code}，绑定设备。" + await self._send_stt_message(context, transport, text) + + # 获取TTS组件 + tts_component = await self._get_component(context, ComponentType.TTS) + if not tts_component or not hasattr(tts_component, 'tts_instance'): + return + + tts_instance = tts_component.tts_instance + if not hasattr(tts_instance, 'tts_audio_queue'): + return + + # 播放提示音 + from core.providers.tts.dto.dto import SentenceType + music_path = "config/assets/bind_code.wav" + opus_packets = await audio_to_data(music_path) + tts_instance.tts_audio_queue.put( + (SentenceType.FIRST, opus_packets, text, context.sentence_id) + ) + + # 逐个播放数字 + for i in range(6): # 确保只播放6位数字 + try: + digit = bind_code[i] + num_path = f"config/assets/bind_code/{digit}.wav" + num_packets = await audio_to_data(num_path) + tts_instance.tts_audio_queue.put( + (SentenceType.MIDDLE, num_packets, None, context.sentence_id) + ) + except Exception as e: + logger.error(f"播放数字音频失败: {e}") + continue + tts_instance.tts_audio_queue.put( + (SentenceType.LAST, [], None, context.sentence_id) + ) + else: + # 播放未绑定提示 + context.abort_requested = False + text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。" + await self._send_stt_message(context, transport, text) + + # 获取TTS组件 + tts_component = await self._get_component(context, ComponentType.TTS) + if tts_component and hasattr(tts_component, 'tts_instance'): + tts_instance = tts_component.tts_instance + if hasattr(tts_instance, 'tts_audio_queue'): + from core.providers.tts.dto.dto import SentenceType + music_path = "config/assets/bind_not_found.wav" + opus_packets = await audio_to_data(music_path) + tts_instance.tts_audio_queue.put( + (SentenceType.LAST, opus_packets, text, context.sentence_id) + ) + + async def _handle_abort_message(self, context: SessionContext, transport: TransportInterface): + """处理中断消息""" + from core.processors.abort_processor import AbortProcessor + + await AbortProcessor().handle_abort_message( + context, + transport, + {"type": "abort", "session_id": context.session_id}, + ) + + async def _clear_queues(self, context: SessionContext): + """清理所有队列""" + # 清理TTS音频队列 + tts_component = await self._get_component(context, ComponentType.TTS) + if tts_component and hasattr(tts_component, 'tts_instance'): + tts_instance = tts_component.tts_instance + for queue_name in ('tts_text_queue', 'tts_audio_queue'): + pending_queue = getattr(tts_instance, queue_name, None) + if pending_queue is None: + continue + try: + while not pending_queue.empty(): + pending_queue.get_nowait() + except Exception: + pass + + # Invalidate late TTS text/audio generated by the interrupted turn. + context.sentence_id = uuid.uuid4().hex + + # 清理ASR音频队列 + context.clear_audio_buffer() + + async def _get_component(self, context: SessionContext, component_type: ComponentType): + if not context.component_manager: + return None + return await context.component_manager.get_component(component_type, context) + + async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str): + """发送STT消息""" + from core.processors.audio_send_processor import AudioSendProcessor + + await AudioSendProcessor().send_stt_message( + context, transport, text + ) diff --git a/main/xiaozhi-server/core/processors/audio_send_processor.py b/main/xiaozhi-server/core/processors/audio_send_processor.py new file mode 100644 index 00000000..28f9884d --- /dev/null +++ b/main/xiaozhi-server/core/processors/audio_send_processor.py @@ -0,0 +1,578 @@ +import asyncio +import json +import time +from typing import Any, List +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from core.providers.tts.dto.dto import SentenceType +from core.utils import textUtils +from core.components.component_manager import ComponentType +from core.services.audio_ingress_service import AudioIngressService +from config.logger import setup_logging + +logger = setup_logging() + + +class AudioSendProcessor(MessageProcessor): + """音频发送处理器：完整迁移sendAudioHandle.py的所有功能""" + + def __init__(self, audio_ingress_service=None): + self.audio_ingress_service = audio_ingress_service or AudioIngressService() + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """这个处理器不直接处理消息，而是被其他处理器调用""" + return False + + async def send_audio_message(self, context: SessionContext, transport: TransportInterface, + sentence_type: SentenceType, audios: bytes, text: str, + sentence_id: str = None): + """发送音频消息 - 完整迁移自sendAudioMessage""" + if not self._is_current_turn(context, sentence_id): + return + + tts_component = await self._get_component(context, ComponentType.TTS) + if not tts_component or not hasattr(tts_component, 'tts_instance'): + return + + tts_instance = tts_component.tts_instance + + if hasattr(tts_instance, 'tts_audio_first_sentence') and tts_instance.tts_audio_first_sentence: + logger.info(f"发送第一段语音: {text}") + tts_instance.tts_audio_first_sentence = False + await self.start_tts_stream(context, transport) + + if sentence_type == SentenceType.FIRST: + await self.send_tts_message( + context, transport, "sentence_start", text, sentence_id=sentence_id + ) + + await self.send_audio( + context, transport, audios, sentence_id=sentence_id + ) + if not self._is_current_turn(context, sentence_id): + return + + # 发送句子开始消息 + if sentence_type is not SentenceType.MIDDLE: + logger.info(f"发送音频消息: {sentence_type}, {text}") + + # 发送结束消息（如果是最后一个文本） + if ( + not getattr(context, "calling", False) + and context.llm_finish_task + and sentence_type == SentenceType.LAST + ): + # Latch the terminal action before sending tts:stop. The device can + # immediately answer with listen:start, whose handler resets the + # mutable turn flags while this coroutine is still awaiting I/O. + close_after_chat = bool(context.close_after_chat) + closing_session_id = ( + getattr(transport, "session_id", None) or context.session_id + ) + if close_after_chat: + context.close_after_chat = False + await self.send_tts_message( + context, transport, "stop", None, sentence_id=sentence_id + ) + if not self._is_current_turn(context, sentence_id): + return + context.is_speaking = False + if close_after_chat: + # MQTT/UDP：结束后回到Idle，不关闭连接 + if transport.keeps_connection_between_sessions: + end_conversation = getattr(context, "end_conversation", None) + if callable(end_conversation): + await end_conversation(closing_session_id) + await transport.end_session(closing_session_id) + else: + await transport.close() + + async def send_audio( + self, + context: SessionContext, + transport: TransportInterface, + audios: bytes, + frame_duration: int = 60, + sentence_id: str = None, + ): + """发送单个opus包，支持流控 - 完整迁移自sendAudio""" + if audios is None or len(audios) == 0: + return + if getattr(context, "audio_flow_control", {}).get("send_failed"): + return + + # MQTT/UDP：等待UDP远端地址就绪，避免首包丢失 + if transport.has_datagram_audio: + if not await transport.wait_audio_ready(timeout=2): + logger.warning("UDP远端地址未就绪，跳过音频发送") + return + + audio_list = [audios] if isinstance(audios, bytes) else audios + if not isinstance(audio_list, list): + return + + flow = context.audio_flow_control + pre_buffer_count = max( + 0, int(context.config.get("tts_pre_buffer_count", 5)) + ) + for audio in audio_list: + if context.abort_requested or not self._is_current_turn(context, sentence_id): + break + + # Match the legacy AudioRateController contract: send only the + # initial pre-buffer inline, then enqueue the remaining frames so + # the TTS consumer can prefetch later sentence chunks. + packet_count = int(flow.get("packet_count", 0)) + if packet_count < pre_buffer_count and "_send_queue" not in flow: + sent = await self._send_audio_packet( + context, + transport, + flow, + audio, + frame_duration, + sentence_id, + paced=False, + ) + if not sent: + break + continue + + queue = self._ensure_audio_sender( + context, transport, flow, sentence_id + ) + await queue.put(("audio", audio, frame_duration, sentence_id)) + + @staticmethod + async def _pace_audio_send( + context: SessionContext, + frame_duration: int, + flow: dict = None, + ) -> None: + flow = flow if flow is not None else context.audio_flow_control + packet_count = int(flow.get("packet_count", 0)) + pre_buffer_count = max(0, int(context.config.get("tts_pre_buffer_count", 5))) + if packet_count < pre_buffer_count: + return + + configured_delay = int(context.config.get("tts_audio_send_delay", -1)) + if configured_delay > 0: + await asyncio.sleep(configured_delay / 1000.0) + return + + delay_ms = max(0, int(frame_duration)) + if delay_ms <= 0: + return + + now = time.monotonic() + # packet_count is the number already sent. The first packet after the + # pre-buffer must therefore wait one complete frame, not become an + # extra immediate packet. + paced_packet_index = packet_count - pre_buffer_count + 1 + pacing_started_at = flow.get("pacing_started_at") + pacing_delay_ms = flow.get("pacing_delay_ms") + if pacing_started_at is None or pacing_delay_ms != delay_ms: + pacing_started_at = now + flow["pacing_started_at"] = pacing_started_at + flow["pacing_delay_ms"] = delay_ms + + delay_seconds = delay_ms / 1000.0 + target = pacing_started_at + paced_packet_index * delay_seconds + + # TTS producers can pause between sentence chunks. Rebase after a + # full-frame gap instead of bursting stale deadlines to catch up. + if now - target >= delay_seconds: + pacing_started_at = now - paced_packet_index * delay_seconds + flow["pacing_started_at"] = pacing_started_at + target = now + + wait_seconds = target - now + if wait_seconds > 0: + await asyncio.sleep(wait_seconds) + + def _ensure_audio_sender( + self, + context: SessionContext, + transport: TransportInterface, + flow: dict, + sentence_id: str = None, + ) -> asyncio.Queue: + queue = flow.get("_send_queue") + task = flow.get("_send_task") + if queue is not None and task is not None and not task.done(): + return queue + + queue_size = max( + 1, int(context.config.get("tts_audio_queue_size", 256)) + ) + queue = asyncio.Queue(maxsize=queue_size) + sender = self._audio_send_loop( + context, transport, flow, queue, sentence_id + ) + create_task = getattr(context, "create_background_task", None) + if callable(create_task): + task = create_task(sender, turn_scoped=True) + else: + task = asyncio.create_task(sender) + flow["_send_queue"] = queue + flow["_send_task"] = task + return queue + + async def _audio_send_loop( + self, + context: SessionContext, + transport: TransportInterface, + flow: dict, + queue: asyncio.Queue, + sentence_id: str = None, + ) -> None: + try: + while True: + item = await queue.get() + try: + item_type = item[0] + if not self._is_current_turn(context, sentence_id): + continue + if item_type == "audio": + _, audio, frame_duration, item_sentence_id = item + await self._send_audio_packet( + context, + transport, + flow, + audio, + frame_duration, + item_sentence_id, + paced=True, + ) + elif item_type == "json": + _, message, item_sentence_id = item + if self._is_current_turn(context, item_sentence_id): + await transport.send_json(message) + finally: + queue.task_done() + except asyncio.CancelledError: + raise + except Exception as error: + flow["send_failed"] = True + logger.error("后台音频发送循环失败: {}", error) + finally: + while True: + try: + queue.get_nowait() + except asyncio.QueueEmpty: + break + else: + queue.task_done() + + async def _send_audio_packet( + self, + context: SessionContext, + transport: TransportInterface, + flow: dict, + audio: bytes, + frame_duration: int, + sentence_id: str = None, + *, + paced: bool, + ) -> bool: + if paced: + await self._pace_audio_send(context, frame_duration, flow) + if context.abort_requested or not self._is_current_turn(context, sentence_id): + return False + + context.update_activity() + timestamp = self._next_audio_timestamp( + context, frame_duration, flow + ) + sent = await self._send_audio_with_retry( + context, + transport, + audio, + timestamp, + sentence_id, + flow, + ) + if not sent or not self._is_current_turn(context, sentence_id): + return False + self.audio_ingress_service.cache_output_reference( + context, audio, timestamp + ) + flow["packet_count"] = int(flow.get("packet_count", 0)) + 1 + return True + + async def _send_audio_with_retry( + self, + context: SessionContext, + transport: TransportInterface, + audio: bytes, + timestamp: int, + sentence_id: str = None, + flow: dict = None, + ) -> bool: + flow = flow if flow is not None else context.audio_flow_control + retries = max(0, int(context.config.get("audio_send_retries", 2))) + retry_delay = max( + 0, int(context.config.get("audio_send_retry_delay_ms", 20)) + ) / 1000.0 + for attempt in range(retries + 1): + if context.abort_requested or not self._is_current_turn(context, sentence_id): + return False + try: + await transport.send_audio(audio, timestamp) + return self._is_current_turn(context, sentence_id) + except Exception as error: + if attempt >= retries: + flow["send_failed"] = True + logger.error( + "Audio send failed after {} attempts: {}", + attempt + 1, + error, + ) + raise + await asyncio.sleep(retry_delay) + return False + + @staticmethod + def _next_audio_timestamp( + context: SessionContext, + frame_duration: int, + flow: dict = None, + ) -> int: + flow = flow if flow is not None else getattr( + context, "audio_flow_control", None + ) + if flow is None: + flow = {} + context.audio_flow_control = flow + timestamp_step = int(frame_duration) or int( + getattr(context, "output_frame_duration", 60) or 60 + ) + timestamp = int(flow.get("output_timestamp", 0)) + timestamp_step + timestamp %= 2 ** 32 + flow["output_timestamp"] = timestamp + return timestamp + + async def send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str): + """发送STT消息 - 完整迁移自send_stt_message""" + end_prompt = getattr(context, "config", {}).get("end_prompt", {}).get("prompt") + if end_prompt and text == end_prompt: + await self.start_tts_stream(context, transport) + return + + display_text = text + parsed_data = None + if isinstance(text, dict): + parsed_data = text + elif isinstance(text, str): + stripped = text.strip() + if stripped.startswith("{") and stripped.endswith("}"): + try: + parsed_data = json.loads(stripped) + except json.JSONDecodeError: + pass + + if isinstance(parsed_data, dict) and "content" in parsed_data: + display_text = parsed_data["content"] + if "speaker" in parsed_data: + context.current_speaker = parsed_data["speaker"] + + stt_text = textUtils.get_string_no_punctuation_or_emoji( + str(display_text) + ) + await self._send_json(transport, { + "type": "stt", + "text": stt_text, + "session_id": context.session_id + }) + logger.info(f"发送STT消息: {stt_text}") + # Legacy WS sends TTS start together with STT, before synthesis begins. + # This lead time is required by MQTT/UDP clients because the device + # changes to Speaking asynchronously and drops UDP audio beforehand. + await self.start_tts_stream(context, transport) + + async def start_tts_stream( + self, + context: SessionContext, + transport: TransportInterface, + ) -> bool: + """Open one device-side TTS stream without sending duplicate starts.""" + if getattr(context, "is_speaking", False): + return False + + context.is_speaking = True + flow_control = getattr(context, "audio_flow_control", None) or {} + await self._stop_audio_sender(flow_control) + context.audio_flow_control = {} + try: + await self._send_json(transport, { + "type": "tts", + "state": "start", + "session_id": context.session_id, + }) + except Exception: + context.is_speaking = False + raise + return True + + @staticmethod + async def _send_json( + transport: TransportInterface, + message: dict, + ) -> None: + send_json = getattr(transport, "send_json", None) + if callable(send_json): + await send_json(message) + return + await transport.send(json.dumps(message)) + + async def send_tts_message( + self, + context: SessionContext, + transport: TransportInterface, + state: str, + text: str = None, + sentence_id: str = None, + ): + """发送TTS消息 - 完整迁移自send_tts_message""" + if not self._is_current_turn(context, sentence_id): + return False + if state == "sentence_start": + flow = getattr(context, "audio_flow_control", {}) + queue = flow.get("_send_queue") + task = flow.get("_send_task") + if queue is not None and task is not None and not task.done(): + message = { + "type": "tts", + "state": state, + "session_id": context.session_id, + } + if text: + message["text"] = text + await queue.put(("json", message, sentence_id)) + return True + if state == "stop": + if context.config.get("enable_stop_tts_notify", False): + from core.utils.util import audio_to_data + + notify_path = context.config.get( + "stop_tts_notify_voice", "config/assets/tts_notify.mp3" + ) + notify_audio = await audio_to_data(notify_path, is_opus=True) + if notify_audio: + await self.send_audio( + context, + transport, + notify_audio, + sentence_id=sentence_id, + ) + if not await self._wait_for_audio_completion(context, sentence_id): + return False + + if not self._is_current_turn(context, sentence_id): + return False + + message = { + "type": "tts", + "state": state, + "session_id": context.session_id + } + if text: + message["text"] = text + + await transport.send_json(message) + logger.debug(f"发送TTS消息: state={state}, text={text}") + return True + + @staticmethod + async def _wait_for_audio_completion( + context: SessionContext, sentence_id: str = None + ) -> bool: + flow = context.audio_flow_control + queue = flow.get("_send_queue") + sender_task = flow.get("_send_task") + if queue is not None and sender_task is not None: + join_task = asyncio.create_task(queue.join()) + done, _ = await asyncio.wait( + {join_task, sender_task}, + return_when=asyncio.FIRST_COMPLETED, + ) + if sender_task in done and not join_task.done(): + join_task.cancel() + await asyncio.gather(join_task, return_exceptions=True) + return False + await join_task + await AudioSendProcessor._stop_audio_sender(flow) + if flow.get("send_failed"): + return False + + packet_count = int(flow.get("packet_count", 0)) + if packet_count <= 0: + return AudioSendProcessor._is_current_turn(context, sentence_id) + frame_duration = max(1, int(getattr(context, "output_frame_duration", 60))) + pre_buffer_count = max(0, int(context.config.get("tts_pre_buffer_count", 5))) + tail_frames = max( + 0, + int(context.config.get("tts_stop_buffer_frames", pre_buffer_count + 2)), + ) + if tail_frames: + await asyncio.sleep(tail_frames * frame_duration / 1000.0) + return AudioSendProcessor._is_current_turn(context, sentence_id) + + @staticmethod + async def _stop_audio_sender(flow: dict) -> None: + task = flow.pop("_send_task", None) + flow.pop("_send_queue", None) + if task is not None and not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + @staticmethod + def _is_current_turn(context: SessionContext, sentence_id: str = None) -> bool: + if sentence_id is None: + return bool( + getattr(context, "conversation_active", True) + and getattr(context, "sentence_id", None) is None + ) + return sentence_id == context.sentence_id + + async def send_music_message(self, context: SessionContext, transport: TransportInterface, + music_path: str, text: str): + """发送音乐消息 - 完整迁移自send_music_message""" + from core.utils.util import audio_to_data + + try: + # 获取音频数据 + opus_packets = await audio_to_data(music_path) + if opus_packets: + # 发送音乐开始消息 + await self.send_tts_message(context, transport, "start", text) + + # 发送音频数据 + await self.send_audio(context, transport, opus_packets) + + # 发送音乐结束消息 + await self.send_tts_message(context, transport, "stop", None) + + logger.info(f"发送音乐: {music_path}") + else: + logger.warning(f"无法加载音乐文件: {music_path}") + + except Exception as e: + logger.error(f"发送音乐失败: {e}") + + async def _get_component(self, context: SessionContext, component_type: ComponentType): + if not context.component_manager: + return None + return await context.component_manager.get_component(component_type, context) + + async def send_welcome_audio(self, context: SessionContext, transport: TransportInterface): + """发送欢迎音频""" + welcome_audio_path = context.config.get("welcome_audio_path") + if welcome_audio_path: + await self.send_music_message(context, transport, welcome_audio_path, "欢迎使用小智助手") + + async def send_goodbye_audio(self, context: SessionContext, transport: TransportInterface): + """发送告别音频""" + goodbye_audio_path = context.config.get("goodbye_audio_path") + if goodbye_audio_path: + await self.send_music_message(context, transport, goodbye_audio_path, "再见，期待下次相遇") diff --git a/main/xiaozhi-server/core/processors/auth_processor.py b/main/xiaozhi-server/core/processors/auth_processor.py new file mode 100644 index 00000000..e131349d --- /dev/null +++ b/main/xiaozhi-server/core/processors/auth_processor.py @@ -0,0 +1,55 @@ +from typing import Any +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from core.auth import AuthMiddleware, AuthenticationError +from config.logger import setup_logging + +logger = setup_logging() + + +class AuthProcessor(MessageProcessor): + """认证处理器：处理连接认证逻辑""" + + def __init__(self): + self.auth_middleware = None + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """处理认证相关逻辑""" + # 服务器管理连接走server消息处理（基于secret校验），跳过认证 + if isinstance(message, str): + try: + import json + msg_json = json.loads(message) + if isinstance(msg_json, dict) and msg_json.get("type") == "server": + return False + except json.JSONDecodeError: + pass + elif isinstance(message, dict) and message.get("type") == "server": + return False + + # 如果已经认证，跳过 + if context.is_authenticated: + return False + + # 初始化认证中间件（延迟初始化） + if self.auth_middleware is None: + self.auth_middleware = AuthMiddleware(context.config) + + # 检查是否为认证消息（通过headers进行认证） + if context.headers: + try: + await self.auth_middleware.authenticate_async(context.headers) + context.is_authenticated = True + logger.info(f"设备认证成功: {context.device_id}") + return False # 认证成功，继续处理其他消息 + except AuthenticationError as e: + logger.error(f"设备认证失败: {e}") + # 发送认证失败消息 + await transport.send("Authentication failed") + await transport.close() + return True # 认证失败，停止处理 + + # 如果没有认证信息，要求认证 + await transport.send("Authentication required") + return True # 停止后续处理 diff --git a/main/xiaozhi-server/core/processors/chat_processor.py b/main/xiaozhi-server/core/processors/chat_processor.py new file mode 100644 index 00000000..2dc853bd --- /dev/null +++ b/main/xiaozhi-server/core/processors/chat_processor.py @@ -0,0 +1,1042 @@ +import json +import uuid +import asyncio +import re +from typing import Any, Dict +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from core.utils.dialogue import Message, Dialogue +from core.utils.util import remove_punctuation_and_length +from core.utils import textUtils +from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType +from core.components.component_manager import ComponentType +from plugins_func.register import Action, ActionResponse +from config.logger import setup_logging + +logger = setup_logging() + +DIRECT_ANSWER_TOOL = { + "type": "function", + "function": { + "name": "direct_answer", + "description": "当请求不匹配其他工具时直接回复用户。", + "parameters": { + "type": "object", + "properties": {"response": {"type": "string"}}, + "required": ["response"], + }, + }, +} + + +class ChatProcessor(MessageProcessor): + """聊天处理器：完整迁移intentHandler.py的所有功能""" + + def __init__(self): + # 会话对话历史管理 + self._dialogues: Dict[str, Dialogue] = {} + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """处理聊天消息""" + # 这个处理器不直接处理原始消息，而是被其他处理器调用 + return False + + async def handle_chat(self, context: SessionContext, transport: TransportInterface, text: str, skip_intent: bool = False, depth: int = 0): + """处理聊天请求 - 完整迁移自handle_user_intent""" + try: + logger.info(f"开始处理聊天: session_id={context.session_id}, text={text[:50]}...") + if not skip_intent: + # 首先进行意图处理 + intent_handled = await self.handle_user_intent(context, transport, text) + if intent_handled: + logger.info("意图已处理，结束聊天流程") + return + else: + logger.info("跳过意图处理，直接进入常规聊天") + + # 如果意图未处理，进行常规聊天 + logger.info("进入常规聊天流程") + await self._regular_chat(context, transport, text, depth=depth) + + except Exception as e: + logger.error(f"处理聊天失败: {e}") + await self._send_error(transport, "聊天处理失败，请重试") + await self._finish_tts_turn(context, transport) + + async def handle_user_intent(self, context: SessionContext, transport: TransportInterface, text: str): + """处理用户意图 - 完整迁移自intentHandler.py""" + # 预处理输入文本，处理可能的JSON格式 + try: + if text.strip().startswith('{') and text.strip().endswith('}'): + parsed_data = json.loads(text) + if isinstance(parsed_data, dict) and "content" in parsed_data: + text = parsed_data["content"] # 提取content用于意图分析 + context.current_speaker = parsed_data.get("speaker") # 保留说话人信息 + except (json.JSONDecodeError, TypeError): + pass + + # 检查是否有明确的退出命令 + _, filtered_text = remove_punctuation_and_length(text) + if await self._check_direct_exit(context, transport, filtered_text): + return True + + # 检查是否是唤醒词 + if await self._check_wakeup_words(context, transport, filtered_text): + return True + + if context.intent_type == "function_call": + # 使用支持function calling的聊天方法,不再进行意图分析 + return False + + # 使用LLM进行意图分析 + intent_result = await self._analyze_intent_with_llm(context, text) + if not intent_result: + return False + + # 会话开始时生成sentence_id + context.sentence_id = str(uuid.uuid4().hex) + + # 处理各种意图 + return await self._process_intent_result(context, transport, intent_result, text) + + def _get_dialogue(self, session_id: str) -> Dialogue: + """获取或创建对话历史""" + if session_id not in self._dialogues: + self._dialogues[session_id] = Dialogue() + return self._dialogues[session_id] + + async def _get_memory_context(self, context: SessionContext, query: str) -> str: + """获取记忆上下文""" + try: + memory_component = await self._get_component(context, ComponentType.MEMORY) + if memory_component and hasattr(memory_component, 'memory_instance'): + memory_instance = memory_component.memory_instance + if hasattr(memory_instance, 'query_memory'): + return await memory_instance.query_memory(query) + except Exception as e: + logger.warning(f"获取记忆上下文失败: {e}") + + return None + + async def _generate_llm_response(self, context: SessionContext, transport: TransportInterface, + llm_instance, dialogue_context: list, dialogue: Dialogue): + """生成LLM回复""" + try: + logger.info("开始生成LLM回复") + # 初始化sentence_id并发送TTS FIRST标记（模拟原connection.py第692-700行） + if not context.sentence_id: + context.sentence_id = str(uuid.uuid4().hex) + + # 发送TTS开始标记 + await self._send_tts_first_marker(context) + + # 检查是否支持流式响应 + if hasattr(llm_instance, 'response'): + # 使用流式响应 + response_generator = llm_instance.response(context.session_id, dialogue_context) + + response_parts = [] + first_chunk = True + async for response_part in self._async_generator_wrapper(response_generator): + if context.abort_requested: + break + + if response_part and len(response_part) > 0: + response_parts.append(response_part) + if first_chunk: + logger.info("收到LLM首段回复") + self._schedule_emotion(context, response_part) + first_chunk = False + + # 原架构不发送流式响应给前端，直接进行TTS处理 + # 将响应片段放入TTS队列进行语音合成 + await self._process_response_part_for_tts(context, response_part) + + # 完整回复 + full_response = "".join(response_parts) + if full_response: + # 添加助手回复到对话历史 + dialogue.put(Message(role="assistant", content=full_response)) + + # 原架构不发送response_complete给前端，只进行TTS处理 + # 发送TTS结束标记 + await self._finalize_tts_response(context, full_response) + + logger.info(f"LLM回复完成: {full_response[:100]}...") + elif not context.abort_requested: + logger.warning("LLM未生成可播放内容，结束当前TTS流") + await self._finish_tts_turn(context, transport) + + else: + logger.warning("LLM实例不支持流式响应") + await self._finish_tts_turn(context, transport) + + except Exception as e: + logger.error(f"生成LLM回复失败: {e}") + await self._send_error(transport, "生成回复失败") + await self._finish_tts_turn(context, transport) + + async def _generate_llm_response_with_tools( + self, + context: SessionContext, + transport: TransportInterface, + llm_instance, + dialogue_context: list, + dialogue: Dialogue, + depth: int = 0, + ): + """生成LLM回复（支持function_call）""" + from core.utils.util import extract_json_from_string + + MAX_DEPTH = 5 + force_final_answer = depth >= MAX_DEPTH + + functions = None + if context.intent_type == "function_call" and context.func_handler and not force_final_answer: + functions = list(context.func_handler.get_functions() or []) + if depth == 0: + functions.append(DIRECT_ANSWER_TOOL) + + try: + # 工具结果递归沿用同一轮 TTS，仅最外层发送 FIRST。 + if depth == 0: + await self._send_tts_first_marker(context) + + responses = llm_instance.response_with_functions( + context.session_id, + dialogue_context, + functions=functions, + ) + except Exception as e: + logger.error(f"LLM工具调用失败: {e}") + await self._finish_tts_turn(context, transport) + return + + response_message = [] + tool_call_flag = False + tool_calls_list = [] + content_arguments = "" + first_chunk = True + + async for response in self._async_generator_wrapper(responses): + if context.abort_requested: + break + if context.intent_type == "function_call" and functions is not None: + content, tools_call = response + if isinstance(response, dict) and "content" in response: + content = response.get("content") + tools_call = None + if content: + content_arguments += content + if not tool_call_flag and content_arguments.startswith(""): + tool_call_flag = True + if tools_call: + tool_call_flag = True + self._merge_tool_calls(tool_calls_list, tools_call) + for tool_call in tool_calls_list: + if tool_call.get("name") != "direct_answer": + continue + direct_text = self._extract_direct_answer_response( + tool_call.get("arguments", "") + ) + sent_len = tool_call.get("_da_sent", 0) + safe_end = max(sent_len, len(direct_text) - 5) + if safe_end > sent_len: + delta = self._clean_response_garbage( + direct_text[sent_len:safe_end] + ) + if delta: + await self._process_response_part_for_tts( + context, delta + ) + tool_call["_da_sent"] = safe_end + else: + content = response + + if content: + if first_chunk: + logger.info("收到LLM首段回复") + self._schedule_emotion(context, content) + first_chunk = False + if not tool_call_flag: + response_message.append(content) + await self._process_response_part_for_tts(context, content) + + # 处理function_call + if tool_call_flag: + if not tool_calls_list and content_arguments: + a = extract_json_from_string(content_arguments) + if a: + try: + content_arguments_json = json.loads(a) + tool_calls_list.append( + { + "id": str(uuid.uuid4().hex), + "name": content_arguments_json["name"], + "arguments": json.dumps( + content_arguments_json["arguments"], + ensure_ascii=False, + ), + } + ) + except Exception: + response_message.append(a) + else: + response_message.append(content_arguments) + + if response_message: + text_buff = "".join(response_message) + dialogue.put(Message(role="assistant", content=text_buff)) + response_message.clear() + + direct_answer_calls = [ + call for call in tool_calls_list if call.get("name") == "direct_answer" + ] + tool_calls_list = [ + call for call in tool_calls_list if call.get("name") != "direct_answer" + ] + for direct_call in direct_answer_calls: + direct_text = self._clean_response_garbage( + self._extract_direct_answer_response( + direct_call.get("arguments", "") + ) + ) + if direct_text: + dialogue.put(Message(role="assistant", content=direct_text)) + remaining = direct_text[direct_call.get("_da_sent", 0):] + if remaining: + await self._process_response_part_for_tts( + context, remaining + ) + + if direct_answer_calls and not tool_calls_list: + direct_text = "".join( + self._clean_response_garbage( + self._extract_direct_answer_response( + call.get("arguments", "") + ) + ) + for call in direct_answer_calls + ) + await self._finalize_tts_response(context, direct_text) + return + + if not tool_calls_list: + fallback_text = "工具调用格式有误，请再试一次。" + dialogue.put(Message(role="assistant", content=fallback_text)) + await self._process_response_part_for_tts(context, fallback_text) + await self._finalize_tts_response(context, fallback_text) + return + + from core.processors.report_processor import ReportProcessor + + reporter = ReportProcessor() + timeout = max(1, int(context.config.get("tool_call_timeout", 30))) + + async def execute_tool(tool_call_data): + try: + tool_input = json.loads(tool_call_data.get("arguments") or "{}") + except (json.JSONDecodeError, TypeError): + tool_input = {} + reporter.enqueue_tool_report( + context, tool_call_data.get("name", ""), tool_input + ) + try: + result = await asyncio.wait_for( + context.func_handler.handle_llm_function_call( + context, tool_call_data + ), + timeout=timeout, + ) + except Exception as error: + logger.error( + f"工具调用失败: {tool_call_data.get('name')}, error={error}" + ) + result = ActionResponse( + action=Action.ERROR, + result="工具调用超时或失败，请稍后再试。", + response="工具调用超时或失败，请稍后再试。", + ) + reporter.enqueue_tool_report( + context, + tool_call_data.get("name", ""), + tool_input, + str(result.result) if result.result is not None else None, + report_tool_call=False, + ) + return result, tool_call_data + + tool_results = await asyncio.gather( + *(execute_tool(tool_call_data) for tool_call_data in tool_calls_list) + ) + + if tool_results: + await self._handle_function_result(context, transport, tool_results, depth=depth) + return + + # 无工具调用，正常结束 + if response_message: + full_response = "".join(response_message) + dialogue.put(Message(role="assistant", content=full_response)) + await self._finalize_tts_response(context, full_response) + elif not context.abort_requested: + logger.warning("LLM工具流程未生成可播放内容，结束当前TTS流") + await self._finish_tts_turn(context, transport) + + async def _handle_function_result(self, context: SessionContext, transport: TransportInterface, tool_results, depth: int): + need_llm_tools = [] + completed_text = [] + record_tools = [] + for result, tool_call_data in tool_results: + if result.action in [Action.RESPONSE, Action.NOTFOUND, Action.ERROR]: + text = result.response if result.response else result.result + if text: + completed_text.append(text) + context.dialogue.put(Message(role="assistant", content=text)) + await self._process_response_part_for_tts(context, text) + elif result.action == Action.REQLLM: + need_llm_tools.append((result, tool_call_data)) + elif result.action == Action.RECORD: + record_tools.append((result, tool_call_data)) + + if record_tools: + context.dialogue.put( + Message( + role="assistant", + tool_calls=[ + { + "id": tool_call_data["id"], + "function": { + "arguments": tool_call_data.get("arguments") or "{}", + "name": tool_call_data["name"], + }, + "type": "function", + "index": index, + } + for index, (_, tool_call_data) in enumerate(record_tools) + ], + ) + ) + for result, tool_call_data in record_tools: + context.dialogue.put( + Message( + role="tool", + tool_call_id=tool_call_data["id"], + content=result.result or "", + ) + ) + if result.response: + completed_text.append(result.response) + context.dialogue.put( + Message(role="assistant", content=result.response) + ) + await self._process_response_part_for_tts( + context, result.response + ) + + if not need_llm_tools: + await self._finalize_tts_response( + context, "".join(completed_text) + ) + return + + all_tool_calls = [ + { + "id": tool_call_data["id"], + "function": { + "arguments": "{}" if tool_call_data["arguments"] == "" else tool_call_data["arguments"], + "name": tool_call_data["name"], + }, + "type": "function", + "index": idx, + } + for idx, (_, tool_call_data) in enumerate(need_llm_tools) + ] + context.dialogue.put(Message(role="assistant", tool_calls=all_tool_calls)) + + for result, tool_call_data in need_llm_tools: + text = result.result + if text: + context.dialogue.put( + Message( + role="tool", + tool_call_id=tool_call_data.get("id", str(uuid.uuid4())), + content=text, + ) + ) + + # 继续下一轮工具结果回答 + await self._regular_chat(context, transport, "", depth=depth + 1) + + def _merge_tool_calls(self, tool_calls_list, tools_call): + """合并工具调用列表（从旧版迁移）""" + for tool_call in tools_call: + tool_index = getattr(tool_call, "index", None) + if tool_index is None: + if tool_call.function.name: + tool_index = len(tool_calls_list) + else: + tool_index = len(tool_calls_list) - 1 if tool_calls_list else 0 + + if tool_index >= len(tool_calls_list): + tool_calls_list.append({"id": "", "name": "", "arguments": ""}) + + if tool_call.id: + tool_calls_list[tool_index]["id"] = tool_call.id + if tool_call.function.name: + tool_calls_list[tool_index]["name"] = tool_call.function.name + if tool_call.function.arguments: + tool_calls_list[tool_index]["arguments"] += tool_call.function.arguments + + @staticmethod + def _extract_direct_answer_response(arguments): + if not arguments: + return "" + try: + parsed = json.loads(arguments) + if isinstance(parsed, dict): + return str(parsed.get("response") or "") + except (json.JSONDecodeError, TypeError): + pass + + for marker in ('"response": "', '"response":"'): + start = arguments.find(marker) + if start >= 0: + value = arguments[start + len(marker):] + if value.endswith('"}'): + value = value[:-2] + elif value.endswith('"'): + value = value[:-1] + return value.replace('\\"', '"').replace('\\n', '\n').replace('\\\\', '\\') + return "" + + @staticmethod + def _clean_response_garbage(text): + if not text: + return text + garbage_chars = frozenset('\")\'}）') + cleaned = [] + for line in text.split("\n"): + stripped = line.strip() + if stripped and len(stripped) <= 8 and all( + char in garbage_chars for char in stripped + ): + continue + cleaned.append(line) + return re.sub(r'["\'}\]]+$', "", "\n".join(cleaned).rstrip()).rstrip() + + @staticmethod + def _schedule_emotion(context: SessionContext, content: str) -> None: + if not content or not (getattr(context, "features", None) or {}).get( + "emoji", True + ): + return + coroutine = textUtils.get_emotion(context, content) + if hasattr(context, "create_background_task"): + context.create_background_task(coroutine, turn_scoped=True) + else: + asyncio.create_task(coroutine) + + async def _async_generator_wrapper(self, generator): + """将同步生成器包装为异步生成器""" + try: + def next_item(): + try: + return True, next(generator) + except StopIteration: + return False, None + + while True: + has_item, item = await asyncio.to_thread(next_item) + if not has_item: + break + yield item + except Exception as e: + logger.error(f"生成器包装失败: {e}") + + async def _send_tts_first_marker(self, context: SessionContext): + """发送TTS开始标记""" + try: + tts_component = await self._get_component(context, ComponentType.TTS) + if not tts_component or not hasattr(tts_component, 'tts_instance'): + return + + tts_instance = tts_component.tts_instance + if not tts_instance or not hasattr(tts_instance, 'tts_text_queue'): + return + + # 发送TTS开始标记（模拟原connection.py第694-700行） + tts_instance.tts_text_queue.put(TTSMessageDTO( + sentence_id=context.sentence_id, + sentence_type=SentenceType.FIRST, + content_type=ContentType.ACTION + )) + + except Exception as e: + logger.error(f"发送TTS开始标记失败: {e}") + + async def _process_response_part_for_tts(self, context: SessionContext, response_part: str): + """处理响应片段进行TTS - 模拟原架构逻辑""" + try: + tts_component = await self._get_component(context, ComponentType.TTS) + if not tts_component or not hasattr(tts_component, 'tts_instance'): + return + + tts_instance = tts_component.tts_instance + if not tts_instance or not hasattr(tts_instance, 'tts_text_queue'): + return + + # 将响应片段放入TTS队列（模拟原connection.py第782-789行逻辑） + tts_instance.tts_text_queue.put(TTSMessageDTO( + sentence_id=context.sentence_id, + sentence_type=SentenceType.MIDDLE, + content_type=ContentType.TEXT, + content_detail=response_part + )) + + except Exception as e: + logger.error(f"处理TTS响应片段失败: {e}") + + async def _finalize_tts_response(self, context: SessionContext, full_response: str): + """完成TTS响应 - 发送结束标记""" + try: + tts_component = await self._get_component(context, ComponentType.TTS) + if not tts_component or not hasattr(tts_component, 'tts_instance'): + return False + + tts_instance = tts_component.tts_instance + if not tts_instance or not hasattr(tts_instance, 'tts_text_queue'): + return False + + # 发送TTS结束标记（模拟原speak_txt函数逻辑） + tts_instance.tts_text_queue.put(TTSMessageDTO( + sentence_id=context.sentence_id, + sentence_type=SentenceType.LAST, + content_type=ContentType.ACTION + )) + + # 设置LLM完成标记 + context.llm_finish_task = True + return True + + except Exception as e: + logger.error(f"完成TTS响应失败: {e}") + return False + + async def _finish_tts_turn( + self, + context: SessionContext, + transport: TransportInterface, + ) -> None: + """Guarantee that every opened device TTS stream reaches a terminal state.""" + if await self._finalize_tts_response(context, ""): + return + + if not getattr(context, "is_speaking", False): + return + from core.processors.audio_send_processor import AudioSendProcessor + + sentence_id = getattr(context, "sentence_id", None) + try: + await AudioSendProcessor().send_tts_message( + context, + transport, + "stop", + sentence_id=sentence_id, + ) + finally: + context.is_speaking = False + + async def _trigger_tts(self, context: SessionContext, transport: TransportInterface, text: str): + """触发TTS语音合成 - 完整迁移自原chat方法的TTS处理""" + try: + tts_component = await self._get_component(context, ComponentType.TTS) + if not tts_component or not hasattr(tts_component, 'tts_instance'): + logger.warning("TTS组件未初始化") + return + + tts_instance = tts_component.tts_instance + + # 确保有sentence_id + if not context.sentence_id: + context.sentence_id = str(uuid.uuid4().hex) + + logger.info(f"触发TTS合成: {text[:50]}...") + + # 使用原来的TTS处理方式 + if hasattr(tts_instance, 'tts_text_queue') and hasattr(tts_instance, 'tts_one_sentence'): + # 发送FIRST消息到TTS队列 + tts_instance.tts_text_queue.put( + TTSMessageDTO( + sentence_id=context.sentence_id, + sentence_type=SentenceType.FIRST, + content_type=ContentType.ACTION, + ) + ) + + # 合成一句话 + tts_instance.tts_one_sentence(context, ContentType.TEXT, content_detail=text) + + # 发送LAST消息到TTS队列 + tts_instance.tts_text_queue.put( + TTSMessageDTO( + sentence_id=context.sentence_id, + sentence_type=SentenceType.LAST, + content_type=ContentType.ACTION, + ) + ) + + logger.info("TTS合成任务已提交到队列") + else: + logger.warning("TTS实例不支持队列处理") + + except Exception as e: + logger.error(f"TTS合成失败: {e}") + + async def _send_error(self, transport: TransportInterface, error_message: str): + """发送错误消息""" + try: + await transport.send(json.dumps({ + "type": "error", + "message": error_message + })) + except Exception as e: + logger.error(f"发送错误消息失败: {e}") + + # === 意图处理相关方法：完整迁移自intentHandler.py === + + async def _check_direct_exit(self, context: SessionContext, transport: TransportInterface, text: str): + """检查是否有明确的退出命令 - 完整迁移自check_direct_exit""" + _, text = remove_punctuation_and_length(text) + cmd_exit = list(context.cmd_exit or []) + extra_exit = ["退下吧", "退下", "再见", "下次聊", "闭嘴", "停止", "停下", "别说了", "不要说了"] + for cmd in extra_exit: + if cmd not in cmd_exit: + cmd_exit.append(cmd) + for cmd in cmd_exit: + if text == cmd: + logger.info(f"识别到明确的退出命令: {text}") + await self._send_stt_message(context, transport, text) + # 走结束提示语，随后回到Idle（不关闭连接） + context.close_after_chat = True + end_prompt = context.config.get("end_prompt", {}) + if end_prompt and end_prompt.get("enable", True) is False: + # 即使不播结束语，也必须发出 LAST，驱动设备回到 Idle。 + await self._finalize_tts_response(context, "") + return True + prompt = end_prompt.get("prompt") + if not prompt: + prompt = "请你以“时间过得真快”开头，用富有感情、依依不舍的话来结束这场对话吧！" + await self._regular_chat(context, transport, prompt, depth=0) + return True + return False + + async def _check_wakeup_words(self, context: SessionContext, transport: TransportInterface, text: str): + """Use the single wake-word cache implementation shared with Hello.""" + from core.processors.hello_processor import HelloProcessor + + return await HelloProcessor().check_wakeup_words( + context, transport, text + ) + + async def _analyze_intent_with_llm(self, context: SessionContext, text: str): + """使用LLM分析用户意图 - 完整迁移自analyze_intent_with_llm""" + intent_component = await self._get_component(context, ComponentType.INTENT) + if not intent_component or not hasattr(intent_component, 'intent_instance'): + logger.warning("意图识别服务未初始化") + return None + + intent_instance = intent_component.intent_instance + + # 对话历史记录 + dialogue = context.dialogue + if not dialogue: + return None + + try: + intent_result = await intent_instance.detect_intent(context, dialogue.dialogue, text) + return intent_result + except Exception as e: + logger.error(f"意图识别失败: {str(e)}") + + return None + + async def _process_intent_result(self, context: SessionContext, transport: TransportInterface, intent_result: str, original_text: str): + """处理意图识别结果 - 完整迁移自process_intent_result""" + try: + # 尝试将结果解析为JSON + intent_data = json.loads(intent_result) + + function_call = intent_data.get("function_call") + if not isinstance(function_call, dict): + return False + + function_name = function_call.get("name") + if not function_name: + return False + logger.debug(f"检测到function_call格式的意图结果: {function_name}") + if function_name == "continue_chat": + return False + + if function_name == "result_for_context": + await self._send_stt_message(context, transport, original_text) + context.abort_requested = False + dialogue = context.dialogue + if dialogue: + dialogue.put(Message(role="user", content=original_text)) + + from core.utils.current_time import get_current_time_info + + current_time, today_date, today_weekday, lunar_date = ( + get_current_time_info() + ) + context_prompt = ( + f"当前时间：{current_time}\n" + f"今天日期：{today_date} ({today_weekday})\n" + f"今天农历：{lunar_date}\n\n" + f"请根据以上信息回答用户的问题：{original_text}" + ) + intent_component = await self._get_component( + context, ComponentType.INTENT + ) + intent_instance = getattr(intent_component, "intent_instance", None) + if intent_instance and hasattr(intent_instance, "replyResult"): + response = await asyncio.to_thread( + intent_instance.replyResult, context_prompt, original_text + ) + if response: + await self._speak_txt(context, response) + return True + + function_args = function_call.get("arguments") or {} + if isinstance(function_args, dict): + function_args = json.dumps(function_args, ensure_ascii=False) + + function_call_data = { + "name": function_name, + "id": str(uuid.uuid4().hex), + "arguments": function_args, + } + + await self._send_stt_message(context, transport, original_text) + context.abort_requested = False + await self._process_function_call( + context, transport, function_call_data, original_text + ) + return True + except (json.JSONDecodeError, TypeError) as e: + logger.error(f"处理意图结果时出错: {e}") + return False + + async def _process_function_call(self, context: SessionContext, transport: TransportInterface, function_call_data: dict, original_text: str): + """处理函数调用 - 完整迁移自process_function_call""" + async def async_process(): + # 添加用户消息到对话历史 + dialogue = context.dialogue + if dialogue: + dialogue.put(Message(role="user", content=original_text)) + + from core.processors.report_processor import ReportProcessor + + try: + tool_input = json.loads(function_call_data.get("arguments") or "{}") + except (json.JSONDecodeError, TypeError): + tool_input = {} + reporter = ReportProcessor() + reporter.enqueue_tool_report( + context, function_call_data.get("name", ""), tool_input + ) + + # 使用统一工具处理器处理所有工具调用 + try: + func_handler = context.func_handler + if not func_handler: + raise Exception("函数处理器未初始化") + + result = await asyncio.wait_for( + func_handler.handle_llm_function_call( + context, function_call_data + ), + timeout=max(1, int(context.config.get("tool_call_timeout", 30))), + ) + except Exception as e: + logger.error(f"工具调用失败: {e}") + result = ActionResponse( + action=Action.ERROR, result=str(e), response=str(e) + ) + + reporter.enqueue_tool_report( + context, + function_call_data.get("name", ""), + tool_input, + str(result.result) if result and result.result is not None else None, + report_tool_call=False, + ) + + if result: + function_name = function_call_data.get("name", "") + if result.action == Action.RESPONSE: # 直接回复前端 + text = result.response + if text is not None: + await self._speak_txt(context, text) + elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 + text = result.result + if dialogue: + dialogue.put(Message(role="tool", content=text)) + intent_component = await self._get_component(context, ComponentType.INTENT) + if intent_component and hasattr(intent_component, 'intent_instance'): + intent_instance = intent_component.intent_instance + if hasattr(intent_instance, 'replyResult'): + llm_result = await asyncio.to_thread( + intent_instance.replyResult, text, original_text + ) + if llm_result is None: + llm_result = text + await self._speak_txt(context, llm_result) + elif ( + result.action == Action.NOTFOUND + or result.action == Action.ERROR + ): + text = result.response if result.response else result.result + if text is not None: + await self._speak_txt(context, text) + elif function_name != "play_music": + # For backward compatibility with original code + # 获取当前最新的文本索引 + text = result.response + if text is None: + text = result.result + if text is not None: + await self._speak_txt(context, text) + + # 将函数执行放在线程池中 + if hasattr(context, "create_background_task"): + context.create_background_task(async_process(), turn_scoped=True) + else: + # 如果没有executor，直接执行 + await async_process() + + async def _speak_txt(self, context: SessionContext, text: str): + """语音合成文本 - 完整迁移自speak_txt""" + tts_component = await self._get_component(context, ComponentType.TTS) + if not tts_component or not hasattr(tts_component, 'tts_instance'): + return + + tts_instance = tts_component.tts_instance + sentence_id = context.sentence_id or str(uuid.uuid4().hex) + context.sentence_id = sentence_id + + if hasattr(tts_instance, "store_tts_text"): + tts_instance.store_tts_text(sentence_id, text) + + # 发送TTS消息队列 + if hasattr(tts_instance, 'tts_text_queue'): + tts_instance.tts_text_queue.put( + TTSMessageDTO( + sentence_id=sentence_id, + sentence_type=SentenceType.FIRST, + content_type=ContentType.ACTION, + ) + ) + + # 合成一句话 + if hasattr(tts_instance, 'tts_one_sentence'): + tts_instance.tts_one_sentence(context, ContentType.TEXT, content_detail=text) + + tts_instance.tts_text_queue.put( + TTSMessageDTO( + sentence_id=sentence_id, + sentence_type=SentenceType.LAST, + content_type=ContentType.ACTION, + ) + ) + + # 添加到对话历史 + dialogue = context.dialogue + if dialogue: + dialogue.put(Message(role="assistant", content=text)) + + async def _regular_chat(self, context: SessionContext, transport: TransportInterface, text: str, depth: int = 0): + """常规聊天处理""" + # Match the legacy chat lifecycle: one stable ID per top-level turn. + # Tool-call recursion keeps the same ID and its queued TTS stream. + if depth == 0: + context.sentence_id = uuid.uuid4().hex + + # 使用SessionContext的对话历史 + dialogue = context.dialogue + if not dialogue: + from core.utils.dialogue import Dialogue + dialogue = Dialogue() + context.dialogue = dialogue + + # 获取LLM组件 + llm_component = await self._get_component(context, ComponentType.LLM) + if not llm_component: + await self._send_error(transport, "LLM组件未初始化") + await self._finish_tts_turn(context, transport) + return + + llm_instance = getattr(llm_component, 'llm_instance', None) + if not llm_instance: + await self._send_error(transport, "LLM实例未就绪") + await self._finish_tts_turn(context, transport) + return + + logger.info("LLM实例已就绪，开始构建对话上下文") + + # 添加用户消息到对话历史（工具结果回合可能为空） + if text: + dialogue.put(Message(role="user", content=text)) + if depth >= 5: + dialogue.put( + Message( + role="user", + content="[系统提示] 已达到最大工具调用次数限制，请基于现有信息直接回答，不要再调用工具。", + ) + ) + + # 原架构不发送thinking状态给前端，直接开始处理 + + # 获取记忆上下文 + memory_context = await self._get_memory_context(context, text) + + # 构建对话上下文 + dialogue_context = dialogue.get_llm_dialogue_with_memory( + memory_context, + context.config.get("voiceprint", {}), + current_speaker=context.current_speaker, + ) + + # 调用LLM生成回复 + if context.intent_type == "function_call" and context.func_handler and hasattr(llm_instance, "response_with_functions"): + await self._generate_llm_response_with_tools( + context, + transport, + llm_instance, + dialogue_context, + dialogue, + depth=depth, + ) + else: + await self._generate_llm_response(context, transport, llm_instance, dialogue_context, dialogue) + + async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str): + """发送STT消息""" + from core.processors.audio_send_processor import AudioSendProcessor + + await AudioSendProcessor().send_stt_message( + context, transport, text + ) + + async def _get_component(self, context: SessionContext, component_type: ComponentType): + if not context.component_manager: + return None + return await context.component_manager.get_component(component_type, context) + + def cleanup_session(self, session_id: str): + """清理会话对话历史""" + if session_id in self._dialogues: + del self._dialogues[session_id] + logger.info(f"已清理会话对话历史: {session_id}") diff --git a/main/xiaozhi-server/core/processors/goodbye_processor.py b/main/xiaozhi-server/core/processors/goodbye_processor.py new file mode 100644 index 00000000..186300ca --- /dev/null +++ b/main/xiaozhi-server/core/processors/goodbye_processor.py @@ -0,0 +1,34 @@ +import json +from typing import Any +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from config.logger import setup_logging + +logger = setup_logging() + + +class GoodbyeProcessor(MessageProcessor): + """Goodbye消息处理器：处理会话结束""" + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + msg_json = None + if isinstance(message, str): + try: + msg_json = json.loads(message) + except json.JSONDecodeError: + msg_json = None + elif isinstance(message, dict): + msg_json = message + + if isinstance(msg_json, dict) and msg_json.get("type") == "goodbye": + logger.info(f"收到goodbye: session_id={msg_json.get('session_id')}") + # 长连接传输仅结束逻辑会话，短连接传输关闭连接。 + if transport.keeps_connection_between_sessions: + end_conversation = getattr(context, "end_conversation", None) + if callable(end_conversation): + await end_conversation(msg_json.get("session_id")) + else: + await transport.close() + return True + return False diff --git a/main/xiaozhi-server/core/processors/hello_processor.py b/main/xiaozhi-server/core/processors/hello_processor.py new file mode 100644 index 00000000..fd966b14 --- /dev/null +++ b/main/xiaozhi-server/core/processors/hello_processor.py @@ -0,0 +1,351 @@ +import time +import json +import random +import asyncio +import uuid +from typing import Any +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from core.utils.dialogue import Message +from core.utils.util import audio_to_data, remove_punctuation_and_length, opus_datas_to_wav_bytes +from core.providers.tts.dto.dto import SentenceType +from core.processors.audio_receive_processor import arm_wake_audio_suppression +from core.utils.wakeup_word import WakeupWordsConfig +from core.components.component_manager import ComponentType +from core.providers.tools.device_mcp import ( + MCPClient, + send_mcp_initialize_message, +) +from config.logger import setup_logging + +logger = setup_logging() + +# 唤醒词配置 +WAKEUP_CONFIG = { + "refresh_time": 5, + "words": ["你好", "你好啊", "嘿，你好", "嗨"], +} + +# 创建全局的唤醒词配置管理器 +wakeup_words_config = WakeupWordsConfig() + +# 用于防止并发调用wakeupWordsResponse的锁 +_wakeup_response_lock = asyncio.Lock() + + +class HelloProcessor(MessageProcessor): + """Hello消息处理器：完整迁移helloHandle.py的所有功能""" + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """处理hello类型的消息""" + msg_json = None + if isinstance(message, str): + try: + msg_json = json.loads(message) + except json.JSONDecodeError: + msg_json = None + elif isinstance(message, dict): + msg_json = message + + if isinstance(msg_json, dict) and msg_json.get("type") == "hello": + await self.handle_hello_message(context, transport, msg_json) + return True + return False + + async def handle_hello_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict): + """处理hello消息 - 完整迁移自handleHelloMessage""" + pending_tasks = [] + for task_name in ( + "listen_stop_task", + "listen_start_task", + ): + pending_task = getattr(context, task_name, None) + if pending_task and not pending_task.done(): + pending_task.cancel() + pending_tasks.append(pending_task) + setattr(context, task_name, None) + if pending_tasks: + await asyncio.gather(*pending_tasks, return_exceptions=True) + + transport_session_id = getattr(transport, "session_id", None) + if transport_session_id: + if ( + context.session_id != transport_session_id + and callable(getattr(context, "end_conversation", None)) + ): + # A previous finalizer may already have published + # conversation_active=False while persistence is still in + # progress. Always join it before exposing the new session. + await context.end_conversation(context.session_id) + context.session_id = transport_session_id + if context.welcome_msg: + context.welcome_msg["session_id"] = context.session_id + dialogue = getattr(context, "dialogue", None) + if getattr(context, "prompt", None) and dialogue and not dialogue.dialogue: + dialogue.update_system_message(context.prompt) + inject_fewshot = getattr(context, "inject_tool_call_fewshot", None) + if callable(inject_fewshot): + inject_fewshot() + context.conversation_active = True + context.listen_stop_pending = False + context.listen_stop_deadline = 0.0 + if transport.requires_audio_tail_grace: + context.accepting_input_audio = False + + # 处理音频参数 + audio_params = msg_json.get("audio_params") + if audio_params: + format = audio_params.get("format") + logger.info(f"客户端音频格式: {format}") + context.audio_format = format + context.input_sample_rate = int( + audio_params.get("sample_rate", getattr(context, "input_sample_rate", 16000)) + ) + context.input_channels = int( + audio_params.get("channels", getattr(context, "input_channels", 1)) + ) + context.input_frame_duration = int( + audio_params.get( + "frame_duration", getattr(context, "input_frame_duration", 60) + ) + ) + + # 处理客户端特性 + features = msg_json.get("features") or {} + context.features = dict(features) + context.client_aec = bool(features.get("aec")) + mcp_enabled = bool(features.get("mcp")) + previous_mcp_client = getattr(context, "mcp_client", None) + if not mcp_enabled and previous_mcp_client: + initialize_task = getattr( + context, "mcp_initialize_task", None + ) + if initialize_task and not initialize_task.done(): + initialize_task.cancel() + await asyncio.gather( + initialize_task, return_exceptions=True + ) + context.mcp_initialize_task = None + previous_mcp_cleanup = getattr( + context, "_mcp_cleanup_callback", None + ) + if previous_mcp_cleanup: + context.unregister_cleanup(previous_mcp_cleanup) + context._mcp_cleanup_callback = None + if hasattr(previous_mcp_client, "close"): + await previous_mcp_client.close() + context.mcp_client = None + if features: + logger.info(f"客户端特性: {features}") + if mcp_enabled: + logger.info("客户端支持MCP") + if context.mcp_client is None: + context.mcp_client = MCPClient() + context._mcp_cleanup_callback = ( + context.mcp_client.close + ) + context.register_cleanup( + context._mcp_cleanup_callback + ) + initialize_task = getattr( + context, "mcp_initialize_task", None + ) + if ( + not await context.mcp_client.is_ready() + and ( + initialize_task is None + or initialize_task.done() + ) + ): + context.mcp_initialize_task = ( + context.create_background_task( + send_mcp_initialize_message( + context, transport + ), + conversation_scoped=False, + ) + ) + if features.get("aec"): + logger.info("客户端启用了服务端AEC") + + # MQTT/UDP 场景已由MQTT连接层发送hello回复，避免重复发送websocket格式 + if transport.has_datagram_audio: + return + + # 发送欢迎消息 + if context.welcome_msg: + await transport.send_json(context.welcome_msg) + else: + # 默认欢迎消息 + welcome_msg = { + "type": "hello", + "session_id": context.session_id, + "version": 1, + "transport": "websocket" + } + await transport.send_json(welcome_msg) + + async def check_wakeup_words(self, context: SessionContext, transport: TransportInterface, text: str) -> bool: + """检查唤醒词 - 完整迁移自checkWakeupWords""" + enable_wakeup_words_response_cache = context.config.get("enable_wakeup_words_response_cache", False) + + # 等待tts初始化，最多等待3秒 + tts_component = await self._get_component(context, ComponentType.TTS) + start_time = time.time() + while time.time() - start_time < 3: + if tts_component and hasattr(tts_component, 'tts_instance'): + break + await asyncio.sleep(0.1) + else: + return False + + if not enable_wakeup_words_response_cache: + return False + + _, filtered_text = remove_punctuation_and_length(text) + if filtered_text not in context.config.get("wakeup_words", []): + return False + + sentence_id = uuid.uuid4().hex + context.sentence_id = sentence_id + context.just_woken_up = True + await self._send_stt_message(context, transport, text) + + # 获取当前音色 + tts_instance = getattr(tts_component, 'tts_instance', None) if tts_component else None + voice = getattr(tts_instance, "voice", "default") if tts_instance else "default" + if not voice: + voice = "default" + + # 获取唤醒词回复配置 + response = wakeup_words_config.get_wakeup_response(voice) + if not response or not response.get("file_path"): + response = { + "voice": "default", + "file_path": "config/assets/wakeup_words.wav", + "time": 0, + "text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦", + } + + # 获取音频数据 + opus_packets = await audio_to_data(response.get("file_path")) + # 播放唤醒词回复 + context.abort_requested = False + context.llm_finish_task = True + if tts_instance is not None: + # A prior turn may have consumed the one-shot start marker. Cached + # playback is a new TTS stream and must always emit start first. + tts_instance.tts_audio_first_sentence = True + + logger.info(f"播放唤醒词回复: {response.get('text')}") + try: + await self._send_audio_message( + context, + transport, + SentenceType.FIRST, + opus_packets, + response.get("text"), + sentence_id, + ) + await self._send_audio_message( + context, + transport, + SentenceType.LAST, + [], + None, + sentence_id, + ) + finally: + # Cached wake playback bypasses ListenProcessor, so it must arm + # the same bounded release used by the regular wake-word path. + arm_wake_audio_suppression(context) + + # 补充对话 + if context.dialogue: + context.dialogue.put(Message(role="assistant", content=response.get("text"))) + + # 检查是否需要更新唤醒词回复 + if time.time() - response.get("time", 0) > WAKEUP_CONFIG["refresh_time"]: + if not _wakeup_response_lock.locked(): + context.create_background_task( + self._wakeup_words_response(context, transport) + ) + return True + + async def _wakeup_words_response(self, context: SessionContext, transport: TransportInterface): + """生成唤醒词回复 - 完整迁移自wakeupWordsResponse""" + tts_component = await self._get_component(context, ComponentType.TTS) + llm_component = await self._get_component(context, ComponentType.LLM) + + tts_instance = getattr(tts_component, 'tts_instance', None) if tts_component else None + llm_instance = getattr(llm_component, 'llm_instance', None) if llm_component else None + + if not tts_instance or not llm_instance or not hasattr(llm_instance, 'response_no_stream'): + return + + try: + # 尝试获取锁，如果获取不到就返回 + async with _wakeup_response_lock: + # 生成唤醒词回复 + wakeup_word = random.choice(WAKEUP_CONFIG["words"]) + question = ( + "此刻用户正在和你说```" + + wakeup_word + + "```。\n请你根据以上用户的内容进行20-30字回复。要符合系统设置的角色情感和态度，不要像机器人一样说话。\n" + + "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。" + ) + + result = await asyncio.to_thread( + llm_instance.response_no_stream, + context.config.get("prompt", ""), + question, + ) + if not result or len(result) == 0: + return + + # 生成TTS音频 + tts_result = await asyncio.to_thread(tts_instance.to_tts, result) + if not tts_result: + return + + # 获取当前音色 + voice = getattr(tts_instance, "voice", "default") + + wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000) + file_path = wakeup_words_config.generate_file_path(voice) + with open(file_path, "wb") as f: + f.write(wav_bytes) + # 更新配置 + wakeup_words_config.update_wakeup_response(voice, file_path, result) + except Exception as e: + logger.error(f"生成唤醒词回复失败: {e}") + + async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str): + """发送STT消息""" + from core.processors.audio_send_processor import AudioSendProcessor + + await AudioSendProcessor().send_stt_message( + context, transport, text + ) + + async def _send_audio_message(self, context: SessionContext, transport: TransportInterface, + sentence_type: SentenceType, audios: bytes, text: str, + sentence_id: str = None): + """发送音频消息""" + # 这里应该调用AudioSendProcessor + from core.processors.audio_send_processor import AudioSendProcessor + audio_send_processor = AudioSendProcessor() + await audio_send_processor.send_audio_message( + context, + transport, + sentence_type, + audios, + text, + sentence_id=sentence_id, + ) + + async def _get_component(self, context: SessionContext, component_type: ComponentType): + if not context.component_manager: + return None + return await context.component_manager.get_component(component_type, context) diff --git a/main/xiaozhi-server/core/processors/iot_processor.py b/main/xiaozhi-server/core/processors/iot_processor.py new file mode 100644 index 00000000..348d64d8 --- /dev/null +++ b/main/xiaozhi-server/core/processors/iot_processor.py @@ -0,0 +1,124 @@ +import json +from typing import Any +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from core.providers.tools.device_iot import handleIotStatus, handleIotDescriptors +from config.logger import setup_logging + +logger = setup_logging() + + +class IotProcessor(MessageProcessor): + """IoT消息处理器：完整迁移iotMessageHandler.py的所有功能""" + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """处理iot类型的消息""" + msg_json = None + if isinstance(message, str): + try: + msg_json = json.loads(message) + except json.JSONDecodeError: + msg_json = None + elif isinstance(message, dict): + msg_json = message + + if isinstance(msg_json, dict) and msg_json.get("type") == "iot": + await self.handle_iot_message(context, transport, msg_json) + return True + return False + + async def handle_iot_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict): + """处理IoT消息 - 完整迁移自iotMessageHandler.py""" + tasks = [] + + # 处理设备描述符 - 完整迁移原逻辑 + if "descriptors" in msg_json: + logger.debug("处理IoT设备描述符") + task = context.create_background_task( + self._handle_iot_descriptors(context, transport, msg_json["descriptors"]) + ) + tasks.append(task) + + # 处理设备状态 - 完整迁移原逻辑 + if "states" in msg_json: + logger.debug("处理IoT设备状态") + task = context.create_background_task( + self._handle_iot_status(context, transport, msg_json["states"]) + ) + tasks.append(task) + + # 如果没有有效的IoT数据 + if not tasks: + logger.warning("IoT消息缺少descriptors或states字段") + await self._send_error_response( + transport, + context.session_id, + "IoT消息格式错误：缺少descriptors或states字段" + ) + return + + # 任务由 SessionContext 持有，结束会话时统一取消。 + + async def _handle_iot_descriptors(self, context: SessionContext, transport: TransportInterface, descriptors: Any): + """处理IoT设备描述符 - 包装原handleIotDescriptors函数""" + try: + # 调用原有的handleIotDescriptors函数 + # 注意：这里需要传入context而不是conn，因为handleIotDescriptors可能需要适配 + await handleIotDescriptors(context, descriptors) + logger.debug("IoT设备描述符处理完成") + + except Exception as e: + logger.error(f"处理IoT设备描述符失败: {e}", exc_info=True) + await self._send_error_response( + transport, + context.session_id, + f"IoT设备描述符处理失败: {str(e)}" + ) + + async def _handle_iot_status(self, context: SessionContext, transport: TransportInterface, states: Any): + """处理IoT设备状态 - 包装原handleIotStatus函数""" + try: + # 调用原有的handleIotStatus函数 + # 注意：这里需要传入context而不是conn，因为handleIotStatus可能需要适配 + await handleIotStatus(context, states) + logger.debug("IoT设备状态处理完成") + + except Exception as e: + logger.error(f"处理IoT设备状态失败: {e}", exc_info=True) + await self._send_error_response( + transport, + context.session_id, + f"IoT设备状态处理失败: {str(e)}" + ) + + async def _send_error_response(self, transport: TransportInterface, session_id: str, message: str): + """发送IoT错误响应""" + response = { + "type": "iot", + "status": "error", + "message": message, + "session_id": session_id + } + + try: + await transport.send(json.dumps(response)) + except Exception as e: + logger.error(f"发送IoT错误响应失败: {e}") + + async def _send_success_response(self, transport: TransportInterface, session_id: str, + message: str, data: dict = None): + """发送IoT成功响应""" + response = { + "type": "iot", + "status": "success", + "message": message, + "session_id": session_id + } + if data: + response["data"] = data + + try: + await transport.send(json.dumps(response)) + except Exception as e: + logger.error(f"发送IoT成功响应失败: {e}") diff --git a/main/xiaozhi-server/core/processors/listen_processor.py b/main/xiaozhi-server/core/processors/listen_processor.py new file mode 100644 index 00000000..6e9782cc --- /dev/null +++ b/main/xiaozhi-server/core/processors/listen_processor.py @@ -0,0 +1,433 @@ +import time +import json +import asyncio +import uuid +from typing import Any +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from core.utils.util import remove_punctuation_and_length +from core.utils.dialogue import Message +from core.components.component_manager import ComponentType +from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType +from core.processors.audio_receive_processor import arm_wake_audio_suppression +from config.logger import setup_logging + +logger = setup_logging() + + +class ListenProcessor(MessageProcessor): + """Listen消息处理器：完整迁移listenMessageHandler.py的所有功能""" + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """处理listen类型的消息""" + msg_json = None + if isinstance(message, str): + try: + msg_json = json.loads(message) + except json.JSONDecodeError: + msg_json = None + elif isinstance(message, dict): + msg_json = message + + if isinstance(msg_json, dict) and msg_json.get("type") == "listen": + await self.handle_listen_message(context, transport, msg_json) + return True + return False + + async def handle_listen_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict): + """处理listen消息 - 完整迁移自listenMessageHandler.py""" + msg_session_id = msg_json.get("session_id") + if msg_session_id and msg_session_id != context.session_id: + logger.warning( + f"忽略非当前会话的listen消息: " + f"msg={msg_session_id}, current={context.session_id}" + ) + return + + state = msg_json.get("state") + has_datagram_audio = getattr( + transport, "has_datagram_audio", False + ) + if ( + has_datagram_audio + and state in {"start", "detect"} + and not context.conversation_active + ): + if transport.keeps_connection_between_sessions: + await transport.send_json( + { + "type": "goodbye", + "session_id": msg_session_id or context.session_id, + } + ) + logger.info( + "忽略已结束会话的listen {}: session_id={}", + state, + context.session_id, + ) + return + + # 设置拾音模式 + if "mode" in msg_json: + context.listen_mode = msg_json["mode"] + logger.debug(f"客户端拾音模式：{context.listen_mode}") + + # 处理不同的状态 + if state == "start": + pending_start = getattr(context, "listen_start_task", None) + if pending_start and not pending_start.done(): + logger.debug("忽略尾包隔离期内的重复listen start") + return + tail_quarantine = await self._flush_pending_listen_stop( + context, transport + ) + # 开始监听语音 + logger.info(f"listen start: session_id={context.session_id}, mode={context.listen_mode}") + context.reset_audio_states() + if ( + getattr(transport, "requires_audio_tail_grace", False) + and tail_quarantine > 0 + ): + context.accepting_input_audio = False + context.listen_start_task = context.create_background_task( + self._open_input_after_tail_quarantine( + context, + context.session_id, + tail_quarantine, + ), + turn_scoped=True, + ) + else: + context.accepting_input_audio = True + context.abort_requested = False + context.close_after_chat = False + logger.debug("开始语音监听") + + elif state == "stop": + # 停止监听语音 + logger.info(f"listen stop: session_id={context.session_id}") + context.client_have_voice = True + context.listen_stop_pending = True + pending_start = getattr(context, "listen_start_task", None) + if pending_start and not pending_start.done(): + pending_start.cancel() + context.listen_start_task = None + + if getattr(transport, "requires_audio_tail_grace", False): + pending = getattr(context, "listen_stop_task", None) + if pending and not pending.done(): + logger.debug("忽略重复的listen stop") + return + delay_ms = max( + 0, + min( + 1000, + int( + context.config.get( + "mqtt_udp_tail_grace_ms", 180 + ) + ), + ), + ) + context.listen_stop_deadline = ( + time.monotonic() + delay_ms / 1000 + ) + context.listen_stop_task = context.create_background_task( + self._finalize_listen_stop( + context, + transport, + context.session_id, + delay_ms / 1000, + ), + turn_scoped=True, + ) + else: + context.listen_stop_deadline = 0.0 + await self._finalize_listen_stop( + context, + transport, + context.session_id, + 0, + ) + logger.debug("停止语音监听") + + elif state == "detect": + # 检测到文本输入 + logger.info(f"listen detect: session_id={context.session_id}, text={msg_json.get('text')}") + context.client_have_voice = False + context.asr_audio.clear() + + if "text" in msg_json: + context.update_activity() + original_text = msg_json["text"] # 保留原始文本 + filtered_len, filtered_text = remove_punctuation_and_length(original_text) + + if original_text.startswith("[device_call]"): + await self._handle_device_call( + context, + transport, + original_text[len("[device_call]"):].strip(), + ) + return + + # 识别是否是唤醒词 + is_wakeup_words = filtered_text in context.config.get("wakeup_words", []) + # 是否开启唤醒词回复 + enable_greeting = context.config.get("enable_greeting", True) + + if is_wakeup_words and not enable_greeting: + # 如果是唤醒词，且关闭了唤醒词回复，就不用回答 + # Native already drops wake history until listen/start. + # Keeping the greeting suppression window armed here would + # discard the beginning of the user's first real utterance. + await self._send_stt_message(context, transport, original_text) + await self._send_tts_message(context, transport, "stop", None) + context.is_speaking = False + + elif is_wakeup_words: + # 处理唤醒词 + self._arm_wake_audio_suppression(context) + # 上报纯文字数据（复用ASR上报功能，但不提供音频数据） + await self._enqueue_asr_report(context, "嘿，你好呀", []) + await self._start_to_chat(context, transport, "嘿，你好呀", skip_intent=True) + + else: + # 处理普通文本 + # 上报纯文字数据（复用ASR上报功能，但不提供音频数据） + await self._enqueue_asr_report(context, original_text, []) + # 否则需要LLM对文字内容进行答复 + await self._start_to_chat(context, transport, original_text) + + async def _flush_pending_listen_stop( + self, + context: SessionContext, + transport: TransportInterface, + ) -> float: + """Finalize buffered speech before a new listen/start resets the turn.""" + pending = getattr(context, "listen_stop_task", None) + had_pending_stop = bool(getattr(context, "listen_stop_pending", False)) + tail_quarantine = max( + 0.0, + float(getattr(context, "listen_stop_deadline", 0.0) or 0.0) + - time.monotonic(), + ) + if pending and not pending.done(): + pending.cancel() + try: + await pending + except asyncio.CancelledError: + pass + context.listen_stop_task = None + if had_pending_stop: + await self._finalize_listen_stop( + context, + transport, + context.session_id, + 0, + ) + else: + context.listen_stop_pending = False + if tail_quarantine <= 0: + context.listen_stop_deadline = 0.0 + return tail_quarantine + + @staticmethod + async def _open_input_after_tail_quarantine( + context: SessionContext, + session_id: str, + delay_seconds: float, + ) -> None: + current_task = asyncio.current_task() + try: + await asyncio.sleep(delay_seconds) + if ( + context.session_id == session_id + and context.conversation_active + and not context.listen_stop_pending + ): + context.accepting_input_audio = True + finally: + if getattr(context, "listen_start_task", None) is current_task: + context.listen_start_task = None + context.listen_stop_deadline = 0.0 + + async def _finalize_listen_stop( + self, + context: SessionContext, + transport: TransportInterface, + session_id: str, + delay_seconds: float, + ) -> None: + """Finalize ASR after datagram tail frames have crossed the router.""" + current_task = asyncio.current_task() + try: + if delay_seconds > 0: + await asyncio.sleep(delay_seconds) + if context.session_id != session_id: + return + if ( + getattr(transport, "has_datagram_audio", False) + and not context.conversation_active + ): + return + + # Close the tail-grace ingress gate before resolving ASR. + # Component lookup may fail; leaving it open would leak subsequent + # packets into a turn that has already stopped. + context.listen_stop_pending = False + context.client_voice_stop = True + if getattr(transport, "requires_audio_tail_grace", False): + context.accepting_input_audio = False + + asr_component = await self._get_component( + context, ComponentType.ASR + ) + if context.session_id != session_id: + return + asr_instance = ( + getattr(asr_component, "asr_instance", None) + if asr_component + else None + ) + + if not asr_instance or not hasattr( + asr_instance, "interface_type" + ): + return + + from core.providers.asr.dto.dto import InterfaceType + + if asr_instance.interface_type == InterfaceType.STREAM: + if hasattr(asr_instance, "_send_stop_request"): + await asr_instance._send_stop_request() + return + + if context.asr_audio: + asr_audio_task = context.asr_audio.copy() + context.asr_audio.clear() + context.reset_vad_states() + await asr_instance.handle_voice_stop( + context, asr_audio_task + ) + except asyncio.CancelledError: + raise + except Exception as error: + logger.error( + "listen stop收尾失败: session_id={}, error={}", + session_id, + error, + ) + finally: + if getattr(context, "listen_stop_task", None) is current_task: + context.listen_stop_task = None + context.listen_stop_deadline = 0.0 + if context.session_id == session_id: + context.listen_stop_pending = False + if getattr(transport, "requires_audio_tail_grace", False): + context.accepting_input_audio = False + + def _arm_wake_audio_suppression(self, context: SessionContext) -> None: + """Bound the UDP/TCP reorder window used by the legacy gateway flow.""" + arm_wake_audio_suppression(context) + + async def _handle_device_call( + self, + context: SessionContext, + transport: TransportInterface, + call_text: str, + ) -> None: + """Restore the legacy device-call announcement and call-state handoff.""" + logger.info(f"收到设备呼叫指令: {call_text}") + context.incoming_call = True + context.sentence_id = uuid.uuid4().hex + await self._send_stt_message(context, transport, call_text) + + tts_component = await self._get_component(context, ComponentType.TTS) + tts_instance = ( + getattr(tts_component, "tts_instance", None) + if tts_component + else None + ) + if tts_instance: + tts_instance.store_tts_text(context.sentence_id, call_text) + tts_instance.tts_text_queue.put( + TTSMessageDTO( + sentence_id=context.sentence_id, + sentence_type=SentenceType.FIRST, + content_type=ContentType.ACTION, + ) + ) + tts_instance.tts_one_sentence( + context, + ContentType.TEXT, + content_detail=call_text, + ) + tts_instance.tts_text_queue.put( + TTSMessageDTO( + sentence_id=context.sentence_id, + sentence_type=SentenceType.LAST, + content_type=ContentType.ACTION, + ) + ) + + context.dialogue.put(Message(role="assistant", content=call_text)) + + async def _handle_audio_message(self, context: SessionContext, transport: TransportInterface, audio: bytes): + """处理音频消息 - 调用AudioReceiveProcessor""" + # 这里应该调用AudioReceiveProcessor来处理音频 + from core.processors.audio_receive_processor import AudioReceiveProcessor + audio_processor = AudioReceiveProcessor() + await audio_processor.handle_audio_message(context, transport, audio) + + async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str): + """发送STT消息""" + from core.processors.audio_send_processor import AudioSendProcessor + + await AudioSendProcessor().send_stt_message( + context, transport, text + ) + + async def _send_tts_message(self, context: SessionContext, transport: TransportInterface, state: str, text: str = None): + """发送TTS消息""" + message = { + "type": "tts", + "state": state, + "session_id": context.session_id + } + if text: + message["text"] = text + + await transport.send(json.dumps(message)) + logger.debug(f"发送TTS消息: state={state}, text={text}") + + async def _get_component(self, context: SessionContext, component_type: ComponentType): + if not context.component_manager: + return None + return await context.component_manager.get_component(component_type, context) + + async def _enqueue_asr_report(self, context: SessionContext, text: str, audio_data: list): + """ASR上报队列""" + if context.report_asr_enable: + from core.processors.report_processor import ReportProcessor + report_processor = ReportProcessor() + report_processor.enqueue_asr_report(context, text, audio_data) + + async def _start_to_chat(self, context: SessionContext, transport: TransportInterface, text: str, skip_intent: bool = False): + """开始聊天 - 调用ChatProcessor""" + # 与旧架构一致：先发送STT，再异步触发聊天，避免阻塞事件循环 + await self._send_stt_message(context, transport, text) + + from core.processors.chat_processor import ChatProcessor + chat_processor = ChatProcessor() + + if hasattr(context, "create_background_task"): + context.create_background_task( + chat_processor.handle_chat( + context, transport, text, skip_intent=skip_intent + ), + turn_scoped=True, + ) + logger.info(f"chat任务已提交: session_id={context.session_id}") + else: + await chat_processor.handle_chat(context, transport, text, skip_intent=skip_intent) diff --git a/main/xiaozhi-server/core/processors/mcp_processor.py b/main/xiaozhi-server/core/processors/mcp_processor.py new file mode 100644 index 00000000..67bbba07 --- /dev/null +++ b/main/xiaozhi-server/core/processors/mcp_processor.py @@ -0,0 +1,96 @@ +import json +from typing import Any +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from core.providers.tools.device_mcp import handle_mcp_message +from config.logger import setup_logging + +logger = setup_logging() + + +class McpProcessor(MessageProcessor): + """MCP消息处理器：完整迁移mcpMessageHandler.py的所有功能""" + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """处理mcp类型的消息""" + msg_json = None + if isinstance(message, str): + try: + msg_json = json.loads(message) + except json.JSONDecodeError: + msg_json = None + elif isinstance(message, dict): + msg_json = message + + if isinstance(msg_json, dict) and msg_json.get("type") == "mcp": + await self.handle_mcp_message(context, transport, msg_json) + return True + return False + + async def handle_mcp_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict): + """处理MCP消息 - 完整迁移自mcpMessageHandler.py""" + if "payload" in msg_json: + # 检查MCP客户端是否存在 + if not context.mcp_client: + logger.warning("MCP客户端未初始化，无法处理MCP消息") + await self._send_error_response(transport, context.session_id, "MCP客户端未初始化") + return + + # 创建异步任务处理MCP消息 - 完整迁移原逻辑 + context.create_background_task( + self._handle_mcp_payload( + context, transport, msg_json["payload"] + ), + conversation_scoped=False, + ) + else: + logger.warning("MCP消息缺少payload字段") + await self._send_error_response(transport, context.session_id, "MCP消息格式错误：缺少payload") + + async def _handle_mcp_payload(self, context: SessionContext, transport: TransportInterface, payload: dict): + """处理MCP payload - 包装原handle_mcp_message函数""" + try: + # 调用原有的handle_mcp_message函数 + # 注意：这里需要传入context而不是conn，因为handle_mcp_message可能需要适配 + await handle_mcp_message(context, context.mcp_client, payload, transport) + logger.debug("MCP消息处理完成") + + except Exception as e: + logger.error(f"处理MCP消息失败: {e}", exc_info=True) + await self._send_error_response( + transport, + context.session_id, + f"MCP消息处理失败: {str(e)}" + ) + + async def _send_error_response(self, transport: TransportInterface, session_id: str, message: str): + """发送MCP错误响应""" + response = { + "type": "mcp", + "status": "error", + "message": message, + "session_id": session_id + } + + try: + await transport.send(json.dumps(response)) + except Exception as e: + logger.error(f"发送MCP错误响应失败: {e}") + + async def _send_success_response(self, transport: TransportInterface, session_id: str, + message: str, data: dict = None): + """发送MCP成功响应""" + response = { + "type": "mcp", + "status": "success", + "message": message, + "session_id": session_id + } + if data: + response["data"] = data + + try: + await transport.send(json.dumps(response)) + except Exception as e: + logger.error(f"发送MCP成功响应失败: {e}") diff --git a/main/xiaozhi-server/core/processors/message_router.py b/main/xiaozhi-server/core/processors/message_router.py new file mode 100644 index 00000000..2737c423 --- /dev/null +++ b/main/xiaozhi-server/core/processors/message_router.py @@ -0,0 +1,124 @@ +import json +from typing import Any, List +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from core.processors.hello_processor import HelloProcessor +from core.processors.listen_processor import ListenProcessor +from core.processors.audio_receive_processor import AudioReceiveProcessor +from core.processors.auth_processor import AuthProcessor +from core.processors.timeout_processor import TimeoutProcessor +from core.processors.server_processor import ServerProcessor +from core.processors.mcp_processor import McpProcessor +from core.processors.iot_processor import IotProcessor +from core.processors.abort_processor import AbortProcessor +from core.processors.goodbye_processor import GoodbyeProcessor +from core.processors.text_processor import TextProcessor +from core.processors.ping_processor import PingProcessor +from config.logger import setup_logging + +logger = setup_logging() + + +class MessageRouter(MessageProcessor): + """ + 消息路由器：协调所有独立的processor + 按功能职责分离，避免耦合，每个processor专注单一职责 + """ + + def __init__(self): + # 初始化所有独立的processor + self.auth_processor = AuthProcessor() + self.timeout_processor = TimeoutProcessor() + self.abort_processor = AbortProcessor() + self.goodbye_processor = GoodbyeProcessor() + self.hello_processor = HelloProcessor() + self.listen_processor = ListenProcessor() + self.server_processor = ServerProcessor() + self.mcp_processor = McpProcessor() + self.iot_processor = IotProcessor() + self.audio_receive_processor = AudioReceiveProcessor() + self.text_processor = TextProcessor() + self.ping_processor = PingProcessor() + + # 按优先级排序的processor列表 + self.processors: List[MessageProcessor] = [ + self.timeout_processor, # 首先检查超时 + self.auth_processor, # 然后检查认证 + self.server_processor, # 服务器消息（管理端下发动作） + self.abort_processor, # 中断消息 + self.goodbye_processor, # goodbye消息 + self.hello_processor, # hello消息 + self.ping_processor, # 可选JSON ping/pong心跳 + self.listen_processor, # listen消息 + self.mcp_processor, # MCP消息 + self.iot_processor, # IoT消息 + self.audio_receive_processor, # 音频消息 + self.text_processor, # 纯文本消息（放在最后，作为兜底处理） + ] + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """ + 路由消息到合适的processor + 每个processor专注处理自己的消息类型，避免耦合 + """ + # Audio activity is updated only after VAD confirms voice. Treating + # every silent frame as activity prevents logical-session timeouts. + is_audio = isinstance(message, bytes) or ( + isinstance(message, dict) and message.get("type") == "audio" + ) + if not is_audio: + context.update_activity() + + # 按优先级顺序尝试每个processor + for processor in self.processors: + try: + if await processor.process(context, transport, message): + # 消息已被处理，记录日志并返回 + logger.debug(f"消息被 {processor.__class__.__name__} 处理") + return True + except Exception as e: + logger.error(f"{processor.__class__.__name__} 处理消息时出错: {e}", exc_info=True) + continue + + # 如果没有processor处理该消息，记录警告 + if isinstance(message, str): + try: + msg_json = json.loads(message) + msg_type = msg_json.get("type", "unknown") if isinstance(msg_json, dict) else "non-dict" + logger.warning(f"未处理的消息类型: {msg_type}, 内容: {message[:100]}...") + except json.JSONDecodeError: + logger.warning(f"未处理的非JSON消息: {message[:100]}...") + elif isinstance(message, bytes): + logger.warning(f"未处理的二进制消息，大小: {len(message)} bytes") + else: + logger.warning(f"未处理的消息类型: {type(message)}, {message}") + + return False + + def add_processor(self, processor: MessageProcessor, priority: int = None): + """ + 添加新的processor + priority: 优先级，数字越小优先级越高，None表示添加到末尾 + """ + if priority is None: + self.processors.append(processor) + else: + self.processors.insert(priority, processor) + logger.info(f"添加processor: {processor.__class__.__name__}") + + def remove_processor(self, processor_class): + """移除指定类型的processor""" + self.processors = [p for p in self.processors if not isinstance(p, processor_class)] + logger.info(f"移除processor: {processor_class.__name__}") + + def get_processor(self, processor_class): + """获取指定类型的processor""" + for processor in self.processors: + if isinstance(processor, processor_class): + return processor + return None + + def list_processors(self) -> List[str]: + """列出所有processor的名称""" + return [processor.__class__.__name__ for processor in self.processors] diff --git a/main/xiaozhi-server/core/processors/ping_processor.py b/main/xiaozhi-server/core/processors/ping_processor.py new file mode 100644 index 00000000..309faf4b --- /dev/null +++ b/main/xiaozhi-server/core/processors/ping_processor.py @@ -0,0 +1,40 @@ +import json +import time +from typing import Any + +from core.context.session_context import SessionContext +from core.pipeline.message_pipeline import MessageProcessor +from core.transport.transport_interface import TransportInterface +from config.logger import setup_logging + +logger = setup_logging() + + +class PingProcessor(MessageProcessor): + """Handle the optional legacy JSON ping/pong control contract.""" + + async def process( + self, + context: SessionContext, + transport: TransportInterface, + message: Any, + ) -> bool: + if isinstance(message, str): + try: + message = json.loads(message) + except json.JSONDecodeError: + return False + if not isinstance(message, dict) or message.get("type") != "ping": + return False + + if not context.config.get("enable_websocket_ping", False): + logger.debug("WebSocket心跳功能未启用，忽略PING消息") + return True + + await transport.send_json( + { + "type": "pong", + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), + } + ) + return True diff --git a/main/xiaozhi-server/core/processors/report_processor.py b/main/xiaozhi-server/core/processors/report_processor.py new file mode 100644 index 00000000..fda9c70d --- /dev/null +++ b/main/xiaozhi-server/core/processors/report_processor.py @@ -0,0 +1,301 @@ +import asyncio +import time +import queue +import threading +import json +from typing import Any, List +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from config.manage_api_client import ManageApiClient, report as manage_report +from config.logger import setup_logging + +logger = setup_logging() + + +class ReportProcessor(MessageProcessor): + """上报处理器：完整迁移reportHandle.py的所有功能""" + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """这个处理器不直接处理消息，而是被其他处理器调用""" + return False + + def enqueue_asr_report(self, context: SessionContext, text: str, audio_data: List[bytes]): + """ASR上报队列 - 完整迁移自enqueue_asr_report""" + if not self._should_report(context, context.report_asr_enable): + return + + report_time = int(time.time() * 1000) + if context.chat_history_conf != 2: + audio_data = None + + # 将上报任务放入队列 + self._enqueue_report(context, { + "type": 1, # 用户类型 + "text": text, + "audio_data": audio_data, + "report_time": report_time, + "session_id": context.session_id, + "device_id": context.device_id, + }) + + # 确保上报线程已启动 + self._ensure_report_thread(context) + + def enqueue_tts_report(self, context: SessionContext, text: str, opus_data: bytes): + """TTS上报队列 - 完整迁移自enqueue_tts_report""" + if not self._should_report(context, context.report_tts_enable): + return + + report_time = int(time.time() * 1000) + if context.chat_history_conf != 2: + opus_data = None + + # 将上报任务放入队列 + self._enqueue_report(context, { + "type": 2, # 智能体类型 + "text": text, + "audio_data": opus_data, + "report_time": report_time, + "session_id": context.session_id, + "device_id": context.device_id, + }) + + # 确保上报线程已启动 + self._ensure_report_thread(context) + + def enqueue_tool_report( + self, + context: SessionContext, + tool_name: str, + tool_input: dict, + tool_result: str = None, + report_tool_call: bool = True, + ): + if not self._should_report(context, True): + return + + timestamp = int(time.time() * 1000) + entries = [] + if report_tool_call: + entries.append( + ( + json.dumps( + [{"type": "tool", "text": f"{tool_name}({json.dumps(tool_input, ensure_ascii=False)})"}], + ensure_ascii=False, + ), + timestamp, + ) + ) + if tool_result is not None: + entries.append( + ( + json.dumps( + [{"type": "tool_result", "text": json.dumps({"result": str(tool_result)}, ensure_ascii=False)}], + ensure_ascii=False, + ), + timestamp + 1, + ) + ) + + for text, report_time in entries: + self._enqueue_report( + context, + { + "type": 3, + "text": text, + "audio_data": None, + "report_time": report_time, + "session_id": context.session_id, + "device_id": context.device_id, + }, + ) + if entries: + self._ensure_report_thread(context) + + def _ensure_report_thread(self, context: SessionContext): + """确保上报线程已启动""" + if context.report_thread is None or not context.report_thread.is_alive(): + if not getattr(context, "_report_cleanup_registered", False): + context.register_cleanup( + lambda: asyncio.to_thread(self.cleanup_session, context) + ) + context._report_cleanup_registered = True + context.report_thread = threading.Thread( + target=self._report_worker, + args=(context,), + daemon=True + ) + context.report_thread.start() + logger.info(f"上报线程已启动: {context.session_id}") + + def _enqueue_report(self, context: SessionContext, report_task: dict) -> None: + try: + context.report_queue.put_nowait(report_task) + except queue.Full: + try: + context.report_queue.get_nowait() + context.report_queue.put_nowait(report_task) + logger.warning("上报队列已满，丢弃最旧任务: {}", context.session_id) + except (queue.Empty, queue.Full): + logger.warning("上报队列持续拥塞，丢弃当前任务: {}", context.session_id) + + def _should_report(self, context: SessionContext, enabled: bool) -> bool: + return bool( + enabled + and context.read_config_from_api + and not context.need_bind + and context.chat_history_conf != 0 + ) + + def _report_worker(self, context: SessionContext): + """上报工作线程 - 完整迁移自ConnectionHandler中的上报逻辑""" + logger.info(f"上报工作线程启动: {context.session_id}") + event_loop = asyncio.new_event_loop() + asyncio.set_event_loop(event_loop) + + try: + while not context.stop_event.is_set(): + try: + # 从队列获取上报任务 + report_task = context.report_queue.get(timeout=1) + if report_task is None: + break + + # 执行上报 + self._execute_report(context, report_task, event_loop) + + except queue.Empty: + continue + except Exception as e: + logger.error(f"上报工作线程异常: {e}") + finally: + try: + event_loop.run_until_complete( + ManageApiClient.close_current_loop_client() + ) + except Exception as exc: + logger.debug("关闭上报HTTP客户端失败: {}", exc) + event_loop.close() + logger.info(f"上报工作线程退出: {context.session_id}") + + def _execute_report( + self, context: SessionContext, report_task: dict, + event_loop=None + ): + """执行聊天记录上报操作 - 完整迁移自report函数""" + try: + report_type = report_task["type"] + text = report_task["text"] + audio_data = report_task["audio_data"] + report_time = report_task["report_time"] + + # 处理音频数据 + processed_audio = None + if audio_data: + if isinstance(audio_data, list): + # ASR音频数据（多个音频片段） + processed_audio = self._process_asr_audio(audio_data) + elif isinstance(audio_data, bytes): + # TTS音频数据（opus格式） + processed_audio = self._opus_to_wav(audio_data) + + # 执行上报 + coroutine = manage_report( + mac_address=report_task["device_id"], + session_id=report_task["session_id"], + chat_type=report_type, + content=text, + audio=processed_audio, + report_time=report_time, + ) + if event_loop is None: + asyncio.run(coroutine) + else: + event_loop.run_until_complete(coroutine) + + logger.debug(f"上报成功: type={report_type}, text={text[:50]}...") + + except Exception as e: + logger.error(f"聊天记录上报失败: {e}") + + def _process_asr_audio(self, audio_data_list: List[bytes]) -> bytes: + """处理ASR音频数据""" + try: + # 将多个音频片段合并 + combined_audio = b''.join(audio_data_list) + return self._pcm_to_wav(combined_audio) + except Exception as e: + logger.error(f"处理ASR音频数据失败: {e}") + return b'' + + def _pcm_to_wav(self, pcm_data: bytes) -> bytes: + import io + import wave + + wav_buffer = io.BytesIO() + with wave.open(wav_buffer, 'wb') as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(pcm_data) + return wav_buffer.getvalue() + + def _opus_to_wav(self, opus_data: bytes) -> bytes: + """将Opus数据转换为WAV格式的字节流 - 完整迁移自opus_to_wav""" + try: + import opuslib_next + import io + import wave + + # Opus解码器配置 + sample_rate = 16000 + channels = 1 + + # 创建Opus解码器 + decoder = opuslib_next.Decoder(sample_rate, channels) + + # 解码Opus数据 + pcm_data = decoder.decode(opus_data, frame_size=960) + + # 创建WAV文件 + wav_buffer = io.BytesIO() + with wave.open(wav_buffer, 'wb') as wav_file: + wav_file.setnchannels(channels) + wav_file.setsampwidth(2) # 16-bit + wav_file.setframerate(sample_rate) + wav_file.writeframes(pcm_data) + + return wav_buffer.getvalue() + + except Exception as e: + logger.error(f"Opus转WAV失败: {e}") + return b'' + + def cleanup_session(self, context: SessionContext): + """清理会话上报资源""" + # 停止上报线程 + if context.report_thread and context.report_thread.is_alive(): + context.stop_event.set() + try: + context.report_queue.put_nowait(None) + except queue.Full: + try: + context.report_queue.get_nowait() + context.report_queue.put_nowait(None) + except (queue.Empty, queue.Full): + pass + context.report_thread.join(timeout=5) + if context.report_thread.is_alive(): + logger.warning(f"上报线程未能按时退出: {context.session_id}") + else: + context.report_thread = None + + # 清理上报队列 + try: + while not context.report_queue.empty(): + context.report_queue.get_nowait() + except queue.Empty: + pass + + logger.info(f"上报资源清理完成: {context.session_id}") diff --git a/main/xiaozhi-server/core/processors/server_processor.py b/main/xiaozhi-server/core/processors/server_processor.py new file mode 100644 index 00000000..a15c3bb8 --- /dev/null +++ b/main/xiaozhi-server/core/processors/server_processor.py @@ -0,0 +1,177 @@ +import json +import os +import sys +import time +import threading +import subprocess +from pathlib import Path +from typing import Any +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from config.logger import setup_logging +from core.utils.restart import restart_server + +logger = setup_logging() + + +class ServerProcessor(MessageProcessor): + """服务器消息处理器：完整迁移serverMessageHandler.py的所有功能""" + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """处理server类型的消息""" + msg_json = None + if isinstance(message, str): + try: + msg_json = json.loads(message) + except json.JSONDecodeError: + msg_json = None + elif isinstance(message, dict): + msg_json = message + + if isinstance(msg_json, dict) and msg_json.get("type") == "server": + await self.handle_server_message(context, transport, msg_json) + return True + return False + + async def handle_server_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict): + """处理server消息 - 完整迁移自serverMessageHandler.py""" + # 如果配置是从API读取的，则需要验证secret + if not context.read_config_from_api: + return + + # 获取post请求的secret + post_secret = msg_json.get("content", {}).get("secret", "") + secret = context.config.get("manager-api", {}).get("secret", "") + + # 如果secret不匹配，则返回 + if post_secret != secret: + await self._send_error_response( + transport, + context.session_id, + "服务器密钥验证失败" + ) + await self._close_transport(transport) + return + + # 处理不同的action + action = msg_json.get("action") + + if action == "update_config": + await self._handle_update_config(context, transport, msg_json) + elif action == "restart": + await self._handle_restart(context, transport, msg_json) + else: + await self._send_error_response( + transport, + context.session_id, + f"未知的服务器操作: {action}" + ) + await self._close_transport(transport) + + async def _handle_update_config(self, context: SessionContext, transport: TransportInterface, msg_json: dict): + """处理配置更新 - 完整迁移自update_config逻辑""" + try: + # 检查是否有服务器实例 + if not context.server: + await self._send_error_response( + transport, + context.session_id, + "无法获取服务器实例", + {"action": "update_config"} + ) + await self._close_transport(transport) + return + + # 更新WebSocketServer的配置 + if not await context.server.update_config(): + error_msg = "" + if hasattr(context.server, "get_last_update_error"): + error_msg = context.server.get_last_update_error() + await self._send_error_response( + transport, + context.session_id, + error_msg or "更新服务器配置失败", + {"action": "update_config"} + ) + await self._close_transport(transport) + return + + # 发送成功响应 + await self._send_success_response( + transport, + context.session_id, + "配置更新成功", + {"action": "update_config"} + ) + await self._close_transport(transport) + + except Exception as e: + logger.error(f"更新配置失败: {str(e)}") + await self._send_error_response( + transport, + context.session_id, + f"更新配置失败: {str(e)}", + {"action": "update_config"} + ) + await self._close_transport(transport) + + async def _handle_restart(self, context: SessionContext, transport: TransportInterface, msg_json: dict): + """处理服务器重启 - 完整迁移自handle_restart逻辑""" + try: + # 发送确认响应 + await self._send_success_response( + transport, + context.session_id, + "服务器重启中...", + {"action": "restart"} + ) + await self._close_transport(transport) + + # 异步执行重启操作 + threading.Thread(target=lambda : restart_server(logger), daemon=True).start() + except Exception as e: + logger.error(f"处理重启请求失败: {str(e)}") + await self._send_error_response( + transport, + context.session_id, + f"重启失败: {str(e)}", + {"action": "restart"} + ) + await self._close_transport(transport) + + async def _send_success_response(self, transport: TransportInterface, session_id: str, + message: str, content: dict = None): + """发送成功响应""" + response = { + "type": "server", + "status": "success", + "message": message, + "session_id": session_id + } + if content: + response["content"] = content + + await transport.send(json.dumps(response)) + logger.info(f"服务器操作成功: {message}") + + async def _send_error_response(self, transport: TransportInterface, session_id: str, + message: str, content: dict = None): + """发送错误响应""" + response = { + "type": "server", + "status": "fail", + "message": message, + "session_id": session_id + } + if content: + response["content"] = content + + await transport.send(json.dumps(response)) + logger.error(f"服务器操作失败: {message}") + + async def _close_transport(self, transport: TransportInterface): + try: + await transport.close() + except Exception: + pass diff --git a/main/xiaozhi-server/core/processors/text_processor.py b/main/xiaozhi-server/core/processors/text_processor.py new file mode 100644 index 00000000..d4cb1a1a --- /dev/null +++ b/main/xiaozhi-server/core/processors/text_processor.py @@ -0,0 +1,54 @@ +import json +from typing import Any +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from config.logger import setup_logging + +logger = setup_logging() + + +class TextProcessor(MessageProcessor): + """ + 纯文本消息处理器：处理非JSON格式的文本消息 + 这是新架构中缺失的重要组件，用于处理直接发送的文本聊天内容 + """ + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """处理纯文本消息""" + if isinstance(message, str): + try: + # 尝试解析为JSON，如果成功则不是纯文本消息 + json.loads(message) + return False # JSON消息由其他processor处理 + except json.JSONDecodeError: + # 确实是纯文本消息，进行聊天处理 + await self.handle_text_message(context, transport, message) + return True + return False + + async def handle_text_message(self, context: SessionContext, transport: TransportInterface, text: str): + """处理纯文本消息 - 直接调用ChatProcessor进行聊天""" + try: + # 记录收到纯文本消息 + logger.info(f"收到纯文本消息: {text[:100]}...") + + # 使用ChatProcessor处理聊天 + from core.processors.chat_processor import ChatProcessor + chat_processor = ChatProcessor() + await chat_processor.handle_chat(context, transport, text) + + except Exception as e: + logger.error(f"处理纯文本消息失败: {e}") + # 发送错误响应 + await self._send_error_response(transport, "文本处理失败，请重试") + + async def _send_error_response(self, transport: TransportInterface, error_message: str): + """发送错误响应""" + try: + await transport.send(json.dumps({ + "type": "error", + "message": error_message + })) + except Exception as e: + logger.error(f"发送错误响应失败: {e}") diff --git a/main/xiaozhi-server/core/processors/timeout_processor.py b/main/xiaozhi-server/core/processors/timeout_processor.py new file mode 100644 index 00000000..81d760e2 --- /dev/null +++ b/main/xiaozhi-server/core/processors/timeout_processor.py @@ -0,0 +1,48 @@ +from typing import Any +from core.pipeline.message_pipeline import MessageProcessor +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from config.logger import setup_logging + +logger = setup_logging() + + +class TimeoutProcessor(MessageProcessor): + """超时检查处理器：检查会话是否超时""" + + async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool: + """检查会话超时""" + return await self.handle_timeout(context, transport) + + async def handle_timeout( + self, + context: SessionContext, + transport: TransportInterface, + ) -> bool: + """End an expired logical conversation without closing a long connection.""" + if not getattr(context, "conversation_active", False): + return False + + timeout_seconds = context.config.get("close_connection_no_voice_time", 120) + if not context.is_timeout(timeout_seconds): + return False + + try: + if transport.keeps_connection_between_sessions: + logger.info(f"会话超时，结束长连接逻辑会话: {context.session_id}") + from core.processors.audio_receive_processor import ( + AudioReceiveProcessor, + ) + + await AudioReceiveProcessor()._no_voice_close_connect( + context, + transport, + have_voice=False, + ) + else: + logger.info(f"会话超时，准备关闭连接: {context.session_id}") + await transport.close() + except Exception as e: + logger.error(f"处理超时连接失败: {e}") + + return True diff --git a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py index 74bf6137..81e9c6a2 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py @@ -109,7 +109,7 @@ class ASRProvider(ASRProviderBase): self.token, expire_time_str = AccessToken.create_token(self.access_key_id, self.access_key_secret) if not self.token: raise ValueError("无法获取有效的访问Token") - + try: expire_str = str(expire_time_str).strip() if expire_str.isdigit(): @@ -151,7 +151,7 @@ class ASRProvider(ASRProviderBase): """开始识别会话""" if self._is_token_expired(): self._refresh_token() - + # 建立连接 headers = {"X-NLS-Token": self.token} self.asr_ws = await websockets.connect( @@ -169,7 +169,10 @@ class ASRProvider(ASRProviderBase): self.is_processing = True self.server_ready = False # 重置服务器准备状态 - self.forward_task = asyncio.create_task(self._forward_results(conn)) + session_id = getattr(conn, "session_id", None) + self.forward_task = self._create_session_task( + conn, self._forward_results(conn, session_id) + ) # 发送开始请求 start_request = { @@ -193,10 +196,13 @@ class ASRProvider(ASRProviderBase): await self.asr_ws.send(json.dumps(start_request, ensure_ascii=False)) logger.bind(tag=TAG).debug("已发送开始请求，等待服务器准备...") - async def _forward_results(self, conn: "ConnectionHandler"): + async def _forward_results(self, conn: "ConnectionHandler", session_id=None): """转发识别结果""" try: - while not conn.stop_event.is_set(): + while ( + not conn.stop_event.is_set() + and self._session_is_current(conn, session_id) + ): # 获取当前连接的音频数据 audio_data = conn.asr_audio try: @@ -276,7 +282,7 @@ class ASRProvider(ASRProviderBase): finally: # 清理连接的音频缓存 await self._cleanup() - conn.reset_audio_states() + self._reset_audio_if_current(conn, session_id) async def _send_stop_request(self): """发送停止识别请求（不关闭连接）""" @@ -308,6 +314,21 @@ class ASRProvider(ASRProviderBase): self.server_ready = False logger.bind(tag=TAG).debug("ASR状态已重置") + forward_task = self.forward_task + current_task = asyncio.current_task() + if ( + forward_task + and forward_task is not current_task + and not forward_task.done() + ): + forward_task.cancel() + try: + await forward_task + except asyncio.CancelledError: + pass + except Exception as e: + logger.bind(tag=TAG).warning(f"等待ASR转发任务退出失败: {e}") + # 关闭连接 if self.asr_ws: try: @@ -319,8 +340,10 @@ class ASRProvider(ASRProviderBase): finally: self.asr_ws = None - # 清理任务引用 - self.forward_task = None + # Never discard a live task reference. The current forward task reaches + # this branch from its own finally block and is already completing. + if self.forward_task is forward_task: + self.forward_task = None logger.bind(tag=TAG).debug("ASR会话清理完成") diff --git a/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py b/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py index a7c2953a..8c88335c 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py +++ b/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py @@ -103,7 +103,10 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).debug("WebSocket连接建立成功") self.server_ready = False - self.forward_task = asyncio.create_task(self._forward_results(conn)) + session_id = getattr(conn, "session_id", None) + self.forward_task = self._create_session_task( + conn, self._forward_results(conn, session_id) + ) # 发送run-task指令 run_task_msg = self._build_run_task_message() @@ -154,10 +157,13 @@ class ASRProvider(ASRProviderBase): return message - async def _forward_results(self, conn: "ConnectionHandler"): + async def _forward_results(self, conn: "ConnectionHandler", session_id=None): """转发识别结果""" try: - while not conn.stop_event.is_set(): + while ( + not conn.stop_event.is_set() + and self._session_is_current(conn, session_id) + ): # 获取当前连接的音频数据 audio_data = conn.asr_audio try: @@ -243,7 +249,7 @@ class ASRProvider(ASRProviderBase): finally: # 清理连接的音频缓存 await self._cleanup() - conn.reset_audio_states() + self._reset_audio_if_current(conn, session_id) async def _send_stop_request(self): """发送停止请求(用于手动模式停止录音)""" @@ -285,6 +291,21 @@ class ASRProvider(ASRProviderBase): self.server_ready = False logger.bind(tag=TAG).debug("ASR状态已重置") + forward_task = self.forward_task + current_task = asyncio.current_task() + if ( + forward_task + and forward_task is not current_task + and not forward_task.done() + ): + forward_task.cancel() + try: + await forward_task + except asyncio.CancelledError: + pass + except Exception as e: + logger.bind(tag=TAG).warning(f"等待ASR转发任务退出失败: {e}") + # 关闭连接 if self.asr_ws: try: @@ -301,8 +322,8 @@ class ASRProvider(ASRProviderBase): finally: self.asr_ws = None - # 清理任务引用 - self.forward_task = None + if self.forward_task is forward_task: + self.forward_task = None self.task_id = None logger.bind(tag=TAG).debug("ASR会话清理完成") @@ -315,4 +336,4 @@ class ASRProvider(ASRProviderBase): async def close(self): """关闭资源""" - await self._cleanup() \ No newline at end of file + await self._cleanup() diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index dcfaf3a7..33c3a100 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -32,8 +32,37 @@ class ASRProviderBase(ABC): def __init__(self): pass + @staticmethod + def _session_is_current(conn: "ConnectionHandler", session_id: str) -> bool: + """Return whether an async ASR result still belongs to this session.""" + return session_id is None or getattr(conn, "session_id", None) == session_id + + def _create_session_task(self, conn: "ConnectionHandler", coroutine): + """Bind streaming ASR work to the logical session when supported.""" + creator = getattr(conn, "create_background_task", None) + if callable(creator): + try: + return creator(coroutine, turn_scoped=True) + except TypeError: + return creator(coroutine) + return asyncio.create_task(coroutine) + + def _reset_audio_if_current( + self, conn: "ConnectionHandler", session_id: str + ) -> None: + if not self._session_is_current(conn, session_id): + return + reset_audio_states = getattr(conn, "reset_audio_states", None) + if callable(reset_audio_states): + reset_audio_states() + # 打开音频通道 async def open_audio_channels(self, conn: "ConnectionHandler"): + # The pipeline runtime feeds PCM directly and does not use the legacy + # priority queue. Starting that worker would route frames back through + # ConnectionHandler-only functions. + if getattr(conn, "uses_pipeline_runtime", False): + return conn.asr_priority_thread = threading.Thread( target=self.asr_text_priority_thread, args=(conn,), daemon=True ) @@ -72,18 +101,26 @@ class ASRProviderBase(ABC): return # 自动模式下通过VAD检测到语音停止时触发识别 - if conn.asr.interface_type != InterfaceType.STREAM and conn.client_voice_stop: + interface_type = getattr( + self, + "interface_type", + getattr(getattr(conn, "asr", None), "interface_type", None), + ) + if interface_type != InterfaceType.STREAM and conn.client_voice_stop: # 直接使用asr_audio中的PCM数据 pcm_bytes = b"".join(conn.asr_audio) # 检查是否有足够的音频数据（每帧1920字节，15帧约28800字节） if len(pcm_bytes) > 1920 * 15: await self.handle_voice_stop(conn, [pcm_bytes]) - conn.reset_audio_states() + reset_audio_states = getattr(conn, "reset_audio_states", None) + if callable(reset_audio_states): + reset_audio_states() # 处理语音停止 async def handle_voice_stop(self, conn: "ConnectionHandler", asr_audio_task: List[bytes]): """并行处理ASR和声纹识别""" try: + session_id = getattr(conn, "session_id", None) total_start_time = time.monotonic() # 数据已经是PCM直接使用 @@ -96,13 +133,11 @@ class ASRProviderBase(ABC): wav_data = self._pcm_to_wav(combined_pcm_data) # 定义ASR任务 - asr_task = self.speech_to_text_wrapper( - asr_audio_task, conn.session_id - ) + asr_task = self.speech_to_text_wrapper(asr_audio_task, session_id) if conn.voiceprint_provider and wav_data: voiceprint_task = conn.voiceprint_provider.identify_speaker( - wav_data, conn.session_id + wav_data, session_id ) # 并发等待两个结果 asr_result, voiceprint_result = await asyncio.gather( @@ -112,6 +147,12 @@ class ASRProviderBase(ABC): asr_result = await asr_task voiceprint_result = None + if not self._session_is_current(conn, session_id): + logger.bind(tag=TAG).info( + f"丢弃旧会话ASR结果: session_id={session_id}" + ) + return + # 记录识别结果 - 检查是否为异常 if isinstance(asr_result, Exception): logger.bind(tag=TAG).error(f"ASR识别失败: {asr_result}") @@ -165,9 +206,13 @@ class ASRProviderBase(ABC): if text_len > 0: audio_snapshot = asr_audio_task.copy() - enqueue_asr_report(conn, enhanced_text, audio_snapshot) - # 使用自定义模块进行上报 - await startToChat(conn, enhanced_text) + result_handler = getattr(conn, "asr_result_handler", None) + if callable(result_handler): + await result_handler(enhanced_text, audio_snapshot) + else: + enqueue_asr_report(conn, enhanced_text, audio_snapshot) + # Legacy ConnectionHandler path. + await startToChat(conn, enhanced_text) except Exception as e: logger.bind(tag=TAG).error(f"处理语音停止失败: {e}") import traceback diff --git a/main/xiaozhi-server/core/providers/asr/doubao_stream.py b/main/xiaozhi-server/core/providers/asr/doubao_stream.py index 130729ba..53f6a1c2 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao_stream.py +++ b/main/xiaozhi-server/core/providers/asr/doubao_stream.py @@ -117,7 +117,10 @@ class ASRProvider(ASRProviderBase): raise e # 启动接收ASR结果的异步任务 - self.forward_task = asyncio.create_task(self._forward_asr_results(conn)) + session_id = getattr(conn, "session_id", None) + self.forward_task = self._create_session_task( + conn, self._forward_asr_results(conn, session_id) + ) # 发送缓存的音频数据 if conn.asr_audio and len(conn.asr_audio) > 0: @@ -156,9 +159,13 @@ class ASRProvider(ASRProviderBase): except Exception as e: logger.bind(tag=TAG).info(f"发送音频数据时发生错误: {e}") - async def _forward_asr_results(self, conn: "ConnectionHandler"): + async def _forward_asr_results(self, conn: "ConnectionHandler", session_id=None): try: - while self.asr_ws and not conn.stop_event.is_set(): + while ( + self.asr_ws + and not conn.stop_event.is_set() + and self._session_is_current(conn, session_id) + ): # 获取当前连接的音频数据 audio_data = conn.asr_audio try: @@ -249,21 +256,47 @@ class ASRProvider(ASRProviderBase): if hasattr(e, "__cause__") and e.__cause__: logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}") finally: - if self.asr_ws: - await self.asr_ws.close() - self.asr_ws = None - self.is_processing = False - self._is_stopping = False + await self._cleanup() # 重置所有音频相关状态 - conn.reset_audio_states() + self._reset_audio_if_current(conn, session_id) def stop_ws_connection(self): - if self.asr_ws: - asyncio.create_task(self.asr_ws.close()) - self.asr_ws = None + # The forward task owns the WebSocket and closes it from _cleanup(). + # Scheduling an untracked close here races with that cleanup path. self.is_processing = False self._is_stopping = False + async def _cleanup(self): + """取消转发任务并关闭流式 ASR 连接。""" + self.is_processing = False + self._is_stopping = False + + forward_task = self.forward_task + current_task = asyncio.current_task() + if ( + forward_task + and forward_task is not current_task + and not forward_task.done() + ): + forward_task.cancel() + try: + await forward_task + except asyncio.CancelledError: + pass + except Exception as e: + logger.bind(tag=TAG).warning(f"等待ASR转发任务退出失败: {e}") + + if self.asr_ws: + try: + await asyncio.wait_for(self.asr_ws.close(), timeout=2.0) + except Exception as e: + logger.bind(tag=TAG).warning(f"关闭ASR WebSocket连接失败: {e}") + finally: + self.asr_ws = None + + if self.forward_task is forward_task: + self.forward_task = None + async def _send_stop_request(self): """发送最后一个音频帧以通知服务器结束""" self._is_stopping = True # 先标记为停止状态，阻止后续音频发送 @@ -417,14 +450,4 @@ class ASRProvider(ASRProviderBase): async def close(self): """资源清理方法""" - if self.asr_ws: - await self.asr_ws.close() - self.asr_ws = None - if self.forward_task: - self.forward_task.cancel() - try: - await self.forward_task - except asyncio.CancelledError: - pass - self.forward_task = None - self.is_processing = False + await self._cleanup() diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 5547104d..ea58ea60 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -90,8 +90,11 @@ class ASRProvider(ASRProviderBase): batch_size_s=60, ) text = lang_tag_filter(result[0]["text"]) + recognized_content = ( + text.get("content", "") if isinstance(text, dict) else text + ) logger.bind(tag=TAG).debug( - f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}" + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {recognized_content}" ) return text, artifacts.file_path diff --git a/main/xiaozhi-server/core/providers/asr/shared_asr_manager.py b/main/xiaozhi-server/core/providers/asr/shared_asr_manager.py new file mode 100644 index 00000000..37cc9ce5 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/shared_asr_manager.py @@ -0,0 +1,522 @@ +""" +SharedASRManager: 全局 ASR 管理器 +实现单例模型 + 单推理执行器 + 队列限流。 +单例的原因是：推理是 CPU/GPU-bound，不是 I/O-bound，多实例不仅会占用内存，还会降低吞吐能力 +""" + +import asyncio +import copy +import json +from concurrent.futures import ThreadPoolExecutor +from typing import Dict, Any, Optional, Tuple, List +from config.logger import setup_logging + +logger = setup_logging() +TAG = __name__ + + +class SharedASRManager: + """ + 全局共享 ASR 管理器 + """ + + # 支持预加载的本地模型类型 + LOCAL_MODEL_TYPES = [ + "fun_local", # FunASR 本地 + "sherpa_onnx_local", # Sherpa ONNX + "sense_voice" # SenseVoice + ] + + def __init__(self, config: Dict[str, Any], asr_type: str = None): + """ + 初始化 ASR 管理器 + Args: + config: 服务器配置 + asr_type: ASR 类型（Optional，用于显式指定） + """ + self.config = config + self.asr_type = asr_type + + # 模型实例（全局单例） + self.model_instance = None + + # 任务队列（限流） + queue_max_size = self._get_queue_max_size() + self.task_queue: asyncio.Queue = asyncio.Queue(maxsize=queue_max_size) + + # 推理锁（使得推理串行化） + self.inference_lock = asyncio.Lock() + # 线程池执行器，用于阻塞调用 + self.executor: Optional[ThreadPoolExecutor] = None + + # 运行状态 + self.running = False + self._inference_task: Optional[asyncio.Task] = None + self.is_local_model = self._check_local_model() + self._variant_lock = asyncio.Lock() + self._variant_managers = {} + self._max_shared_models = max( + 1, int(config.get("shared_asr_max_models", 3) or 3) + ) + + logger.bind(tag=TAG).info( + f"SharedASRManager 初始化完成, " + f"类型: {self.asr_type}, " + f"本地模型: {self.is_local_model}, " + f"队列大小: {queue_max_size}" + ) + + def _get_queue_max_size(self) -> int: + """获取队列最大大小""" + # 尝试从配置获取 + selected_asr = self.config.get("selected_module", {}).get("ASR") + if selected_asr: + asr_config = self.config.get("ASR", {}).get(selected_asr, {}) + return asr_config.get("queue_max_size", 100) + return 100 + + def _check_local_model(self) -> bool: + """检查是否为本地模型""" + if self.asr_type: + return self.asr_type in self.LOCAL_MODEL_TYPES + + # 从配置推断 + selected_asr = self.config.get("selected_module", {}).get("ASR") + if not selected_asr: + return False + + asr_config = self.config.get("ASR", {}).get(selected_asr, {}) + asr_type = asr_config.get("type", selected_asr) + self.asr_type = asr_type + + return asr_type in self.LOCAL_MODEL_TYPES + + async def initialize(self): + """ + 初始化管理器 + - 预加载模型 + - 启动推理执行器 + """ + if not self.is_local_model: + logger.bind(tag=TAG).info("非本地模型，跳过预加载") + return + + if self.running: + logger.bind(tag=TAG).warning("管理器已在运行中") + return + + try: + logger.bind(tag=TAG).info(f"开始预加载 ASR 模型: {self.asr_type}") + + # 预加载模型 + await self._preload_model() + + # 启动推理执行器 + self.running = True + self._inference_task = asyncio.create_task(self._inference_loop()) + + logger.bind(tag=TAG).info("ASR 模型预加载完成，推理执行器已启动") + + except Exception as e: + logger.bind(tag=TAG).error(f"ASR 模型预加载失败: {e}") + raise + + async def _preload_model(self): + """在线程池中预加载模型""" + # 创建线程池 + self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="asr_worker") + + loop = asyncio.get_event_loop() + self.model_instance = await loop.run_in_executor( + self.executor, + self._create_model_instance + ) + + logger.bind(tag=TAG).info("模型实例创建完成") + + def _create_model_instance(self): + """ + 实际创建模型实例（在线程中执行） + + Returns: + ASR Provider 实例 + """ + from core.utils.modules_initialize import initialize_asr + + logger.bind(tag=TAG).info("正在创建 ASR 模型实例...") + instance = initialize_asr(self.config) + logger.bind(tag=TAG).info("ASR 模型实例创建成功") + + return instance + + async def submit_task( + self, + opus_data: List[bytes], + session_id: str, + audio_format: str = "opus" + ) -> Tuple[Optional[str], Optional[str]]: + """ + 提交推理任务 + + Args: + opus_data: 音频数据 + session_id: 会话 ID + audio_format: 音频格式 + + Returns: + (识别文本, 文件路径) + + Raises: + RuntimeError: 队列满或服务未运行 + """ + if not self.running: + raise RuntimeError("ASR 服务未运行") + + # 检查队列是否满（限流） + if self.task_queue.full(): + queue_status = self.get_queue_status() + logger.bind(tag=TAG).warning( + f"ASR 队列已满: {queue_status}" + ) + raise RuntimeError("ASR 服务繁忙，请稍后重试") + + # 创建 Future 用于返回结果 + result_future: asyncio.Future = asyncio.Future() + + # 构造任务 + task = { + 'opus_data': opus_data, + 'session_id': session_id, + 'audio_format': audio_format, + 'future': result_future + } + + # 放入队列 + await self.task_queue.put(task) + + logger.bind(tag=TAG).debug( + f"任务已提交, session: {session_id}, " + f"队列大小: {self.task_queue.qsize()}" + ) + + # 等待结果 + return await result_future + + async def _inference_loop(self): + """ + 单个推理执行器循环 + + 核心原则： + - 只有一个执行器 + - 串行处理任务 + - 带超时的队列获取 + """ + logger.bind(tag=TAG).info("推理执行器启动") + + while self.running: + task = None + try: + # 带超时的队列获取，避免关闭时卡住 + try: + task = await asyncio.wait_for( + self.task_queue.get(), + timeout=1.0 + ) + except asyncio.TimeoutError: + # 超时后检查 running 状态，继续循环 + continue + + if task['future'].done(): + continue + + # 执行推理（加锁保证串行） + async with self.inference_lock: + if task['future'].done(): + continue + result = await self._run_inference( + task['opus_data'], + task['session_id'], + task['audio_format'] + ) + + # 返回结果 + if not task['future'].done(): + task['future'].set_result(result) + + logger.bind(tag=TAG).debug( + f"推理完成, session: {task['session_id']}" + ) + + except asyncio.CancelledError: + logger.bind(tag=TAG).info("推理执行器被取消") + break + except Exception as e: + logger.bind(tag=TAG).error(f"推理执行失败: {e}") + if task and 'future' in task and not task['future'].done(): + task['future'].set_exception(e) + finally: + if task is not None: + self.task_queue.task_done() + + logger.bind(tag=TAG).info("推理执行器已停止") + + async def _run_inference( + self, + opus_data: List[bytes], + session_id: str, + audio_format: str + ) -> Tuple[Optional[str], Optional[str]]: + """ + 执行实际推理（在线程池中） + + Args: + opus_data: 音频数据 + session_id: 会话 ID + audio_format: 音频格式 + + Returns: + (识别文本, 文件路径) + """ + loop = asyncio.get_event_loop() + + # 在线程池中执行推理 + result = await loop.run_in_executor( + self.executor, + lambda: self._sync_wrapper(opus_data, session_id) + ) + + return result + + def _sync_wrapper( + self, + opus_data: List[bytes], + session_id: str, + ) -> Tuple[Optional[str], Optional[str]]: + """Run the provider's complete artifact wrapper in the worker thread.""" + import asyncio + + async def _call(): + return await self.model_instance.speech_to_text_wrapper( + opus_data, session_id + ) + + # 创建新的事件循环执行 + loop = None + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + result = loop.run_until_complete(_call()) + return result + finally: + if loop: + loop.close() + + def matches_config(self, config: Dict[str, Any]) -> bool: + """Return whether this manager owns the exact selected ASR config.""" + selected = config.get("selected_module", {}).get("ASR") + manager_selected = self.config.get("selected_module", {}).get("ASR") + if not selected or selected != manager_selected: + return False + return ( + config.get("ASR", {}).get(selected) + == self.config.get("ASR", {}).get(manager_selected) + and bool(config.get("delete_audio", True)) + == bool(self.config.get("delete_audio", True)) + ) + + @classmethod + def _config_fingerprint(cls, config: Dict[str, Any]) -> str: + selected = config.get("selected_module", {}).get("ASR") + payload = { + "selected": selected, + "config": config.get("ASR", {}).get(selected), + "delete_audio": bool(config.get("delete_audio", True)), + } + return json.dumps(payload, sort_keys=True, separators=(",", ":")) + + @classmethod + def _config_is_local(cls, config: Dict[str, Any]) -> bool: + selected = config.get("selected_module", {}).get("ASR") + asr_config = config.get("ASR", {}).get(selected, {}) + asr_type = asr_config.get("type", selected) + return asr_type in cls.LOCAL_MODEL_TYPES + + async def acquire_for_config( + self, config: Dict[str, Any] + ) -> Optional["SharedASRManager"]: + """Acquire a bounded shared local model matching one Agent config.""" + if self.matches_config(config): + return self if self.is_ready() else None + if not self._config_is_local(config): + return None + + fingerprint = self._config_fingerprint(config) + async with self._variant_lock: + entry = self._variant_managers.get(fingerprint) + if entry is not None: + entry["references"] += 1 + return entry["manager"] + + if len(self._variant_managers) >= self._max_shared_models - 1: + idle_fingerprint = next( + ( + key + for key, candidate in self._variant_managers.items() + if candidate["references"] == 0 + ), + None, + ) + if idle_fingerprint is None: + raise RuntimeError( + "ASR共享模型容量已满，请提高shared_asr_max_models或统一Agent ASR配置" + ) + idle_manager = self._variant_managers.pop( + idle_fingerprint + )["manager"] + # Keep the lock until shutdown completes so a replacement can + # never overlap the retiring model and exceed the hard limit. + await idle_manager.shutdown() + + manager = SharedASRManager(copy.deepcopy(config)) + try: + await manager.initialize() + except Exception: + await manager.shutdown() + raise + self._variant_managers[fingerprint] = { + "manager": manager, + "references": 1, + } + logger.bind(tag=TAG).info( + "已加载Agent专用共享ASR模型，当前模型数: {}", + len(self._variant_managers) + 1, + ) + return manager + + async def release_for_config(self, manager: "SharedASRManager") -> None: + """Release an Agent model; idle variants stay cached for safe reuse.""" + if manager is self: + return + async with self._variant_lock: + for entry in self._variant_managers.values(): + if entry["manager"] is not manager: + continue + entry["references"] = max(0, entry["references"] - 1) + break + + async def shutdown(self): + """ + 优雅停机 + + 步骤： + 1. 停止接收新任务 + 2. 等待当前任务完成（带超时） + 3. 取消未完成的任务 + 4. 关闭线程池 + """ + async with self._variant_lock: + variants = [ + entry["manager"] for entry in self._variant_managers.values() + ] + self._variant_managers.clear() + if variants: + await asyncio.gather( + *(manager.shutdown() for manager in variants), + return_exceptions=True, + ) + + if ( + not self.running + and self.executor is None + and self.model_instance is None + ): + return + + logger.bind(tag=TAG).info("开始关闭 ASR 管理器...") + + # 停止接收新任务 + self.running = False + + # 等待推理任务完成 + if self._inference_task and not self._inference_task.done(): + try: + # 最多等待 5 秒 + await asyncio.wait_for( + self._inference_task, + timeout=5.0 + ) + except asyncio.TimeoutError: + logger.bind(tag=TAG).warning("推理任务超时，强制取消") + self._inference_task.cancel() + try: + await self._inference_task + except asyncio.CancelledError: + pass + except asyncio.CancelledError: + pass + + # 取消所有队列中未完成的任务 + cancelled_count = 0 + while not self.task_queue.empty(): + try: + task = self.task_queue.get_nowait() + if not task['future'].done(): + task['future'].set_exception( + RuntimeError("ASR 服务正在关闭") + ) + cancelled_count += 1 + except asyncio.QueueEmpty: + break + + if cancelled_count > 0: + logger.bind(tag=TAG).info(f"已取消 {cancelled_count} 个待处理任务") + + # 关闭线程池 + if self.executor: + self.executor.shutdown(wait=False) + self.executor = None + logger.bind(tag=TAG).info("线程池已关闭") + + # 清理模型实例 + self.model_instance = None + + logger.bind(tag=TAG).info("ASR 管理器已关闭") + + def get_queue_status(self) -> Dict[str, Any]: + """ + 获取队列状态（用于监控） + + Returns: + 队列状态字典 + """ + queue_size = self.task_queue.qsize() + queue_max = self.task_queue.maxsize + + return { + 'queue_size': queue_size, + 'queue_max': queue_max, + 'is_busy': queue_size > queue_max * 0.8, + 'utilization': queue_size / queue_max if queue_max > 0 else 0, + 'running': self.running + } + + def is_ready(self) -> bool: + """检查管理器是否就绪""" + return ( + self.running and + self.model_instance is not None and + self.executor is not None + ) + + @classmethod + def is_local_model_type(cls, asr_type: str) -> bool: + """ + 检查 ASR 类型是否为本地模型 + + Args: + asr_type: ASR 类型 + + Returns: + 是否为本地模型 + """ + return asr_type in cls.LOCAL_MODEL_TYPES diff --git a/main/xiaozhi-server/core/providers/asr/shared_asr_proxy.py b/main/xiaozhi-server/core/providers/asr/shared_asr_proxy.py new file mode 100644 index 00000000..83f8bd8f --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/shared_asr_proxy.py @@ -0,0 +1,119 @@ +""" +SharedASRProxy: 共享 ASR 管理器的代理类 + +功能： +- 包装 SharedASRManager +- 提供与原 ASR Provider 相同的接口 +- 处理队列满等异常情况 +""" + +from typing import List, Tuple, Optional, Dict, Any +from core.providers.asr.base import ASRProviderBase +from core.providers.asr.dto.dto import InterfaceType +from config.logger import setup_logging + +logger = setup_logging() +TAG = __name__ + + +class SharedASRProxy(ASRProviderBase): + """ + 共享 ASR 管理器的代理类 + + 该类提供与原 ASR Provider 相同的接口， + 但实际推理工作由 SharedASRManager 完成。 + """ + + def __init__(self, manager): + """ + 初始化代理 + + Args: + manager: SharedASRManager 实例 + """ + super().__init__() + self.manager = manager + + # 从共享管理器获取接口类型 + if manager.model_instance and hasattr(manager.model_instance, 'interface_type'): + self.interface_type = manager.model_instance.interface_type + else: + self.interface_type = InterfaceType.LOCAL + + logger.bind(tag=TAG).info("SharedASRProxy 初始化完成") + + async def speech_to_text( + self, + opus_data: List[bytes], + session_id: str, + audio_format: str = "opus" + ) -> Tuple[Optional[str], Optional[str]]: + """ + 语音转文本（通过共享管理器） + + Args: + opus_data: 音频数据（Opus 编码的字节列表） + session_id: 会话 ID + audio_format: 音频格式，默认 "opus" + + Returns: + Tuple[str, str]: (识别的文本, 音频文件路径) + """ + try: + # 检查管理器状态 + if not self.manager.is_ready(): + logger.bind(tag=TAG).error("ASR 管理器未就绪") + return "", None + + # 提交任务到共享管理器 + result = await self.manager.submit_task( + opus_data, + session_id, + audio_format + ) + + return result + + except RuntimeError as e: + # 队列满或服务未运行 + logger.bind(tag=TAG).warning(f"ASR 服务繁忙: {e}") + # 返回友好提示，而不是空字符串 + return "服务繁忙，请稍后重试", None + + except Exception as e: + logger.bind(tag=TAG).error(f"ASR 推理失败: {e}") + return "", None + + async def speech_to_text_wrapper( + self, pcm_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: + """Delegate PCM inference without using uninitialized proxy file state.""" + return await self.manager.submit_task(pcm_data, session_id, "pcm") + + def get_queue_status(self) -> Dict[str, Any]: + """ + 获取队列状态 + + Returns: + 队列状态字典 + """ + return self.manager.get_queue_status() + + def is_ready(self) -> bool: + """ + 检查代理是否就绪 + + Returns: + 是否就绪 + """ + return self.manager.is_ready() + + async def close(self): + """ + 关闭代理 + + 注意：不关闭共享管理器，由服务器统一管理 + """ + logger.bind(tag=TAG).debug("SharedASRProxy 关闭") + # 代理不负责关闭共享管理器 + pass diff --git a/main/xiaozhi-server/core/providers/asr/xunfei_stream.py b/main/xiaozhi-server/core/providers/asr/xunfei_stream.py index fa59ca34..c7460e0f 100644 --- a/main/xiaozhi-server/core/providers/asr/xunfei_stream.py +++ b/main/xiaozhi-server/core/providers/asr/xunfei_stream.py @@ -141,7 +141,10 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).info("ASR WebSocket连接已建立") self.server_ready = False - self.forward_task = asyncio.create_task(self._forward_results(conn)) + session_id = getattr(conn, "session_id", None) + self.forward_task = self._create_session_task( + conn, self._forward_results(conn, session_id) + ) # 发送首帧音频 if conn.asr_audio and len(conn.asr_audio) > 0: @@ -185,10 +188,13 @@ class ASRProvider(ASRProviderBase): await self.asr_ws.send(json.dumps(frame_data, ensure_ascii=False)) - async def _forward_results(self, conn: "ConnectionHandler"): + async def _forward_results(self, conn: "ConnectionHandler", session_id=None): """转发识别结果""" try: - while not conn.stop_event.is_set(): + while ( + not conn.stop_event.is_set() + and self._session_is_current(conn, session_id) + ): try: response = await asyncio.wait_for(self.asr_ws.recv(), timeout=60) result = json.loads(response) @@ -247,7 +253,7 @@ class ASRProvider(ASRProviderBase): finally: # 清理连接资源 await self._cleanup() - conn.reset_audio_states() + self._reset_audio_if_current(conn, session_id) async def handle_voice_stop( self, conn: "ConnectionHandler", asr_audio_task: List[bytes] @@ -272,9 +278,8 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}") def stop_ws_connection(self): - if self.asr_ws: - asyncio.create_task(self.asr_ws.close()) - self.asr_ws = None + # The forward task owns the WebSocket and closes it from _cleanup(). + # Scheduling an untracked close here races with that cleanup path. self.is_processing = False async def _send_stop_request(self): @@ -299,6 +304,21 @@ class ASRProvider(ASRProviderBase): self.server_ready = False logger.bind(tag=TAG).debug("ASR状态已重置") + forward_task = self.forward_task + current_task = asyncio.current_task() + if ( + forward_task + and forward_task is not current_task + and not forward_task.done() + ): + forward_task.cancel() + try: + await forward_task + except asyncio.CancelledError: + pass + except Exception as e: + logger.bind(tag=TAG).warning(f"等待ASR转发任务退出失败: {e}") + # 关闭连接 if self.asr_ws: try: @@ -310,8 +330,8 @@ class ASRProvider(ASRProviderBase): finally: self.asr_ws = None - # 清理任务引用 - self.forward_task = None + if self.forward_task is forward_task: + self.forward_task = None logger.bind(tag=TAG).debug("ASR会话清理完成") @@ -323,15 +343,4 @@ class ASRProvider(ASRProviderBase): async def close(self): """资源清理方法""" - if self.asr_ws: - await self.asr_ws.close() - self.asr_ws = None - if self.forward_task: - self.forward_task.cancel() - try: - await self.forward_task - except asyncio.CancelledError: - pass - self.forward_task = None - self.is_processing = False - + await self._cleanup() diff --git a/main/xiaozhi-server/core/providers/tools/device_iot/iot_executor.py b/main/xiaozhi-server/core/providers/tools/device_iot/iot_executor.py index 7be9e112..920edc49 100644 --- a/main/xiaozhi-server/core/providers/tools/device_iot/iot_executor.py +++ b/main/xiaozhi-server/core/providers/tools/device_iot/iot_executor.py @@ -127,7 +127,16 @@ class DeviceIoTExecutor(ToolExecutor): send_message = json.dumps( {"type": "iot", "commands": [command]} ) - await self.conn.websocket.send(send_message) + + # 使用transport接口发送消息 + if hasattr(self.conn, 'transport') and self.conn.transport: + await self.conn.transport.send(send_message) + elif hasattr(self.conn, 'websocket') and self.conn.websocket: + # 兼容旧版本 + logger.warning("未找到SessionContext的传输层接口, 回退使用旧版conn.websocket发送消息") + await self.conn.websocket.send(send_message) + else: + raise AttributeError("无法找到可用的传输层接口") return raise Exception(f"未找到设备{device_name}的方法{method_name}") diff --git a/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_client.py b/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_client.py index 75aa22b5..3a3104f7 100644 --- a/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_client.py +++ b/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_client.py @@ -17,7 +17,7 @@ class MCPClient: self.name_mapping = {} self.ready = False self.call_results = {} # To store Futures for tool call responses - self.next_id = 1 + self.next_id = 10000 self.lock = asyncio.Lock() self._cached_available_tools = None # Cache for get_available_tools @@ -91,3 +91,12 @@ class MCPClient: async with self.lock: if id in self.call_results: self.call_results.pop(id) + + async def close(self): + async with self.lock: + pending = list(self.call_results.values()) + self.call_results.clear() + self.ready = False + for future in pending: + if not future.done(): + future.set_exception(ConnectionError("MCP会话已关闭")) diff --git a/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_handler.py b/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_handler.py index 76994cab..4f3d4daa 100644 --- a/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_handler.py +++ b/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_handler.py @@ -3,11 +3,11 @@ import json import asyncio import re -from concurrent.futures import Future -from core.utils.util import get_vision_url, sanitize_tool_name +from core.utils.util import get_vision_url from core.utils.auth import AuthToken from config.logger import setup_logging from typing import TYPE_CHECKING +from .mcp_client import MCPClient if TYPE_CHECKING: from core.connection import ConnectionHandler @@ -15,108 +15,45 @@ if TYPE_CHECKING: TAG = __name__ logger = setup_logging() - -class MCPClient: - """设备端MCP客户端，用于管理MCP状态和工具""" - - def __init__(self): - self.tools = {} # sanitized_name -> tool_data - self.name_mapping = {} - self.ready = False - self.call_results = {} # To store Futures for tool call responses - self.next_id = 1 - self.lock = asyncio.Lock() - self._cached_available_tools = None # Cache for get_available_tools - - def has_tool(self, name: str) -> bool: - return name in self.tools - - def get_available_tools(self) -> list: - # Check if the cache is valid - if self._cached_available_tools is not None: - return self._cached_available_tools - - # If cache is not valid, regenerate the list - result = [] - for tool_name, tool_data in self.tools.items(): - function_def = { - "name": tool_name, - "description": tool_data["description"], - "parameters": { - "type": tool_data["inputSchema"].get("type", "object"), - "properties": tool_data["inputSchema"].get("properties", {}), - "required": tool_data["inputSchema"].get("required", []), - }, - } - result.append({"type": "function", "function": function_def}) - - self._cached_available_tools = result # Store the generated list in cache - return result - - async def is_ready(self) -> bool: - async with self.lock: - return self.ready - - async def set_ready(self, status: bool): - async with self.lock: - self.ready = status - - async def add_tool(self, tool_data: dict): - async with self.lock: - sanitized_name = sanitize_tool_name(tool_data["name"]) - self.tools[sanitized_name] = tool_data - self.name_mapping[sanitized_name] = tool_data["name"] - self._cached_available_tools = ( - None # Invalidate the cache when a tool is added - ) - - async def get_next_id(self) -> int: - async with self.lock: - current_id = self.next_id - self.next_id += 1 - return current_id - - async def register_call_result_future(self, id: int, future: Future): - async with self.lock: - self.call_results[id] = future - - async def resolve_call_result(self, id: int, result: any): - async with self.lock: - if id in self.call_results: - future = self.call_results.pop(id) - if not future.done(): - future.set_result(result) - - async def reject_call_result(self, id: int, exception: Exception): - async with self.lock: - if id in self.call_results: - future = self.call_results.pop(id) - if not future.done(): - future.set_exception(exception) - - async def cleanup_call_result(self, id: int): - async with self.lock: - if id in self.call_results: - self.call_results.pop(id) - - -async def send_mcp_message(conn: "ConnectionHandler", payload: dict): +async def send_mcp_message( + conn: "ConnectionHandler", + payload: dict, + transport=None, + *, + raise_on_error: bool = False, +): """Helper to send MCP messages, encapsulating common logic.""" - if not conn.features.get("mcp"): + features = getattr(conn, "features", {}) or {} + if not features.get("mcp"): logger.bind(tag=TAG).warning("客户端不支持MCP，无法发送MCP消息") return message = json.dumps({"type": "mcp", "payload": payload}) try: - await conn.websocket.send(message) + # 优先使用传入的transport，否则尝试从conn获取 + if transport: + await transport.send(message) + elif getattr(conn, "transport", None): + # 新架构 + await conn.transport.send(message) + elif getattr(conn, "websocket", None): + # 兼容旧版本 + await conn.websocket.send(message) + else: + raise AttributeError("无法找到可用的传输层接口") logger.bind(tag=TAG).debug(f"成功发送MCP消息: {message}") except Exception as e: logger.bind(tag=TAG).error(f"发送MCP消息失败: {e}") + if raise_on_error: + raise async def handle_mcp_message( - conn: "ConnectionHandler", mcp_client: MCPClient, payload: dict + conn: "ConnectionHandler", + mcp_client: MCPClient, + payload: dict, + transport=None, ): """处理MCP消息,包括初始化、工具列表和工具调用响应等""" logger.bind(tag=TAG).debug(f"处理MCP消息: {str(payload)[:100]}") @@ -150,7 +87,7 @@ async def handle_mcp_message( await asyncio.sleep(1) logger.bind(tag=TAG).debug("初始化完成，开始请求MCP工具列表") - await send_mcp_tools_list_request(conn) + await send_mcp_tools_list_request(conn, transport) return @@ -207,7 +144,7 @@ async def handle_mcp_message( next_cursor = result.get("nextCursor", "") if next_cursor: logger.bind(tag=TAG).debug(f"有更多工具，nextCursor: {next_cursor}") - await send_mcp_tools_list_continue_request(conn, next_cursor) + await send_mcp_tools_list_continue_request(conn, next_cursor, transport) else: await mcp_client.set_ready(True) logger.bind(tag=TAG).debug("所有工具已获取，MCP客户端准备就绪") @@ -235,7 +172,9 @@ async def handle_mcp_message( ) -async def send_mcp_initialize_message(conn: "ConnectionHandler"): +async def send_mcp_initialize_message( + conn: "ConnectionHandler", transport=None +): """发送MCP初始化消息""" vision_url = get_vision_url(conn.config) @@ -267,10 +206,12 @@ async def send_mcp_initialize_message(conn: "ConnectionHandler"): }, } logger.bind(tag=TAG).debug("发送MCP初始化消息") - await send_mcp_message(conn, payload) + await send_mcp_message(conn, payload, transport) -async def send_mcp_tools_list_request(conn: "ConnectionHandler"): +async def send_mcp_tools_list_request( + conn: "ConnectionHandler", transport=None +): """发送MCP工具列表请求""" payload = { "jsonrpc": "2.0", @@ -278,10 +219,12 @@ async def send_mcp_tools_list_request(conn: "ConnectionHandler"): "method": "tools/list", } logger.bind(tag=TAG).debug("发送MCP工具列表请求") - await send_mcp_message(conn, payload) + await send_mcp_message(conn, payload, transport) -async def send_mcp_tools_list_continue_request(conn: "ConnectionHandler", cursor: str): +async def send_mcp_tools_list_continue_request( + conn: "ConnectionHandler", cursor: str, transport=None +): """发送带有cursor的MCP工具列表请求""" payload = { "jsonrpc": "2.0", @@ -290,7 +233,7 @@ async def send_mcp_tools_list_continue_request(conn: "ConnectionHandler", cursor "params": {"cursor": cursor}, } logger.bind(tag=TAG).info(f"发送带cursor的MCP工具列表请求: {cursor}") - await send_mcp_message(conn, payload) + await send_mcp_message(conn, payload, transport) async def call_mcp_tool( @@ -299,6 +242,7 @@ async def call_mcp_tool( tool_name: str, args: str = "{}", timeout: int = 30, + return_raw: bool = False, ): """ 调用指定的工具，并等待响应 @@ -359,6 +303,7 @@ async def call_mcp_tool( raise ValueError(f"参数必须是字典类型，实际类型: {type(arguments)}") except Exception as e: + await mcp_client.cleanup_call_result(tool_call_id) if not isinstance(e, ValueError): raise ValueError(f"参数处理失败: {str(e)}") raise e @@ -372,7 +317,11 @@ async def call_mcp_tool( } logger.bind(tag=TAG).info(f"发送客户端mcp工具调用请求: {actual_name}，参数: {args}") - await send_mcp_message(conn, payload) + try: + await send_mcp_message(conn, payload, raise_on_error=True) + except Exception: + await mcp_client.cleanup_call_result(tool_call_id) + raise try: # Wait for response or timeout @@ -388,13 +337,21 @@ async def call_mcp_tool( ) raise RuntimeError(f"工具调用错误: {error_msg}") + if return_raw: + return raw_result + content = raw_result.get("content") if isinstance(content, list) and len(content) > 0: if isinstance(content[0], dict) and "text" in content[0]: # 直接返回文本内容，不进行JSON解析 return content[0]["text"] # 如果结果不是预期的格式，将其转换为字符串 + if return_raw: + return raw_result return str(raw_result) + except asyncio.CancelledError: + await mcp_client.cleanup_call_result(tool_call_id) + raise except asyncio.TimeoutError: await mcp_client.cleanup_call_result(tool_call_id) raise TimeoutError("工具调用请求超时") diff --git a/main/xiaozhi-server/core/providers/tools/mcp_endpoint/mcp_endpoint_client.py b/main/xiaozhi-server/core/providers/tools/mcp_endpoint/mcp_endpoint_client.py index 4e9e5303..8271c4f7 100644 --- a/main/xiaozhi-server/core/providers/tools/mcp_endpoint/mcp_endpoint_client.py +++ b/main/xiaozhi-server/core/providers/tools/mcp_endpoint/mcp_endpoint_client.py @@ -22,6 +22,7 @@ class MCPEndpointClient: self.lock = asyncio.Lock() self._cached_available_tools = None # Cache for get_available_tools self.websocket = None # WebSocket连接 + self.listener_task = None def has_tool(self, name: str) -> bool: return name in self.tools @@ -107,6 +108,18 @@ class MCPEndpointClient: async def close(self): """关闭WebSocket连接""" + current_task = asyncio.current_task() + if ( + self.listener_task + and self.listener_task is not current_task + and not self.listener_task.done() + ): + self.listener_task.cancel() + try: + await self.listener_task + except asyncio.CancelledError: + pass + self.listener_task = None if self.websocket: await self.websocket.close() self.websocket = None diff --git a/main/xiaozhi-server/core/providers/tools/mcp_endpoint/mcp_endpoint_handler.py b/main/xiaozhi-server/core/providers/tools/mcp_endpoint/mcp_endpoint_handler.py index 8d9921cd..ae541a07 100644 --- a/main/xiaozhi-server/core/providers/tools/mcp_endpoint/mcp_endpoint_handler.py +++ b/main/xiaozhi-server/core/providers/tools/mcp_endpoint/mcp_endpoint_handler.py @@ -16,6 +16,8 @@ async def connect_mcp_endpoint(mcp_endpoint_url: str, conn=None) -> MCPEndpointC if not mcp_endpoint_url or "你的" in mcp_endpoint_url or mcp_endpoint_url == "null": return None + websocket = None + mcp_client = None try: websocket = await websockets.connect(mcp_endpoint_url) @@ -23,7 +25,9 @@ async def connect_mcp_endpoint(mcp_endpoint_url: str, conn=None) -> MCPEndpointC mcp_client.set_websocket(websocket) # 启动消息监听器 - asyncio.create_task(_message_listener(mcp_client)) + mcp_client.listener_task = asyncio.create_task( + _message_listener(mcp_client), name="xiaozhi-mcp-endpoint-listener" + ) # 发送初始化消息 await send_mcp_endpoint_initialize(mcp_client) @@ -39,6 +43,10 @@ async def connect_mcp_endpoint(mcp_endpoint_url: str, conn=None) -> MCPEndpointC except Exception as e: logger.bind(tag=TAG).error(f"连接MCP接入点失败: {e}") + if mcp_client is not None: + await mcp_client.close() + elif websocket is not None: + await websocket.close() return None diff --git a/main/xiaozhi-server/core/providers/tools/server_mcp/mcp_client.py b/main/xiaozhi-server/core/providers/tools/server_mcp/mcp_client.py index c8c42d79..1edcdc28 100644 --- a/main/xiaozhi-server/core/providers/tools/server_mcp/mcp_client.py +++ b/main/xiaozhi-server/core/providers/tools/server_mcp/mcp_client.py @@ -72,16 +72,25 @@ class ServerMCPClient: async def cleanup(self): """清理MCP客户端资源""" - if not self._worker_task: + task = self._worker_task + if not task: return self._shutdown_evt.set() try: - await asyncio.wait_for(self._worker_task, timeout=20) - except (asyncio.TimeoutError, Exception) as e: + await asyncio.wait_for(asyncio.shield(task), timeout=20) + except asyncio.TimeoutError: + self.logger.bind(tag=TAG).warning("服务端MCP关闭超时，取消工作任务") + task.cancel() + done, _ = await asyncio.wait({task}, timeout=5) + if task not in done: + self.logger.bind(tag=TAG).error("服务端MCP工作任务取消超时") + return + except Exception as e: self.logger.bind(tag=TAG).error(f"服务端MCP客户端关闭错误: {e}") finally: - self._worker_task = None + if task.done(): + self._worker_task = None def has_tool(self, name: str) -> bool: """检查是否包含指定工具 @@ -197,7 +206,7 @@ class ServerMCPClient: if "API_ACCESS_TOKEN" in self.config: headers["Authorization"] = f"Bearer {self.config['API_ACCESS_TOKEN']}" self.logger.bind(tag=TAG).warning(f"你正在使用旧过时的配置 API_ACCESS_TOKEN ，请在.mcp_server_settings.json中将API_ACCESS_TOKEN直接设置在headers中，例如 'Authorization': 'Bearer API_ACCESS_TOKEN'") - + # 根据transport类型选择不同的客户端，默认为SSE transport_type = self.config.get("transport", "sse") diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index f9b2e02e..a2107124 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -1,4 +1,5 @@ import os +import json import re import uuid import queue @@ -16,8 +17,6 @@ from config.logger import setup_logging from core.utils import opus_encoder_utils from core.utils.tts import MarkdownCleaner, convert_percentage_to_range from core.utils.output_counter import add_device_output -from core.handle.reportHandle import enqueue_tts_report -from core.handle.sendAudioHandle import sendAudioMessage from core.utils.util import audio_bytes_to_data_stream, audio_to_data_stream from core.providers.tts.dto.dto import ( TTSMessageDTO, @@ -30,6 +29,58 @@ TAG = __name__ logger = setup_logging() +async def sendAudioMessage(conn, sentenceType, audios, text, sentence_id=None): + """兼容函数：使用新的processor发送音频消息""" + try: + # 获取transport接口 + transport = getattr(conn, 'transport', None) + if not transport: + logger.error("SessionContext中没有transport接口") + return + + # 使用AudioSendProcessor发送音频 + from core.processors.audio_send_processor import AudioSendProcessor + processor = AudioSendProcessor() + + await processor.send_audio_message( + conn, + transport, + sentenceType, + audios, + text, + sentence_id=sentence_id, + ) + + except Exception as e: + logger.error(f"发送音频消息失败: {e}") + import traceback + traceback.print_exc() + + +def enqueue_tts_report(conn, text, audio_data): + """兼容函数：使用新的processor处理TTS报告""" + try: + # 获取transport接口 + transport = getattr(conn, 'transport', None) + if not transport: + logger.error("SessionContext中没有transport接口") + return + + # 使用ReportProcessor处理报告 + from core.processors.report_processor import ReportProcessor + processor = ReportProcessor() + + # 异步执行报告 + if hasattr(conn, 'loop') and conn.loop: + # 直接调用同步方法 + processor.enqueue_tts_report(conn, text, audio_data) + else: + logger.warning("SessionContext中没有事件循环，跳过TTS报告") + + except Exception as e: + logger.error(f"TTS报告处理失败: {e}") + + class TTSProviderBase(ABC): def __init__(self, config, delete_audio_file): self.interface_type = InterfaceType.NON_STREAM @@ -189,7 +240,7 @@ class TTSProviderBase(ABC): except Exception as e: logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}") return None - + def to_tts(self, text): # 保留原始文本用于日志/显示 original_text = text @@ -312,13 +363,17 @@ class TTSProviderBase(ABC): # tts 消化线程 self.tts_priority_thread = threading.Thread( - target=self.tts_text_priority_thread, daemon=True + target=self.tts_text_priority_thread, + name=f"xiaozhi-tts-text-{id(self)}", + daemon=True, ) self.tts_priority_thread.start() # 音频播放 消化线程 self.audio_play_priority_thread = threading.Thread( - target=self._audio_play_priority_thread, daemon=True + target=self._audio_play_priority_thread, + name=f"xiaozhi-tts-audio-{id(self)}", + daemon=True, ) self.audio_play_priority_thread.start() @@ -369,6 +424,8 @@ class TTSProviderBase(ABC): while not self.conn.stop_event.is_set(): try: message = self.tts_text_queue.get(timeout=1) + if message is None: + break if self.conn.client_abort: logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") continue @@ -417,11 +474,15 @@ class TTSProviderBase(ABC): try: try: item = self.tts_audio_queue.get(timeout=0.1) + if item is None: + break if len(item) == 4: sentence_type, audio_datas, text, sentence_id = item else: sentence_type, audio_datas, text = item - sentence_id = None + sentence_id = getattr( + self, "current_sentence_id", None + ) or getattr(self.conn, "sentence_id", None) except queue.Empty: if self.conn.stop_event.is_set(): break @@ -459,7 +520,19 @@ class TTSProviderBase(ABC): sendAudioMessage(self.conn, sentence_type, audio_datas, text, sentence_id), self.conn.loop, ) - future.result() + self._pending_audio_future = future + try: + future.result(timeout=max(1, self.tts_timeout)) + except concurrent.futures.CancelledError: + break + except concurrent.futures.TimeoutError: + future.cancel() + logger.bind(tag=TAG).warning( + "TTS音频发送超时，取消当前发送任务" + ) + finally: + if getattr(self, "_pending_audio_future", None) is future: + self._pending_audio_future = None # 记录输出和报告 if self.conn.max_output_size > 0 and text: @@ -476,6 +549,22 @@ class TTSProviderBase(ABC): async def close(self): """资源清理方法""" + self.tts_stop_request = True + pending_future = getattr(self, "_pending_audio_future", None) + if pending_future and not pending_future.done(): + pending_future.cancel() + # Wake workers immediately; relying on queue timeouts leaves Provider + # threads alive after the component has released its ownership. + self.tts_text_queue.put(None) + self.tts_audio_queue.put(None) + for thread_name in ("tts_priority_thread", "audio_play_priority_thread"): + thread = getattr(self, thread_name, None) + if thread and thread.is_alive() and thread is not threading.current_thread(): + await asyncio.to_thread(thread.join, 2) + if thread.is_alive(): + logger.bind(tag=TAG).warning( + "TTS工作线程未能按时退出: {}", thread.name + ) self._sentence_text_map.clear() if hasattr(self, "ws") and self.ws: await self.ws.close() diff --git a/main/xiaozhi-server/core/servers/multi_protocol_server.py b/main/xiaozhi-server/core/servers/multi_protocol_server.py new file mode 100644 index 00000000..9720f505 --- /dev/null +++ b/main/xiaozhi-server/core/servers/multi_protocol_server.py @@ -0,0 +1,450 @@ +import asyncio +from typing import Dict, Any, List, Optional +from config.logger import setup_logging +from core.websocket_server_new import NewWebSocketServer + +logger = setup_logging() + + +class MultiProtocolServer: + """ + 协议服务器管理器 + 提供统一的启动、停止和状态监控接口 + """ + + def __init__(self, config: Dict[str, Any]): + self.config = config + self.logger = setup_logging() + + # 服务器实例 + self.servers: Dict[str, Any] = {} + self.server_tasks: Dict[str, asyncio.Task] = {} + self.monitor_task: Optional[asyncio.Task] = None + self._lifecycle_lock = asyncio.Lock() + self.management_owner = None + + # 服务器状态 + self.is_running = False + self.startup_complete = False + + # 初始化服务器 + self._initialize_servers() + + def _initialize_servers(self): + """初始化所有协议服务器""" + try: + self.servers.clear() + # 检查配置中启用的协议 + enabled_protocols = self.config.get('enabled_protocols', ['websocket']) + + # 初始化WebSocket服务器 + if 'websocket' in enabled_protocols: + self.servers['websocket'] = NewWebSocketServer(self.config) + logger.info("WebSocket服务器已初始化") + + if not self.servers: + logger.warning("没有启用任何协议服务器") + + for server in self.servers.values(): + self._bind_management_owner(server) + + except Exception as e: + logger.error(f"初始化服务器失败: {e}") + raise + + def set_management_owner(self, owner: Any) -> None: + """Route management commands through the facade that owns all protocols.""" + self.management_owner = owner + for server in self.servers.values(): + self._bind_management_owner(server) + + def _bind_management_owner(self, server: Any) -> None: + owner = self.management_owner or server + server.management_owner = owner + connection_service = getattr(server, "connection_service", None) + if connection_service is not None: + connection_service.server = owner + + async def start(self): + """启动所有服务器，并由管理器持有协议任务。""" + async with self._lifecycle_lock: + await self._start_unlocked() + + async def _start_unlocked(self): + if self.is_running: + logger.warning("服务器已经在运行中") + return + if self.server_tasks: + raise RuntimeError("上次协议停止尚未完成，请先重试 stop 清理残留资源") + + try: + logger.info("开始启动多协议服务器...") + self.is_running = True + self.startup_complete = False + + # 启动所有服务器 + for protocol, server in self.servers.items(): + try: + logger.info(f"启动{protocol}服务器...") + task = asyncio.create_task( + server.start(), name=f"xiaozhi-{protocol}-server" + ) + self.server_tasks[protocol] = task + await self._wait_for_server_started(protocol, server, task) + + logger.info(f"{protocol}服务器启动成功") + + except Exception as e: + logger.error(f"启动{protocol}服务器失败: {e}") + raise + + self.startup_complete = True + logger.info("多协议服务器启动完成") + + # 启动监控任务；必须保存引用，以便停止和重启时回收。 + self.monitor_task = asyncio.create_task( + self._monitor_servers(), name="xiaozhi-protocol-monitor" + ) + + except Exception as e: + logger.error(f"启动多协议服务器失败: {e}") + try: + await self._stop_unlocked() + except Exception as cleanup_error: + logger.error(f"启动失败后的协议清理失败: {cleanup_error}") + raise + + async def _wait_for_server_started(self, protocol, server, task): + """等待协议监听器真正就绪，而不是依赖固定 sleep。""" + started_event = getattr(server, "_started_event", None) + if not isinstance(started_event, asyncio.Event): + await asyncio.sleep(0) + if task.done(): + await task + return + + timeout = float(self.config.get("server_startup_timeout", 10)) + event_waiter = asyncio.create_task(started_event.wait()) + try: + done, _ = await asyncio.wait( + {task, event_waiter}, + timeout=timeout, + return_when=asyncio.FIRST_COMPLETED, + ) + if started_event.is_set(): + if task.done(): + await task + return + if task in done: + await task + raise RuntimeError(f"{protocol}服务器在就绪前退出") + raise TimeoutError(f"等待{protocol}服务器启动超时") + finally: + if not event_waiter.done(): + event_waiter.cancel() + try: + await event_waiter + except asyncio.CancelledError: + pass + + async def stop(self): + """停止所有服务器""" + async with self._lifecycle_lock: + await self._stop_unlocked() + + async def _stop_unlocked(self): + if not self.is_running and not self.server_tasks and not self.monitor_task: + return + + logger.info("开始停止多协议服务器...") + self.is_running = False + self.startup_complete = False + + if self.monitor_task and not self.monitor_task.done(): + self.monitor_task.cancel() + try: + await self.monitor_task + except asyncio.CancelledError: + pass + self.monitor_task = None + + # 停止所有服务器 + errors = [] + stopped_protocols = [] + for protocol, server in self.servers.items(): + try: + logger.info(f"停止{protocol}服务器...") + stopped = await server.stop() + if stopped is False: + raise RuntimeError("协议仍有取消不响应的后台任务") + logger.info(f"{protocol}服务器已停止") + stopped_protocols.append(protocol) + except Exception as e: + logger.error(f"停止{protocol}服务器失败: {e}") + errors.append((protocol, e)) + + # 只释放已确认停止的任务。失败协议保留所有权，允许再次 stop。 + for protocol in stopped_protocols: + task = self.server_tasks.get(protocol) + if task is None: + continue + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception as e: + errors.append((f"{protocol}任务", e)) + logger.error(f"回收{protocol}服务器任务失败: {e}") + continue + self.server_tasks.pop(protocol, None) + + if errors: + details = ", ".join( + f"{protocol}: {error}" for protocol, error in errors + ) + raise RuntimeError(f"停止协议服务器时发生错误: {details}") + logger.info("多协议服务器已停止") + + async def restart(self): + """重启所有服务器""" + async with self._lifecycle_lock: + logger.info("重启多协议服务器...") + await self._stop_unlocked() + await self._start_unlocked() + + async def restart_server(self, protocol: str): + """重启指定协议的服务器""" + async with self._lifecycle_lock: + return await self._restart_server_unlocked(protocol) + + async def _restart_server_unlocked(self, protocol: str): + if protocol not in self.servers: + logger.error(f"未找到协议服务器: {protocol}") + return False + + try: + logger.info(f"重启{protocol}服务器...") + + # 停止指定服务器 + server = self.servers[protocol] + stopped = await server.stop() + if stopped is False: + raise RuntimeError("协议仍有取消不响应的后台任务") + + # 取消任务 + if protocol in self.server_tasks: + task = self.server_tasks[protocol] + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + # 重新启动 + task = asyncio.create_task( + server.start(), name=f"xiaozhi-{protocol}-server" + ) + self.server_tasks[protocol] = task + await self._wait_for_server_started(protocol, server, task) + + logger.info(f"{protocol}服务器重启成功") + return True + + except Exception as e: + logger.error(f"重启{protocol}服务器失败: {e}") + return False + + async def update_config(self, new_config: Dict[str, Any]): + """更新配置""" + async with self._lifecycle_lock: + return await self._update_config_unlocked(new_config) + + async def _update_config_unlocked(self, new_config: Dict[str, Any]): + old_config = self.config + was_running = self.is_running + config_changed = False + old_runtime_stopped = False + try: + logger.info("更新多协议服务器配置...") + # 检查配置变化 + config_changed = self._check_config_changes(self.config, new_config) + + # 如果配置有重大变化，重新初始化服务器 + if config_changed: + logger.info("配置有重大变化，重新初始化服务器...") + await self._stop_unlocked() + old_runtime_stopped = True + self.config = new_config + self._initialize_servers() + if was_running: + await self._start_unlocked() + else: + # 更新各个服务器的配置 + for protocol, server in self.servers.items(): + updated = True + if hasattr(server, 'apply_config'): + updated = await server.apply_config(new_config) + elif hasattr(server, 'update_config'): + updated = await server.update_config(new_config) + if updated is False: + raise RuntimeError(f"{protocol}配置更新失败") + self._bind_management_owner(server) + self.config = new_config + + logger.info("配置更新完成") + return True + + except Exception as e: + logger.error(f"更新配置失败: {e}") + if config_changed and not old_runtime_stopped: + # A listener still owns resources. Preserve the old server + # objects/tasks so stop can be retried safely. + self.config = old_config + return False + try: + await self._stop_unlocked() + self.config = old_config + self._initialize_servers() + if was_running: + await self._start_unlocked() + except Exception: + logger.exception("回滚多协议服务器配置失败") + return False + + def _check_config_changes(self, old_config: Dict[str, Any], new_config: Dict[str, Any]) -> bool: + """检查配置是否有重大变化""" + # 检查启用的协议是否变化 + old_protocols = set(old_config.get('enabled_protocols', ['websocket'])) + new_protocols = set(new_config.get('enabled_protocols', ['websocket'])) + + if old_protocols != new_protocols: + logger.info(f"启用协议发生变化: {old_protocols} -> {new_protocols}") + return True + + if old_config.get("_shared_asr_manager") is not new_config.get( + "_shared_asr_manager" + ): + logger.info("共享ASR管理器发生变化") + return True + + # 检查服务器端口配置 + server_configs = { + 'server': ('port', 'host', 'ip'), + } + for config_key, listener_keys in server_configs.items(): + old_server_config = old_config.get(config_key, {}) + new_server_config = new_config.get(config_key, {}) + + # 检查端口和主机配置 + for key in listener_keys: + if old_server_config.get(key) != new_server_config.get(key): + logger.info(f"服务器配置{config_key}.{key}发生变化") + return True + + return False + + async def _monitor_servers(self): + """监控服务器状态""" + try: + while self.is_running: + await asyncio.sleep(30) # 每30秒检查一次 + + # 检查服务器任务状态 + for protocol, task in list(self.server_tasks.items()): + if task.done(): + exception = None if task.cancelled() else task.exception() + if exception: + logger.error(f"{protocol}服务器异常退出: {exception}") + else: + logger.warning(f"{protocol}服务器任务意外退出") + # 监控任务本身不能重入 lifecycle lock。 + async with self._lifecycle_lock: + if self.is_running: + await self._restart_server_unlocked(protocol) + + except asyncio.CancelledError: + pass + except Exception as e: + logger.error(f"服务器监控任务出错: {e}") + + def get_server_status(self) -> Dict[str, Any]: + """获取所有服务器状态""" + status = { + 'is_running': self.is_running, + 'startup_complete': self.startup_complete, + 'enabled_protocols': list(self.servers.keys()), + 'servers': {} + } + + # 获取各个服务器的状态 + for protocol, server in self.servers.items(): + try: + if hasattr(server, 'get_server_status'): + server_status = server.get_server_status() + else: + server_status = {'type': protocol, 'status': 'unknown'} + + # 添加任务状态 + task = self.server_tasks.get(protocol) + if task: + server_status['task_status'] = 'running' if not task.done() else 'stopped' + if task.done() and not task.cancelled() and task.exception(): + server_status['task_error'] = str(task.exception()) + + status['servers'][protocol] = server_status + + except Exception as e: + status['servers'][protocol] = { + 'type': protocol, + 'status': 'error', + 'error': str(e) + } + + return status + + def get_active_connections_count(self) -> Dict[str, int]: + """获取各协议的活跃连接数""" + connections = {} + + for protocol, server in self.servers.items(): + try: + if hasattr(server, 'get_active_connections_count'): + connections[protocol] = server.get_active_connections_count() + elif hasattr(server, 'connections'): + connections[protocol] = len(server.connections) + else: + connections[protocol] = 0 + except Exception as e: + logger.error(f"获取{protocol}连接数失败: {e}") + connections[protocol] = -1 + + return connections + + async def broadcast_message(self, message: Dict[str, Any], protocol: Optional[str] = None): + """向所有连接广播消息""" + try: + if protocol: + # 向指定协议广播 + if protocol in self.servers: + server = self.servers[protocol] + if hasattr(server, 'broadcast_message'): + await server.broadcast_message(message) + else: + # 向所有协议广播 + for server in self.servers.values(): + if hasattr(server, 'broadcast_message'): + await server.broadcast_message(message) + + except Exception as e: + logger.error(f"广播消息失败: {e}") + + def get_supported_protocols(self) -> List[str]: + """获取支持的协议列表""" + return ['websocket'] + + def is_protocol_enabled(self, protocol: str) -> bool: + """检查协议是否启用""" + return protocol in self.servers diff --git a/main/xiaozhi-server/core/services/audio_ingress_service.py b/main/xiaozhi-server/core/services/audio_ingress_service.py new file mode 100644 index 00000000..324d96e5 --- /dev/null +++ b/main/xiaozhi-server/core/services/audio_ingress_service.py @@ -0,0 +1,229 @@ +import time +from typing import Any, Optional + +import numpy as np +import opuslib_next + +from config.logger import setup_logging + + +logger = setup_logging() + + +class AudioIngressService: + """Decode transport audio once and apply optional server-side AEC.""" + + def __init__(self, decoder_factory=None): + self._decoder_factory = decoder_factory or opuslib_next.Decoder + + def process(self, context: Any, opus_packet: bytes, timestamp: int = 0) -> Optional[bytes]: + pcm_frame = self.decode(context, opus_packet) + if pcm_frame and timestamp > 0 and getattr(context, "client_aec", False): + return self.apply_aec(context, timestamp, pcm_frame) + return pcm_frame + + def decode(self, context: Any, opus_packet: bytes) -> Optional[bytes]: + if not opus_packet: + return None + + sample_rate = int(getattr(context, "input_sample_rate", 16000) or 16000) + channels = int(getattr(context, "input_channels", 1) or 1) + frame_duration = int( + getattr(context, "input_frame_duration", 60) or 60 + ) + decoder_config = (sample_rate, channels) + + try: + if getattr(context, "_audio_ingress_decoder_config", None) != decoder_config: + context._audio_ingress_decoder = self._decoder_factory(sample_rate, channels) + context._audio_ingress_decoder_config = decoder_config + self._register_cleanup(context) + + frame_size = max(1, sample_rate * frame_duration // 1000) + pcm_frame = context._audio_ingress_decoder.decode(bytes(opus_packet), frame_size) + return bytes(pcm_frame) + except Exception as exc: + logger.debug("Opus decode failed: {}", exc) + return None + + def apply_aec(self, context: Any, timestamp: int, pcm_frame: bytes) -> bytes: + """Apply the AEC algorithm used by the upstream gateway path.""" + try: + self._expire_aec_references(context) + audio_cache = getattr(context, "aec_audio_cache", None) + if not pcm_frame or not audio_cache: + return pcm_frame + + mic_audio = np.frombuffer(pcm_frame, dtype=np.int16).astype(np.float32) + mic_rms = np.sqrt(np.mean(mic_audio**2)) + if mic_rms < 100: + return pcm_frame + + sorted_timestamps = sorted(audio_cache.keys()) + if len(sorted_timestamps) < 2: + return pcm_frame + + sample_count = len(mic_audio) + closest_idx = min( + range(len(sorted_timestamps)), + key=lambda index: abs(sorted_timestamps[index] - timestamp), + ) + + mic_window = np.hanning(sample_count) + mic_fft = np.fft.rfft(mic_audio * mic_window) + mic_psd = np.abs(mic_fft) ** 2 + mic_log_psd = 10 * np.log10(mic_psd + 1e-8) + mic_power = np.dot(mic_log_psd, mic_log_psd) + + best_corr = -1 + best_ref_idx = closest_idx + best_ref_rms = 0.0 + for offset in range(-2, 3): + test_idx = closest_idx + offset + if test_idx < 0 or test_idx >= len(sorted_timestamps): + continue + + test_ref = np.frombuffer( + audio_cache[sorted_timestamps[test_idx]], dtype=np.int16 + ).astype(np.float32) + test_ref_rms = np.sqrt(np.mean(test_ref**2)) + if test_ref_rms < 50: + continue + + test_fft = np.fft.rfft(test_ref * np.hanning(len(test_ref))) + test_log_psd = 10 * np.log10(np.abs(test_fft) ** 2 + 1e-8) + cross_power = np.dot(mic_log_psd, test_log_psd) + reference_power = np.dot(test_log_psd, test_log_psd) + corr = abs(cross_power) / ( + np.sqrt(mic_power) * np.sqrt(reference_power) + 1e-8 + ) + if corr > best_corr: + best_corr = corr + best_ref_idx = test_idx + best_ref_rms = test_ref_rms + + best_ref = np.frombuffer( + audio_cache[sorted_timestamps[best_ref_idx]], dtype=np.int16 + ).astype(np.float32) + if best_ref_rms < 50: + return pcm_frame + + aligned_ref = best_ref[:sample_count] + if len(aligned_ref) < sample_count: + aligned_ref = np.pad(aligned_ref, (0, sample_count - len(aligned_ref))) + + mic_mag = np.abs(mic_fft) + mic_phase = np.angle(mic_fft) + ref_mag = np.abs(np.fft.rfft(aligned_ref * np.hanning(sample_count))) + scale = np.sum(mic_mag * ref_mag) / (np.dot(ref_mag, ref_mag) + 1e-8) + coefficient = max( + 0.5, min(3.0, 1.0 + scale * 3 + (best_corr - 0.97) * 30) + ) + result_mag = np.maximum( + mic_mag - ref_mag * scale * coefficient * 1.5, mic_mag * 0.1 + ) + output = np.fft.irfft(result_mag * np.exp(1j * mic_phase), sample_count) + if best_corr >= 0.97 and best_ref_rms > 500: + output *= 0.3 + + return np.clip(output, -32768, 32767).astype(np.int16).tobytes() + except Exception as exc: + logger.warning("AEC processing failed: {}", exc) + return pcm_frame + + def cache_output_reference( + self, context: Any, opus_packet: bytes, timestamp: int + ) -> None: + if not getattr(context, "client_aec", False) or timestamp <= 0: + return + + try: + sample_rate = int( + getattr( + context, + "output_sample_rate", + getattr(context, "sample_rate", 24000), + ) + or 24000 + ) + channels = int(getattr(context, "output_channels", 1) or 1) + frame_duration = int( + getattr(context, "output_frame_duration", 60) or 60 + ) + decoder_config = (sample_rate, channels) + decoder = getattr(context, "_audio_reference_decoder", None) + if ( + decoder is None + or getattr(context, "_audio_reference_decoder_config", None) + != decoder_config + ): + decoder = self._decoder_factory(sample_rate, channels) + context._audio_reference_decoder = decoder + context._audio_reference_decoder_config = decoder_config + self._register_cleanup(context) + frame_size = max(1, sample_rate * frame_duration // 1000) + pcm_data = bytes(decoder.decode(bytes(opus_packet), frame_size)) + input_rate = int(getattr(context, "input_sample_rate", 16000) or 16000) + if sample_rate != input_rate: + pcm_data = self._resample_pcm(pcm_data, sample_rate, input_rate) + self._expire_aec_references(context) + context.aec_audio_cache[timestamp] = pcm_data + context.aec_audio_cache_time[timestamp] = time.time() + except Exception as exc: + logger.debug("AEC reference decode failed: {}", exc) + + @staticmethod + def _expire_aec_references(context: Any) -> None: + cache = getattr(context, "aec_audio_cache", {}) + cache_times = getattr(context, "aec_audio_cache_time", {}) + config = getattr(context, "config", {}) or {} + max_age = max(1, int(config.get("aec_reference_max_age_seconds", 120))) + max_frames = max(2, int(config.get("aec_reference_max_frames", 256))) + now = time.time() + + expired = [ + timestamp + for timestamp, cached_at in list(cache_times.items()) + if now - cached_at > max_age + ] + overflow = max(0, len(cache) - max_frames + 1) + if overflow: + expired.extend(sorted(cache_times, key=cache_times.get)[:overflow]) + for timestamp in set(expired): + cache.pop(timestamp, None) + cache_times.pop(timestamp, None) + + def cleanup(self, context: Any) -> None: + for name in ( + "_audio_ingress_decoder", + "_audio_ingress_decoder_config", + "_audio_reference_decoder", + "_audio_reference_decoder_config", + ): + if hasattr(context, name): + delattr(context, name) + if hasattr(context, "aec_audio_cache"): + context.aec_audio_cache.clear() + if hasattr(context, "aec_audio_cache_time"): + context.aec_audio_cache_time.clear() + + def _register_cleanup(self, context: Any) -> None: + if getattr(context, "_audio_ingress_cleanup_registered", False): + return + register_cleanup = getattr(context, "register_cleanup", None) + if callable(register_cleanup): + register_cleanup(lambda: self.cleanup(context)) + context._audio_ingress_cleanup_registered = True + + @staticmethod + def _resample_pcm(pcm_data: bytes, source_rate: int, target_rate: int) -> bytes: + if not pcm_data or source_rate == target_rate: + return pcm_data + source = np.frombuffer(pcm_data, dtype=np.int16) + if len(source) < 2: + return pcm_data + target_size = max(1, round(len(source) * target_rate / source_rate)) + source_positions = np.linspace(0.0, 1.0, len(source), endpoint=False) + target_positions = np.linspace(0.0, 1.0, target_size, endpoint=False) + target = np.interp(target_positions, source_positions, source) + return np.clip(target, -32768, 32767).astype(np.int16).tobytes() diff --git a/main/xiaozhi-server/core/services/connection_service.py b/main/xiaozhi-server/core/services/connection_service.py new file mode 100644 index 00000000..4693a602 --- /dev/null +++ b/main/xiaozhi-server/core/services/connection_service.py @@ -0,0 +1,1140 @@ +import asyncio +import copy +import json +import re +import time +from typing import Dict, Any +from urllib.parse import parse_qs, urlparse + +from core.context.session_context import SessionContext +from core.transport.transport_interface import TransportInterface +from core.components.component_registry import ComponentRegistry +from core.components.component_manager import ComponentType +from core.pipeline.message_pipeline import MessagePipeline +from core.processors.message_router import MessageRouter +from config.logger import setup_logging +from config.config_loader import get_private_config_from_api +from config.manage_api_client import ( + DeviceNotFoundException, + DeviceBindException, + generate_and_save_chat_title, +) +from core.utils.util import check_vad_update, check_asr_update, filter_sensitive_info +from core.utils.dialogue import Dialogue, Message +from core.utils.voiceprint_provider import VoiceprintProvider + +logger = setup_logging() +TAG = __name__ + + +class ConnectionService: + """连接服务：统一管理连接生命周期，替代ConnectionHandler""" + + def __init__(self, config: Dict[str, Any]): + self.config = config + self.logger = setup_logging() + + # 创建统一消息路由器 + self.message_router = MessageRouter() + + # 创建消息处理管道 + self.message_pipeline = MessagePipeline() + self._setup_pipeline() + + def _setup_pipeline(self): + """设置消息处理管道""" + # 使用统一的MessageRouter替代单独的processor + self.message_pipeline.add_processor(self.message_router) + + async def handle_connection(self, transport: TransportInterface, headers: Dict[str, str]): + """处理新连接""" + # 创建会话上下文 + base_config = dict(self.config) + if "_shared_asr_manager" in base_config: + del base_config["_shared_asr_manager"] + context = SessionContext(config=base_config) + context.headers = headers + + # 设置transport接口 + context.transport = transport + + async def handle_asr_result(text, audio_snapshot): + from core.processors.audio_receive_processor import AudioReceiveProcessor + from core.processors.report_processor import ReportProcessor + + ReportProcessor().enqueue_asr_report(context, text, audio_snapshot) + await AudioReceiveProcessor().start_to_chat( + context, transport, text + ) + + context.asr_result_handler = handle_asr_result + + async def end_conversation(session_id=None): + await self._finalize_conversation_session(context, session_id) + + context.end_conversation = end_conversation + + # 传入共享 ASR 管理器 + # 这使得 ASRAdapter 可以使用预加载的模型实例 + if '_shared_asr_manager' in self.config: + context.shared_asr_manager = self.config['_shared_asr_manager'] + logger.bind(tag=TAG).debug("连接使用共享 ASR 实例") + + # 兼容性：设置websocket属性（如果transport是WebSocket） + if transport.transport_type in ("websocket", "gateway"): + context.websocket = transport.raw_connection + + # 绑定服务器实例（用于管理端下发动作） + context.server = getattr(self, "server", None) + + # 初始化 welcome_msg（关键！设备需要这些信息来确定协议配置） + # 从 config.xiaozhi 读取欢迎消息配置 + if 'xiaozhi' in self.config: + import copy + context.welcome_msg = copy.deepcopy(self.config['xiaozhi']) + context.welcome_msg['session_id'] = context.session_id + else: + # 默认欢迎消息 + context.welcome_msg = { + "type": "hello", + "version": 1, + "transport": "websocket", + "session_id": context.session_id, + "audio_params": { + "format": "opus", + "sample_rate": 16000, + "channels": 1, + "frame_duration": 60 + } + } + + output_audio = context.welcome_msg.get("audio_params", {}) + context.output_sample_rate = int(output_audio.get("sample_rate", 24000)) + context.output_channels = int(output_audio.get("channels", 1)) + context.output_frame_duration = int(output_audio.get("frame_duration", 60)) + context.sample_rate = context.output_sample_rate + context.channels = context.output_channels + context.frame_duration = context.output_frame_duration + + # 从headers或URL参数中提取设备信息 + await self._extract_device_info(context, headers) + if transport.is_protocol_authenticated: + context.is_authenticated = True + + management_owner = context.server + management_registered = False + + # 识别服务器管理连接（非设备连接） + context.is_server_action_conn = self._is_server_action_connection(context, headers) + + # 创建组件管理器（仅在需要时） + component_manager = None + if not context.is_server_action_conn: + component_manager = ComponentRegistry.create_component_manager(self.config) + context.component_manager = component_manager + + # 设置绑定检查事件 + bind_completed_event = asyncio.Event() + last_bind_prompt_time = 0 + bind_prompt_interval = 60 # 绑定提示播放间隔(秒) + + # 启动超时检查任务 + timeout_task = None + initialization_task = None + + try: + logger.bind(tag=TAG).info(f"新连接建立: {context.device_id} from {context.client_ip}") + register_context = getattr( + management_owner, "register_connection_context", None + ) + if callable(register_context): + try: + management_registered = bool( + await register_context(context, transport) + ) + except Exception as e: + logger.bind(tag=TAG).warning( + "注册连接管理上下文失败: {}", e + ) + + if not context.is_server_action_conn: + initialization = self._background_initialize( + context, component_manager, bind_completed_event + ) + if transport.keeps_connection_between_sessions: + # Native MQTT owns Hello at the protocol layer. Keep its + # readiness barrier until the business runtime can accept + # the negotiated UDP session. + await initialization + else: + # WS and Gateway clients expect the Hello response before + # slow Agent/provider initialization completes. + initialization_task = asyncio.create_task( + initialization, + name="xiaozhi-connection-initialize", + ) + # Start the owner before entering receive() so immediate + # disconnect cleanup can still observe partial resources. + await asyncio.sleep(0) + else: + context.need_bind = False + bind_completed_event.set() + + # Native MQTT may have received Hello while private configuration + # was loading. Release that handshake only after the runtime can + # consume the session boundary and subsequent audio. + await transport.mark_business_ready() + + # 启动超时检查 + timeout_task = asyncio.create_task(self._check_timeout(context, transport)) + context.timeout_task = timeout_task + + # 处理消息流 + async for message in transport.receive(): + try: + msg_json = self._parse_control_message(message) + is_hello = ( + isinstance(msg_json, dict) + and msg_json.get("type") == "hello" + ) + if initialization_task is not None: + if is_hello and not initialization_task.done(): + # Acknowledge transport negotiation first, then + # join initialization before admitting listen/audio. + await self.message_pipeline.process_message( + context, transport, message + ) + await initialization_task + initialization_task = None + + if getattr(context, "init_error", None): + await self._send_config_error_audio( + context, + transport, + context.init_error, + send_hello=False, + ) + context.init_error_notified = True + break + if context.need_bind: + await self._prompt_bind_if_needed( + context, + transport, + last_bind_prompt_time, + bind_prompt_interval, + ) + last_bind_prompt_time = time.time() + continue + + await initialization_task + initialization_task = None + + if getattr(context, "init_error", None): + # 配置错误时，允许hello/listen触发默认语音（节流） + msg_type = msg_json.get("type") if isinstance(msg_json, dict) else None + if msg_type == "hello": + # 配置错误场景：默认语音限流，避免唤醒循环 + session_id = msg_json.get("session_id") if isinstance(msg_json, dict) else None + if session_id and session_id != context.session_id: + context.session_id = session_id + now = time.time() + last_audio_ts = getattr(context, "_init_error_last_audio_ts", 0.0) + + audio_params = msg_json.get("audio_params") + if audio_params: + context.audio_format = audio_params.get("format", context.audio_format) + if not transport.has_datagram_audio: + hello_resp = { + "type": "hello", + "session_id": context.session_id, + "version": 1, + "transport": "websocket", + } + if audio_params: + hello_resp["audio_params"] = audio_params + await transport.send_json(hello_resp) + # 60秒冷却：期间只发goodbye让设备回Idle，不重复播报 + if now - last_audio_ts < 60: + session_id = getattr(transport, "session_id", None) or context.session_id + if transport.keeps_connection_between_sessions: + await transport.end_session(session_id) + else: + await self._send_config_error_audio( + context, + transport, + context.init_error, + send_hello=False, + ) + context.init_error_notified = True + context._init_error_last_audio_ts = now + # MQTT/UDP 连接保持，不主动断开 + if not transport.keeps_connection_between_sessions: + break + continue + if getattr(context, "init_error_notified", False): + continue + # 检查绑定状态 + should_process = await self._check_bind_status( + context, transport, bind_completed_event, + last_bind_prompt_time, bind_prompt_interval + ) + if not should_process: + last_bind_prompt_time = time.time() + continue + + await self.message_pipeline.process_message(context, transport, message) + except Exception as e: + logger.bind(tag=TAG).error(f"处理消息时出错: {e}") + # 继续处理其他消息，不中断连接 + + except Exception as e: + logger.bind(tag=TAG).error(f"连接处理出错: {e}") + finally: + # 清理资源 + if management_registered: + unregister_context = getattr( + management_owner, "unregister_connection_context", None + ) + if callable(unregister_context): + try: + await unregister_context(context, transport) + except Exception as e: + logger.bind(tag=TAG).warning( + "注销连接管理上下文失败: {}", e + ) + + if initialization_task and not initialization_task.done(): + initialization_task.cancel() + await asyncio.gather( + initialization_task, return_exceptions=True + ) + + if timeout_task and not timeout_task.done(): + timeout_task.cancel() + try: + await timeout_task + except asyncio.CancelledError: + pass + + # init_task 已移除（同步初始化），这里不再取消 + + await self._finalize_conversation_session(context) + + # MQTT/UDP 连接关闭前通知设备回到Idle + try: + if ( + transport.keeps_connection_between_sessions + and transport.is_connected + ): + session_id = getattr(transport, "session_id", None) or context.session_id + await transport.end_session(session_id) + except Exception as e: + logger.bind(tag=TAG).warning(f"关闭前发送goodbye失败: {e}") + + # 先停止所有生产任务和会话回调，再关闭其依赖的 Provider。 + try: + await context.run_cleanup() + except Exception as e: + logger.bind(tag=TAG).error(f"会话清理失败: {e}") + + # Clean the manager currently owned by the context. + active_component_manager = ( + getattr(context, "component_manager", None) or component_manager + ) + if active_component_manager: + try: + await active_component_manager.cleanup_all() + except Exception as e: + logger.bind(tag=TAG).error(f"组件清理失败: {e}") + + try: + await transport.close() + except Exception as e: + logger.bind(tag=TAG).error(f"关闭传输层失败: {e}") + + logger.bind(tag=TAG).info(f"连接已关闭: {context.device_id}") + + async def _extract_device_info(self, context: SessionContext, headers: Dict[str, str]): + """ + 从headers中提取设备信息，必要时从transport补齐 + """ + context.device_id = headers.get("device-id") + context.headers["client-id"] = headers.get("client-id") + context.client_ip = headers.get("x-real-ip") or headers.get("x-forwarded-for", "unknown") + + if context.client_ip and "," in context.client_ip: + context.client_ip = context.client_ip.split(",")[0].strip() + + # MQTT连接可能在connect后才有device_id/client_id，这里等待短时间补齐 + if not context.device_id and hasattr(context.transport, "device_id"): + for _ in range(30): + transport_device_id = getattr(context.transport, "device_id", None) + if transport_device_id: + context.device_id = transport_device_id + break + await asyncio.sleep(0.1) + + if not context.device_id and hasattr(context.transport, "username"): + transport_username = getattr(context.transport, "username", None) + if transport_username: + context.device_id = transport_username + + if context.device_id and not context.headers.get("device-id"): + context.headers["device-id"] = context.device_id + + if not context.headers.get("client-id") and hasattr(context.transport, "client_id"): + transport_client_id = getattr(context.transport, "client_id", None) + if transport_client_id: + context.headers["client-id"] = transport_client_id + + if ( + not context.transport.is_protocol_authenticated + and not context.headers.get("authorization") + and hasattr(context.transport, "password") + ): + transport_password = getattr(context.transport, "password", None) + if transport_password: + context.headers["authorization"] = transport_password + + def _is_server_action_connection(self, context: SessionContext, headers: Dict[str, str]) -> bool: + """判断是否为管理端下发动作的临时连接""" + if not context.read_config_from_api: + return False + device_id = headers.get("device-id") or "" + if not device_id: + return False + # 设备ID是MAC则认为是设备连接 + mac_pattern = r"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$" + if re.match(mac_pattern, device_id): + return False + # 非MAC且带Authorization头，视为管理端连接 + return bool(headers.get("authorization")) + + async def _initialize_components(self, context: SessionContext, component_manager): + """初始化必要的组件""" + try: + # 设置组件管理器到上下文 + context.component_manager = component_manager + + # 根据配置确定需要初始化的组件 + required_components = ComponentRegistry.get_required_components(context.config) + if not required_components: + logger.bind(tag=TAG).warning("未配置任何组件，跳过初始化") + return + + # 按需初始化组件 + for component_type in required_components: + component = await component_manager.get_component(component_type, context) + if component: + logger.bind(tag=TAG).info(f"组件初始化成功: {component_type.value}") + else: + raise RuntimeError(f"组件初始化失败: {component_type.value}") + + component_manager.lazy_enabled = False + + await self._initialize_voiceprint(context) + + # 对齐旧版：初始化意图类型与工具处理器 + selected_intent = context.config.get("selected_module", {}).get("Intent") + intent_cfg = context.config.get("Intent", {}) + if selected_intent and selected_intent in intent_cfg: + context.intent_type = intent_cfg[selected_intent].get("type", "nointent") + if context.intent_type in ("function_call", "intent_llm"): + context.load_function_plugin = True + try: + from core.providers.tools.unified_tool_handler import UnifiedToolHandler + context.func_handler = UnifiedToolHandler(context) + context.register_cleanup(context.func_handler.cleanup) + await context.func_handler._initialize() + logger.bind(tag=TAG).info( + f"统一工具处理器初始化完成: intent_type={context.intent_type}" + ) + except Exception as e: + logger.bind(tag=TAG).error(f"初始化工具处理器失败: {e}") + + await self._initialize_prompt(context) + context.inject_tool_call_fewshot = lambda: self._inject_tool_call_fewshot( + context + ) + self._inject_tool_call_fewshot(context) + + except Exception as e: + logger.bind(tag=TAG).error(f"组件初始化出错: {e}") + raise + + async def _initialize_voiceprint(self, context: SessionContext) -> None: + """Initialize the connection-scoped voiceprint provider after config merge.""" + voiceprint_config = context.config.get("voiceprint") or {} + context.voiceprint_provider = None + if not voiceprint_config: + logger.bind(tag=TAG).info("声纹识别功能未启用") + return + try: + provider = await asyncio.to_thread(VoiceprintProvider, voiceprint_config) + if provider.enabled: + context.voiceprint_provider = provider + logger.bind(tag=TAG).info("声纹识别功能已在连接时动态启用") + else: + logger.bind(tag=TAG).warning("声纹识别功能启用但配置不完整") + except Exception as e: + logger.bind(tag=TAG).warning(f"声纹识别初始化失败: {e}") + + @staticmethod + def _inject_tool_call_fewshot(context: SessionContext) -> None: + if context.intent_type != "function_call" or not context.func_handler: + return + if any( + getattr(message, "is_temporary", False) + for message in context.dialogue.dialogue + ): + return + tools = context.func_handler.get_functions() or [] + if not tools: + return + + tool_names = { + tool.get("function", {}).get("name") for tool in tools + } + direct_id = "fewshot_da_001" + context.dialogue.put( + Message(role="user", content="给我讲个故事吧", is_temporary=True) + ) + context.dialogue.put( + Message( + role="assistant", + tool_calls=[ + { + "id": direct_id, + "function": { + "arguments": '{"response":"好呀，你想听什么类型的呀？"}', + "name": "direct_answer", + }, + "type": "function", + "index": 0, + } + ], + is_temporary=True, + ) + ) + context.dialogue.put( + Message( + role="tool", + tool_call_id=direct_id, + content="已直接回复", + is_temporary=True, + ) + ) + + if "handle_exit_intent" in tool_names: + exit_id = "fewshot_exit_001" + context.dialogue.put( + Message(role="user", content="拜拜", is_temporary=True) + ) + context.dialogue.put( + Message( + role="assistant", + tool_calls=[ + { + "id": exit_id, + "function": { + "arguments": '{"say_goodbye":"再见，下次再聊~"}', + "name": "handle_exit_intent", + }, + "type": "function", + "index": 0, + } + ], + is_temporary=True, + ) + ) + context.dialogue.put( + Message( + role="tool", + tool_call_id=exit_id, + content="退出意图已处理", + is_temporary=True, + ) + ) + context.dialogue.put( + Message( + role="assistant", + content="再见，下次再聊~", + is_temporary=True, + ) + ) + + async def _initialize_prompt(self, context: SessionContext): + user_prompt = context.config.get("prompt") + if user_prompt is None or not context.prompt_manager: + return + + quick_prompt = context.prompt_manager.get_quick_prompt( + user_prompt, context.device_id + ) + context.prompt = quick_prompt + context.dialogue.update_system_message(quick_prompt) + + try: + await asyncio.to_thread( + context.prompt_manager.update_context_info, + context, + context.client_ip, + ) + enhanced_prompt = await asyncio.to_thread( + context.prompt_manager.build_enhanced_prompt, + user_prompt, + context.device_id, + context.client_ip, + emoji_enabled=(context.features or {}).get("emoji", True), + ) + if enhanced_prompt: + context.prompt = enhanced_prompt + context.dialogue.update_system_message(enhanced_prompt) + except Exception as e: + logger.bind(tag=TAG).warning(f"增强系统提示词失败，使用快速提示词: {e}") + + def _collect_missing_modules(self, config: Dict[str, Any]) -> list[str]: + selected = config.get("selected_module", {}) or {} + missing = [] + module_keys = { + "ASR": "语音识别", + "LLM": "大模型", + "TTS": "语音合成", + "Memory": "记忆功能", + "Intent": "意图识别", + "VAD": "语音活动检测" + } + for key, name in module_keys.items(): + module_name = selected.get(key) + if not module_name: + missing.append(name) + continue + section = config.get(key, {}) + if module_name not in section: + missing.append(name) + return missing + + async def _send_config_error_audio( + self, + context: SessionContext, + transport: TransportInterface, + error: Exception, + send_hello: bool = True, + ): + logger.bind(tag=TAG).error("配置错误，发送默认语音") + missing = self._collect_missing_modules(context.config) + if missing: + missing_text = "、".join(missing) + text = f"你好啊，很高兴认识你。在我们开始聊天之前，{missing_text}未配置完成，请前往控制台完成配置" + else: + text = f"配置加载失败，请前往控制台检查配置。" + + try: + if send_hello: + # 先发送最小 hello，避免客户端等待超时 + await transport.send_json({ + "type": "hello", + "session_id": context.session_id, + "version": 1, + "transport": "websocket" + }) + + # MQTT/UDP 场景需要等待设备先发送UDP以拿到远端地址 + if transport.has_datagram_audio: + await transport.prepare_audio_channel( + context.welcome_msg.get("audio_params", {}), 3 + ) + await transport.wait_audio_ready(timeout=3) + + from core.providers.tts.edge import TTSProvider as EdgeTTSProvider + from core.utils.util import audio_bytes_to_data_stream + + # 使用 EdgeTTS 兜底提示 + tts_cfg = { + "voice": "zh-CN-XiaoxiaoNeural", + "format": "mp3", + "output_dir": "tmp", + } + tts = EdgeTTSProvider(tts_cfg, delete_audio_file=True) + audio_bytes = await tts.text_to_speak(text, None) + if not audio_bytes: + return + + # 转成 Opus 帧 + opus_frames: list[bytes] = [] + loop = asyncio.get_running_loop() + + def _convert(): + audio_bytes_to_data_stream(audio_bytes, "mp3", True, lambda f: opus_frames.append(f)) + + await loop.run_in_executor(None, _convert) + + session_id = getattr(transport, "session_id", None) or context.session_id + # 发送 TTS 状态与音频 + await transport.send_json({ + "type": "tts", + "state": "start", + "session_id": session_id, + }) + await transport.send_json({ + "type": "tts", + "state": "sentence_start", + "text": text, + "session_id": session_id, + }) + for frame in opus_frames: + await transport.send_audio(frame) + await asyncio.sleep(0.06) + await transport.send_json({ + "type": "tts", + "state": "stop", + "session_id": session_id, + }) + # MQTT/UDP：默认语音结束后主动发送goodbye，设备回到Idle但保持MQTT连接 + if transport.keeps_connection_between_sessions: + await transport.end_session(session_id) + # MQTT/UDP 需要保持连接，避免发送完默认语音后断链 + if not transport.keeps_connection_between_sessions: + await transport.close() + except Exception as e: + logger.bind(tag=TAG).error(f"发送配置错误提示语音失败: {e}") + + async def _background_initialize( + self, + context: SessionContext, + component_manager, + bind_completed_event: asyncio.Event + ): + """在后台初始化配置和组件""" + try: + await self._initialize_private_config(context, bind_completed_event) + await self._initialize_components(context, component_manager) + except asyncio.CancelledError: + raise + except Exception as e: + logger.bind(tag=TAG).error(f"后台初始化失败: {e}") + context.init_error = e + # 即使初始化失败，也要设置绑定完成事件，避免消息一直被丢弃 + bind_completed_event.set() + + async def _initialize_private_config( + self, + context: SessionContext, + bind_completed_event: asyncio.Event + ): + """从API异步获取差异化配置""" + if not context.read_config_from_api: + context.need_bind = False + bind_completed_event.set() + return + + try: + begin_time = time.time() + + private_config = await get_private_config_from_api( + context.config, + context.device_id, + context.headers.get("client-id", context.device_id), + ) + + if private_config: + private_config["delete_audio"] = bool(context.config.get("delete_audio", True)) + logger.bind(tag=TAG).info( + f"{time.time() - begin_time:.2f}秒，获取差异化配置成功" + ) + + # 合并私有配置并重算所有派生字段，避免长连接刷新后残留旧值。 + self._apply_private_runtime_config(context, private_config) + if context.component_manager is not None: + context.component_manager._config = context.config + selected = context.config.get("selected_module", {}) + llm_cfg = context.config.get("LLM", {}) + logger.bind(tag=TAG).info( + f"私有配置模块: selected={list(selected.keys())}, LLM_selected={selected.get('LLM')}, " + f"LLM_keys={list(llm_cfg.keys())}" + ) + + context.need_bind = False + else: + context.need_bind = False + + bind_completed_event.set() + + except DeviceNotFoundException: + logger.bind(tag=TAG).warning(f"设备 {context.device_id} 未找到，需要绑定") + context.need_bind = True + bind_completed_event.set() + except DeviceBindException as e: + logger.bind(tag=TAG).warning(f"设备绑定异常: {e}") + context.need_bind = True + context.bind_code = getattr(e, 'bind_code', None) + bind_completed_event.set() + except Exception as e: + logger.bind(tag=TAG).error(f"获取差异化配置失败: {e}") + context.need_bind = True + bind_completed_event.set() + + @staticmethod + def _parse_control_message(message: Any): + if isinstance(message, dict): + return message + if isinstance(message, str): + try: + return json.loads(message) + except json.JSONDecodeError: + return None + return None + + @staticmethod + def _reset_config_derived_state(context: SessionContext) -> None: + """Reset fields derived from config before applying a replacement.""" + context.max_output_size = int( + context.config.get( + "device_max_output_size", + context.config.get("max_output_size", 0), + ) + or 0 + ) + context.chat_history_conf = int( + context.config.get("chat_history_conf", 0) or 0 + ) + context.cmd_exit = list(context.config.get("exit_commands", []) or []) + context.intent_type = "nointent" + context.load_function_plugin = False + + def _apply_private_runtime_config( + self, context: SessionContext, private_config: Dict[str, Any] + ) -> None: + """Replace common/private config and recompute connection derivatives.""" + context.config = copy.deepcopy(context.common_config) + context.private_config = copy.deepcopy(private_config) + self._reset_config_derived_state(context) + applied_private_config = copy.deepcopy(private_config) + context.config.update(applied_private_config) + self._reset_config_derived_state(context) + self._merge_private_modules(context, applied_private_config) + + def _merge_private_modules(self, context: SessionContext, private_config: Dict[str, Any]) -> None: + """合并私有模块配置（与旧服务保持一致）""" + selected = context.config.setdefault("selected_module", {}) + for module_name in ("VAD", "ASR", "TTS", "LLM", "VLLM", "Memory", "Intent"): + module_cfg = private_config.get(module_name) + module_selected = private_config.get("selected_module", {}).get(module_name) + if module_cfg is not None: + context.config[module_name] = module_cfg + if module_selected: + selected[module_name] = module_selected + + # 私有配置未包含VAD/ASR时，保留公共配置的选择与配置 + common_selected = context.common_config.get("selected_module", {}) if context.common_config else {} + for module_name in ("VAD", "ASR"): + if not selected.get(module_name) and common_selected.get(module_name): + selected[module_name] = common_selected.get(module_name) + if module_name not in context.config and context.common_config.get(module_name) is not None: + context.config[module_name] = context.common_config.get(module_name) + + # Intent插件配置处理 + intent_selected = private_config.get("selected_module", {}).get("Intent") + if intent_selected and intent_selected != "Intent_nointent": + plugins = private_config.get("plugins", {}) + if plugins: + for plugin, config_str in plugins.items(): + if isinstance(config_str, str): + try: + plugins[plugin] = json.loads(config_str) + except json.JSONDecodeError: + pass + context.config["plugins"] = plugins + if "Intent" in context.config and intent_selected in context.config["Intent"]: + context.config["Intent"][intent_selected]["functions"] = plugins.keys() + + # 更新会话级配置 + if private_config.get("device_max_output_size") is not None: + context.max_output_size = int(private_config["device_max_output_size"]) + if private_config.get("chat_history_conf") is not None: + context.chat_history_conf = int(private_config["chat_history_conf"]) + + # TTS providers consume replacement words from the selected module + # configuration, matching the legacy connection runtime. + correct_words = private_config.get("correct_words") + selected_tts = selected.get("TTS") + if ( + correct_words is not None + and selected_tts + and selected_tts in context.config.get("TTS", {}) + ): + context.config["TTS"][selected_tts]["correct_words"] = correct_words + + async def _check_bind_status( + self, + context: SessionContext, + transport: TransportInterface, + bind_completed_event: asyncio.Event, + last_bind_prompt_time: float, + bind_prompt_interval: int + ) -> bool: + """ + 检查设备绑定状态 + + Returns: + bool: True 表示可以处理消息，False 表示需要丢弃消息 + """ + # 如果还没获取到真实绑定状态，等待一下 + if not bind_completed_event.is_set(): + try: + await asyncio.wait_for(bind_completed_event.wait(), timeout=1) + except asyncio.TimeoutError: + # 超时仍未获取到真实状态，丢弃消息并提示绑定 + await self._prompt_bind_if_needed( + context, transport, last_bind_prompt_time, bind_prompt_interval + ) + return False + + # 检查是否需要绑定 + if context.need_bind: + await self._prompt_bind_if_needed( + context, transport, last_bind_prompt_time, bind_prompt_interval + ) + return False + + return True + + async def _prompt_bind_if_needed( + self, + context: SessionContext, + transport: TransportInterface, + last_prompt_time: float, + prompt_interval: int + ): + """如果需要，播放绑定提示""" + current_time = time.time() + if current_time - last_prompt_time >= prompt_interval: + try: + from core.processors.audio_receive_processor import ( + AudioReceiveProcessor, + ) + + # Binding checks run before the normal Hello pipeline. Give the + # prompt an explicit transport session and turn owner so the + # TTS worker does not classify its queued audio as stale. + transport_session_id = getattr(transport, "session_id", None) + if transport_session_id: + context.session_id = transport_session_id + context.sentence_id = ( + f"bind:{context.session_id}:{time.time_ns()}" + ) + context.abort_requested = False + + context.create_background_task( + AudioReceiveProcessor().prompt_bind_device(context, transport), + conversation_scoped=False, + ) + except Exception as e: + logger.bind(tag=TAG).error(f"播放绑定提示失败: {e}") + + async def _save_memory_async( + self, + context: SessionContext, + session_id: str = None, + dialogue_snapshot=None, + ): + """Persist title and memory before releasing session-owned resources.""" + try: + session_id = session_id or context.session_id + dialogue_snapshot = ( + list(dialogue_snapshot) + if dialogue_snapshot is not None + else list(context.dialogue.dialogue) + ) + tasks = [] + if session_id: + tasks.append(generate_and_save_chat_title(session_id)) + + memory_instance = None + if context.component_manager: + memory_component = await context.component_manager.get_component( + ComponentType.MEMORY, context + ) + memory_instance = getattr(memory_component, "memory_instance", None) + + if ( + memory_instance + and context.dialogue + and hasattr(memory_instance, "save_memory") + ): + tasks.append( + memory_instance.save_memory( + dialogue_snapshot, session_id + ) + ) + + if tasks: + results = await asyncio.gather(*tasks, return_exceptions=True) + for result in results: + if isinstance(result, Exception): + logger.bind(tag=TAG).error(f"保存会话数据失败: {result}") + + except Exception as e: + logger.bind(tag=TAG).error(f"保存会话数据失败: {e}") + + async def _finalize_conversation_session( + self, context: SessionContext, session_id: str = None + ): + session_id = session_id or context.session_id + if not session_id: + return + + if session_id != context.session_id: + if context.last_finalized_session_id != session_id: + logger.bind(tag=TAG).warning( + "忽略非当前会话的结束请求: requested={}, current={}", + session_id, + context.session_id, + ) + return + + if not hasattr(context, "_conversation_finalize_lock"): + context._conversation_finalize_lock = asyncio.Lock() + + async with context._conversation_finalize_lock: + context.accepting_input_audio = False + if context.last_finalized_session_id == session_id: + return + if not context.conversation_active and not context.dialogue.dialogue: + return + + # Publish the terminal state before the first await. MQTT control + # frames can be dispatched while persistence/cleanup is running; + # delayed start/detect must not revive this logical session. + context.conversation_active = False + context.listen_stop_pending = False + context.abort_requested = True + await context.cancel_conversation_tasks() + context.listen_stop_task = None + context.listen_start_task = None + context.listen_stop_deadline = 0.0 + + if context.component_manager and hasattr(context.component_manager, "get"): + vad_component = context.component_manager.get( + ComponentType.VAD.value + ) + vad_instance = getattr(vad_component, "vad_instance", None) + if vad_instance and hasattr(vad_instance, "release_conn_resources"): + vad_instance.release_conn_resources(context) + + dialogue_snapshot = list(context.dialogue.dialogue) + try: + persistence_timeout = max( + 0.01, + float( + context.config.get( + "session_persistence_timeout", 5 + ) + or 5 + ), + ) + except (TypeError, ValueError): + persistence_timeout = 5.0 + try: + await asyncio.wait_for( + self._save_memory_async( + context, session_id, dialogue_snapshot + ), + timeout=persistence_timeout, + ) + except asyncio.TimeoutError: + logger.bind(tag=TAG).warning( + "会话持久化超时，继续释放会话资源: session_id={}, timeout={}s", + session_id, + persistence_timeout, + ) + + tts_component = None + if context.component_manager: + tts_component = await context.component_manager.get_component( + ComponentType.TTS, context + ) + tts_instance = getattr(tts_component, "tts_instance", None) + if tts_instance: + for queue_name in ("tts_text_queue", "tts_audio_queue"): + pending_queue = getattr(tts_instance, queue_name, None) + if pending_queue: + while True: + try: + pending_queue.get_nowait() + except Exception: + break + tts_instance.tts_audio_first_sentence = True + + keeps_connection = bool( + getattr( + getattr(context, "transport", None), + "keeps_connection_between_sessions", + False, + ) + ) + mcp_client = context.mcp_client + if mcp_client and not keeps_connection: + mcp_cleanup = getattr( + context, "_mcp_cleanup_callback", None + ) + if mcp_cleanup: + context.unregister_cleanup(mcp_cleanup) + context._mcp_cleanup_callback = None + if hasattr(mcp_client, "close"): + await mcp_client.close() + + if context.func_handler: + iot_executor = getattr( + context.func_handler, "device_iot_executor", None + ) + if iot_executor: + iot_executor.iot_tools.clear() + tool_manager = getattr(context.func_handler, "tool_manager", None) + if tool_manager: + tool_manager.refresh_tools() + + context.clear_audio_buffer() + context.reset_voice_state() + context.dialogue = Dialogue() + context.current_speaker = None + context.introduced_speakers.clear() + context.system_introduced_speakers.clear() + context.iot_descriptors.clear() + context.aec_audio_cache.clear() + context.aec_audio_cache_time.clear() + context.audio_flow_control.clear() + if not keeps_connection: + context.mcp_client = None + context.sentence_id = f"ended:{session_id}:{time.time_ns()}" + if tts_instance: + tts_instance.current_sentence_id = context.sentence_id + context.just_woken_up = False + context.is_speaking = False + context.close_after_chat = False + context.llm_finish_task = True + context.abort_requested = False + context.accepting_input_audio = False + context.last_finalized_session_id = session_id + logger.bind(tag=TAG).info(f"会话资源已结束: {session_id}") + + async def _check_timeout(self, context: SessionContext, transport: TransportInterface): + """定期检查连接超时""" + timeout_seconds = context.config.get("close_connection_no_voice_time", 120) + check_interval = max(1, min(30, timeout_seconds // 4)) + + try: + while transport.is_connected: + await asyncio.sleep(check_interval) + + timed_out = await self.message_router.timeout_processor.handle_timeout( + context, + transport, + ) + if timed_out and not transport.keeps_connection_between_sessions: + break + + except asyncio.CancelledError: + pass + except Exception as e: + logger.bind(tag=TAG).error(f"超时检查出错: {e}") diff --git a/main/xiaozhi-server/core/transport/transport_interface.py b/main/xiaozhi-server/core/transport/transport_interface.py new file mode 100644 index 00000000..c1bcd7c4 --- /dev/null +++ b/main/xiaozhi-server/core/transport/transport_interface.py @@ -0,0 +1,90 @@ +from abc import ABC, abstractmethod +import json +from typing import Any, AsyncGenerator + + +class TransportInterface(ABC): + """ + 传输层抽象接口。 + """ + + @abstractmethod + async def send(self, data: Any) -> None: + """发送一条消息。""" + raise NotImplementedError + + async def send_json(self, message: Any) -> None: + """Send a control message without exposing transport framing to callers.""" + payload = message if isinstance(message, str) else json.dumps(message) + await self.send(payload) + + async def send_audio(self, audio: bytes, timestamp: int = 0) -> None: + """Send one encoded audio frame over the transport's audio channel.""" + await self.send(audio) + + async def prepare_audio_channel(self, audio_params=None, version: int = 3) -> None: + """Prepare transport-specific audio negotiation when required.""" + + async def wait_audio_ready(self, timeout: float = 0) -> bool: + """Wait until encoded audio can be delivered to the peer.""" + return True + + async def mark_business_ready(self) -> None: + """Allow a transport handshake to proceed after runtime initialization.""" + + async def mark_session_ready(self, session_id: str = None) -> None: + """Release a logical-session handshake after its runtime is ready.""" + + async def end_session(self, session_id: str) -> None: + """Tell the device to return to Idle without closing the connection.""" + await self.send_json({"type": "goodbye", "session_id": session_id}) + + @abstractmethod + async def receive(self) -> AsyncGenerator[Any, None]: + """异步消息流。""" + yield # pragma: no cover + + @abstractmethod + async def close(self) -> None: + """关闭底层连接。""" + raise NotImplementedError + + @property + @abstractmethod + def is_connected(self) -> bool: + """连接是否存活。""" + raise NotImplementedError + + @property + def transport_type(self) -> str: + """Stable transport identifier used by shared connection logic.""" + return "unknown" + + @property + def has_datagram_audio(self) -> bool: + """Whether audio is carried by a channel separate from control messages.""" + return False + + @property + def requires_audio_tail_grace(self) -> bool: + """Whether control can overtake the last audio frames of a turn.""" + return False + + @property + def keeps_connection_between_sessions(self) -> bool: + """Whether ending a conversation should leave the transport connected.""" + return False + + @property + def is_protocol_authenticated(self) -> bool: + """Whether the transport already authenticated the peer during handshake.""" + return False + + @property + def raw_connection(self): + """Underlying connection for temporary compatibility with legacy code.""" + return None + + @property + def session_id(self): + return None diff --git a/main/xiaozhi-server/core/transport/websocket_transport.py b/main/xiaozhi-server/core/transport/websocket_transport.py new file mode 100644 index 00000000..adedd17a --- /dev/null +++ b/main/xiaozhi-server/core/transport/websocket_transport.py @@ -0,0 +1,154 @@ +import struct +import time +from typing import Any, AsyncGenerator +from .transport_interface import TransportInterface + + +class WebSocketTransport(TransportInterface): + """ + WebSocket 传输实现：包装 websockets 库的协议对象， + 提供统一的 send/receive/close 接口。 + """ + + def __init__( + self, + websocket, + from_mqtt_gateway: bool = False, + protocol_version: int = 1, + ): + self._ws = websocket + self._from_mqtt_gateway = from_mqtt_gateway + self._protocol_version = protocol_version if protocol_version in (1, 2, 3) else 1 + self._gateway_sequence = 0 + + async def send(self, data: Any) -> None: + if isinstance(data, bytes): + await self.send_audio(data) + return + + if isinstance(data, (str, bytes)): + await self._ws.send(data) + else: + await self._ws.send(str(data)) + + async def send_audio(self, audio: bytes, timestamp: int = 0) -> None: + if self._from_mqtt_gateway: + await self._ws.send(self._frame_gateway_audio(audio, timestamp)) + return + await self._ws.send(self._frame_websocket_audio(audio, timestamp)) + + def _frame_websocket_audio(self, audio: bytes, timestamp: int = 0) -> bytes: + if self._protocol_version == 2: + return struct.pack("!HHIII", 2, 0, 0, timestamp, len(audio)) + audio + if self._protocol_version == 3: + return struct.pack("!BBH", 0, 0, len(audio)) + audio + return audio + + def _parse_websocket_audio(self, message: bytes): + if self._protocol_version == 2: + if len(message) < 16: + return None + version, message_type, _, timestamp, audio_length = struct.unpack( + "!HHIII", message[:16] + ) + if ( + version != 2 + or message_type != 0 + or audio_length <= 0 + or len(message) != 16 + audio_length + ): + return None + return { + "type": "audio", + "data": message[16:], + "timestamp": timestamp, + "_transport_type": "websocket", + } + if self._protocol_version == 3: + if len(message) < 4: + return None + message_type, _, audio_length = struct.unpack("!BBH", message[:4]) + if ( + message_type != 0 + or audio_length <= 0 + or len(message) != 4 + audio_length + ): + return None + return { + "type": "audio", + "data": message[4:], + "timestamp": 0, + "_transport_type": "websocket", + } + return message + + def _frame_gateway_audio(self, audio: bytes, timestamp: int = 0) -> bytes: + self._gateway_sequence += 1 + if timestamp <= 0: + timestamp = int(time.time() * 1000) % (2 ** 32) + header = bytearray(16) + header[0] = 1 + header[2:4] = len(audio).to_bytes(2, "big") + header[4:8] = self._gateway_sequence.to_bytes(4, "big") + header[8:12] = timestamp.to_bytes(4, "big") + header[12:16] = len(audio).to_bytes(4, "big") + return bytes(header) + audio + + async def receive(self) -> AsyncGenerator[Any, None]: + async for message in self._ws: + if self._from_mqtt_gateway and isinstance(message, bytes): + if len(message) < 16: + continue + if message[:8] != b"\x00" * 8: + continue + timestamp = int.from_bytes(message[8:12], "big") + audio_length = int.from_bytes(message[12:16], "big") + if audio_length <= 0 or len(message) != 16 + audio_length: + continue + message = { + "type": "audio", + "data": message[16:], + "timestamp": timestamp, + "_transport_type": "gateway", + } + elif isinstance(message, bytes): + message = self._parse_websocket_audio(message) + if message is None: + continue + yield message + + async def close(self) -> None: + try: + if hasattr(self._ws, "closed") and not self._ws.closed: + await self._ws.close() + elif hasattr(self._ws, "state") and self._ws.state.name != "CLOSED": + await self._ws.close() + else: + await self._ws.close() + except Exception: + raise RuntimeError("WebSocket close failed") + + @property + def is_connected(self) -> bool: + try: + if hasattr(self._ws, "closed"): + return not self._ws.closed + if hasattr(self._ws, "state"): + return getattr(self._ws.state, "name", "CLOSED") != "CLOSED" + except Exception: + raise RuntimeError("WebSocket connection check failed") + return False + + @property + def transport_type(self) -> str: + return "gateway" if self._from_mqtt_gateway else "websocket" + + @property + def requires_audio_tail_grace(self) -> bool: + # The gateway receives MQTT control and UDP audio independently before + # serializing both streams onto this WebSocket. + return self._from_mqtt_gateway + + @property + def raw_connection(self): + return self._ws diff --git a/main/xiaozhi-server/core/utils/config_validation.py b/main/xiaozhi-server/core/utils/config_validation.py new file mode 100644 index 00000000..2a25334f --- /dev/null +++ b/main/xiaozhi-server/core/utils/config_validation.py @@ -0,0 +1,106 @@ +import asyncio +from typing import Any, Dict, Tuple + +from config.logger import setup_logging +from core.utils.modules_initialize import initialize_modules +from core.providers.asr.shared_asr_manager import SharedASRManager + + +def _validate_selected_modules(config: Dict[str, Any]) -> Tuple[bool, str]: + selected = config.get("selected_module", {}) + required_sections = { + "VAD": "VAD", + "ASR": "ASR", + "LLM": "LLM", + "TTS": "TTS", + "Memory": "Memory", + "Intent": "Intent", + } + for module_key, section_key in required_sections.items(): + module_name = selected.get(module_key) + if not module_name: + continue + section = config.get(section_key, {}) + if module_name not in section: + return False, f"{section_key}配置缺失: {module_name}" + return True, "" + + +async def _cleanup_instance(instance: Any) -> None: + if instance is None: + return + try: + if hasattr(instance, "close"): + result = instance.close() + if asyncio.iscoroutine(result): + await result + if hasattr(instance, "cleanup"): + result = instance.cleanup() + if asyncio.iscoroutine(result): + await result + if hasattr(instance, "cleanup_audio_files"): + instance.cleanup_audio_files() + except Exception: + # 校验阶段的清理异常不影响结果 + pass + + +async def _cleanup_modules(modules: Dict[str, Any]) -> None: + for instance in modules.values(): + await _cleanup_instance(instance) + + +async def validate_config_components( + config: Dict[str, Any], logger=None +) -> Tuple[bool, str]: + """ + 预初始化选中组件用于校验配置，捕获配置/密钥类错误。 + 返回 (是否通过, 错误信息)。 + """ + logger = logger or setup_logging() + ok, msg = _validate_selected_modules(config) + if not ok: + return False, msg + + selected = config.get("selected_module", {}) + init_vad = bool(selected.get("VAD")) + init_asr = bool(selected.get("ASR")) + asr_type = None + selected_asr = selected.get("ASR") + if selected_asr: + asr_config = config.get("ASR", {}).get(selected_asr, {}) + asr_type = asr_config.get("type", selected_asr) + if SharedASRManager.is_local_model_type(asr_type): + init_asr = False + init_llm = bool(selected.get("LLM")) + init_tts = bool(selected.get("TTS")) + init_memory = bool(selected.get("Memory")) + init_intent = bool(selected.get("Intent")) + + modules: Dict[str, Any] = {} + manager = None + try: + modules = initialize_modules( + logger, + config, + init_vad, + init_asr, + init_llm, + init_tts, + init_memory, + init_intent, + ) + if selected_asr and asr_type and SharedASRManager.is_local_model_type(asr_type): + manager = SharedASRManager(config, asr_type) + await manager.initialize() + return True, "" + except Exception as e: + logger.error(f"配置校验失败: {e}") + return False, str(e) + finally: + if manager: + try: + await manager.shutdown() + except Exception: + pass + await _cleanup_modules(modules) diff --git a/main/xiaozhi-server/core/utils/restart.py b/main/xiaozhi-server/core/utils/restart.py new file mode 100644 index 00000000..063b6bf2 --- /dev/null +++ b/main/xiaozhi-server/core/utils/restart.py @@ -0,0 +1,18 @@ +import os +import subprocess +import sys +import time + + +def restart_server(logger): + """实际执行重启的方法""" + time.sleep(1) + logger.info("执行服务器重启...") + subprocess.Popen( + [sys.executable, "app.py"], + stdin=sys.stdin, + stdout=sys.stdout, + stderr=sys.stderr, + start_new_session=True, + ) + os._exit(0) \ No newline at end of file diff --git a/main/xiaozhi-server/core/utils/textUtils.py b/main/xiaozhi-server/core/utils/textUtils.py index 11dee51f..cb0f84a0 100644 --- a/main/xiaozhi-server/core/utils/textUtils.py +++ b/main/xiaozhi-server/core/utils/textUtils.py @@ -1,5 +1,6 @@ import json from typing import TYPE_CHECKING +from config.logger import setup_logging if TYPE_CHECKING: from core.connection import ConnectionHandler @@ -91,18 +92,27 @@ async def get_emotion(conn: "ConnectionHandler", text): emotion = EMOJI_MAP[char] break try: - await conn.websocket.send( - json.dumps( - { - "type": "llm", - "text": emoji, - "emotion": emotion, - "session_id": conn.session_id, - } - ) + message = json.dumps( + { + "type": "llm", + "text": emoji, + "emotion": emotion, + "session_id": conn.session_id, + } ) + + # 使用transport接口发送消息 + if hasattr(conn, 'transport') and conn.transport: + await conn.transport.send(message) + elif hasattr(conn, 'websocket') and conn.websocket: + # 兼容旧版本 + await conn.websocket.send(message) + else: + raise AttributeError("无法找到可用的传输层接口") + except Exception as e: - conn.logger.bind(tag=TAG).warning(f"发送情绪表情失败，错误:{e}") + logger = setup_logging() + logger.warning(f"发送情绪表情失败，错误:{e}") return diff --git a/main/xiaozhi-server/core/websocket_server_new.py b/main/xiaozhi-server/core/websocket_server_new.py new file mode 100644 index 00000000..71142d13 --- /dev/null +++ b/main/xiaozhi-server/core/websocket_server_new.py @@ -0,0 +1,452 @@ +import asyncio +import logging +import json +import websockets +from typing import Dict, Any, Optional +from config.logger import setup_logging +from core.services.connection_service import ConnectionService +from core.transport.websocket_transport import WebSocketTransport +from config.config_loader import get_config_from_api_async +from core.utils.util import check_vad_update, check_asr_update +from core.auth import AuthMiddleware, AuthenticationError +from core.utils.config_validation import validate_config_components +from core.providers.asr.shared_asr_manager import SharedASRManager + + +class SuppressInvalidHandshakeFilter(logging.Filter): + """过滤掉无效握手错误日志（如HTTPS访问WS端口）""" + + def filter(self, record): + msg = record.getMessage() + suppress_keywords = [ + "opening handshake failed", + "did not receive a valid HTTP request", + "connection closed while reading HTTP request", + "line without CRLF", + ] + return not any(keyword in msg for keyword in suppress_keywords) + + +def _setup_websockets_logger(): + """配置 websockets 相关的所有 logger，过滤无效握手错误""" + filter_instance = SuppressInvalidHandshakeFilter() + for logger_name in ["websockets", "websockets.server", "websockets.client"]: + ws_logger = logging.getLogger(logger_name) + ws_logger.addFilter(filter_instance) + + +_setup_websockets_logger() + + +logger = setup_logging() +TAG = __name__ + + +class NewWebSocketServer: + """ + 新的WebSocket服务器：使用新架构替代旧的ConnectionHandler + 集成ConnectionService、MessageRouter和新的Processor架构 + """ + + def __init__(self, config: Dict[str, Any]): + self.config = config + self.logger = setup_logging() + self.config_lock = asyncio.Lock() + self.last_update_error = None + + # 创建连接服务 + self.connection_service = ConnectionService(config) + self.connection_service.server = self + + # 活跃连接管理 + self.active_connections = set() + + # 认证中间件 + self.auth_middleware = AuthMiddleware(config) + + # 服务器实例和控制 + self._server = None + self._stop_event = asyncio.Event() + self._started_event = asyncio.Event() + self._is_running = False + + async def start(self): + """启动WebSocket服务器""" + server_config = self.config["server"] + host = server_config.get("ip", "0.0.0.0") + port = int(server_config.get("port", 8000)) + + logger.bind(tag=TAG).info(f"启动新架构WebSocket服务器: {host}:{port}") + + self._stop_event.clear() + self._started_event.clear() + + try: + self._server = await websockets.serve( + self._handle_connection, + host, + port, + process_request=self._http_response + ) + self._is_running = True + self._started_event.set() + logger.bind(tag=TAG).info("WebSocket服务器启动成功") + + # 等待停止信号 + await self._stop_event.wait() + + except Exception as e: + logger.bind(tag=TAG).error(f"WebSocket服务器启动失败: {e}") + raise + finally: + self._is_running = False + self._started_event.clear() + + async def stop(self): + """停止WebSocket服务器""" + if not self._is_running: + logger.bind(tag=TAG).debug("WebSocket服务器未运行，无需停止") + return + + logger.bind(tag=TAG).info("正在停止WebSocket服务器...") + + # 关闭所有活跃连接 + for transport in list(self.active_connections): + try: + await transport.close() + except Exception as e: + logger.bind(tag=TAG).error(f"关闭连接失败: {e}") + self.active_connections.clear() + + # 关闭服务器 + if self._server: + self._server.close() + try: + await asyncio.wait_for(self._server.wait_closed(), timeout=5.0) + except asyncio.TimeoutError: + logger.bind(tag=TAG).warning("等待服务器关闭超时") + self._server = None + + # 发送停止信号 + self._stop_event.set() + self._is_running = False + self._started_event.clear() + + logger.bind(tag=TAG).info("WebSocket服务器已停止") + + async def _handle_connection(self, websocket): + """处理新连接 - 使用新架构""" + # 提取连接头信息 + headers = self._extract_headers(websocket) + device_id = headers.get('device-id') + + # 如果没有 device-id，提示并关闭连接 + if not device_id: + await websocket.send("端口正常，如需测试连接，请使用test_page.html") + await websocket.close() + return + + # 连接时认证 + try: + await self._handle_auth(headers) + except AuthenticationError as e: + logger.bind(tag=TAG).warning(f"认证失败: {e}") + await websocket.send("认证失败") + await websocket.close() + return + + # 创建WebSocket传输层 + try: + protocol_version = int(headers.get('protocol-version', 1) or 1) + except (TypeError, ValueError): + protocol_version = 1 + transport = WebSocketTransport( + websocket, + from_mqtt_gateway=headers.get('from_mqtt_gateway') == 'true', + protocol_version=protocol_version, + ) + + # 记录活跃连接 + self.active_connections.add(transport) + + try: + logger.bind(tag=TAG).info( + f"新连接建立: {device_id} from {headers.get('x-real-ip', 'unknown')}" + ) + + # 使用ConnectionService处理连接 + await self.connection_service.handle_connection(transport, headers) + + except websockets.exceptions.ConnectionClosed: + logger.bind(tag=TAG).info("WebSocket连接正常关闭") + except Exception as e: + logger.bind(tag=TAG).error(f"处理WebSocket连接时出错: {e}", exc_info=True) + # 将错误反馈给管理端，避免长时间等待 + try: + if hasattr(websocket, "closed") and not websocket.closed: + await websocket.send( + json.dumps( + { + "type": "server", + "status": "error", + "message": f"Server error: {e}", + "content": {"action": "unknown"}, + } + ) + ) + except Exception: + pass + finally: + # 确保从活动连接集合中移除 + self.active_connections.discard(transport) + + # 强制关闭连接（如果还没有关闭的话） + try: + if hasattr(websocket, "closed") and not websocket.closed: + await websocket.close() + elif hasattr(websocket, "state") and websocket.state.name != "CLOSED": + await websocket.close() + except Exception as close_error: + logger.bind(tag=TAG).error(f"强制关闭WebSocket连接时出错: {close_error}") + + async def _handle_auth(self, headers: Dict[str, str]): + """ + 连接时认证 + + Args: + headers: HTTP 请求头 + + Raises: + AuthenticationError: 认证失败时抛出 + """ + await self.auth_middleware.authenticate_async(headers) + + def _extract_headers(self, websocket) -> Dict[str, str]: + """ + 从WebSocket请求中提取头信息 + + 支持从以下来源提取信息： + 1. HTTP 请求头 + 2. URL 查询参数（device-id, client-id, authorization） + 3. 路径参数（如 ?from=mqtt_gateway） + """ + headers = {} + + # 1. 提取 HTTP 请求头 + if hasattr(websocket, 'request') and hasattr(websocket.request, 'headers'): + for name, value in websocket.request.headers.items(): + headers[name.lower()] = value + elif hasattr(websocket, 'request_headers'): + for name, value in websocket.request_headers.items(): + headers[name.lower()] = value + + # 2. 提取路径参数（如果有的话） + request_path = None + if hasattr(websocket, 'request') and hasattr(websocket.request, 'path'): + request_path = websocket.request.path + elif hasattr(websocket, 'path'): + request_path = websocket.path + + if request_path: + from urllib.parse import urlparse, parse_qs + parsed = urlparse(request_path) + query_params = parse_qs(parsed.query) + + # 处理关键参数：device-id, client-id, authorization + key_params = ['device-id', 'client-id', 'authorization'] + for key in key_params: + if key in query_params and query_params[key]: + # URL 参数优先级低于 header + if key not in headers or not headers[key]: + headers[key] = query_params[key][0] + + # 处理其他参数 + for key, values in query_params.items(): + if values and key not in headers: + headers[key] = values[0] + + # 检查是否来自 MQTT 网关 + if request_path.endswith("?from=mqtt_gateway") or "from=mqtt_gateway" in request_path: + headers['from_mqtt_gateway'] = 'true' + + # 3. 提取远程地址 + if hasattr(websocket, 'remote_address'): + # 如果 headers 中没有 x-real-ip，使用 remote_address + if 'x-real-ip' not in headers: + headers['x-real-ip'] = websocket.remote_address[0] + + return headers + + async def _http_response(self, websocket, request_headers): + """处理HTTP请求""" + # 检查是否为 WebSocket 升级请求 + if request_headers.headers.get("connection", "").lower() == "upgrade": + # 如果是 WebSocket 请求，返回 None 允许握手继续 + return None + else: + # 如果是普通 HTTP 请求，返回服务器状态 + return websocket.respond(200, "New Architecture WebSocket Server is running\n") + + async def apply_config(self, new_config: Dict[str, Any]) -> bool: + """Apply a facade-validated config to future WebSocket connections.""" + self.config = new_config + self.connection_service = ConnectionService(new_config) + self.connection_service.server = getattr(self, "management_owner", self) + self.auth_middleware = AuthMiddleware(new_config) + return True + + async def update_config( + self, new_config: Optional[Dict[str, Any]] = None + ) -> bool: + """ + 更新服务器配置并重新初始化组件 + + Returns: + bool: 更新是否成功 + """ + try: + async with self.config_lock: + logger.bind(tag=TAG).info("开始更新服务器配置") + self.last_update_error = None + old_config = self.config + old_connection_service = self.connection_service + old_auth_middleware = self.auth_middleware + + # 管理命令可自行拉取配置；多协议管理器则直接传入已合并配置。 + if new_config is None: + new_config = await get_config_from_api_async(self.config) + if new_config is None: + logger.bind(tag=TAG).error("获取新配置失败") + self.last_update_error = "获取新配置失败" + return False + + logger.bind(tag=TAG).info("获取新配置成功") + + new_shared_manager = None + reuse_manager = False + + # 校验新配置（预初始化组件以发现配置错误） + ok, error_msg = await validate_config_components(new_config, logger) + if not ok: + logger.bind(tag=TAG).error(f"配置校验失败: {error_msg}") + self.last_update_error = f"配置校验失败: {error_msg}" + return False + + # 准备共享 ASR 管理器（本地模型走共享预加载） + old_shared_manager = old_config.get("_shared_asr_manager") + selected_asr = new_config.get("selected_module", {}).get("ASR") + if selected_asr: + asr_config = new_config.get("ASR", {}).get(selected_asr, {}) + asr_type = asr_config.get("type", selected_asr) + if SharedASRManager.is_local_model_type(asr_type): + if ( + old_shared_manager + and getattr(old_shared_manager, "asr_type", None) == asr_type + and old_shared_manager.is_ready() + ): + new_shared_manager = old_shared_manager + reuse_manager = True + else: + new_shared_manager = SharedASRManager(new_config, asr_type) + await new_shared_manager.initialize() + # 非本地模型时不立即关闭旧共享管理器，待更新成功后统一处理 + + # 检查 VAD 和 ASR 类型是否需要更新 + update_vad = check_vad_update(self.config, new_config) + update_asr = check_asr_update(self.config, new_config) + logger.bind(tag=TAG).info( + f"检查VAD和ASR类型是否需要更新: VAD={update_vad}, ASR={update_asr}" + ) + + # 检查配置是否有重大变化 + changed_configs = self._get_changed_configs(self.config, new_config) + + # 更新配置 + self.config = new_config + if new_shared_manager: + self.config["_shared_asr_manager"] = new_shared_manager + elif "_shared_asr_manager" in self.config: + del self.config["_shared_asr_manager"] + + # 根据变化类型进行更新 + if changed_configs: + logger.bind(tag=TAG).info(f"配置项变化: {', '.join(changed_configs)}") + + # 重新创建连接服务，使用新配置 + # 注意：已建立的连接会继续使用旧配置，只有新连接使用新配置 + self.connection_service = ConnectionService(new_config) + self.connection_service.server = self + + # 如果 ASR 配置变化且复用旧共享管理器，提示重启 + if update_asr and reuse_manager: + logger.bind(tag=TAG).warning( + "ASR 配置已变化，但仍复用已有共享 ASR 管理器，建议重启服务" + ) + else: + # 即使没有重大变化，也更新 ConnectionService 的配置引用 + self.connection_service.config = new_config + self.connection_service.server = self + + # 更新认证中间件 + self.auth_middleware = AuthMiddleware(new_config) + + # 更新成功后再关闭旧共享管理器（避免失败回滚时不可用） + if old_shared_manager and old_shared_manager is not new_shared_manager: + await old_shared_manager.shutdown() + + logger.bind(tag=TAG).info("配置更新任务执行完毕") + return True + + except Exception as e: + logger.bind(tag=TAG).error(f"更新服务器配置失败: {str(e)}", exc_info=True) + self.last_update_error = f"更新服务器配置失败: {str(e)}" + try: + if new_shared_manager and not reuse_manager: + await new_shared_manager.shutdown() + self.config = old_config + self.connection_service = old_connection_service + self.auth_middleware = old_auth_middleware + except Exception: + pass + return False + + def get_last_update_error(self) -> str: + return self.last_update_error or "" + + def _get_changed_configs(self, old_config: Dict[str, Any], new_config: Dict[str, Any]) -> list: + """ + 获取变化的配置项列表 + + Returns: + list: 变化的配置项名称列表 + """ + changed = [] + key_configs = [ + "selected_module", + "VAD", + "ASR", + "LLM", + "TTS", + "Memory", + "Intent" + ] + + for key in key_configs: + old_value = old_config.get(key) + new_value = new_config.get(key) + if old_value != new_value: + changed.append(key) + + return changed + + def get_active_connections_count(self) -> int: + """获取活跃连接数""" + return len(self.active_connections) + + def get_server_status(self) -> Dict[str, Any]: + """获取服务器状态""" + return { + "active_connections": self.get_active_connections_count(), + "server_type": "new_architecture", + "processors": self.connection_service.message_router.list_processors() + } diff --git a/main/xiaozhi-server/core/xiaozhi_server_facade.py b/main/xiaozhi-server/core/xiaozhi_server_facade.py new file mode 100644 index 00000000..feb424b3 --- /dev/null +++ b/main/xiaozhi-server/core/xiaozhi_server_facade.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 +""" +小智服务器门面类 +统一管理所有协议服务器的启动和停止 +""" + +import asyncio +from typing import Dict, Any, Optional +from config.logger import setup_logging +from core.servers.multi_protocol_server import MultiProtocolServer + +logger = setup_logging() +TAG = __name__ + + +class XiaozhiServerFacade: + """ + 小智服务器门面类 + 提供统一的服务器管理接口，屏蔽内部协议复杂性 + + 功能： + - 协议管理 + - 本地 ASR 模型预加载 + - 优雅启动和停止 + """ + + def __init__(self, config: Dict[str, Any]): + """ + 初始化服务器门面 + + Args: + config: 服务器配置字典 + """ + self.config = config + self.multi_protocol_server: Optional[MultiProtocolServer] = None + self.shared_asr_manager = None # 共享 ASR 管理器 + self._retired_shared_asr_managers = [] + self.is_initialized = False + self.is_running = False + self.last_update_error = None + self._cleanup_pending = False + + # 处理协议配置 + self._setup_protocol_config() + + def _setup_protocol_config(self): + """设置协议配置""" + try: + protocols = self.config.get("protocols", {}) + if not isinstance(protocols, dict): + protocols = {} + requested = protocols.get("enabled_protocols") + requested = requested if isinstance(requested, list) else [] + websocket_enabled = protocols.get("websocket_enabled") + if websocket_enabled is None: + websocket_enabled = not protocols or "websocket" in requested + + enabled_protocols = [] + if websocket_enabled: + enabled_protocols.append("websocket") + + self.config["enabled_protocols"] = enabled_protocols + logger.info(f"启用的协议: {enabled_protocols}") + + except Exception as e: + logger.error(f"设置协议配置失败: {e}") + # 使用最基本的配置 + self.config["enabled_protocols"] = ["websocket"] + + async def initialize(self): + """初始化服务器""" + if self.is_initialized: + logger.bind(tag=TAG).warning("服务器已经初始化") + return + + try: + logger.bind(tag=TAG).info("正在初始化小智服务器...") + + # 检查并预加载本地 ASR 模型（关键步骤） + await self._preload_asr_if_needed() + + # 创建多协议服务器 + self.multi_protocol_server = MultiProtocolServer(self.config) + self.multi_protocol_server.set_management_owner(self) + + self.is_initialized = True + logger.bind(tag=TAG).info("小智服务器初始化完成") + + except Exception as e: + logger.bind(tag=TAG).error(f"初始化服务器失败: {e}") + raise + + async def _preload_asr_if_needed(self): + """ + 检查并预加载本地 ASR 模型 + + 如果配置使用本地 ASR 模型（如 FunASR），则在服务器启动时预加载， + 避免首次语音识别时的延迟导致客户端超时。 + """ + try: + # 获取 ASR 配置 + selected_asr = self.config.get("selected_module", {}).get("ASR") + if not selected_asr: + logger.bind(tag=TAG).info("未配置 ASR 模块，跳过预加载") + return + + # 获取 ASR 类型 + asr_config = self.config.get("ASR", {}).get(selected_asr, {}) + asr_type = asr_config.get("type", selected_asr) + + # 导入 SharedASRManager 检查是否为本地模型 + from core.providers.asr.shared_asr_manager import SharedASRManager + + if SharedASRManager.is_local_model_type(asr_type): + logger.bind(tag=TAG).info( + f"检测到本地 ASR 模型: {asr_type}，开始预加载..." + ) + + # 创建全局 ASR 管理器 + self.shared_asr_manager = SharedASRManager(self.config, asr_type) + + # 预加载模型 + await self.shared_asr_manager.initialize() + + # 将管理器放入配置中供后续使用 + self.config['_shared_asr_manager'] = self.shared_asr_manager + + logger.bind(tag=TAG).info( + f"ASR 模型预加载完成，类型: {asr_type}" + ) + else: + logger.bind(tag=TAG).info( + f"ASR 类型为远程服务: {asr_type}，无需预加载" + ) + + except Exception as e: + logger.bind(tag=TAG).error(f"ASR 预加载失败: {e}") + # 预加载失败不影响服务器启动，继续使用懒加载模式 + logger.bind(tag=TAG).warning("将回退到懒加载模式") + + async def start(self): + """启动服务器""" + if self._cleanup_pending: + raise RuntimeError("上次停止尚未完成，请先重试 stop 清理残留资源") + if not self.is_initialized: + await self.initialize() + + if self.is_running: + logger.warning("服务器已经在运行中") + return + + try: + logger.info("正在启动小智服务器...") + + # 启动多协议服务器 + await self.multi_protocol_server.start() + + self.is_running = True + logger.info("小智服务器启动成功") + + except Exception as e: + logger.error(f"启动服务器失败: {e}") + self.is_running = False + # initialize() may already own a shared ASR manager and partially + # started listeners. Release both before propagating startup failure. + try: + await self.stop() + except Exception as cleanup_error: + logger.bind(tag=TAG).error( + f"启动失败后的资源清理失败: {cleanup_error}" + ) + raise + + async def stop(self): + """停止服务器""" + if ( + not self.is_running + and self.multi_protocol_server is None + and self.shared_asr_manager is None + and not self._retired_shared_asr_managers + ): + logger.bind(tag=TAG).info("服务器未在运行") + return + + logger.bind(tag=TAG).info("正在停止小智服务器...") + errors = [] + protocols_stopped = self.multi_protocol_server is None + if self.multi_protocol_server: + try: + await self.multi_protocol_server.stop() + except Exception as e: + errors.append(("多协议服务器", e)) + logger.bind(tag=TAG).error(f"停止多协议服务器失败: {e}") + else: + self.multi_protocol_server = None + protocols_stopped = True + + if self.shared_asr_manager and protocols_stopped: + logger.bind(tag=TAG).info("正在关闭共享 ASR 管理器...") + try: + await self.shared_asr_manager.shutdown() + except Exception as e: + errors.append(("共享 ASR", e)) + logger.bind(tag=TAG).error(f"关闭共享 ASR 管理器失败: {e}") + else: + self.shared_asr_manager = None + self.config.pop('_shared_asr_manager', None) + elif self.shared_asr_manager: + logger.bind(tag=TAG).warning( + "协议服务器仍持有连接，延后关闭共享 ASR 管理器" + ) + + if protocols_stopped and self._retired_shared_asr_managers: + remaining_retired = [] + for manager in self._retired_shared_asr_managers: + try: + await manager.shutdown() + except Exception as e: + remaining_retired.append(manager) + errors.append(("旧共享 ASR", e)) + logger.bind(tag=TAG).error( + f"关闭旧共享ASR管理器失败: {e}" + ) + self._retired_shared_asr_managers = remaining_retired + + self.is_running = False + self.is_initialized = bool( + self.multi_protocol_server + or self.shared_asr_manager + or self._retired_shared_asr_managers + ) + if not self.is_initialized: + self.shared_asr_manager = None + self.config.pop('_shared_asr_manager', None) + logger.bind(tag=TAG).info("小智服务器已停止") + else: + logger.bind(tag=TAG).warning( + "服务器部分资源停止失败，已保留所有权供重试清理" + ) + + if errors: + self._cleanup_pending = True + details = ", ".join(f"{owner}: {error}" for owner, error in errors) + raise RuntimeError(f"停止服务器时发生错误: {details}") + self._cleanup_pending = False + + async def restart(self): + """重启服务器""" + logger.info("重启小智服务器...") + await self.stop() + await asyncio.sleep(1) # 等待清理完成 + await self.start() + + async def update_config( + self, new_config: Optional[Dict[str, Any]] = None + ) -> bool: + """ + 更新服务器配置 + + Args: + new_config: 新的配置字典 + + Returns: + bool: 更新是否成功 + """ + old_config = self.config + old_shared_manager = self.shared_asr_manager + prepared_shared_manager = old_shared_manager + owns_prepared_manager = False + try: + logger.info("更新服务器配置...") + self.last_update_error = None + + if new_config is None: + from config.config_loader import get_config_from_api_async + + new_config = await get_config_from_api_async(self.config) + if new_config is None: + raise RuntimeError("获取新配置失败") + + # 使用新顶层对象，避免 MultiProtocolServer 的旧配置引用被原地改写， + # 从而导致协议/端口变化无法被检测。 + merged_config = dict(self.config) + merged_config.update(new_config) + self.config = merged_config + self._setup_protocol_config() + + from core.utils.config_validation import validate_config_components + from core.utils.util import check_asr_update + from core.providers.asr.shared_asr_manager import SharedASRManager + + ok, error_msg = await validate_config_components(self.config, logger) + if not ok: + raise RuntimeError(f"配置校验失败: {error_msg}") + + selected_asr = self.config.get("selected_module", {}).get("ASR") + asr_config = self.config.get("ASR", {}).get(selected_asr, {}) + asr_type = asr_config.get("type", selected_asr) if selected_asr else None + needs_new_asr = check_asr_update(old_config, self.config) + + if asr_type and SharedASRManager.is_local_model_type(asr_type): + if not ( + old_shared_manager + and not needs_new_asr + and old_shared_manager.is_ready() + ): + prepared_shared_manager = SharedASRManager(self.config, asr_type) + await prepared_shared_manager.initialize() + owns_prepared_manager = True + self.config["_shared_asr_manager"] = prepared_shared_manager + else: + prepared_shared_manager = None + self.config.pop("_shared_asr_manager", None) + + # 已初始化时即更新实例集合；运行中会完成监听器切换。 + if self.multi_protocol_server: + success = await self.multi_protocol_server.update_config(self.config) + if success: + logger.info("服务器配置更新成功") + else: + raise RuntimeError("多协议服务器配置更新失败") + + self.shared_asr_manager = prepared_shared_manager + if ( + old_shared_manager + and old_shared_manager is not prepared_shared_manager + ): + try: + await old_shared_manager.shutdown() + except Exception as e: + # 新配置已经提交，保留旧资源所有权供 stop 重试。 + self._retired_shared_asr_managers.append( + old_shared_manager + ) + logger.bind(tag=TAG).error(f"关闭旧共享ASR管理器失败: {e}") + + logger.info("配置更新完成") + return True + + except Exception as e: + logger.error(f"更新配置失败: {e}") + self.last_update_error = str(e) + if owns_prepared_manager and prepared_shared_manager: + try: + await prepared_shared_manager.shutdown() + except Exception as cleanup_error: + self._retired_shared_asr_managers.append( + prepared_shared_manager + ) + logger.bind(tag=TAG).error( + f"回滚新共享ASR管理器失败: {cleanup_error}" + ) + self.config = old_config + self.shared_asr_manager = old_shared_manager + if self.multi_protocol_server: + self.is_running = self.multi_protocol_server.is_running + if ( + self.multi_protocol_server.server_tasks + and not self.multi_protocol_server.is_running + ): + self._cleanup_pending = True + return False + + def get_last_update_error(self) -> str: + return self.last_update_error or "" + + def get_server_status(self) -> Dict[str, Any]: + """获取服务器状态""" + base_status = { + 'is_initialized': self.is_initialized, + 'is_running': self.is_running, + 'enabled_protocols': self.config.get('enabled_protocols', []) + } + + if self.multi_protocol_server: + server_status = self.multi_protocol_server.get_server_status() + base_status.update(server_status) + + # 添加 ASR 状态 + if self.shared_asr_manager: + base_status['asr'] = { + 'mode': 'shared', + 'ready': self.shared_asr_manager.is_ready(), + 'queue_status': self.shared_asr_manager.get_queue_status() + } + else: + base_status['asr'] = {'mode': 'lazy_load'} + + return base_status + + def get_active_connections_count(self) -> Dict[str, int]: + """获取各协议的活跃连接数""" + if self.multi_protocol_server: + return self.multi_protocol_server.get_active_connections_count() + return {} + + def get_supported_protocols(self) -> list: + """获取支持的协议列表""" + if self.multi_protocol_server: + return self.multi_protocol_server.get_supported_protocols() + return ['websocket'] + + def is_protocol_enabled(self, protocol: str) -> bool: + """检查协议是否启用""" + enabled_protocols = self.config.get('enabled_protocols', []) + return protocol in enabled_protocols + + async def broadcast_message(self, message: Dict[str, Any], protocol: Optional[str] = None): + """ + 向所有连接广播消息 + + Args: + message: 要广播的消息 + protocol: 指定协议，None表示向所有协议广播 + """ + if self.multi_protocol_server: + await self.multi_protocol_server.broadcast_message(message, protocol) + + def get_websocket_info(self) -> Dict[str, Any]: + """获取WebSocket连接信息""" + if not self.is_protocol_enabled('websocket'): + return {'enabled': False} + + server_config = self.config.get('server', {}) + return { + 'enabled': True, + 'host': server_config.get('ip', '0.0.0.0'), + 'port': server_config.get('port', 8000), + 'path': '/xiaozhi/v1/' + } + + def get_connection_info(self) -> Dict[str, Any]: + """获取所有协议的连接信息""" + return { + 'websocket': self.get_websocket_info(), + 'active_connections': self.get_active_connections_count() + }