mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-28 10:03:54 +08:00
Merge pull request #3307 from CaixyPromise/feature/connection-runtime-pr
refactor: 传输协议无关的连接运行时
This commit is contained in:
CGD
@@ -10,6 +10,164 @@ class AuthenticationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class AuthMiddleware:
|
||||
"""
|
||||
认证中间件
|
||||
用于 WebSocket/MQTT 连接认证
|
||||
集成 AuthManager 的 token 验证逻辑，支持多种认证方式
|
||||
"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
"""
|
||||
初始化认证中间件
|
||||
|
||||
Args:
|
||||
config: 配置字典，包含认证相关配置
|
||||
"""
|
||||
self.config = config
|
||||
server_config = config.get("server", {})
|
||||
auth_config = server_config.get("auth", {})
|
||||
|
||||
self.enabled = auth_config.get("enabled", False)
|
||||
self.tokens = auth_config.get("tokens", [])
|
||||
self.allowed_devices = set(auth_config.get("allowed_devices", []))
|
||||
|
||||
# 获取 auth_key 用于 HMAC token 验证
|
||||
self.auth_key = server_config.get("auth_key", "")
|
||||
expire_seconds = auth_config.get("expire_seconds", None)
|
||||
|
||||
# 创建 AuthManager 实例用于 HMAC token 验证
|
||||
if self.auth_key:
|
||||
self._auth_manager = AuthManager(
|
||||
secret_key=self.auth_key,
|
||||
expire_seconds=expire_seconds
|
||||
)
|
||||
else:
|
||||
self._auth_manager = None
|
||||
|
||||
def authenticate(self, device_id: str, token: str = None, client_id: str = None) -> bool:
|
||||
"""
|
||||
验证设备认证（同步方法）
|
||||
|
||||
Args:
|
||||
device_id: 设备 ID
|
||||
token: 认证令牌（可以是静态 token 或 HMAC token）
|
||||
client_id: 客户端 ID（用于 HMAC token 验证）
|
||||
|
||||
Returns:
|
||||
bool: 认证是否通过
|
||||
"""
|
||||
from config.logger import setup_logging
|
||||
logger = setup_logging()
|
||||
|
||||
if not self.enabled:
|
||||
logger.debug("[AuthMiddleware.authenticate] 认证未启用")
|
||||
return True
|
||||
|
||||
# 1. 检查白名单
|
||||
if device_id and device_id in self.allowed_devices:
|
||||
logger.debug(f"[AuthMiddleware.authenticate] 设备 {device_id} 在白名单中，放行")
|
||||
return True
|
||||
|
||||
# 2. 检查静态 token
|
||||
if token:
|
||||
# 移除 Bearer 前缀（如果有）
|
||||
if token.startswith("Bearer "):
|
||||
token = token[7:]
|
||||
|
||||
logger.debug(f"[AuthMiddleware.authenticate] 检查静态token, 配置了 {len(self.tokens)} 个token")
|
||||
for token_config in self.tokens:
|
||||
configured_token = token_config.get("token")
|
||||
if configured_token == token:
|
||||
logger.debug(f"[AuthMiddleware.authenticate] 静态token匹配成功")
|
||||
return True
|
||||
logger.debug(f"[AuthMiddleware.authenticate] 静态token不匹配")
|
||||
|
||||
# 3. 检查 HMAC token（需要 AuthManager）
|
||||
if token and self._auth_manager and client_id and device_id:
|
||||
logger.debug(f"[AuthMiddleware.authenticate] 尝试HMAC token验证")
|
||||
if self._auth_manager.verify_token(token, client_id, device_id):
|
||||
logger.debug(f"[AuthMiddleware.authenticate] HMAC token验证成功")
|
||||
return True
|
||||
logger.debug(f"[AuthMiddleware.authenticate] HMAC token验证失败")
|
||||
|
||||
logger.debug(f"[AuthMiddleware.authenticate] 所有认证方式均失败")
|
||||
return False
|
||||
|
||||
async def authenticate_async(self, headers: dict) -> bool:
|
||||
"""
|
||||
从 headers 中提取信息并进行异步认证
|
||||
|
||||
Args:
|
||||
headers: HTTP 请求头字典
|
||||
|
||||
Returns:
|
||||
bool: 认证是否通过
|
||||
|
||||
Raises:
|
||||
AuthenticationError: 认证失败时抛出
|
||||
"""
|
||||
from config.logger import setup_logging
|
||||
logger = setup_logging()
|
||||
|
||||
logger.debug(f"[AuthMiddleware] 开始认证, enabled={self.enabled}")
|
||||
|
||||
if not self.enabled:
|
||||
logger.debug("[AuthMiddleware] 认证未启用，跳过认证")
|
||||
return True
|
||||
|
||||
device_id = headers.get("device-id")
|
||||
client_id = headers.get("client-id")
|
||||
authorization = headers.get("authorization", "")
|
||||
|
||||
logger.debug(f"[AuthMiddleware] device_id={device_id}, client_id={client_id}, has_auth={bool(authorization)}")
|
||||
|
||||
# 提取 token
|
||||
token = None
|
||||
if authorization:
|
||||
if authorization.startswith("Bearer "):
|
||||
token = authorization[7:]
|
||||
else:
|
||||
token = authorization
|
||||
|
||||
# 执行认证
|
||||
auth_result = self.authenticate(device_id, token, client_id)
|
||||
logger.debug(f"[AuthMiddleware] 认证结果: {auth_result}")
|
||||
|
||||
if auth_result:
|
||||
return True
|
||||
|
||||
raise AuthenticationError(f"认证失败: device_id={device_id}")
|
||||
|
||||
def authenticate_websocket(self, websocket) -> bool:
|
||||
"""
|
||||
WebSocket 连接认证
|
||||
|
||||
Args:
|
||||
websocket: WebSocket 连接对象
|
||||
|
||||
Returns:
|
||||
bool: 认证是否通过
|
||||
"""
|
||||
if not self.enabled:
|
||||
return True
|
||||
|
||||
headers = dict(websocket.request.headers)
|
||||
device_id = headers.get("device-id")
|
||||
client_id = headers.get("client-id")
|
||||
authorization = headers.get("authorization", "")
|
||||
|
||||
# 提取 token
|
||||
token = None
|
||||
if authorization:
|
||||
if authorization.startswith("Bearer "):
|
||||
token = authorization[7:]
|
||||
else:
|
||||
token = authorization
|
||||
|
||||
return self.authenticate(device_id, token, client_id)
|
||||
|
||||
|
||||
class AuthManager:
|
||||
"""
|
||||
统一授权认证管理器
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
from typing import Any, Dict
|
||||
from core.components.component_manager import Component, ComponentType, ComponentFactory
|
||||
from core.utils import asr
|
||||
from core.utils.modules_initialize import initialize_asr
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
TAG = __name__
|
||||
|
||||
|
||||
class ASRAdapter(Component):
|
||||
"""
|
||||
ASR组件适配器：将现有ASR组件包装为新的组件接口
|
||||
|
||||
支持两种模式：
|
||||
1. 共享实例模式：使用 SharedASRManager 的全局共享实例
|
||||
2. 独立实例模式：每个连接创建独立的 ASR 实例（原有逻辑）
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
super().__init__(ComponentType.ASR, config)
|
||||
self._asr_instance = None
|
||||
self._delete_audio = config.get("delete_audio", True)
|
||||
self._using_shared = False # 是否使用共享实例
|
||||
self._shared_owner = None
|
||||
|
||||
async def _do_initialize(self, context: Any) -> None:
|
||||
"""初始化ASR组件"""
|
||||
try:
|
||||
# 获取ASR配置
|
||||
selected_module = self.config.get("selected_module", {}).get("ASR")
|
||||
if not selected_module:
|
||||
raise ValueError("未配置ASR模块")
|
||||
|
||||
# 检查是否有全局共享 ASR 管理器
|
||||
shared_manager = getattr(context, 'shared_asr_manager', None)
|
||||
|
||||
acquired_manager = None
|
||||
if shared_manager and hasattr(shared_manager, "acquire_for_config"):
|
||||
acquired_manager = await shared_manager.acquire_for_config(
|
||||
self.config
|
||||
)
|
||||
elif (
|
||||
shared_manager
|
||||
and shared_manager.is_ready()
|
||||
and shared_manager.matches_config(self.config)
|
||||
):
|
||||
acquired_manager = shared_manager
|
||||
|
||||
if acquired_manager is not None:
|
||||
# 使用共享实例模式
|
||||
logger.bind(tag=TAG).info(f"使用共享 ASR 实例: {selected_module}")
|
||||
from core.providers.asr.shared_asr_proxy import SharedASRProxy
|
||||
self._asr_instance = SharedASRProxy(acquired_manager)
|
||||
self._using_shared = True
|
||||
self._shared_owner = shared_manager
|
||||
else:
|
||||
# 使用独立实例模式（原有逻辑）
|
||||
if shared_manager and shared_manager.is_ready():
|
||||
logger.bind(tag=TAG).info(
|
||||
f"连接ASR配置与共享实例不匹配，使用独立实例: {selected_module}"
|
||||
)
|
||||
logger.bind(tag=TAG).info(f"使用独立 ASR 实例: {selected_module}")
|
||||
self._asr_instance = initialize_asr(self.config)
|
||||
self._using_shared = False
|
||||
|
||||
# Legacy providers inspect conn.asr while handling stream state.
|
||||
context.asr = self._asr_instance
|
||||
|
||||
# 打开音频通道（如果需要）
|
||||
if hasattr(self._asr_instance, 'open_audio_channels'):
|
||||
await self._asr_instance.open_audio_channels(context)
|
||||
|
||||
logger.bind(tag=TAG).info(
|
||||
f"ASR组件初始化完成: {selected_module}, "
|
||||
f"共享模式: {self._using_shared}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR组件初始化失败: {e}")
|
||||
raise
|
||||
|
||||
async def _do_cleanup(self) -> None:
|
||||
"""清理ASR组件"""
|
||||
if self._asr_instance:
|
||||
instance = self._asr_instance
|
||||
if self._using_shared:
|
||||
await self._close_resource(instance)
|
||||
if self._shared_owner and hasattr(
|
||||
self._shared_owner, "release_for_config"
|
||||
):
|
||||
await self._shared_owner.release_for_config(instance.manager)
|
||||
logger.bind(tag=TAG).debug("共享 ASR 代理已释放")
|
||||
else:
|
||||
await self._close_resource(instance)
|
||||
if hasattr(instance, 'cleanup_audio_files'):
|
||||
instance.cleanup_audio_files()
|
||||
logger.bind(tag=TAG).info("ASR组件清理完成")
|
||||
self._asr_instance = None
|
||||
self._using_shared = False
|
||||
self._shared_owner = None
|
||||
|
||||
@property
|
||||
def asr_instance(self):
|
||||
"""获取ASR实例"""
|
||||
return self._asr_instance
|
||||
|
||||
|
||||
class ASRFactory(ComponentFactory):
|
||||
"""ASR组件工厂"""
|
||||
|
||||
def create(self, config: Dict[str, Any]) -> Component:
|
||||
return ASRAdapter(config)
|
||||
|
||||
def get_component_type(self) -> ComponentType:
|
||||
return ComponentType.ASR
|
||||
@@ -0,0 +1,92 @@
|
||||
from typing import Any, Dict
|
||||
from core.components.component_manager import Component, ComponentType, ComponentFactory
|
||||
from core.utils import intent, llm
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class IntentAdapter(Component):
|
||||
"""Intent组件适配器：将现有Intent组件包装为新的组件接口"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
super().__init__(ComponentType.INTENT, config)
|
||||
self._intent_instance = None
|
||||
self._owned_llm_instance = None
|
||||
|
||||
async def _do_initialize(self, context: Any) -> None:
|
||||
"""初始化Intent组件"""
|
||||
try:
|
||||
# 获取Intent配置
|
||||
selected_module = self.config.get("selected_module", {}).get("Intent")
|
||||
if not selected_module:
|
||||
raise ValueError("未配置Intent模块")
|
||||
|
||||
# 获取Intent类型
|
||||
intent_type = (
|
||||
selected_module
|
||||
if "type" not in self.config["Intent"][selected_module]
|
||||
else self.config["Intent"][selected_module]["type"]
|
||||
)
|
||||
|
||||
# 创建Intent实例
|
||||
self._intent_instance = intent.create_instance(
|
||||
intent_type,
|
||||
self.config["Intent"][selected_module],
|
||||
)
|
||||
|
||||
# intent_llm 可配置独立模型；未配置时回退主 LLM。
|
||||
if intent_type == "intent_llm":
|
||||
llm_component = None
|
||||
if getattr(context, "component_manager", None):
|
||||
llm_component = await context.component_manager.get_component(ComponentType.LLM, context)
|
||||
main_llm = getattr(llm_component, "llm_instance", None)
|
||||
intent_config = self.config["Intent"][selected_module]
|
||||
dedicated_llm_name = intent_config.get("llm")
|
||||
selected_llm = main_llm
|
||||
if (
|
||||
dedicated_llm_name
|
||||
and dedicated_llm_name in self.config.get("LLM", {})
|
||||
):
|
||||
dedicated_config = self.config["LLM"][dedicated_llm_name]
|
||||
dedicated_type = dedicated_config.get("type", dedicated_llm_name)
|
||||
self._owned_llm_instance = llm.create_instance(
|
||||
dedicated_type, dedicated_config
|
||||
)
|
||||
selected_llm = self._owned_llm_instance
|
||||
logger.info(f"为意图识别创建专用LLM: {dedicated_llm_name}")
|
||||
if selected_llm and hasattr(self._intent_instance, "set_llm"):
|
||||
self._intent_instance.set_llm(selected_llm)
|
||||
|
||||
logger.info(f"Intent组件初始化完成: {intent_type}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Intent组件初始化失败: {e}")
|
||||
raise
|
||||
|
||||
async def _do_cleanup(self) -> None:
|
||||
"""清理Intent组件"""
|
||||
if self._intent_instance:
|
||||
instance = self._intent_instance
|
||||
await self._close_resource(instance)
|
||||
self._intent_instance = None
|
||||
logger.info("Intent组件清理完成")
|
||||
if self._owned_llm_instance:
|
||||
instance = self._owned_llm_instance
|
||||
await self._close_resource(instance)
|
||||
self._owned_llm_instance = None
|
||||
|
||||
@property
|
||||
def intent_instance(self):
|
||||
"""获取Intent实例"""
|
||||
return self._intent_instance
|
||||
|
||||
|
||||
class IntentFactory(ComponentFactory):
|
||||
"""Intent组件工厂"""
|
||||
|
||||
def create(self, config: Dict[str, Any]) -> Component:
|
||||
return IntentAdapter(config)
|
||||
|
||||
def get_component_type(self) -> ComponentType:
|
||||
return ComponentType.INTENT
|
||||
@@ -0,0 +1,64 @@
|
||||
from typing import Any, Dict
|
||||
from core.components.component_manager import Component, ComponentType, ComponentFactory
|
||||
from core.utils import llm
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class LLMAdapter(Component):
|
||||
"""LLM组件适配器：将现有LLM组件包装为新的组件接口"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
super().__init__(ComponentType.LLM, config)
|
||||
self._llm_instance = None
|
||||
|
||||
async def _do_initialize(self, context: Any) -> None:
|
||||
"""初始化LLM组件"""
|
||||
try:
|
||||
# 获取LLM配置
|
||||
selected_module = self.config.get("selected_module", {}).get("LLM")
|
||||
if not selected_module:
|
||||
raise ValueError("未配置LLM模块")
|
||||
|
||||
# 获取LLM类型
|
||||
llm_type = (
|
||||
selected_module
|
||||
if "type" not in self.config["LLM"][selected_module]
|
||||
else self.config["LLM"][selected_module]["type"]
|
||||
)
|
||||
|
||||
# 创建LLM实例
|
||||
self._llm_instance = llm.create_instance(
|
||||
llm_type,
|
||||
self.config["LLM"][selected_module],
|
||||
)
|
||||
|
||||
logger.info(f"LLM组件初始化完成: {llm_type}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"LLM组件初始化失败: {e}")
|
||||
raise
|
||||
|
||||
async def _do_cleanup(self) -> None:
|
||||
"""清理LLM组件"""
|
||||
if self._llm_instance:
|
||||
instance = self._llm_instance
|
||||
await self._close_resource(instance)
|
||||
self._llm_instance = None
|
||||
logger.info("LLM组件清理完成")
|
||||
|
||||
@property
|
||||
def llm_instance(self):
|
||||
"""获取LLM实例"""
|
||||
return self._llm_instance
|
||||
|
||||
|
||||
class LLMFactory(ComponentFactory):
|
||||
"""LLM组件工厂"""
|
||||
|
||||
def create(self, config: Dict[str, Any]) -> Component:
|
||||
return LLMAdapter(config)
|
||||
|
||||
def get_component_type(self) -> ComponentType:
|
||||
return ComponentType.LLM
|
||||
@@ -0,0 +1,103 @@
|
||||
from typing import Any, Dict
|
||||
from core.components.component_manager import Component, ComponentType, ComponentFactory
|
||||
from core.utils import llm, memory
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class MemoryAdapter(Component):
|
||||
"""Memory组件适配器：将现有Memory组件包装为新的组件接口"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
super().__init__(ComponentType.MEMORY, config)
|
||||
self._memory_instance = None
|
||||
self._owned_llm_instance = None
|
||||
|
||||
async def _do_initialize(self, context: Any) -> None:
|
||||
"""初始化Memory组件"""
|
||||
try:
|
||||
# 获取Memory配置
|
||||
selected_module = self.config.get("selected_module", {}).get("Memory")
|
||||
if not selected_module:
|
||||
raise ValueError("未配置Memory模块")
|
||||
|
||||
# 获取Memory类型
|
||||
memory_type = (
|
||||
selected_module
|
||||
if "type" not in self.config["Memory"][selected_module]
|
||||
else self.config["Memory"][selected_module]["type"]
|
||||
)
|
||||
|
||||
# 创建Memory实例
|
||||
self._memory_instance = memory.create_instance(
|
||||
memory_type,
|
||||
self.config["Memory"][selected_module],
|
||||
self.config.get("summaryMemory", None),
|
||||
)
|
||||
|
||||
# 初始化记忆模块
|
||||
main_llm = None
|
||||
if hasattr(self._memory_instance, 'init_memory'):
|
||||
# 需要LLM实例来初始化记忆
|
||||
llm_component = None
|
||||
if getattr(context, "component_manager", None):
|
||||
llm_component = await context.component_manager.get_component(ComponentType.LLM, context)
|
||||
if llm_component and hasattr(llm_component, 'llm_instance'):
|
||||
main_llm = llm_component.llm_instance
|
||||
self._memory_instance.init_memory(
|
||||
role_id=context.device_id,
|
||||
llm=main_llm,
|
||||
summary_memory=self.config.get("summaryMemory", None),
|
||||
save_to_file=not self.config.get("read_config_from_api", False),
|
||||
)
|
||||
|
||||
memory_config = self.config["Memory"][selected_module]
|
||||
dedicated_llm_name = memory_config.get("llm")
|
||||
if (
|
||||
memory_type == "mem_local_short"
|
||||
and dedicated_llm_name
|
||||
and dedicated_llm_name in self.config.get("LLM", {})
|
||||
):
|
||||
dedicated_config = self.config["LLM"][dedicated_llm_name]
|
||||
dedicated_type = dedicated_config.get("type", dedicated_llm_name)
|
||||
self._owned_llm_instance = llm.create_instance(
|
||||
dedicated_type, dedicated_config
|
||||
)
|
||||
self._memory_instance.set_llm(self._owned_llm_instance)
|
||||
logger.info(f"为记忆总结创建专用LLM: {dedicated_llm_name}")
|
||||
elif main_llm and hasattr(self._memory_instance, "set_llm"):
|
||||
self._memory_instance.set_llm(main_llm)
|
||||
|
||||
logger.info(f"Memory组件初始化完成: {memory_type}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Memory组件初始化失败: {e}")
|
||||
raise
|
||||
|
||||
async def _do_cleanup(self) -> None:
|
||||
"""清理Memory组件"""
|
||||
if self._memory_instance:
|
||||
instance = self._memory_instance
|
||||
await self._close_resource(instance)
|
||||
self._memory_instance = None
|
||||
logger.info("Memory组件清理完成")
|
||||
if self._owned_llm_instance:
|
||||
instance = self._owned_llm_instance
|
||||
await self._close_resource(instance)
|
||||
self._owned_llm_instance = None
|
||||
|
||||
@property
|
||||
def memory_instance(self):
|
||||
"""获取Memory实例"""
|
||||
return self._memory_instance
|
||||
|
||||
|
||||
class MemoryFactory(ComponentFactory):
|
||||
"""Memory组件工厂"""
|
||||
|
||||
def create(self, config: Dict[str, Any]) -> Component:
|
||||
return MemoryAdapter(config)
|
||||
|
||||
def get_component_type(self) -> ComponentType:
|
||||
return ComponentType.MEMORY
|
||||
@@ -0,0 +1,76 @@
|
||||
from typing import Any, Dict
|
||||
from core.components.component_manager import Component, ComponentType, ComponentFactory
|
||||
from core.utils import tts
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TTSAdapter(Component):
|
||||
"""TTS组件适配器：将现有TTS组件包装为新的组件接口"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
super().__init__(ComponentType.TTS, config)
|
||||
self._tts_instance = None
|
||||
self._delete_audio = config.get("delete_audio", True)
|
||||
|
||||
async def _do_initialize(self, context: Any) -> None:
|
||||
"""初始化TTS组件"""
|
||||
try:
|
||||
# 获取TTS配置
|
||||
selected_module = self.config.get("selected_module", {}).get("TTS")
|
||||
if not selected_module:
|
||||
raise ValueError("未配置TTS模块")
|
||||
|
||||
# 获取TTS类型
|
||||
tts_type = (
|
||||
selected_module
|
||||
if "type" not in self.config["TTS"][selected_module]
|
||||
else self.config["TTS"][selected_module]["type"]
|
||||
)
|
||||
|
||||
# 创建TTS实例
|
||||
self._tts_instance = tts.create_instance(
|
||||
tts_type,
|
||||
self.config["TTS"][selected_module],
|
||||
str(self._delete_audio).lower() in ("true", "1", "yes"),
|
||||
)
|
||||
|
||||
# 打开音频通道
|
||||
if hasattr(self._tts_instance, 'open_audio_channels'):
|
||||
await self._tts_instance.open_audio_channels(context)
|
||||
|
||||
# 设置兼容属性（用于向后兼容）
|
||||
if hasattr(context, 'tts'):
|
||||
context.tts = self._tts_instance
|
||||
|
||||
logger.info(f"TTS组件初始化完成: {tts_type}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"TTS组件初始化失败: {e}")
|
||||
raise
|
||||
|
||||
async def _do_cleanup(self) -> None:
|
||||
"""清理TTS组件"""
|
||||
if self._tts_instance:
|
||||
instance = self._tts_instance
|
||||
await self._close_resource(instance)
|
||||
if hasattr(instance, 'cleanup_audio_files'):
|
||||
instance.cleanup_audio_files()
|
||||
self._tts_instance = None
|
||||
logger.info("TTS组件清理完成")
|
||||
|
||||
@property
|
||||
def tts_instance(self):
|
||||
"""获取TTS实例"""
|
||||
return self._tts_instance
|
||||
|
||||
|
||||
class TTSFactory(ComponentFactory):
|
||||
"""TTS组件工厂"""
|
||||
|
||||
def create(self, config: Dict[str, Any]) -> Component:
|
||||
return TTSAdapter(config)
|
||||
|
||||
def get_component_type(self) -> ComponentType:
|
||||
return ComponentType.TTS
|
||||
@@ -0,0 +1,64 @@
|
||||
from typing import Any, Dict
|
||||
from core.components.component_manager import Component, ComponentType, ComponentFactory
|
||||
from core.utils import vad
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class VADAdapter(Component):
|
||||
"""VAD组件适配器：将现有VAD组件包装为新的组件接口"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
super().__init__(ComponentType.VAD, config)
|
||||
self._vad_instance = None
|
||||
|
||||
async def _do_initialize(self, context: Any) -> None:
|
||||
"""初始化VAD组件"""
|
||||
try:
|
||||
# 获取VAD配置
|
||||
selected_module = self.config.get("selected_module", {}).get("VAD")
|
||||
if not selected_module:
|
||||
raise ValueError("未配置VAD模块")
|
||||
|
||||
# 获取VAD类型
|
||||
vad_type = (
|
||||
selected_module
|
||||
if "type" not in self.config["VAD"][selected_module]
|
||||
else self.config["VAD"][selected_module]["type"]
|
||||
)
|
||||
|
||||
# 创建VAD实例
|
||||
self._vad_instance = vad.create_instance(
|
||||
vad_type,
|
||||
self.config["VAD"][selected_module],
|
||||
)
|
||||
|
||||
logger.info(f"VAD组件初始化完成: {vad_type}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"VAD组件初始化失败: {e}")
|
||||
raise
|
||||
|
||||
async def _do_cleanup(self) -> None:
|
||||
"""清理VAD组件"""
|
||||
if self._vad_instance:
|
||||
instance = self._vad_instance
|
||||
await self._close_resource(instance)
|
||||
self._vad_instance = None
|
||||
logger.info("VAD组件清理完成")
|
||||
|
||||
@property
|
||||
def vad_instance(self):
|
||||
"""获取VAD实例"""
|
||||
return self._vad_instance
|
||||
|
||||
|
||||
class VADFactory(ComponentFactory):
|
||||
"""VAD组件工厂"""
|
||||
|
||||
def create(self, config: Dict[str, Any]) -> Component:
|
||||
return VADAdapter(config)
|
||||
|
||||
def get_component_type(self) -> ComponentType:
|
||||
return ComponentType.VAD
|
||||
@@ -0,0 +1,268 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
import weakref
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Callable, Dict, Optional, Type, TypeVar, Generic
|
||||
from enum import Enum
|
||||
from config.logger import setup_logging
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ComponentType(Enum):
|
||||
"""组件类型枚举"""
|
||||
TTS = "tts"
|
||||
ASR = "asr"
|
||||
VAD = "vad"
|
||||
LLM = "llm"
|
||||
MEMORY = "memory"
|
||||
INTENT = "intent"
|
||||
|
||||
|
||||
class ComponentState(Enum):
|
||||
"""组件状态枚举"""
|
||||
UNINITIALIZED = "uninitialized"
|
||||
INITIALIZING = "initializing"
|
||||
READY = "ready"
|
||||
ERROR = "error"
|
||||
CLEANING = "cleaning"
|
||||
CLEANED = "cleaned"
|
||||
|
||||
|
||||
class Component(ABC):
|
||||
"""组件基类：定义统一的组件接口和生命周期管理"""
|
||||
|
||||
def __init__(self, component_type: ComponentType, config: Dict[str, Any]):
|
||||
self.component_type = component_type
|
||||
self.config = config
|
||||
self.state = ComponentState.UNINITIALIZED
|
||||
self._initialization_lock = asyncio.Lock()
|
||||
self._cleanup_lock = asyncio.Lock()
|
||||
self._dependencies: Dict[str, 'Component'] = {}
|
||||
self._dependents: weakref.WeakSet['Component'] = weakref.WeakSet()
|
||||
self._resources: list = [] # 存储需要清理的资源
|
||||
|
||||
@abstractmethod
|
||||
async def _do_initialize(self, context: Any) -> None:
|
||||
"""子类实现具体的初始化逻辑"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def _do_cleanup(self) -> None:
|
||||
"""子类实现具体的清理逻辑"""
|
||||
pass
|
||||
|
||||
async def initialize(self, context: Any) -> None:
|
||||
"""初始化组件（带锁保护）"""
|
||||
async with self._initialization_lock:
|
||||
if self.state != ComponentState.UNINITIALIZED:
|
||||
return
|
||||
|
||||
try:
|
||||
self.state = ComponentState.INITIALIZING
|
||||
logger.info(f"正在初始化组件: {self.component_type.value}")
|
||||
|
||||
# 初始化依赖组件
|
||||
await self._initialize_dependencies(context)
|
||||
|
||||
# 执行具体初始化
|
||||
await self._do_initialize(context)
|
||||
|
||||
self.state = ComponentState.READY
|
||||
logger.info(f"组件初始化完成: {self.component_type.value}")
|
||||
|
||||
except Exception as e:
|
||||
self.state = ComponentState.ERROR
|
||||
logger.error(f"组件初始化失败: {self.component_type.value}, 错误: {e}")
|
||||
# 初始化可能已经分配线程、连接或文件，失败路径也必须释放。
|
||||
try:
|
||||
await self._do_cleanup()
|
||||
await self._cleanup_resources()
|
||||
except Exception as cleanup_error:
|
||||
logger.error(
|
||||
f"组件初始化回滚失败: {self.component_type.value}, "
|
||||
f"错误: {cleanup_error}"
|
||||
)
|
||||
raise
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
"""清理组件（带锁保护）"""
|
||||
async with self._cleanup_lock:
|
||||
if self.state in [ComponentState.CLEANING, ComponentState.CLEANED]:
|
||||
return
|
||||
|
||||
try:
|
||||
self.state = ComponentState.CLEANING
|
||||
logger.info(f"正在清理组件: {self.component_type.value}")
|
||||
|
||||
# 清理依赖此组件的其他组件
|
||||
await self._cleanup_dependents()
|
||||
|
||||
# 执行具体清理
|
||||
await self._do_cleanup()
|
||||
|
||||
# 清理资源
|
||||
await self._cleanup_resources()
|
||||
|
||||
self.state = ComponentState.CLEANED
|
||||
logger.info(f"组件清理完成: {self.component_type.value}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"组件清理失败: {self.component_type.value}, 错误: {e}")
|
||||
self.state = ComponentState.ERROR
|
||||
raise
|
||||
|
||||
def add_dependency(self, name: str, component: 'Component') -> None:
|
||||
"""添加依赖组件"""
|
||||
self._dependencies[name] = component
|
||||
component._dependents.add(self)
|
||||
|
||||
def add_resource(self, resource: Any) -> None:
|
||||
"""添加需要清理的资源"""
|
||||
self._resources.append(resource)
|
||||
|
||||
async def _initialize_dependencies(self, context: Any) -> None:
|
||||
"""初始化依赖组件"""
|
||||
for name, dep in self._dependencies.items():
|
||||
if dep.state == ComponentState.UNINITIALIZED:
|
||||
await dep.initialize(context)
|
||||
|
||||
async def _cleanup_dependents(self) -> None:
|
||||
"""清理依赖此组件的其他组件"""
|
||||
for dependent in list(self._dependents):
|
||||
await dependent.cleanup()
|
||||
|
||||
async def _cleanup_resources(self) -> None:
|
||||
"""清理所有注册的资源"""
|
||||
for resource in self._resources:
|
||||
try:
|
||||
await self._close_resource(resource)
|
||||
except Exception as e:
|
||||
logger.warning(f"清理资源时出错: {e}")
|
||||
self._resources.clear()
|
||||
|
||||
@staticmethod
|
||||
async def _close_resource(resource: Any) -> None:
|
||||
"""Close either sync or async providers without invoking them twice."""
|
||||
cleanup = getattr(resource, "close", None) or getattr(resource, "cleanup", None)
|
||||
if not cleanup:
|
||||
return
|
||||
result = cleanup()
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
|
||||
|
||||
class ComponentFactory(ABC):
|
||||
"""组件工厂基类"""
|
||||
|
||||
@abstractmethod
|
||||
def create(self, config: Dict[str, Any]) -> Component:
|
||||
"""创建组件实例"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_component_type(self) -> ComponentType:
|
||||
"""获取组件类型"""
|
||||
pass
|
||||
|
||||
|
||||
class ComponentManager:
|
||||
"""
|
||||
组件管理器：统一管理连接期内的组件实例生命周期。
|
||||
支持分类管理、依赖注入、按需懒加载与统一清理。
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
self._config = config
|
||||
self._components: Dict[str, Component] = {}
|
||||
self._factories: Dict[ComponentType, ComponentFactory] = {}
|
||||
self._component_locks: Dict[str, asyncio.Lock] = {}
|
||||
self._initialization_order: list[ComponentType] = []
|
||||
self.lazy_enabled = True
|
||||
|
||||
def register_factory(self, factory: ComponentFactory) -> None:
|
||||
"""注册组件工厂"""
|
||||
component_type = factory.get_component_type()
|
||||
self._factories[component_type] = factory
|
||||
logger.debug(f"注册组件工厂: {component_type.value}")
|
||||
|
||||
def set_initialization_order(self, order: list[ComponentType]) -> None:
|
||||
"""设置组件初始化顺序"""
|
||||
self._initialization_order = order
|
||||
|
||||
async def get_component(self, component_type: ComponentType, context: Any) -> Optional[Component]:
|
||||
"""获取组件实例（按需创建）"""
|
||||
key = component_type.value
|
||||
|
||||
lock = self._component_locks.setdefault(key, asyncio.Lock())
|
||||
async with lock:
|
||||
if key in self._components:
|
||||
return self._components[key]
|
||||
if not self.lazy_enabled:
|
||||
logger.warning(f"组件未初始化且已关闭懒加载: {component_type.value}")
|
||||
return None
|
||||
factory = self._factories.get(component_type)
|
||||
if factory is None:
|
||||
logger.warning(f"未找到组件工厂: {component_type.value}")
|
||||
return None
|
||||
|
||||
try:
|
||||
instance = factory.create(self._config)
|
||||
# 先登记所有权，确保初始化中途失败时资源仍然可达。
|
||||
self._components[key] = instance
|
||||
await instance.initialize(context)
|
||||
logger.info(f"组件创建并初始化完成: {component_type.value}")
|
||||
except Exception as e:
|
||||
logger.error(f"组件创建失败: {component_type.value}, 错误: {e}")
|
||||
failed = self._components.pop(key, None)
|
||||
if failed is not None:
|
||||
await failed.cleanup()
|
||||
return None
|
||||
|
||||
return self._components.get(key)
|
||||
|
||||
def get(self, component_name: str) -> Optional[Component]:
|
||||
"""获取已初始化的组件实例（兼容接口）"""
|
||||
return self._components.get(component_name)
|
||||
|
||||
async def initialize_all(self, context: Any) -> None:
|
||||
"""按顺序初始化所有组件"""
|
||||
for component_type in self._initialization_order:
|
||||
await self.get_component(component_type, context)
|
||||
|
||||
async def cleanup_all(self) -> None:
|
||||
"""清理所有组件（逆序清理）"""
|
||||
errors = []
|
||||
# 按逆序清理，确保依赖关系正确
|
||||
for component_type in reversed(self._initialization_order):
|
||||
key = component_type.value
|
||||
if key in self._components:
|
||||
component = self._components.pop(key)
|
||||
try:
|
||||
await component.cleanup()
|
||||
except Exception as e:
|
||||
errors.append((key, e))
|
||||
|
||||
# 清理可能遗漏的组件
|
||||
remaining_components = list(self._components.items())
|
||||
for key, component in remaining_components:
|
||||
try:
|
||||
await component.cleanup()
|
||||
except Exception as e:
|
||||
errors.append((key, e))
|
||||
|
||||
self._components.clear()
|
||||
self._component_locks.clear()
|
||||
logger.info("所有组件已清理完成")
|
||||
if errors:
|
||||
details = ", ".join(f"{name}: {error}" for name, error in errors)
|
||||
raise RuntimeError(f"部分组件清理失败: {details}")
|
||||
|
||||
def get_component_status(self) -> Dict[str, str]:
|
||||
"""获取所有组件状态"""
|
||||
return {
|
||||
name: component.state.value
|
||||
for name, component in self._components.items()
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
from typing import Dict, Any
|
||||
from core.components.component_manager import ComponentManager, ComponentType
|
||||
from core.components.adapters.tts_adapter import TTSFactory
|
||||
from core.components.adapters.asr_adapter import ASRFactory
|
||||
from core.components.adapters.vad_adapter import VADFactory
|
||||
from core.components.adapters.llm_adapter import LLMFactory
|
||||
from core.components.adapters.memory_adapter import MemoryFactory
|
||||
from core.components.adapters.intent_adapter import IntentFactory
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ComponentRegistry:
|
||||
"""组件注册器：统一管理所有组件工厂的注册"""
|
||||
|
||||
_factories_registered = False
|
||||
|
||||
@classmethod
|
||||
def create_component_manager(cls, config: Dict[str, Any]) -> ComponentManager:
|
||||
"""创建并配置组件管理器"""
|
||||
manager = ComponentManager(config)
|
||||
|
||||
# 只在第一次时记录注册日志
|
||||
if not cls._factories_registered:
|
||||
logger.info("注册组件工厂")
|
||||
cls._factories_registered = True
|
||||
|
||||
# 注册所有组件工厂（每个manager都需要注册，但不重复记录日志）
|
||||
manager.register_factory(TTSFactory())
|
||||
manager.register_factory(ASRFactory())
|
||||
manager.register_factory(VADFactory())
|
||||
manager.register_factory(LLMFactory())
|
||||
manager.register_factory(MemoryFactory())
|
||||
manager.register_factory(IntentFactory())
|
||||
|
||||
# 设置组件初始化顺序（考虑依赖关系）
|
||||
# VAD -> ASR -> LLM -> Memory/Intent -> TTS
|
||||
initialization_order = [
|
||||
ComponentType.VAD,
|
||||
ComponentType.ASR,
|
||||
ComponentType.LLM,
|
||||
ComponentType.MEMORY,
|
||||
ComponentType.INTENT,
|
||||
ComponentType.TTS,
|
||||
]
|
||||
manager.set_initialization_order(initialization_order)
|
||||
|
||||
if not cls._factories_registered:
|
||||
logger.info("组件管理器创建完成，已注册所有组件工厂")
|
||||
|
||||
return manager
|
||||
|
||||
@staticmethod
|
||||
def get_required_components(config: Dict[str, Any]) -> list[ComponentType]:
|
||||
"""根据配置获取需要的组件类型"""
|
||||
required = []
|
||||
selected_modules = config.get("selected_module", {})
|
||||
|
||||
if selected_modules.get("VAD"):
|
||||
required.append(ComponentType.VAD)
|
||||
if selected_modules.get("ASR"):
|
||||
required.append(ComponentType.ASR)
|
||||
if selected_modules.get("LLM"):
|
||||
required.append(ComponentType.LLM)
|
||||
if selected_modules.get("TTS"):
|
||||
required.append(ComponentType.TTS)
|
||||
if selected_modules.get("Memory"):
|
||||
required.append(ComponentType.MEMORY)
|
||||
if selected_modules.get("Intent"):
|
||||
required.append(ComponentType.INTENT)
|
||||
|
||||
return required
|
||||
@@ -0,0 +1,505 @@
|
||||
import copy
|
||||
import uuid
|
||||
import time
|
||||
import queue
|
||||
import asyncio
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, Optional, List, Callable, Awaitable, Union
|
||||
from collections import deque
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from core.utils.dialogue import Dialogue
|
||||
from core.auth import AuthMiddleware
|
||||
from core.utils.prompt_manager import PromptManager
|
||||
from core.utils.voiceprint_provider import VoiceprintProvider
|
||||
from config.logger import setup_logging
|
||||
|
||||
|
||||
@dataclass
|
||||
class SessionContext:
|
||||
"""
|
||||
会话上下文：完全替换ConnectionHandler的所有功能
|
||||
承载单连接生命周期内的状态、组件、资源管理
|
||||
与传输层解耦，支持WebSocket/MQTT/UDP等多协议
|
||||
"""
|
||||
|
||||
# === 基础标识 ===
|
||||
session_id: str = field(default_factory=lambda: str(uuid.uuid4()))
|
||||
device_id: Optional[str] = None
|
||||
client_ip: Optional[str] = None
|
||||
headers: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# === 配置管理 ===
|
||||
config: Dict[str, Any] = field(default_factory=dict)
|
||||
common_config: Dict[str, Any] = field(default_factory=dict)
|
||||
private_config: Dict[str, Any] = field(default_factory=dict)
|
||||
selected_module_str: str = ""
|
||||
|
||||
# === 认证与绑定 ===
|
||||
is_authenticated: bool = False
|
||||
need_bind: bool = False
|
||||
bind_code: Optional[str] = None
|
||||
read_config_from_api: bool = False
|
||||
max_output_size: int = 0
|
||||
chat_history_conf: int = 0
|
||||
|
||||
# === 会话状态 ===
|
||||
is_speaking: bool = False
|
||||
listen_mode: str = "auto"
|
||||
abort_requested: bool = False
|
||||
close_after_chat: bool = False
|
||||
conversation_active: bool = False
|
||||
last_finalized_session_id: Optional[str] = None
|
||||
just_woken_up: bool = False
|
||||
calling: bool = False
|
||||
incoming_call: Optional[Any] = None
|
||||
load_function_plugin: bool = False
|
||||
intent_type: str = "nointent"
|
||||
|
||||
# === 音频相关 ===
|
||||
audio_format: str = "opus"
|
||||
# sample_rate/channels/frame_duration remain the output-side compatibility
|
||||
# fields consumed by existing TTS providers.
|
||||
sample_rate: int = 24000
|
||||
channels: int = 1
|
||||
frame_duration: int = 60
|
||||
input_sample_rate: int = 16000
|
||||
input_channels: int = 1
|
||||
input_frame_duration: int = 60
|
||||
output_sample_rate: int = 24000
|
||||
output_channels: int = 1
|
||||
output_frame_duration: int = 60
|
||||
client_aec: bool = False
|
||||
# Native MQTT control and UDP audio use different channels. Keep UDP
|
||||
# closed until the first listen/start of a logical session. It stays open
|
||||
# through the bounded listen/stop grace period so queued tail frames are
|
||||
# included in the same ASR turn.
|
||||
accepting_input_audio: bool = False
|
||||
wake_audio_suppression_task: Optional[asyncio.Task] = None
|
||||
listen_stop_pending: bool = False
|
||||
listen_stop_task: Optional[asyncio.Task] = None
|
||||
listen_stop_deadline: float = 0.0
|
||||
listen_start_task: Optional[asyncio.Task] = None
|
||||
client_have_voice: bool = False
|
||||
client_voice_stop: bool = False
|
||||
client_audio_buffer: bytearray = field(default_factory=bytearray)
|
||||
client_voice_window: deque = field(default_factory=lambda: deque(maxlen=5))
|
||||
last_is_voice: bool = False
|
||||
audio_flow_control: Dict[str, Any] = field(default_factory=dict)
|
||||
aec_audio_cache: Dict[int, bytes] = field(default_factory=dict)
|
||||
aec_audio_cache_time: Dict[int, float] = field(default_factory=dict)
|
||||
|
||||
# === ASR相关 ===
|
||||
asr_audio: List[bytes] = field(default_factory=list)
|
||||
asr_audio_queue: queue.Queue = field(default_factory=queue.Queue)
|
||||
asr_priority_thread: Optional[threading.Thread] = None
|
||||
asr_result_handler: Optional[Callable[[str, List[bytes]], Awaitable[None]]] = None
|
||||
uses_pipeline_runtime: bool = True
|
||||
|
||||
# === LLM相关 ===
|
||||
llm_finish_task: bool = True
|
||||
dialogue: Optional[Dialogue] = None
|
||||
current_speaker: Optional[str] = None
|
||||
introduced_speakers: set = field(default_factory=set)
|
||||
system_introduced_speakers: set = field(default_factory=set)
|
||||
sentence_id: Optional[str] = None
|
||||
|
||||
# === TTS相关 ===
|
||||
tts_MessageText: str = ""
|
||||
|
||||
# === IoT相关 ===
|
||||
iot_descriptors: Dict[str, Any] = field(default_factory=dict)
|
||||
func_handler: Optional[Any] = None
|
||||
|
||||
# === 时间管理 ===
|
||||
last_activity_time_ms: float = field(default_factory=lambda: time.time() * 1000)
|
||||
created_at: float = field(default_factory=lambda: time.time())
|
||||
timeout_seconds: int = 180 # 默认超时时间
|
||||
timeout_task: Optional[asyncio.Task] = None
|
||||
|
||||
# === 组件实例 ===
|
||||
# components属性通过@property方法提供，指向component_manager
|
||||
|
||||
# === 其他状态 ===
|
||||
welcome_msg: Optional[Dict[str, Any]] = None
|
||||
prompt: Optional[str] = None
|
||||
features: Optional[Dict[str, Any]] = None
|
||||
mcp_client: Optional[Any] = None
|
||||
cmd_exit: List[str] = field(default_factory=list)
|
||||
|
||||
# === 线程与并发 ===
|
||||
loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
stop_event: Optional[threading.Event] = None
|
||||
executor: Optional[ThreadPoolExecutor] = None
|
||||
conversation_tasks: set = field(default_factory=set)
|
||||
turn_tasks: set = field(default_factory=set)
|
||||
background_tasks: set = field(default_factory=set)
|
||||
|
||||
# === 队列管理 ===
|
||||
report_queue: queue.Queue = field(
|
||||
default_factory=lambda: queue.Queue(maxsize=64)
|
||||
)
|
||||
report_thread: Optional[threading.Thread] = None
|
||||
report_asr_enable: bool = False
|
||||
report_tts_enable: bool = False
|
||||
|
||||
# === 组件管理器 ===
|
||||
component_manager: Optional[Any] = None
|
||||
|
||||
# === 兼容属性（用于向后兼容TTS处理） ===
|
||||
tts: Optional[Any] = None
|
||||
websocket: Optional[Any] = None # 兼容旧TTS组件
|
||||
transport: Optional[Any] = None # 新的transport接口
|
||||
|
||||
# === 工具类 ===
|
||||
auth: Optional[AuthMiddleware] = None
|
||||
prompt_manager: Optional[PromptManager] = None
|
||||
voiceprint_provider: Optional[VoiceprintProvider] = None
|
||||
server: Optional[Any] = None # WebSocket服务器引用
|
||||
|
||||
# === 会话级清理回调 ===
|
||||
_cleanup_callbacks: List[Callable[[], Union[None, Awaitable[None]]]] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self):
|
||||
"""初始化后处理"""
|
||||
# 深拷贝配置避免污染
|
||||
if self.config:
|
||||
self.common_config = self.config
|
||||
self.config = copy.deepcopy(self.config)
|
||||
|
||||
# 从配置中读取相关设置
|
||||
self.read_config_from_api = self.config.get("read_config_from_api", False)
|
||||
self.max_output_size = self.config.get("max_output_size", 0)
|
||||
self.chat_history_conf = self.config.get("chat_history_conf", 0)
|
||||
self.cmd_exit = self.config.get("exit_commands", [])
|
||||
self.timeout_seconds = int(self.config.get("close_connection_no_voice_time", 120)) + 60
|
||||
|
||||
# 初始化认证中间件
|
||||
self.auth = AuthMiddleware(self.config)
|
||||
|
||||
# 初始化提示词管理器
|
||||
self.prompt_manager = PromptManager(self.config, setup_logging())
|
||||
|
||||
# 初始化对话管理
|
||||
if not self.dialogue:
|
||||
self.dialogue = Dialogue()
|
||||
|
||||
# 初始化线程相关
|
||||
if not self.loop:
|
||||
try:
|
||||
self.loop = asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
self.loop = asyncio.new_event_loop()
|
||||
|
||||
if not self.stop_event:
|
||||
self.stop_event = threading.Event()
|
||||
|
||||
if not self.executor:
|
||||
self.executor = ThreadPoolExecutor(max_workers=5)
|
||||
|
||||
# 初始化上报设置
|
||||
self.report_asr_enable = self.read_config_from_api
|
||||
self.report_tts_enable = self.read_config_from_api
|
||||
|
||||
def update_activity(self) -> None:
|
||||
"""刷新最后活跃时间"""
|
||||
self.last_activity_time_ms = time.time() * 1000
|
||||
|
||||
def clearSpeakStatus(self) -> None:
|
||||
"""清除服务端讲话状态（兼容方法）"""
|
||||
self.is_speaking = False
|
||||
logger = setup_logging()
|
||||
logger.debug("清除服务端讲话状态")
|
||||
|
||||
def change_system_prompt(self, prompt: str) -> None:
|
||||
"""Update the active role prompt for legacy function plugins."""
|
||||
self.prompt = prompt
|
||||
self.dialogue.update_system_message(prompt)
|
||||
|
||||
def reset_vad_states(self) -> None:
|
||||
"""重置VAD状态（兼容方法）"""
|
||||
self.client_audio_buffer = bytearray()
|
||||
self.client_have_voice = False
|
||||
self.client_voice_stop = False
|
||||
self.last_is_voice = False
|
||||
self.client_voice_window.clear()
|
||||
logger = setup_logging()
|
||||
logger.debug("VAD states reset.")
|
||||
|
||||
def reset_audio_states(self) -> None:
|
||||
"""Reset all turn-scoped input state expected by legacy ASR providers."""
|
||||
self.asr_audio.clear()
|
||||
self.listen_stop_pending = False
|
||||
self.reset_vad_states()
|
||||
|
||||
def is_timeout(self, timeout_seconds: int) -> bool:
|
||||
"""检查是否超时"""
|
||||
now_ms = time.time() * 1000
|
||||
return (now_ms - self.last_activity_time_ms) > (timeout_seconds * 1000)
|
||||
|
||||
def register_cleanup(self, callback: Callable[[], Union[None, Awaitable[None]]]) -> None:
|
||||
"""注册会话结束时需要执行的清理回调"""
|
||||
self._cleanup_callbacks.append(callback)
|
||||
|
||||
def unregister_cleanup(
|
||||
self, callback: Callable[[], Union[None, Awaitable[None]]]
|
||||
) -> None:
|
||||
"""Remove a callback when its connection-scoped owner is replaced."""
|
||||
self._cleanup_callbacks = [
|
||||
registered
|
||||
for registered in self._cleanup_callbacks
|
||||
if registered != callback
|
||||
]
|
||||
|
||||
def create_background_task(
|
||||
self,
|
||||
coroutine,
|
||||
*,
|
||||
conversation_scoped: bool = True,
|
||||
turn_scoped: bool = False,
|
||||
) -> asyncio.Task:
|
||||
"""Create a tracked task with an explicit turn/session/connection owner."""
|
||||
task = asyncio.create_task(coroutine)
|
||||
if turn_scoped:
|
||||
owner = self.turn_tasks
|
||||
elif conversation_scoped:
|
||||
owner = self.conversation_tasks
|
||||
else:
|
||||
owner = self.background_tasks
|
||||
owner.add(task)
|
||||
task.add_done_callback(owner.discard)
|
||||
return task
|
||||
|
||||
async def cancel_turn_tasks(self) -> None:
|
||||
"""Cancel producers belonging only to the active speech/chat turn."""
|
||||
current_task = asyncio.current_task()
|
||||
pending_tasks = [
|
||||
task
|
||||
for task in list(self.turn_tasks)
|
||||
if task is not current_task and not task.done()
|
||||
]
|
||||
for task in pending_tasks:
|
||||
task.cancel()
|
||||
if pending_tasks:
|
||||
await asyncio.gather(*pending_tasks, return_exceptions=True)
|
||||
self.turn_tasks = {
|
||||
task for task in self.turn_tasks if task is current_task
|
||||
}
|
||||
|
||||
async def cancel_conversation_tasks(self) -> None:
|
||||
"""Cancel all producers owned by the current logical conversation."""
|
||||
await self.cancel_turn_tasks()
|
||||
current_task = asyncio.current_task()
|
||||
while True:
|
||||
pending_tasks = [
|
||||
task
|
||||
for task in list(self.conversation_tasks)
|
||||
if task is not current_task and not task.done()
|
||||
]
|
||||
if not pending_tasks:
|
||||
break
|
||||
for task in pending_tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*pending_tasks, return_exceptions=True)
|
||||
self.conversation_tasks = {
|
||||
task for task in self.conversation_tasks if task is current_task
|
||||
}
|
||||
|
||||
async def run_cleanup(self) -> None:
|
||||
"""执行所有注册的清理回调"""
|
||||
logger = setup_logging()
|
||||
logger.info(f"Session {self.session_id} 开始执行会话级清理 ({len(self._cleanup_callbacks)} 个回调)")
|
||||
|
||||
# 停止所有线程
|
||||
if self.stop_event:
|
||||
self.stop_event.set()
|
||||
|
||||
# 取消超时任务
|
||||
if self.timeout_task and not self.timeout_task.done():
|
||||
self.timeout_task.cancel()
|
||||
|
||||
await self.cancel_conversation_tasks()
|
||||
|
||||
current_task = asyncio.current_task()
|
||||
pending_tasks = [
|
||||
task
|
||||
for task in list(self.background_tasks)
|
||||
if task is not current_task and not task.done()
|
||||
]
|
||||
for task in pending_tasks:
|
||||
task.cancel()
|
||||
if pending_tasks:
|
||||
await asyncio.gather(*pending_tasks, return_exceptions=True)
|
||||
self.background_tasks.clear()
|
||||
|
||||
# 执行清理回调
|
||||
for callback in reversed(self._cleanup_callbacks):
|
||||
try:
|
||||
result = callback()
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
except Exception as e:
|
||||
logger.error(f"Session {self.session_id} 清理回调执行失败: {e}", exc_info=True)
|
||||
|
||||
self._cleanup_callbacks.clear()
|
||||
|
||||
# 所有异步生产者退出后再关闭线程池，避免任务继续提交工作。
|
||||
if self.executor:
|
||||
self.executor.shutdown(wait=False)
|
||||
self.executor = None
|
||||
logger.info(f"Session {self.session_id} 会话级清理完成")
|
||||
|
||||
# === 兼容旧代码的属性访问 ===
|
||||
@property
|
||||
def client_is_speaking(self) -> bool:
|
||||
"""兼容旧代码的属性名"""
|
||||
return self.is_speaking
|
||||
|
||||
@client_is_speaking.setter
|
||||
def client_is_speaking(self, value: bool):
|
||||
self.is_speaking = value
|
||||
|
||||
@property
|
||||
def client_listen_mode(self) -> str:
|
||||
"""兼容旧代码的属性名"""
|
||||
return self.listen_mode
|
||||
|
||||
@client_listen_mode.setter
|
||||
def client_listen_mode(self, value: str):
|
||||
self.listen_mode = value
|
||||
|
||||
@property
|
||||
def client_abort(self) -> bool:
|
||||
"""兼容旧代码的属性名"""
|
||||
return self.abort_requested
|
||||
|
||||
@client_abort.setter
|
||||
def client_abort(self, value: bool):
|
||||
self.abort_requested = value
|
||||
|
||||
@property
|
||||
def components(self):
|
||||
"""组件访问器（兼容属性）"""
|
||||
return self.component_manager
|
||||
|
||||
@components.setter
|
||||
def components(self, value):
|
||||
"""组件设置器（兼容属性）- 实际设置到component_manager"""
|
||||
# 如果尝试设置components，我们忽略它或者给出警告
|
||||
# 因为components应该通过component_manager管理
|
||||
logger = setup_logging()
|
||||
logger.warning("尝试直接设置components属性，请使用component_manager")
|
||||
|
||||
@property
|
||||
def last_activity_time(self) -> float:
|
||||
"""兼容旧代码：返回毫秒级时间戳"""
|
||||
return self.last_activity_time_ms
|
||||
|
||||
@last_activity_time.setter
|
||||
def last_activity_time(self, value: float):
|
||||
"""兼容旧代码：接受毫秒级时间戳"""
|
||||
self.last_activity_time_ms = value
|
||||
|
||||
# === 日志相关 ===
|
||||
@property
|
||||
def logger(self):
|
||||
"""获取日志记录器"""
|
||||
return setup_logging()
|
||||
|
||||
# === 工具方法 ===
|
||||
def get_component(self, component_name: str) -> Optional[Any]:
|
||||
"""获取组件实例"""
|
||||
return self.components.get(component_name)
|
||||
|
||||
def set_component(self, component_name: str, component_instance: Any) -> None:
|
||||
"""设置组件实例"""
|
||||
if self.component_manager:
|
||||
self.component_manager._components[component_name] = component_instance
|
||||
|
||||
def has_component(self, component_name: str) -> bool:
|
||||
"""检查是否有指定组件"""
|
||||
return component_name in self.components
|
||||
|
||||
def clear_audio_buffer(self) -> None:
|
||||
"""清空音频缓冲区"""
|
||||
self.client_audio_buffer.clear()
|
||||
self.asr_audio.clear()
|
||||
|
||||
# 清空队列
|
||||
try:
|
||||
while not self.asr_audio_queue.empty():
|
||||
self.asr_audio_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
pass
|
||||
|
||||
def reset_voice_state(self) -> None:
|
||||
"""重置语音状态"""
|
||||
self.client_have_voice = False
|
||||
self.client_voice_stop = False
|
||||
self.last_is_voice = False
|
||||
self.client_voice_window.clear()
|
||||
|
||||
def initialize_private_config(self) -> None:
|
||||
"""初始化差异化配置（从ConnectionHandler迁移）"""
|
||||
from config.config_loader import get_private_config_from_api
|
||||
from config.manage_api_client import DeviceNotFoundException, DeviceBindException
|
||||
|
||||
if not self.read_config_from_api:
|
||||
return
|
||||
|
||||
try:
|
||||
# 获取设备私有配置
|
||||
private_config = get_private_config_from_api(
|
||||
self.config, self.device_id, self.headers.get("client-id")
|
||||
)
|
||||
|
||||
if private_config:
|
||||
self.private_config = private_config
|
||||
# 合并私有配置到主配置
|
||||
self.config.update(private_config)
|
||||
|
||||
except DeviceNotFoundException:
|
||||
self.logger.error(f"设备 {self.device_id} 未找到")
|
||||
self.need_bind = True
|
||||
except DeviceBindException as e:
|
||||
self.logger.error(f"设备绑定异常: {e}")
|
||||
self.need_bind = True
|
||||
self.bind_code = str(e)
|
||||
except Exception as e:
|
||||
self.logger.error(f"获取私有配置失败: {e}")
|
||||
|
||||
async def initialize_components(self) -> None:
|
||||
"""异步初始化组件（从ConnectionHandler迁移）"""
|
||||
if not self.component_manager:
|
||||
return
|
||||
|
||||
try:
|
||||
# 初始化各个组件
|
||||
from core.components.component_registry import ComponentType
|
||||
|
||||
# 按依赖顺序初始化组件
|
||||
component_types = [
|
||||
ComponentType.VAD,
|
||||
ComponentType.ASR,
|
||||
ComponentType.LLM,
|
||||
ComponentType.MEMORY,
|
||||
ComponentType.INTENT,
|
||||
ComponentType.TTS
|
||||
]
|
||||
|
||||
for component_type in component_types:
|
||||
try:
|
||||
component = await self.component_manager.get_component(component_type, self)
|
||||
if component:
|
||||
self.logger.info(f"组件 {component_type} 初始化成功")
|
||||
except Exception as e:
|
||||
self.logger.error(f"组件 {component_type} 初始化失败: {e}")
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"组件初始化失败: {e}")
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"SessionContext(session_id={self.session_id}, device_id={self.device_id})"
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.__str__()
|
||||
@@ -350,4 +350,16 @@ async def send_display_message(conn: "ConnectionHandler", text):
|
||||
"text": text,
|
||||
"session_id": conn.session_id
|
||||
}
|
||||
await conn.websocket.send(json.dumps(message))
|
||||
transport = getattr(conn, "transport", None)
|
||||
if transport is not None:
|
||||
send_json = getattr(transport, "send_json", None)
|
||||
if callable(send_json):
|
||||
await send_json(message)
|
||||
else:
|
||||
await transport.send(json.dumps(message))
|
||||
return
|
||||
|
||||
websocket = getattr(conn, "websocket", None)
|
||||
if websocket is None:
|
||||
raise AttributeError("无法找到可用的传输层接口")
|
||||
await websocket.send(json.dumps(message))
|
||||
|
||||
@@ -13,6 +13,33 @@ class SimpleHttpServer:
|
||||
self.logger = setup_logging()
|
||||
self.ota_handler = OTAHandler(config)
|
||||
self.vision_handler = VisionHandler(config)
|
||||
self._started_event = asyncio.Event()
|
||||
self._stop_event = asyncio.Event()
|
||||
self._runner = None
|
||||
self._start_active = False
|
||||
self._cleanup_lock = asyncio.Lock()
|
||||
|
||||
async def wait_started(self, task: asyncio.Task, timeout: float = 10) -> None:
|
||||
"""Wait until the HTTP listener is bound or surface startup failure."""
|
||||
event_waiter = asyncio.create_task(self._started_event.wait())
|
||||
try:
|
||||
done, _ = await asyncio.wait(
|
||||
{task, event_waiter},
|
||||
timeout=timeout,
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
if self._started_event.is_set():
|
||||
if task.done():
|
||||
await task
|
||||
return
|
||||
if task in done:
|
||||
await task
|
||||
raise RuntimeError("HTTP服务器在就绪前退出")
|
||||
raise TimeoutError("等待HTTP服务器启动超时")
|
||||
finally:
|
||||
if not event_waiter.done():
|
||||
event_waiter.cancel()
|
||||
await asyncio.gather(event_waiter, return_exceptions=True)
|
||||
|
||||
def _get_websocket_url(self, local_ip: str, port: int) -> str:
|
||||
"""获取websocket地址
|
||||
@@ -33,7 +60,11 @@ class SimpleHttpServer:
|
||||
return f"ws://{local_ip}:{port}/xiaozhi/v1/"
|
||||
|
||||
async def start(self):
|
||||
runner = None
|
||||
self._start_active = True
|
||||
try:
|
||||
self._started_event.clear()
|
||||
self._stop_event.clear()
|
||||
server_config = self.config["server"]
|
||||
read_config_from_api = self.config.get("read_config_from_api", False)
|
||||
host = server_config.get("ip", "0.0.0.0")
|
||||
@@ -74,19 +105,39 @@ class SimpleHttpServer:
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
# 运行服务
|
||||
runner = web.AppRunner(app)
|
||||
self._runner = runner
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, host, port)
|
||||
await site.start()
|
||||
|
||||
# 保持服务运行
|
||||
while True:
|
||||
await asyncio.sleep(3600) # 每隔 1 小时检查一次
|
||||
self._started_event.set()
|
||||
await self._stop_event.wait()
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"HTTP服务器启动失败: {e}")
|
||||
import traceback
|
||||
|
||||
self.logger.bind(tag=TAG).error(f"错误堆栈: {traceback.format_exc()}")
|
||||
raise
|
||||
finally:
|
||||
self._started_event.clear()
|
||||
try:
|
||||
if runner is not None:
|
||||
await self._cleanup_runner()
|
||||
finally:
|
||||
self._start_active = False
|
||||
|
||||
async def _cleanup_runner(self):
|
||||
"""Cleanup the owned runner and retain it when cleanup must be retried."""
|
||||
async with self._cleanup_lock:
|
||||
runner = self._runner
|
||||
if runner is None:
|
||||
return
|
||||
await runner.cleanup()
|
||||
self._runner = None
|
||||
|
||||
async def stop(self):
|
||||
"""Stop the HTTP loop and retry cleanup left by a failed start task."""
|
||||
self._stop_event.set()
|
||||
if not self._start_active:
|
||||
await self._cleanup_runner()
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, List
|
||||
|
||||
|
||||
class MessageProcessor(ABC):
|
||||
"""消息处理器接口。返回 True 表示已处理并中止后续处理。"""
|
||||
|
||||
@abstractmethod
|
||||
async def process(self, context: Any, transport: Any, message: Any) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class MessagePipeline:
|
||||
"""责任链式消息处理管道。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._processors: List[MessageProcessor] = []
|
||||
|
||||
def add_processor(self, processor: MessageProcessor) -> None:
|
||||
self._processors.append(processor)
|
||||
|
||||
async def process_message(self, context: Any, transport: Any, message: Any) -> None:
|
||||
for processor in self._processors:
|
||||
handled = await processor.process(context, transport, message)
|
||||
if handled:
|
||||
return
|
||||
@@ -0,0 +1,132 @@
|
||||
import json
|
||||
import uuid
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.components.component_manager import ComponentType
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AbortProcessor(MessageProcessor):
|
||||
"""中断消息处理器：完整迁移abortMessageHandler.py和abortHandle.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理abort类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "abort":
|
||||
await self.handle_abort_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_abort_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理中断消息 - 完整迁移自abortHandle.py的handleAbortMessage"""
|
||||
msg_session_id = msg_json.get("session_id")
|
||||
if msg_session_id and msg_session_id != context.session_id:
|
||||
logger.warning(
|
||||
f"忽略非当前会话的abort消息: "
|
||||
f"msg={msg_session_id}, current={context.session_id}"
|
||||
)
|
||||
return
|
||||
logger.info("Abort message received")
|
||||
|
||||
# 设置成打断状态，会自动打断llm、tts任务 - 完整迁移原逻辑
|
||||
context.abort_requested = True
|
||||
context.close_after_chat = False
|
||||
|
||||
# Stop the old LLM/tool producers before a subsequent listen/start can
|
||||
# clear abort_requested and begin a new turn.
|
||||
cancel_tasks = getattr(context, "cancel_turn_tasks", None)
|
||||
if callable(cancel_tasks):
|
||||
await cancel_tasks()
|
||||
|
||||
# 清理队列 - 完整迁移原逻辑
|
||||
await self._clear_queues(context)
|
||||
|
||||
# 打断客户端说话状态 - 完整迁移原逻辑
|
||||
await transport.send_json({
|
||||
"type": "tts",
|
||||
"state": "stop",
|
||||
"session_id": context.session_id
|
||||
})
|
||||
|
||||
# 清理说话状态 - 完整迁移原逻辑
|
||||
self._clear_speak_status(context)
|
||||
|
||||
logger.info("Abort message received-end")
|
||||
|
||||
async def _clear_queues(self, context: SessionContext):
|
||||
"""清理所有队列 - 完整迁移原clear_queues逻辑"""
|
||||
try:
|
||||
# 清理TTS音频队列
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
for queue_name in ('tts_text_queue', 'tts_audio_queue'):
|
||||
pending_queue = getattr(tts_instance, queue_name, None)
|
||||
if pending_queue is None:
|
||||
continue
|
||||
try:
|
||||
while not pending_queue.empty():
|
||||
pending_queue.get_nowait()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Rotate the turn token so late text/audio from the aborted turn is stale.
|
||||
context.sentence_id = uuid.uuid4().hex
|
||||
|
||||
# 清理ASR音频队列
|
||||
context.clear_audio_buffer()
|
||||
|
||||
# 清理其他可能的队列
|
||||
if hasattr(context, 'clear_queues'):
|
||||
context.clear_queues()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"清理队列时出错: {e}")
|
||||
|
||||
def _clear_speak_status(self, context: SessionContext):
|
||||
"""清理说话状态 - 完整迁移原clearSpeakStatus逻辑"""
|
||||
try:
|
||||
# 清理说话状态
|
||||
context.is_speaking = False
|
||||
|
||||
# 如果有其他说话状态相关的属性，也一并清理
|
||||
if hasattr(context, 'clearSpeakStatus'):
|
||||
context.clearSpeakStatus()
|
||||
|
||||
# 重置相关状态
|
||||
context.client_have_voice = False
|
||||
context.client_voice_stop = True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"清理说话状态时出错: {e}")
|
||||
|
||||
async def _send_abort_confirmation(self, transport: TransportInterface, session_id: str):
|
||||
"""发送中断确认响应（可选）"""
|
||||
response = {
|
||||
"type": "abort",
|
||||
"status": "success",
|
||||
"message": "中断操作已完成",
|
||||
"session_id": session_id
|
||||
}
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送中断确认响应失败: {e}")
|
||||
|
||||
async def _get_component(self, context: SessionContext, component_type: ComponentType):
|
||||
if not context.component_manager:
|
||||
return None
|
||||
return await context.component_manager.get_component(component_type, context)
|
||||
@@ -0,0 +1,439 @@
|
||||
import time
|
||||
import json
|
||||
import asyncio
|
||||
import uuid
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.components.component_manager import ComponentType
|
||||
from core.services.audio_ingress_service import AudioIngressService
|
||||
from core.utils.util import audio_to_data
|
||||
from core.utils.output_counter import check_device_output_limit
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
def arm_wake_audio_suppression(context: SessionContext) -> None:
|
||||
"""Ignore reordered wake-word audio for a bounded interval."""
|
||||
context.just_woken_up = True
|
||||
previous = getattr(context, "wake_audio_suppression_task", None)
|
||||
if previous and not previous.done():
|
||||
previous.cancel()
|
||||
|
||||
delay_ms = max(
|
||||
0,
|
||||
int(context.config.get("wake_audio_suppression_ms", 2000)),
|
||||
)
|
||||
if delay_ms == 0:
|
||||
context.just_woken_up = False
|
||||
context.wake_audio_suppression_task = None
|
||||
return
|
||||
|
||||
session_id = context.session_id
|
||||
context.wake_audio_suppression_task = context.create_background_task(
|
||||
_release_wake_audio_suppression(
|
||||
context,
|
||||
session_id,
|
||||
delay_ms / 1000,
|
||||
),
|
||||
conversation_scoped=True,
|
||||
)
|
||||
|
||||
|
||||
async def _release_wake_audio_suppression(
|
||||
context: SessionContext,
|
||||
session_id: str,
|
||||
delay_seconds: float,
|
||||
) -> None:
|
||||
current_task = asyncio.current_task()
|
||||
try:
|
||||
await asyncio.sleep(delay_seconds)
|
||||
if context.session_id == session_id:
|
||||
context.just_woken_up = False
|
||||
finally:
|
||||
if getattr(context, "wake_audio_suppression_task", None) is current_task:
|
||||
context.wake_audio_suppression_task = None
|
||||
|
||||
|
||||
class AudioReceiveProcessor(MessageProcessor):
|
||||
"""音频接收处理器：完整迁移receiveAudioHandle.py的所有功能"""
|
||||
|
||||
def __init__(self, audio_ingress_service=None):
|
||||
self.audio_ingress_service = audio_ingress_service or AudioIngressService()
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理音频消息"""
|
||||
if isinstance(message, bytes):
|
||||
await self.handle_audio_message(context, transport, message, 0)
|
||||
return True
|
||||
if isinstance(message, dict) and message.get("type") == "audio":
|
||||
audio_data = message.get("data")
|
||||
if isinstance(audio_data, (bytes, bytearray)):
|
||||
await self.handle_audio_message(
|
||||
context,
|
||||
transport,
|
||||
bytes(audio_data),
|
||||
int(message.get("timestamp", 0) or 0),
|
||||
)
|
||||
return True
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_audio_message(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
audio: bytes,
|
||||
timestamp: int = 0,
|
||||
):
|
||||
"""处理音频消息 - 完整迁移自handleAudioMessage"""
|
||||
if (
|
||||
getattr(transport, "requires_audio_tail_grace", False)
|
||||
and not getattr(context, "accepting_input_audio", False)
|
||||
):
|
||||
return
|
||||
|
||||
pcm_frame = self.audio_ingress_service.process(context, audio, timestamp)
|
||||
if not pcm_frame:
|
||||
return
|
||||
|
||||
# Do not feed encoded wake-word history into stateful VAD. Native MQTT
|
||||
# gates the usual pre-start history above; this bounded suppression only
|
||||
# covers frames that crossed the independent control/audio channels.
|
||||
if context.just_woken_up:
|
||||
if not getattr(context, "wake_audio_suppression_task", None):
|
||||
arm_wake_audio_suppression(context)
|
||||
context.asr_audio.clear()
|
||||
context.reset_vad_states()
|
||||
return
|
||||
|
||||
# 获取VAD组件
|
||||
vad_component = await self._get_component(context, ComponentType.VAD)
|
||||
if not vad_component or not hasattr(vad_component, 'vad_instance'):
|
||||
now = time.time()
|
||||
last_log = getattr(context, "_vad_missing_last_log", 0.0)
|
||||
if now - last_log > 10:
|
||||
logger.warning("VAD组件未初始化")
|
||||
context._vad_missing_last_log = now
|
||||
return
|
||||
|
||||
vad_instance = vad_component.vad_instance
|
||||
|
||||
# 当前片段是否有人说话
|
||||
have_voice = vad_instance.is_vad(context, pcm_frame)
|
||||
|
||||
if have_voice:
|
||||
if (
|
||||
getattr(context, "client_aec", False)
|
||||
and context.is_speaking
|
||||
and context.listen_mode != "manual"
|
||||
):
|
||||
await self._handle_abort_message(context, transport)
|
||||
|
||||
# 设备长时间空闲检测，用于say goodbye
|
||||
await self._no_voice_close_connect(context, transport, have_voice)
|
||||
|
||||
# 接收音频
|
||||
asr_component = await self._get_component(context, ComponentType.ASR)
|
||||
asr_instance = asr_component.asr_instance if asr_component and hasattr(asr_component, 'asr_instance') else None
|
||||
|
||||
# 自动模式兜底：没有收到 listen stop 时，基于VAD触发一次停止
|
||||
if (
|
||||
not have_voice
|
||||
and context.client_have_voice
|
||||
and not context.client_voice_stop
|
||||
and not getattr(context, "listen_stop_pending", False)
|
||||
and context.listen_mode != "manual"
|
||||
and asr_instance is not None
|
||||
):
|
||||
context.client_voice_stop = True
|
||||
try:
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
if hasattr(asr_instance, "interface_type") and asr_instance.interface_type == InterfaceType.STREAM:
|
||||
if hasattr(asr_instance, "_send_stop_request"):
|
||||
context.create_background_task(
|
||||
asr_instance._send_stop_request(),
|
||||
turn_scoped=True,
|
||||
)
|
||||
else:
|
||||
if len(context.asr_audio) > 0:
|
||||
asr_audio_task = context.asr_audio.copy()
|
||||
context.asr_audio.clear()
|
||||
context.reset_vad_states()
|
||||
await asr_instance.handle_voice_stop(context, asr_audio_task)
|
||||
except Exception as e:
|
||||
logger.error(f"自动VAD停止处理失败: {e}")
|
||||
|
||||
if asr_instance and hasattr(asr_instance, 'receive_audio'):
|
||||
await asr_instance.receive_audio(context, pcm_frame, have_voice)
|
||||
|
||||
async def start_to_chat(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
text: str,
|
||||
*,
|
||||
preserve_close_after_chat: bool = False,
|
||||
):
|
||||
"""开始聊天 - 完整迁移自startToChat"""
|
||||
# 检查输入是否是JSON格式（包含说话人信息）
|
||||
speaker_name = None
|
||||
actual_text = text
|
||||
|
||||
try:
|
||||
# 尝试解析JSON格式的输入
|
||||
if text.strip().startswith('{') and text.strip().endswith('}'):
|
||||
data = json.loads(text)
|
||||
if 'speaker' in data and 'content' in data:
|
||||
speaker_name = data['speaker']
|
||||
actual_content = data['content']
|
||||
logger.info(f"解析到说话人信息: {speaker_name}")
|
||||
|
||||
if speaker_name not in context.introduced_speakers:
|
||||
context.introduced_speakers.add(speaker_name)
|
||||
actual_text = text
|
||||
else:
|
||||
actual_text = actual_content
|
||||
except (json.JSONDecodeError, KeyError):
|
||||
# 如果解析失败，继续使用原始文本
|
||||
pass
|
||||
|
||||
# 保存说话人信息到上下文
|
||||
if speaker_name:
|
||||
context.current_speaker = speaker_name
|
||||
else:
|
||||
context.current_speaker = None
|
||||
|
||||
# 检查设备绑定
|
||||
if context.need_bind:
|
||||
await self.prompt_bind_device(context, transport)
|
||||
return
|
||||
|
||||
# 如果当日的输出字数大于限定的字数
|
||||
if context.max_output_size > 0:
|
||||
if check_device_output_limit(
|
||||
context.headers.get("device-id"), context.max_output_size
|
||||
):
|
||||
await self._max_out_size(context, transport)
|
||||
return
|
||||
|
||||
if context.is_speaking and getattr(context, "listen_mode", "auto") != "manual":
|
||||
await self._handle_abort_message(context, transport)
|
||||
|
||||
context.abort_requested = False
|
||||
if not preserve_close_after_chat:
|
||||
context.close_after_chat = False
|
||||
|
||||
# 首先进行意图分析，使用实际文本内容
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
intent_handled = await chat_processor.handle_user_intent(context, transport, actual_text)
|
||||
|
||||
if intent_handled:
|
||||
# 如果意图已被处理，不再进行聊天
|
||||
return
|
||||
|
||||
# 意图未被处理，继续常规聊天流程，使用实际文本内容
|
||||
await self._send_stt_message(context, transport, actual_text)
|
||||
|
||||
# 与旧架构一致：将聊天处理放入线程池，避免阻塞事件循环
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
|
||||
if hasattr(context, "create_background_task"):
|
||||
context.create_background_task(
|
||||
chat_processor.handle_chat(
|
||||
context,
|
||||
transport,
|
||||
actual_text,
|
||||
skip_intent=True,
|
||||
),
|
||||
turn_scoped=True,
|
||||
)
|
||||
else:
|
||||
await chat_processor.handle_chat(
|
||||
context, transport, actual_text, skip_intent=True
|
||||
)
|
||||
|
||||
async def _no_voice_close_connect(self, context: SessionContext, transport: TransportInterface, have_voice: bool):
|
||||
"""无声音时关闭连接检测 - 完整迁移自no_voice_close_connect"""
|
||||
if have_voice:
|
||||
context.update_activity()
|
||||
return
|
||||
|
||||
# 只有在已经初始化过时间戳的情况下才进行超时检查
|
||||
if context.last_activity_time_ms > 0.0:
|
||||
no_voice_time = time.time() * 1000 - context.last_activity_time_ms
|
||||
close_connection_no_voice_time = int(
|
||||
context.config.get("close_connection_no_voice_time", 120)
|
||||
)
|
||||
|
||||
if (
|
||||
not context.close_after_chat
|
||||
and no_voice_time > 1000 * close_connection_no_voice_time
|
||||
):
|
||||
context.close_after_chat = True
|
||||
context.abort_requested = False
|
||||
|
||||
end_prompt = context.config.get("end_prompt", {})
|
||||
if end_prompt and end_prompt.get("enable", True) is False:
|
||||
logger.info("结束对话，无需发送结束提示语")
|
||||
if transport.keeps_connection_between_sessions:
|
||||
session_id = context.session_id
|
||||
end_conversation = getattr(context, "end_conversation", None)
|
||||
if callable(end_conversation):
|
||||
await end_conversation(session_id)
|
||||
await transport.end_session(session_id)
|
||||
context.close_after_chat = False
|
||||
context.reset_vad_states()
|
||||
else:
|
||||
await transport.close()
|
||||
return
|
||||
|
||||
prompt = end_prompt.get("prompt")
|
||||
if not prompt:
|
||||
prompt = "请你以```时间过得真快```未来头，用富有感情、依依不舍的话来结束这场对话吧。！"
|
||||
await self.start_to_chat(
|
||||
context,
|
||||
transport,
|
||||
prompt,
|
||||
preserve_close_after_chat=True,
|
||||
)
|
||||
|
||||
async def _max_out_size(self, context: SessionContext, transport: TransportInterface):
|
||||
"""超出最大输出字数处理 - 完整迁移自max_out_size"""
|
||||
# 播放超出最大输出字数的提示
|
||||
context.abort_requested = False
|
||||
text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
file_path = "config/assets/max_output_size.wav"
|
||||
opus_packets = await audio_to_data(file_path)
|
||||
|
||||
# 获取TTS组件并添加到队列
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
if hasattr(tts_instance, 'tts_audio_queue'):
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
tts_instance.tts_audio_queue.put(
|
||||
(SentenceType.LAST, opus_packets, text, context.sentence_id)
|
||||
)
|
||||
|
||||
context.close_after_chat = True
|
||||
|
||||
async def prompt_bind_device(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
):
|
||||
"""检查设备绑定 - 完整迁移自check_bind_device"""
|
||||
bind_code = context.bind_code
|
||||
|
||||
if bind_code:
|
||||
# 确保bind_code是6位数字
|
||||
if len(bind_code) != 6:
|
||||
logger.error(f"无效的绑定码格式: {bind_code}")
|
||||
text = "绑定码格式错误，请检查配置。"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
return
|
||||
|
||||
text = f"请登录控制面板，输入{bind_code}，绑定设备。"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
# 获取TTS组件
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if not tts_component or not hasattr(tts_component, 'tts_instance'):
|
||||
return
|
||||
|
||||
tts_instance = tts_component.tts_instance
|
||||
if not hasattr(tts_instance, 'tts_audio_queue'):
|
||||
return
|
||||
|
||||
# 播放提示音
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
music_path = "config/assets/bind_code.wav"
|
||||
opus_packets = await audio_to_data(music_path)
|
||||
tts_instance.tts_audio_queue.put(
|
||||
(SentenceType.FIRST, opus_packets, text, context.sentence_id)
|
||||
)
|
||||
|
||||
# 逐个播放数字
|
||||
for i in range(6): # 确保只播放6位数字
|
||||
try:
|
||||
digit = bind_code[i]
|
||||
num_path = f"config/assets/bind_code/{digit}.wav"
|
||||
num_packets = await audio_to_data(num_path)
|
||||
tts_instance.tts_audio_queue.put(
|
||||
(SentenceType.MIDDLE, num_packets, None, context.sentence_id)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"播放数字音频失败: {e}")
|
||||
continue
|
||||
tts_instance.tts_audio_queue.put(
|
||||
(SentenceType.LAST, [], None, context.sentence_id)
|
||||
)
|
||||
else:
|
||||
# 播放未绑定提示
|
||||
context.abort_requested = False
|
||||
text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
# 获取TTS组件
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
if hasattr(tts_instance, 'tts_audio_queue'):
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
music_path = "config/assets/bind_not_found.wav"
|
||||
opus_packets = await audio_to_data(music_path)
|
||||
tts_instance.tts_audio_queue.put(
|
||||
(SentenceType.LAST, opus_packets, text, context.sentence_id)
|
||||
)
|
||||
|
||||
async def _handle_abort_message(self, context: SessionContext, transport: TransportInterface):
|
||||
"""处理中断消息"""
|
||||
from core.processors.abort_processor import AbortProcessor
|
||||
|
||||
await AbortProcessor().handle_abort_message(
|
||||
context,
|
||||
transport,
|
||||
{"type": "abort", "session_id": context.session_id},
|
||||
)
|
||||
|
||||
async def _clear_queues(self, context: SessionContext):
|
||||
"""清理所有队列"""
|
||||
# 清理TTS音频队列
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
for queue_name in ('tts_text_queue', 'tts_audio_queue'):
|
||||
pending_queue = getattr(tts_instance, queue_name, None)
|
||||
if pending_queue is None:
|
||||
continue
|
||||
try:
|
||||
while not pending_queue.empty():
|
||||
pending_queue.get_nowait()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Invalidate late TTS text/audio generated by the interrupted turn.
|
||||
context.sentence_id = uuid.uuid4().hex
|
||||
|
||||
# 清理ASR音频队列
|
||||
context.clear_audio_buffer()
|
||||
|
||||
async def _get_component(self, context: SessionContext, component_type: ComponentType):
|
||||
if not context.component_manager:
|
||||
return None
|
||||
return await context.component_manager.get_component(component_type, context)
|
||||
|
||||
async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息"""
|
||||
from core.processors.audio_send_processor import AudioSendProcessor
|
||||
|
||||
await AudioSendProcessor().send_stt_message(
|
||||
context, transport, text
|
||||
)
|
||||
@@ -0,0 +1,578 @@
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from typing import Any, List
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
from core.utils import textUtils
|
||||
from core.components.component_manager import ComponentType
|
||||
from core.services.audio_ingress_service import AudioIngressService
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AudioSendProcessor(MessageProcessor):
|
||||
"""音频发送处理器：完整迁移sendAudioHandle.py的所有功能"""
|
||||
|
||||
def __init__(self, audio_ingress_service=None):
|
||||
self.audio_ingress_service = audio_ingress_service or AudioIngressService()
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""这个处理器不直接处理消息，而是被其他处理器调用"""
|
||||
return False
|
||||
|
||||
async def send_audio_message(self, context: SessionContext, transport: TransportInterface,
|
||||
sentence_type: SentenceType, audios: bytes, text: str,
|
||||
sentence_id: str = None):
|
||||
"""发送音频消息 - 完整迁移自sendAudioMessage"""
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
return
|
||||
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if not tts_component or not hasattr(tts_component, 'tts_instance'):
|
||||
return
|
||||
|
||||
tts_instance = tts_component.tts_instance
|
||||
|
||||
if hasattr(tts_instance, 'tts_audio_first_sentence') and tts_instance.tts_audio_first_sentence:
|
||||
logger.info(f"发送第一段语音: {text}")
|
||||
tts_instance.tts_audio_first_sentence = False
|
||||
await self.start_tts_stream(context, transport)
|
||||
|
||||
if sentence_type == SentenceType.FIRST:
|
||||
await self.send_tts_message(
|
||||
context, transport, "sentence_start", text, sentence_id=sentence_id
|
||||
)
|
||||
|
||||
await self.send_audio(
|
||||
context, transport, audios, sentence_id=sentence_id
|
||||
)
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
return
|
||||
|
||||
# 发送句子开始消息
|
||||
if sentence_type is not SentenceType.MIDDLE:
|
||||
logger.info(f"发送音频消息: {sentence_type}, {text}")
|
||||
|
||||
# 发送结束消息（如果是最后一个文本）
|
||||
if (
|
||||
not getattr(context, "calling", False)
|
||||
and context.llm_finish_task
|
||||
and sentence_type == SentenceType.LAST
|
||||
):
|
||||
# Latch the terminal action before sending tts:stop. The device can
|
||||
# immediately answer with listen:start, whose handler resets the
|
||||
# mutable turn flags while this coroutine is still awaiting I/O.
|
||||
close_after_chat = bool(context.close_after_chat)
|
||||
closing_session_id = (
|
||||
getattr(transport, "session_id", None) or context.session_id
|
||||
)
|
||||
if close_after_chat:
|
||||
context.close_after_chat = False
|
||||
await self.send_tts_message(
|
||||
context, transport, "stop", None, sentence_id=sentence_id
|
||||
)
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
return
|
||||
context.is_speaking = False
|
||||
if close_after_chat:
|
||||
# MQTT/UDP：结束后回到Idle，不关闭连接
|
||||
if transport.keeps_connection_between_sessions:
|
||||
end_conversation = getattr(context, "end_conversation", None)
|
||||
if callable(end_conversation):
|
||||
await end_conversation(closing_session_id)
|
||||
await transport.end_session(closing_session_id)
|
||||
else:
|
||||
await transport.close()
|
||||
|
||||
async def send_audio(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
audios: bytes,
|
||||
frame_duration: int = 60,
|
||||
sentence_id: str = None,
|
||||
):
|
||||
"""发送单个opus包，支持流控 - 完整迁移自sendAudio"""
|
||||
if audios is None or len(audios) == 0:
|
||||
return
|
||||
if getattr(context, "audio_flow_control", {}).get("send_failed"):
|
||||
return
|
||||
|
||||
# MQTT/UDP：等待UDP远端地址就绪，避免首包丢失
|
||||
if transport.has_datagram_audio:
|
||||
if not await transport.wait_audio_ready(timeout=2):
|
||||
logger.warning("UDP远端地址未就绪，跳过音频发送")
|
||||
return
|
||||
|
||||
audio_list = [audios] if isinstance(audios, bytes) else audios
|
||||
if not isinstance(audio_list, list):
|
||||
return
|
||||
|
||||
flow = context.audio_flow_control
|
||||
pre_buffer_count = max(
|
||||
0, int(context.config.get("tts_pre_buffer_count", 5))
|
||||
)
|
||||
for audio in audio_list:
|
||||
if context.abort_requested or not self._is_current_turn(context, sentence_id):
|
||||
break
|
||||
|
||||
# Match the legacy AudioRateController contract: send only the
|
||||
# initial pre-buffer inline, then enqueue the remaining frames so
|
||||
# the TTS consumer can prefetch later sentence chunks.
|
||||
packet_count = int(flow.get("packet_count", 0))
|
||||
if packet_count < pre_buffer_count and "_send_queue" not in flow:
|
||||
sent = await self._send_audio_packet(
|
||||
context,
|
||||
transport,
|
||||
flow,
|
||||
audio,
|
||||
frame_duration,
|
||||
sentence_id,
|
||||
paced=False,
|
||||
)
|
||||
if not sent:
|
||||
break
|
||||
continue
|
||||
|
||||
queue = self._ensure_audio_sender(
|
||||
context, transport, flow, sentence_id
|
||||
)
|
||||
await queue.put(("audio", audio, frame_duration, sentence_id))
|
||||
|
||||
@staticmethod
|
||||
async def _pace_audio_send(
|
||||
context: SessionContext,
|
||||
frame_duration: int,
|
||||
flow: dict = None,
|
||||
) -> None:
|
||||
flow = flow if flow is not None else context.audio_flow_control
|
||||
packet_count = int(flow.get("packet_count", 0))
|
||||
pre_buffer_count = max(0, int(context.config.get("tts_pre_buffer_count", 5)))
|
||||
if packet_count < pre_buffer_count:
|
||||
return
|
||||
|
||||
configured_delay = int(context.config.get("tts_audio_send_delay", -1))
|
||||
if configured_delay > 0:
|
||||
await asyncio.sleep(configured_delay / 1000.0)
|
||||
return
|
||||
|
||||
delay_ms = max(0, int(frame_duration))
|
||||
if delay_ms <= 0:
|
||||
return
|
||||
|
||||
now = time.monotonic()
|
||||
# packet_count is the number already sent. The first packet after the
|
||||
# pre-buffer must therefore wait one complete frame, not become an
|
||||
# extra immediate packet.
|
||||
paced_packet_index = packet_count - pre_buffer_count + 1
|
||||
pacing_started_at = flow.get("pacing_started_at")
|
||||
pacing_delay_ms = flow.get("pacing_delay_ms")
|
||||
if pacing_started_at is None or pacing_delay_ms != delay_ms:
|
||||
pacing_started_at = now
|
||||
flow["pacing_started_at"] = pacing_started_at
|
||||
flow["pacing_delay_ms"] = delay_ms
|
||||
|
||||
delay_seconds = delay_ms / 1000.0
|
||||
target = pacing_started_at + paced_packet_index * delay_seconds
|
||||
|
||||
# TTS producers can pause between sentence chunks. Rebase after a
|
||||
# full-frame gap instead of bursting stale deadlines to catch up.
|
||||
if now - target >= delay_seconds:
|
||||
pacing_started_at = now - paced_packet_index * delay_seconds
|
||||
flow["pacing_started_at"] = pacing_started_at
|
||||
target = now
|
||||
|
||||
wait_seconds = target - now
|
||||
if wait_seconds > 0:
|
||||
await asyncio.sleep(wait_seconds)
|
||||
|
||||
def _ensure_audio_sender(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
flow: dict,
|
||||
sentence_id: str = None,
|
||||
) -> asyncio.Queue:
|
||||
queue = flow.get("_send_queue")
|
||||
task = flow.get("_send_task")
|
||||
if queue is not None and task is not None and not task.done():
|
||||
return queue
|
||||
|
||||
queue_size = max(
|
||||
1, int(context.config.get("tts_audio_queue_size", 256))
|
||||
)
|
||||
queue = asyncio.Queue(maxsize=queue_size)
|
||||
sender = self._audio_send_loop(
|
||||
context, transport, flow, queue, sentence_id
|
||||
)
|
||||
create_task = getattr(context, "create_background_task", None)
|
||||
if callable(create_task):
|
||||
task = create_task(sender, turn_scoped=True)
|
||||
else:
|
||||
task = asyncio.create_task(sender)
|
||||
flow["_send_queue"] = queue
|
||||
flow["_send_task"] = task
|
||||
return queue
|
||||
|
||||
async def _audio_send_loop(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
flow: dict,
|
||||
queue: asyncio.Queue,
|
||||
sentence_id: str = None,
|
||||
) -> None:
|
||||
try:
|
||||
while True:
|
||||
item = await queue.get()
|
||||
try:
|
||||
item_type = item[0]
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
continue
|
||||
if item_type == "audio":
|
||||
_, audio, frame_duration, item_sentence_id = item
|
||||
await self._send_audio_packet(
|
||||
context,
|
||||
transport,
|
||||
flow,
|
||||
audio,
|
||||
frame_duration,
|
||||
item_sentence_id,
|
||||
paced=True,
|
||||
)
|
||||
elif item_type == "json":
|
||||
_, message, item_sentence_id = item
|
||||
if self._is_current_turn(context, item_sentence_id):
|
||||
await transport.send_json(message)
|
||||
finally:
|
||||
queue.task_done()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as error:
|
||||
flow["send_failed"] = True
|
||||
logger.error("后台音频发送循环失败: {}", error)
|
||||
finally:
|
||||
while True:
|
||||
try:
|
||||
queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
else:
|
||||
queue.task_done()
|
||||
|
||||
async def _send_audio_packet(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
flow: dict,
|
||||
audio: bytes,
|
||||
frame_duration: int,
|
||||
sentence_id: str = None,
|
||||
*,
|
||||
paced: bool,
|
||||
) -> bool:
|
||||
if paced:
|
||||
await self._pace_audio_send(context, frame_duration, flow)
|
||||
if context.abort_requested or not self._is_current_turn(context, sentence_id):
|
||||
return False
|
||||
|
||||
context.update_activity()
|
||||
timestamp = self._next_audio_timestamp(
|
||||
context, frame_duration, flow
|
||||
)
|
||||
sent = await self._send_audio_with_retry(
|
||||
context,
|
||||
transport,
|
||||
audio,
|
||||
timestamp,
|
||||
sentence_id,
|
||||
flow,
|
||||
)
|
||||
if not sent or not self._is_current_turn(context, sentence_id):
|
||||
return False
|
||||
self.audio_ingress_service.cache_output_reference(
|
||||
context, audio, timestamp
|
||||
)
|
||||
flow["packet_count"] = int(flow.get("packet_count", 0)) + 1
|
||||
return True
|
||||
|
||||
async def _send_audio_with_retry(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
audio: bytes,
|
||||
timestamp: int,
|
||||
sentence_id: str = None,
|
||||
flow: dict = None,
|
||||
) -> bool:
|
||||
flow = flow if flow is not None else context.audio_flow_control
|
||||
retries = max(0, int(context.config.get("audio_send_retries", 2)))
|
||||
retry_delay = max(
|
||||
0, int(context.config.get("audio_send_retry_delay_ms", 20))
|
||||
) / 1000.0
|
||||
for attempt in range(retries + 1):
|
||||
if context.abort_requested or not self._is_current_turn(context, sentence_id):
|
||||
return False
|
||||
try:
|
||||
await transport.send_audio(audio, timestamp)
|
||||
return self._is_current_turn(context, sentence_id)
|
||||
except Exception as error:
|
||||
if attempt >= retries:
|
||||
flow["send_failed"] = True
|
||||
logger.error(
|
||||
"Audio send failed after {} attempts: {}",
|
||||
attempt + 1,
|
||||
error,
|
||||
)
|
||||
raise
|
||||
await asyncio.sleep(retry_delay)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _next_audio_timestamp(
|
||||
context: SessionContext,
|
||||
frame_duration: int,
|
||||
flow: dict = None,
|
||||
) -> int:
|
||||
flow = flow if flow is not None else getattr(
|
||||
context, "audio_flow_control", None
|
||||
)
|
||||
if flow is None:
|
||||
flow = {}
|
||||
context.audio_flow_control = flow
|
||||
timestamp_step = int(frame_duration) or int(
|
||||
getattr(context, "output_frame_duration", 60) or 60
|
||||
)
|
||||
timestamp = int(flow.get("output_timestamp", 0)) + timestamp_step
|
||||
timestamp %= 2 ** 32
|
||||
flow["output_timestamp"] = timestamp
|
||||
return timestamp
|
||||
|
||||
async def send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息 - 完整迁移自send_stt_message"""
|
||||
end_prompt = getattr(context, "config", {}).get("end_prompt", {}).get("prompt")
|
||||
if end_prompt and text == end_prompt:
|
||||
await self.start_tts_stream(context, transport)
|
||||
return
|
||||
|
||||
display_text = text
|
||||
parsed_data = None
|
||||
if isinstance(text, dict):
|
||||
parsed_data = text
|
||||
elif isinstance(text, str):
|
||||
stripped = text.strip()
|
||||
if stripped.startswith("{") and stripped.endswith("}"):
|
||||
try:
|
||||
parsed_data = json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
if isinstance(parsed_data, dict) and "content" in parsed_data:
|
||||
display_text = parsed_data["content"]
|
||||
if "speaker" in parsed_data:
|
||||
context.current_speaker = parsed_data["speaker"]
|
||||
|
||||
stt_text = textUtils.get_string_no_punctuation_or_emoji(
|
||||
str(display_text)
|
||||
)
|
||||
await self._send_json(transport, {
|
||||
"type": "stt",
|
||||
"text": stt_text,
|
||||
"session_id": context.session_id
|
||||
})
|
||||
logger.info(f"发送STT消息: {stt_text}")
|
||||
# Legacy WS sends TTS start together with STT, before synthesis begins.
|
||||
# This lead time is required by MQTT/UDP clients because the device
|
||||
# changes to Speaking asynchronously and drops UDP audio beforehand.
|
||||
await self.start_tts_stream(context, transport)
|
||||
|
||||
async def start_tts_stream(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
) -> bool:
|
||||
"""Open one device-side TTS stream without sending duplicate starts."""
|
||||
if getattr(context, "is_speaking", False):
|
||||
return False
|
||||
|
||||
context.is_speaking = True
|
||||
flow_control = getattr(context, "audio_flow_control", None) or {}
|
||||
await self._stop_audio_sender(flow_control)
|
||||
context.audio_flow_control = {}
|
||||
try:
|
||||
await self._send_json(transport, {
|
||||
"type": "tts",
|
||||
"state": "start",
|
||||
"session_id": context.session_id,
|
||||
})
|
||||
except Exception:
|
||||
context.is_speaking = False
|
||||
raise
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
async def _send_json(
|
||||
transport: TransportInterface,
|
||||
message: dict,
|
||||
) -> None:
|
||||
send_json = getattr(transport, "send_json", None)
|
||||
if callable(send_json):
|
||||
await send_json(message)
|
||||
return
|
||||
await transport.send(json.dumps(message))
|
||||
|
||||
async def send_tts_message(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
state: str,
|
||||
text: str = None,
|
||||
sentence_id: str = None,
|
||||
):
|
||||
"""发送TTS消息 - 完整迁移自send_tts_message"""
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
return False
|
||||
if state == "sentence_start":
|
||||
flow = getattr(context, "audio_flow_control", {})
|
||||
queue = flow.get("_send_queue")
|
||||
task = flow.get("_send_task")
|
||||
if queue is not None and task is not None and not task.done():
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": context.session_id,
|
||||
}
|
||||
if text:
|
||||
message["text"] = text
|
||||
await queue.put(("json", message, sentence_id))
|
||||
return True
|
||||
if state == "stop":
|
||||
if context.config.get("enable_stop_tts_notify", False):
|
||||
from core.utils.util import audio_to_data
|
||||
|
||||
notify_path = context.config.get(
|
||||
"stop_tts_notify_voice", "config/assets/tts_notify.mp3"
|
||||
)
|
||||
notify_audio = await audio_to_data(notify_path, is_opus=True)
|
||||
if notify_audio:
|
||||
await self.send_audio(
|
||||
context,
|
||||
transport,
|
||||
notify_audio,
|
||||
sentence_id=sentence_id,
|
||||
)
|
||||
if not await self._wait_for_audio_completion(context, sentence_id):
|
||||
return False
|
||||
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
return False
|
||||
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": context.session_id
|
||||
}
|
||||
if text:
|
||||
message["text"] = text
|
||||
|
||||
await transport.send_json(message)
|
||||
logger.debug(f"发送TTS消息: state={state}, text={text}")
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
async def _wait_for_audio_completion(
|
||||
context: SessionContext, sentence_id: str = None
|
||||
) -> bool:
|
||||
flow = context.audio_flow_control
|
||||
queue = flow.get("_send_queue")
|
||||
sender_task = flow.get("_send_task")
|
||||
if queue is not None and sender_task is not None:
|
||||
join_task = asyncio.create_task(queue.join())
|
||||
done, _ = await asyncio.wait(
|
||||
{join_task, sender_task},
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
if sender_task in done and not join_task.done():
|
||||
join_task.cancel()
|
||||
await asyncio.gather(join_task, return_exceptions=True)
|
||||
return False
|
||||
await join_task
|
||||
await AudioSendProcessor._stop_audio_sender(flow)
|
||||
if flow.get("send_failed"):
|
||||
return False
|
||||
|
||||
packet_count = int(flow.get("packet_count", 0))
|
||||
if packet_count <= 0:
|
||||
return AudioSendProcessor._is_current_turn(context, sentence_id)
|
||||
frame_duration = max(1, int(getattr(context, "output_frame_duration", 60)))
|
||||
pre_buffer_count = max(0, int(context.config.get("tts_pre_buffer_count", 5)))
|
||||
tail_frames = max(
|
||||
0,
|
||||
int(context.config.get("tts_stop_buffer_frames", pre_buffer_count + 2)),
|
||||
)
|
||||
if tail_frames:
|
||||
await asyncio.sleep(tail_frames * frame_duration / 1000.0)
|
||||
return AudioSendProcessor._is_current_turn(context, sentence_id)
|
||||
|
||||
@staticmethod
|
||||
async def _stop_audio_sender(flow: dict) -> None:
|
||||
task = flow.pop("_send_task", None)
|
||||
flow.pop("_send_queue", None)
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
@staticmethod
|
||||
def _is_current_turn(context: SessionContext, sentence_id: str = None) -> bool:
|
||||
if sentence_id is None:
|
||||
return bool(
|
||||
getattr(context, "conversation_active", True)
|
||||
and getattr(context, "sentence_id", None) is None
|
||||
)
|
||||
return sentence_id == context.sentence_id
|
||||
|
||||
async def send_music_message(self, context: SessionContext, transport: TransportInterface,
|
||||
music_path: str, text: str):
|
||||
"""发送音乐消息 - 完整迁移自send_music_message"""
|
||||
from core.utils.util import audio_to_data
|
||||
|
||||
try:
|
||||
# 获取音频数据
|
||||
opus_packets = await audio_to_data(music_path)
|
||||
if opus_packets:
|
||||
# 发送音乐开始消息
|
||||
await self.send_tts_message(context, transport, "start", text)
|
||||
|
||||
# 发送音频数据
|
||||
await self.send_audio(context, transport, opus_packets)
|
||||
|
||||
# 发送音乐结束消息
|
||||
await self.send_tts_message(context, transport, "stop", None)
|
||||
|
||||
logger.info(f"发送音乐: {music_path}")
|
||||
else:
|
||||
logger.warning(f"无法加载音乐文件: {music_path}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送音乐失败: {e}")
|
||||
|
||||
async def _get_component(self, context: SessionContext, component_type: ComponentType):
|
||||
if not context.component_manager:
|
||||
return None
|
||||
return await context.component_manager.get_component(component_type, context)
|
||||
|
||||
async def send_welcome_audio(self, context: SessionContext, transport: TransportInterface):
|
||||
"""发送欢迎音频"""
|
||||
welcome_audio_path = context.config.get("welcome_audio_path")
|
||||
if welcome_audio_path:
|
||||
await self.send_music_message(context, transport, welcome_audio_path, "欢迎使用小智助手")
|
||||
|
||||
async def send_goodbye_audio(self, context: SessionContext, transport: TransportInterface):
|
||||
"""发送告别音频"""
|
||||
goodbye_audio_path = context.config.get("goodbye_audio_path")
|
||||
if goodbye_audio_path:
|
||||
await self.send_music_message(context, transport, goodbye_audio_path, "再见，期待下次相遇")
|
||||
@@ -0,0 +1,55 @@
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.auth import AuthMiddleware, AuthenticationError
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AuthProcessor(MessageProcessor):
|
||||
"""认证处理器：处理连接认证逻辑"""
|
||||
|
||||
def __init__(self):
|
||||
self.auth_middleware = None
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理认证相关逻辑"""
|
||||
# 服务器管理连接走server消息处理（基于secret校验），跳过认证
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
import json
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "server":
|
||||
return False
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
elif isinstance(message, dict) and message.get("type") == "server":
|
||||
return False
|
||||
|
||||
# 如果已经认证，跳过
|
||||
if context.is_authenticated:
|
||||
return False
|
||||
|
||||
# 初始化认证中间件（延迟初始化）
|
||||
if self.auth_middleware is None:
|
||||
self.auth_middleware = AuthMiddleware(context.config)
|
||||
|
||||
# 检查是否为认证消息（通过headers进行认证）
|
||||
if context.headers:
|
||||
try:
|
||||
await self.auth_middleware.authenticate_async(context.headers)
|
||||
context.is_authenticated = True
|
||||
logger.info(f"设备认证成功: {context.device_id}")
|
||||
return False # 认证成功，继续处理其他消息
|
||||
except AuthenticationError as e:
|
||||
logger.error(f"设备认证失败: {e}")
|
||||
# 发送认证失败消息
|
||||
await transport.send("Authentication failed")
|
||||
await transport.close()
|
||||
return True # 认证失败，停止处理
|
||||
|
||||
# 如果没有认证信息，要求认证
|
||||
await transport.send("Authentication required")
|
||||
return True # 停止后续处理
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,34 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class GoodbyeProcessor(MessageProcessor):
|
||||
"""Goodbye消息处理器：处理会话结束"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "goodbye":
|
||||
logger.info(f"收到goodbye: session_id={msg_json.get('session_id')}")
|
||||
# 长连接传输仅结束逻辑会话，短连接传输关闭连接。
|
||||
if transport.keeps_connection_between_sessions:
|
||||
end_conversation = getattr(context, "end_conversation", None)
|
||||
if callable(end_conversation):
|
||||
await end_conversation(msg_json.get("session_id"))
|
||||
else:
|
||||
await transport.close()
|
||||
return True
|
||||
return False
|
||||
@@ -0,0 +1,351 @@
|
||||
import time
|
||||
import json
|
||||
import random
|
||||
import asyncio
|
||||
import uuid
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.utils.dialogue import Message
|
||||
from core.utils.util import audio_to_data, remove_punctuation_and_length, opus_datas_to_wav_bytes
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
from core.processors.audio_receive_processor import arm_wake_audio_suppression
|
||||
from core.utils.wakeup_word import WakeupWordsConfig
|
||||
from core.components.component_manager import ComponentType
|
||||
from core.providers.tools.device_mcp import (
|
||||
MCPClient,
|
||||
send_mcp_initialize_message,
|
||||
)
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
# 唤醒词配置
|
||||
WAKEUP_CONFIG = {
|
||||
"refresh_time": 5,
|
||||
"words": ["你好", "你好啊", "嘿，你好", "嗨"],
|
||||
}
|
||||
|
||||
# 创建全局的唤醒词配置管理器
|
||||
wakeup_words_config = WakeupWordsConfig()
|
||||
|
||||
# 用于防止并发调用wakeupWordsResponse的锁
|
||||
_wakeup_response_lock = asyncio.Lock()
|
||||
|
||||
|
||||
class HelloProcessor(MessageProcessor):
|
||||
"""Hello消息处理器：完整迁移helloHandle.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理hello类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "hello":
|
||||
await self.handle_hello_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_hello_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理hello消息 - 完整迁移自handleHelloMessage"""
|
||||
pending_tasks = []
|
||||
for task_name in (
|
||||
"listen_stop_task",
|
||||
"listen_start_task",
|
||||
):
|
||||
pending_task = getattr(context, task_name, None)
|
||||
if pending_task and not pending_task.done():
|
||||
pending_task.cancel()
|
||||
pending_tasks.append(pending_task)
|
||||
setattr(context, task_name, None)
|
||||
if pending_tasks:
|
||||
await asyncio.gather(*pending_tasks, return_exceptions=True)
|
||||
|
||||
transport_session_id = getattr(transport, "session_id", None)
|
||||
if transport_session_id:
|
||||
if (
|
||||
context.session_id != transport_session_id
|
||||
and callable(getattr(context, "end_conversation", None))
|
||||
):
|
||||
# A previous finalizer may already have published
|
||||
# conversation_active=False while persistence is still in
|
||||
# progress. Always join it before exposing the new session.
|
||||
await context.end_conversation(context.session_id)
|
||||
context.session_id = transport_session_id
|
||||
if context.welcome_msg:
|
||||
context.welcome_msg["session_id"] = context.session_id
|
||||
dialogue = getattr(context, "dialogue", None)
|
||||
if getattr(context, "prompt", None) and dialogue and not dialogue.dialogue:
|
||||
dialogue.update_system_message(context.prompt)
|
||||
inject_fewshot = getattr(context, "inject_tool_call_fewshot", None)
|
||||
if callable(inject_fewshot):
|
||||
inject_fewshot()
|
||||
context.conversation_active = True
|
||||
context.listen_stop_pending = False
|
||||
context.listen_stop_deadline = 0.0
|
||||
if transport.requires_audio_tail_grace:
|
||||
context.accepting_input_audio = False
|
||||
|
||||
# 处理音频参数
|
||||
audio_params = msg_json.get("audio_params")
|
||||
if audio_params:
|
||||
format = audio_params.get("format")
|
||||
logger.info(f"客户端音频格式: {format}")
|
||||
context.audio_format = format
|
||||
context.input_sample_rate = int(
|
||||
audio_params.get("sample_rate", getattr(context, "input_sample_rate", 16000))
|
||||
)
|
||||
context.input_channels = int(
|
||||
audio_params.get("channels", getattr(context, "input_channels", 1))
|
||||
)
|
||||
context.input_frame_duration = int(
|
||||
audio_params.get(
|
||||
"frame_duration", getattr(context, "input_frame_duration", 60)
|
||||
)
|
||||
)
|
||||
|
||||
# 处理客户端特性
|
||||
features = msg_json.get("features") or {}
|
||||
context.features = dict(features)
|
||||
context.client_aec = bool(features.get("aec"))
|
||||
mcp_enabled = bool(features.get("mcp"))
|
||||
previous_mcp_client = getattr(context, "mcp_client", None)
|
||||
if not mcp_enabled and previous_mcp_client:
|
||||
initialize_task = getattr(
|
||||
context, "mcp_initialize_task", None
|
||||
)
|
||||
if initialize_task and not initialize_task.done():
|
||||
initialize_task.cancel()
|
||||
await asyncio.gather(
|
||||
initialize_task, return_exceptions=True
|
||||
)
|
||||
context.mcp_initialize_task = None
|
||||
previous_mcp_cleanup = getattr(
|
||||
context, "_mcp_cleanup_callback", None
|
||||
)
|
||||
if previous_mcp_cleanup:
|
||||
context.unregister_cleanup(previous_mcp_cleanup)
|
||||
context._mcp_cleanup_callback = None
|
||||
if hasattr(previous_mcp_client, "close"):
|
||||
await previous_mcp_client.close()
|
||||
context.mcp_client = None
|
||||
if features:
|
||||
logger.info(f"客户端特性: {features}")
|
||||
if mcp_enabled:
|
||||
logger.info("客户端支持MCP")
|
||||
if context.mcp_client is None:
|
||||
context.mcp_client = MCPClient()
|
||||
context._mcp_cleanup_callback = (
|
||||
context.mcp_client.close
|
||||
)
|
||||
context.register_cleanup(
|
||||
context._mcp_cleanup_callback
|
||||
)
|
||||
initialize_task = getattr(
|
||||
context, "mcp_initialize_task", None
|
||||
)
|
||||
if (
|
||||
not await context.mcp_client.is_ready()
|
||||
and (
|
||||
initialize_task is None
|
||||
or initialize_task.done()
|
||||
)
|
||||
):
|
||||
context.mcp_initialize_task = (
|
||||
context.create_background_task(
|
||||
send_mcp_initialize_message(
|
||||
context, transport
|
||||
),
|
||||
conversation_scoped=False,
|
||||
)
|
||||
)
|
||||
if features.get("aec"):
|
||||
logger.info("客户端启用了服务端AEC")
|
||||
|
||||
# MQTT/UDP 场景已由MQTT连接层发送hello回复，避免重复发送websocket格式
|
||||
if transport.has_datagram_audio:
|
||||
return
|
||||
|
||||
# 发送欢迎消息
|
||||
if context.welcome_msg:
|
||||
await transport.send_json(context.welcome_msg)
|
||||
else:
|
||||
# 默认欢迎消息
|
||||
welcome_msg = {
|
||||
"type": "hello",
|
||||
"session_id": context.session_id,
|
||||
"version": 1,
|
||||
"transport": "websocket"
|
||||
}
|
||||
await transport.send_json(welcome_msg)
|
||||
|
||||
async def check_wakeup_words(self, context: SessionContext, transport: TransportInterface, text: str) -> bool:
|
||||
"""检查唤醒词 - 完整迁移自checkWakeupWords"""
|
||||
enable_wakeup_words_response_cache = context.config.get("enable_wakeup_words_response_cache", False)
|
||||
|
||||
# 等待tts初始化，最多等待3秒
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < 3:
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
else:
|
||||
return False
|
||||
|
||||
if not enable_wakeup_words_response_cache:
|
||||
return False
|
||||
|
||||
_, filtered_text = remove_punctuation_and_length(text)
|
||||
if filtered_text not in context.config.get("wakeup_words", []):
|
||||
return False
|
||||
|
||||
sentence_id = uuid.uuid4().hex
|
||||
context.sentence_id = sentence_id
|
||||
context.just_woken_up = True
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
# 获取当前音色
|
||||
tts_instance = getattr(tts_component, 'tts_instance', None) if tts_component else None
|
||||
voice = getattr(tts_instance, "voice", "default") if tts_instance else "default"
|
||||
if not voice:
|
||||
voice = "default"
|
||||
|
||||
# 获取唤醒词回复配置
|
||||
response = wakeup_words_config.get_wakeup_response(voice)
|
||||
if not response or not response.get("file_path"):
|
||||
response = {
|
||||
"voice": "default",
|
||||
"file_path": "config/assets/wakeup_words.wav",
|
||||
"time": 0,
|
||||
"text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",
|
||||
}
|
||||
|
||||
# 获取音频数据
|
||||
opus_packets = await audio_to_data(response.get("file_path"))
|
||||
# 播放唤醒词回复
|
||||
context.abort_requested = False
|
||||
context.llm_finish_task = True
|
||||
if tts_instance is not None:
|
||||
# A prior turn may have consumed the one-shot start marker. Cached
|
||||
# playback is a new TTS stream and must always emit start first.
|
||||
tts_instance.tts_audio_first_sentence = True
|
||||
|
||||
logger.info(f"播放唤醒词回复: {response.get('text')}")
|
||||
try:
|
||||
await self._send_audio_message(
|
||||
context,
|
||||
transport,
|
||||
SentenceType.FIRST,
|
||||
opus_packets,
|
||||
response.get("text"),
|
||||
sentence_id,
|
||||
)
|
||||
await self._send_audio_message(
|
||||
context,
|
||||
transport,
|
||||
SentenceType.LAST,
|
||||
[],
|
||||
None,
|
||||
sentence_id,
|
||||
)
|
||||
finally:
|
||||
# Cached wake playback bypasses ListenProcessor, so it must arm
|
||||
# the same bounded release used by the regular wake-word path.
|
||||
arm_wake_audio_suppression(context)
|
||||
|
||||
# 补充对话
|
||||
if context.dialogue:
|
||||
context.dialogue.put(Message(role="assistant", content=response.get("text")))
|
||||
|
||||
# 检查是否需要更新唤醒词回复
|
||||
if time.time() - response.get("time", 0) > WAKEUP_CONFIG["refresh_time"]:
|
||||
if not _wakeup_response_lock.locked():
|
||||
context.create_background_task(
|
||||
self._wakeup_words_response(context, transport)
|
||||
)
|
||||
return True
|
||||
|
||||
async def _wakeup_words_response(self, context: SessionContext, transport: TransportInterface):
|
||||
"""生成唤醒词回复 - 完整迁移自wakeupWordsResponse"""
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
llm_component = await self._get_component(context, ComponentType.LLM)
|
||||
|
||||
tts_instance = getattr(tts_component, 'tts_instance', None) if tts_component else None
|
||||
llm_instance = getattr(llm_component, 'llm_instance', None) if llm_component else None
|
||||
|
||||
if not tts_instance or not llm_instance or not hasattr(llm_instance, 'response_no_stream'):
|
||||
return
|
||||
|
||||
try:
|
||||
# 尝试获取锁，如果获取不到就返回
|
||||
async with _wakeup_response_lock:
|
||||
# 生成唤醒词回复
|
||||
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
|
||||
question = (
|
||||
"此刻用户正在和你说```"
|
||||
+ wakeup_word
|
||||
+ "```。\n请你根据以上用户的内容进行20-30字回复。要符合系统设置的角色情感和态度，不要像机器人一样说话。\n"
|
||||
+ "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。"
|
||||
)
|
||||
|
||||
result = await asyncio.to_thread(
|
||||
llm_instance.response_no_stream,
|
||||
context.config.get("prompt", ""),
|
||||
question,
|
||||
)
|
||||
if not result or len(result) == 0:
|
||||
return
|
||||
|
||||
# 生成TTS音频
|
||||
tts_result = await asyncio.to_thread(tts_instance.to_tts, result)
|
||||
if not tts_result:
|
||||
return
|
||||
|
||||
# 获取当前音色
|
||||
voice = getattr(tts_instance, "voice", "default")
|
||||
|
||||
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
|
||||
file_path = wakeup_words_config.generate_file_path(voice)
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(wav_bytes)
|
||||
# 更新配置
|
||||
wakeup_words_config.update_wakeup_response(voice, file_path, result)
|
||||
except Exception as e:
|
||||
logger.error(f"生成唤醒词回复失败: {e}")
|
||||
|
||||
async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息"""
|
||||
from core.processors.audio_send_processor import AudioSendProcessor
|
||||
|
||||
await AudioSendProcessor().send_stt_message(
|
||||
context, transport, text
|
||||
)
|
||||
|
||||
async def _send_audio_message(self, context: SessionContext, transport: TransportInterface,
|
||||
sentence_type: SentenceType, audios: bytes, text: str,
|
||||
sentence_id: str = None):
|
||||
"""发送音频消息"""
|
||||
# 这里应该调用AudioSendProcessor
|
||||
from core.processors.audio_send_processor import AudioSendProcessor
|
||||
audio_send_processor = AudioSendProcessor()
|
||||
await audio_send_processor.send_audio_message(
|
||||
context,
|
||||
transport,
|
||||
sentence_type,
|
||||
audios,
|
||||
text,
|
||||
sentence_id=sentence_id,
|
||||
)
|
||||
|
||||
async def _get_component(self, context: SessionContext, component_type: ComponentType):
|
||||
if not context.component_manager:
|
||||
return None
|
||||
return await context.component_manager.get_component(component_type, context)
|
||||
@@ -0,0 +1,124 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.providers.tools.device_iot import handleIotStatus, handleIotDescriptors
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class IotProcessor(MessageProcessor):
|
||||
"""IoT消息处理器：完整迁移iotMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理iot类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "iot":
|
||||
await self.handle_iot_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_iot_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理IoT消息 - 完整迁移自iotMessageHandler.py"""
|
||||
tasks = []
|
||||
|
||||
# 处理设备描述符 - 完整迁移原逻辑
|
||||
if "descriptors" in msg_json:
|
||||
logger.debug("处理IoT设备描述符")
|
||||
task = context.create_background_task(
|
||||
self._handle_iot_descriptors(context, transport, msg_json["descriptors"])
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
# 处理设备状态 - 完整迁移原逻辑
|
||||
if "states" in msg_json:
|
||||
logger.debug("处理IoT设备状态")
|
||||
task = context.create_background_task(
|
||||
self._handle_iot_status(context, transport, msg_json["states"])
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
# 如果没有有效的IoT数据
|
||||
if not tasks:
|
||||
logger.warning("IoT消息缺少descriptors或states字段")
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"IoT消息格式错误：缺少descriptors或states字段"
|
||||
)
|
||||
return
|
||||
|
||||
# 任务由 SessionContext 持有，结束会话时统一取消。
|
||||
|
||||
async def _handle_iot_descriptors(self, context: SessionContext, transport: TransportInterface, descriptors: Any):
|
||||
"""处理IoT设备描述符 - 包装原handleIotDescriptors函数"""
|
||||
try:
|
||||
# 调用原有的handleIotDescriptors函数
|
||||
# 注意：这里需要传入context而不是conn，因为handleIotDescriptors可能需要适配
|
||||
await handleIotDescriptors(context, descriptors)
|
||||
logger.debug("IoT设备描述符处理完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理IoT设备描述符失败: {e}", exc_info=True)
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"IoT设备描述符处理失败: {str(e)}"
|
||||
)
|
||||
|
||||
async def _handle_iot_status(self, context: SessionContext, transport: TransportInterface, states: Any):
|
||||
"""处理IoT设备状态 - 包装原handleIotStatus函数"""
|
||||
try:
|
||||
# 调用原有的handleIotStatus函数
|
||||
# 注意：这里需要传入context而不是conn，因为handleIotStatus可能需要适配
|
||||
await handleIotStatus(context, states)
|
||||
logger.debug("IoT设备状态处理完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理IoT设备状态失败: {e}", exc_info=True)
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"IoT设备状态处理失败: {str(e)}"
|
||||
)
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, session_id: str, message: str):
|
||||
"""发送IoT错误响应"""
|
||||
response = {
|
||||
"type": "iot",
|
||||
"status": "error",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送IoT错误响应失败: {e}")
|
||||
|
||||
async def _send_success_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, data: dict = None):
|
||||
"""发送IoT成功响应"""
|
||||
response = {
|
||||
"type": "iot",
|
||||
"status": "success",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if data:
|
||||
response["data"] = data
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送IoT成功响应失败: {e}")
|
||||
@@ -0,0 +1,433 @@
|
||||
import time
|
||||
import json
|
||||
import asyncio
|
||||
import uuid
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from core.utils.dialogue import Message
|
||||
from core.components.component_manager import ComponentType
|
||||
from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType
|
||||
from core.processors.audio_receive_processor import arm_wake_audio_suppression
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ListenProcessor(MessageProcessor):
|
||||
"""Listen消息处理器：完整迁移listenMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理listen类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "listen":
|
||||
await self.handle_listen_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_listen_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理listen消息 - 完整迁移自listenMessageHandler.py"""
|
||||
msg_session_id = msg_json.get("session_id")
|
||||
if msg_session_id and msg_session_id != context.session_id:
|
||||
logger.warning(
|
||||
f"忽略非当前会话的listen消息: "
|
||||
f"msg={msg_session_id}, current={context.session_id}"
|
||||
)
|
||||
return
|
||||
|
||||
state = msg_json.get("state")
|
||||
has_datagram_audio = getattr(
|
||||
transport, "has_datagram_audio", False
|
||||
)
|
||||
if (
|
||||
has_datagram_audio
|
||||
and state in {"start", "detect"}
|
||||
and not context.conversation_active
|
||||
):
|
||||
if transport.keeps_connection_between_sessions:
|
||||
await transport.send_json(
|
||||
{
|
||||
"type": "goodbye",
|
||||
"session_id": msg_session_id or context.session_id,
|
||||
}
|
||||
)
|
||||
logger.info(
|
||||
"忽略已结束会话的listen {}: session_id={}",
|
||||
state,
|
||||
context.session_id,
|
||||
)
|
||||
return
|
||||
|
||||
# 设置拾音模式
|
||||
if "mode" in msg_json:
|
||||
context.listen_mode = msg_json["mode"]
|
||||
logger.debug(f"客户端拾音模式：{context.listen_mode}")
|
||||
|
||||
# 处理不同的状态
|
||||
if state == "start":
|
||||
pending_start = getattr(context, "listen_start_task", None)
|
||||
if pending_start and not pending_start.done():
|
||||
logger.debug("忽略尾包隔离期内的重复listen start")
|
||||
return
|
||||
tail_quarantine = await self._flush_pending_listen_stop(
|
||||
context, transport
|
||||
)
|
||||
# 开始监听语音
|
||||
logger.info(f"listen start: session_id={context.session_id}, mode={context.listen_mode}")
|
||||
context.reset_audio_states()
|
||||
if (
|
||||
getattr(transport, "requires_audio_tail_grace", False)
|
||||
and tail_quarantine > 0
|
||||
):
|
||||
context.accepting_input_audio = False
|
||||
context.listen_start_task = context.create_background_task(
|
||||
self._open_input_after_tail_quarantine(
|
||||
context,
|
||||
context.session_id,
|
||||
tail_quarantine,
|
||||
),
|
||||
turn_scoped=True,
|
||||
)
|
||||
else:
|
||||
context.accepting_input_audio = True
|
||||
context.abort_requested = False
|
||||
context.close_after_chat = False
|
||||
logger.debug("开始语音监听")
|
||||
|
||||
elif state == "stop":
|
||||
# 停止监听语音
|
||||
logger.info(f"listen stop: session_id={context.session_id}")
|
||||
context.client_have_voice = True
|
||||
context.listen_stop_pending = True
|
||||
pending_start = getattr(context, "listen_start_task", None)
|
||||
if pending_start and not pending_start.done():
|
||||
pending_start.cancel()
|
||||
context.listen_start_task = None
|
||||
|
||||
if getattr(transport, "requires_audio_tail_grace", False):
|
||||
pending = getattr(context, "listen_stop_task", None)
|
||||
if pending and not pending.done():
|
||||
logger.debug("忽略重复的listen stop")
|
||||
return
|
||||
delay_ms = max(
|
||||
0,
|
||||
min(
|
||||
1000,
|
||||
int(
|
||||
context.config.get(
|
||||
"mqtt_udp_tail_grace_ms", 180
|
||||
)
|
||||
),
|
||||
),
|
||||
)
|
||||
context.listen_stop_deadline = (
|
||||
time.monotonic() + delay_ms / 1000
|
||||
)
|
||||
context.listen_stop_task = context.create_background_task(
|
||||
self._finalize_listen_stop(
|
||||
context,
|
||||
transport,
|
||||
context.session_id,
|
||||
delay_ms / 1000,
|
||||
),
|
||||
turn_scoped=True,
|
||||
)
|
||||
else:
|
||||
context.listen_stop_deadline = 0.0
|
||||
await self._finalize_listen_stop(
|
||||
context,
|
||||
transport,
|
||||
context.session_id,
|
||||
0,
|
||||
)
|
||||
logger.debug("停止语音监听")
|
||||
|
||||
elif state == "detect":
|
||||
# 检测到文本输入
|
||||
logger.info(f"listen detect: session_id={context.session_id}, text={msg_json.get('text')}")
|
||||
context.client_have_voice = False
|
||||
context.asr_audio.clear()
|
||||
|
||||
if "text" in msg_json:
|
||||
context.update_activity()
|
||||
original_text = msg_json["text"] # 保留原始文本
|
||||
filtered_len, filtered_text = remove_punctuation_and_length(original_text)
|
||||
|
||||
if original_text.startswith("[device_call]"):
|
||||
await self._handle_device_call(
|
||||
context,
|
||||
transport,
|
||||
original_text[len("[device_call]"):].strip(),
|
||||
)
|
||||
return
|
||||
|
||||
# 识别是否是唤醒词
|
||||
is_wakeup_words = filtered_text in context.config.get("wakeup_words", [])
|
||||
# 是否开启唤醒词回复
|
||||
enable_greeting = context.config.get("enable_greeting", True)
|
||||
|
||||
if is_wakeup_words and not enable_greeting:
|
||||
# 如果是唤醒词，且关闭了唤醒词回复，就不用回答
|
||||
# Native already drops wake history until listen/start.
|
||||
# Keeping the greeting suppression window armed here would
|
||||
# discard the beginning of the user's first real utterance.
|
||||
await self._send_stt_message(context, transport, original_text)
|
||||
await self._send_tts_message(context, transport, "stop", None)
|
||||
context.is_speaking = False
|
||||
|
||||
elif is_wakeup_words:
|
||||
# 处理唤醒词
|
||||
self._arm_wake_audio_suppression(context)
|
||||
# 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
|
||||
await self._enqueue_asr_report(context, "嘿，你好呀", [])
|
||||
await self._start_to_chat(context, transport, "嘿，你好呀", skip_intent=True)
|
||||
|
||||
else:
|
||||
# 处理普通文本
|
||||
# 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
|
||||
await self._enqueue_asr_report(context, original_text, [])
|
||||
# 否则需要LLM对文字内容进行答复
|
||||
await self._start_to_chat(context, transport, original_text)
|
||||
|
||||
async def _flush_pending_listen_stop(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
) -> float:
|
||||
"""Finalize buffered speech before a new listen/start resets the turn."""
|
||||
pending = getattr(context, "listen_stop_task", None)
|
||||
had_pending_stop = bool(getattr(context, "listen_stop_pending", False))
|
||||
tail_quarantine = max(
|
||||
0.0,
|
||||
float(getattr(context, "listen_stop_deadline", 0.0) or 0.0)
|
||||
- time.monotonic(),
|
||||
)
|
||||
if pending and not pending.done():
|
||||
pending.cancel()
|
||||
try:
|
||||
await pending
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
context.listen_stop_task = None
|
||||
if had_pending_stop:
|
||||
await self._finalize_listen_stop(
|
||||
context,
|
||||
transport,
|
||||
context.session_id,
|
||||
0,
|
||||
)
|
||||
else:
|
||||
context.listen_stop_pending = False
|
||||
if tail_quarantine <= 0:
|
||||
context.listen_stop_deadline = 0.0
|
||||
return tail_quarantine
|
||||
|
||||
@staticmethod
|
||||
async def _open_input_after_tail_quarantine(
|
||||
context: SessionContext,
|
||||
session_id: str,
|
||||
delay_seconds: float,
|
||||
) -> None:
|
||||
current_task = asyncio.current_task()
|
||||
try:
|
||||
await asyncio.sleep(delay_seconds)
|
||||
if (
|
||||
context.session_id == session_id
|
||||
and context.conversation_active
|
||||
and not context.listen_stop_pending
|
||||
):
|
||||
context.accepting_input_audio = True
|
||||
finally:
|
||||
if getattr(context, "listen_start_task", None) is current_task:
|
||||
context.listen_start_task = None
|
||||
context.listen_stop_deadline = 0.0
|
||||
|
||||
async def _finalize_listen_stop(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
session_id: str,
|
||||
delay_seconds: float,
|
||||
) -> None:
|
||||
"""Finalize ASR after datagram tail frames have crossed the router."""
|
||||
current_task = asyncio.current_task()
|
||||
try:
|
||||
if delay_seconds > 0:
|
||||
await asyncio.sleep(delay_seconds)
|
||||
if context.session_id != session_id:
|
||||
return
|
||||
if (
|
||||
getattr(transport, "has_datagram_audio", False)
|
||||
and not context.conversation_active
|
||||
):
|
||||
return
|
||||
|
||||
# Close the tail-grace ingress gate before resolving ASR.
|
||||
# Component lookup may fail; leaving it open would leak subsequent
|
||||
# packets into a turn that has already stopped.
|
||||
context.listen_stop_pending = False
|
||||
context.client_voice_stop = True
|
||||
if getattr(transport, "requires_audio_tail_grace", False):
|
||||
context.accepting_input_audio = False
|
||||
|
||||
asr_component = await self._get_component(
|
||||
context, ComponentType.ASR
|
||||
)
|
||||
if context.session_id != session_id:
|
||||
return
|
||||
asr_instance = (
|
||||
getattr(asr_component, "asr_instance", None)
|
||||
if asr_component
|
||||
else None
|
||||
)
|
||||
|
||||
if not asr_instance or not hasattr(
|
||||
asr_instance, "interface_type"
|
||||
):
|
||||
return
|
||||
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
|
||||
if asr_instance.interface_type == InterfaceType.STREAM:
|
||||
if hasattr(asr_instance, "_send_stop_request"):
|
||||
await asr_instance._send_stop_request()
|
||||
return
|
||||
|
||||
if context.asr_audio:
|
||||
asr_audio_task = context.asr_audio.copy()
|
||||
context.asr_audio.clear()
|
||||
context.reset_vad_states()
|
||||
await asr_instance.handle_voice_stop(
|
||||
context, asr_audio_task
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
"listen stop收尾失败: session_id={}, error={}",
|
||||
session_id,
|
||||
error,
|
||||
)
|
||||
finally:
|
||||
if getattr(context, "listen_stop_task", None) is current_task:
|
||||
context.listen_stop_task = None
|
||||
context.listen_stop_deadline = 0.0
|
||||
if context.session_id == session_id:
|
||||
context.listen_stop_pending = False
|
||||
if getattr(transport, "requires_audio_tail_grace", False):
|
||||
context.accepting_input_audio = False
|
||||
|
||||
def _arm_wake_audio_suppression(self, context: SessionContext) -> None:
|
||||
"""Bound the UDP/TCP reorder window used by the legacy gateway flow."""
|
||||
arm_wake_audio_suppression(context)
|
||||
|
||||
async def _handle_device_call(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
call_text: str,
|
||||
) -> None:
|
||||
"""Restore the legacy device-call announcement and call-state handoff."""
|
||||
logger.info(f"收到设备呼叫指令: {call_text}")
|
||||
context.incoming_call = True
|
||||
context.sentence_id = uuid.uuid4().hex
|
||||
await self._send_stt_message(context, transport, call_text)
|
||||
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
tts_instance = (
|
||||
getattr(tts_component, "tts_instance", None)
|
||||
if tts_component
|
||||
else None
|
||||
)
|
||||
if tts_instance:
|
||||
tts_instance.store_tts_text(context.sentence_id, call_text)
|
||||
tts_instance.tts_text_queue.put(
|
||||
TTSMessageDTO(
|
||||
sentence_id=context.sentence_id,
|
||||
sentence_type=SentenceType.FIRST,
|
||||
content_type=ContentType.ACTION,
|
||||
)
|
||||
)
|
||||
tts_instance.tts_one_sentence(
|
||||
context,
|
||||
ContentType.TEXT,
|
||||
content_detail=call_text,
|
||||
)
|
||||
tts_instance.tts_text_queue.put(
|
||||
TTSMessageDTO(
|
||||
sentence_id=context.sentence_id,
|
||||
sentence_type=SentenceType.LAST,
|
||||
content_type=ContentType.ACTION,
|
||||
)
|
||||
)
|
||||
|
||||
context.dialogue.put(Message(role="assistant", content=call_text))
|
||||
|
||||
async def _handle_audio_message(self, context: SessionContext, transport: TransportInterface, audio: bytes):
|
||||
"""处理音频消息 - 调用AudioReceiveProcessor"""
|
||||
# 这里应该调用AudioReceiveProcessor来处理音频
|
||||
from core.processors.audio_receive_processor import AudioReceiveProcessor
|
||||
audio_processor = AudioReceiveProcessor()
|
||||
await audio_processor.handle_audio_message(context, transport, audio)
|
||||
|
||||
async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息"""
|
||||
from core.processors.audio_send_processor import AudioSendProcessor
|
||||
|
||||
await AudioSendProcessor().send_stt_message(
|
||||
context, transport, text
|
||||
)
|
||||
|
||||
async def _send_tts_message(self, context: SessionContext, transport: TransportInterface, state: str, text: str = None):
|
||||
"""发送TTS消息"""
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": context.session_id
|
||||
}
|
||||
if text:
|
||||
message["text"] = text
|
||||
|
||||
await transport.send(json.dumps(message))
|
||||
logger.debug(f"发送TTS消息: state={state}, text={text}")
|
||||
|
||||
async def _get_component(self, context: SessionContext, component_type: ComponentType):
|
||||
if not context.component_manager:
|
||||
return None
|
||||
return await context.component_manager.get_component(component_type, context)
|
||||
|
||||
async def _enqueue_asr_report(self, context: SessionContext, text: str, audio_data: list):
|
||||
"""ASR上报队列"""
|
||||
if context.report_asr_enable:
|
||||
from core.processors.report_processor import ReportProcessor
|
||||
report_processor = ReportProcessor()
|
||||
report_processor.enqueue_asr_report(context, text, audio_data)
|
||||
|
||||
async def _start_to_chat(self, context: SessionContext, transport: TransportInterface, text: str, skip_intent: bool = False):
|
||||
"""开始聊天 - 调用ChatProcessor"""
|
||||
# 与旧架构一致：先发送STT，再异步触发聊天，避免阻塞事件循环
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
|
||||
if hasattr(context, "create_background_task"):
|
||||
context.create_background_task(
|
||||
chat_processor.handle_chat(
|
||||
context, transport, text, skip_intent=skip_intent
|
||||
),
|
||||
turn_scoped=True,
|
||||
)
|
||||
logger.info(f"chat任务已提交: session_id={context.session_id}")
|
||||
else:
|
||||
await chat_processor.handle_chat(context, transport, text, skip_intent=skip_intent)
|
||||
@@ -0,0 +1,96 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.providers.tools.device_mcp import handle_mcp_message
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class McpProcessor(MessageProcessor):
|
||||
"""MCP消息处理器：完整迁移mcpMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理mcp类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "mcp":
|
||||
await self.handle_mcp_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_mcp_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理MCP消息 - 完整迁移自mcpMessageHandler.py"""
|
||||
if "payload" in msg_json:
|
||||
# 检查MCP客户端是否存在
|
||||
if not context.mcp_client:
|
||||
logger.warning("MCP客户端未初始化，无法处理MCP消息")
|
||||
await self._send_error_response(transport, context.session_id, "MCP客户端未初始化")
|
||||
return
|
||||
|
||||
# 创建异步任务处理MCP消息 - 完整迁移原逻辑
|
||||
context.create_background_task(
|
||||
self._handle_mcp_payload(
|
||||
context, transport, msg_json["payload"]
|
||||
),
|
||||
conversation_scoped=False,
|
||||
)
|
||||
else:
|
||||
logger.warning("MCP消息缺少payload字段")
|
||||
await self._send_error_response(transport, context.session_id, "MCP消息格式错误：缺少payload")
|
||||
|
||||
async def _handle_mcp_payload(self, context: SessionContext, transport: TransportInterface, payload: dict):
|
||||
"""处理MCP payload - 包装原handle_mcp_message函数"""
|
||||
try:
|
||||
# 调用原有的handle_mcp_message函数
|
||||
# 注意：这里需要传入context而不是conn，因为handle_mcp_message可能需要适配
|
||||
await handle_mcp_message(context, context.mcp_client, payload, transport)
|
||||
logger.debug("MCP消息处理完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理MCP消息失败: {e}", exc_info=True)
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"MCP消息处理失败: {str(e)}"
|
||||
)
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, session_id: str, message: str):
|
||||
"""发送MCP错误响应"""
|
||||
response = {
|
||||
"type": "mcp",
|
||||
"status": "error",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送MCP错误响应失败: {e}")
|
||||
|
||||
async def _send_success_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, data: dict = None):
|
||||
"""发送MCP成功响应"""
|
||||
response = {
|
||||
"type": "mcp",
|
||||
"status": "success",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if data:
|
||||
response["data"] = data
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送MCP成功响应失败: {e}")
|
||||
@@ -0,0 +1,124 @@
|
||||
import json
|
||||
from typing import Any, List
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.processors.hello_processor import HelloProcessor
|
||||
from core.processors.listen_processor import ListenProcessor
|
||||
from core.processors.audio_receive_processor import AudioReceiveProcessor
|
||||
from core.processors.auth_processor import AuthProcessor
|
||||
from core.processors.timeout_processor import TimeoutProcessor
|
||||
from core.processors.server_processor import ServerProcessor
|
||||
from core.processors.mcp_processor import McpProcessor
|
||||
from core.processors.iot_processor import IotProcessor
|
||||
from core.processors.abort_processor import AbortProcessor
|
||||
from core.processors.goodbye_processor import GoodbyeProcessor
|
||||
from core.processors.text_processor import TextProcessor
|
||||
from core.processors.ping_processor import PingProcessor
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class MessageRouter(MessageProcessor):
|
||||
"""
|
||||
消息路由器：协调所有独立的processor
|
||||
按功能职责分离，避免耦合，每个processor专注单一职责
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# 初始化所有独立的processor
|
||||
self.auth_processor = AuthProcessor()
|
||||
self.timeout_processor = TimeoutProcessor()
|
||||
self.abort_processor = AbortProcessor()
|
||||
self.goodbye_processor = GoodbyeProcessor()
|
||||
self.hello_processor = HelloProcessor()
|
||||
self.listen_processor = ListenProcessor()
|
||||
self.server_processor = ServerProcessor()
|
||||
self.mcp_processor = McpProcessor()
|
||||
self.iot_processor = IotProcessor()
|
||||
self.audio_receive_processor = AudioReceiveProcessor()
|
||||
self.text_processor = TextProcessor()
|
||||
self.ping_processor = PingProcessor()
|
||||
|
||||
# 按优先级排序的processor列表
|
||||
self.processors: List[MessageProcessor] = [
|
||||
self.timeout_processor, # 首先检查超时
|
||||
self.auth_processor, # 然后检查认证
|
||||
self.server_processor, # 服务器消息（管理端下发动作）
|
||||
self.abort_processor, # 中断消息
|
||||
self.goodbye_processor, # goodbye消息
|
||||
self.hello_processor, # hello消息
|
||||
self.ping_processor, # 可选JSON ping/pong心跳
|
||||
self.listen_processor, # listen消息
|
||||
self.mcp_processor, # MCP消息
|
||||
self.iot_processor, # IoT消息
|
||||
self.audio_receive_processor, # 音频消息
|
||||
self.text_processor, # 纯文本消息（放在最后，作为兜底处理）
|
||||
]
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""
|
||||
路由消息到合适的processor
|
||||
每个processor专注处理自己的消息类型，避免耦合
|
||||
"""
|
||||
# Audio activity is updated only after VAD confirms voice. Treating
|
||||
# every silent frame as activity prevents logical-session timeouts.
|
||||
is_audio = isinstance(message, bytes) or (
|
||||
isinstance(message, dict) and message.get("type") == "audio"
|
||||
)
|
||||
if not is_audio:
|
||||
context.update_activity()
|
||||
|
||||
# 按优先级顺序尝试每个processor
|
||||
for processor in self.processors:
|
||||
try:
|
||||
if await processor.process(context, transport, message):
|
||||
# 消息已被处理，记录日志并返回
|
||||
logger.debug(f"消息被 {processor.__class__.__name__} 处理")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"{processor.__class__.__name__} 处理消息时出错: {e}", exc_info=True)
|
||||
continue
|
||||
|
||||
# 如果没有processor处理该消息，记录警告
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
msg_type = msg_json.get("type", "unknown") if isinstance(msg_json, dict) else "non-dict"
|
||||
logger.warning(f"未处理的消息类型: {msg_type}, 内容: {message[:100]}...")
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(f"未处理的非JSON消息: {message[:100]}...")
|
||||
elif isinstance(message, bytes):
|
||||
logger.warning(f"未处理的二进制消息，大小: {len(message)} bytes")
|
||||
else:
|
||||
logger.warning(f"未处理的消息类型: {type(message)}, {message}")
|
||||
|
||||
return False
|
||||
|
||||
def add_processor(self, processor: MessageProcessor, priority: int = None):
|
||||
"""
|
||||
添加新的processor
|
||||
priority: 优先级，数字越小优先级越高，None表示添加到末尾
|
||||
"""
|
||||
if priority is None:
|
||||
self.processors.append(processor)
|
||||
else:
|
||||
self.processors.insert(priority, processor)
|
||||
logger.info(f"添加processor: {processor.__class__.__name__}")
|
||||
|
||||
def remove_processor(self, processor_class):
|
||||
"""移除指定类型的processor"""
|
||||
self.processors = [p for p in self.processors if not isinstance(p, processor_class)]
|
||||
logger.info(f"移除processor: {processor_class.__name__}")
|
||||
|
||||
def get_processor(self, processor_class):
|
||||
"""获取指定类型的processor"""
|
||||
for processor in self.processors:
|
||||
if isinstance(processor, processor_class):
|
||||
return processor
|
||||
return None
|
||||
|
||||
def list_processors(self) -> List[str]:
|
||||
"""列出所有processor的名称"""
|
||||
return [processor.__class__.__name__ for processor in self.processors]
|
||||
@@ -0,0 +1,40 @@
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from core.context.session_context import SessionContext
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class PingProcessor(MessageProcessor):
|
||||
"""Handle the optional legacy JSON ping/pong control contract."""
|
||||
|
||||
async def process(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
message: Any,
|
||||
) -> bool:
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
message = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
if not isinstance(message, dict) or message.get("type") != "ping":
|
||||
return False
|
||||
|
||||
if not context.config.get("enable_websocket_ping", False):
|
||||
logger.debug("WebSocket心跳功能未启用，忽略PING消息")
|
||||
return True
|
||||
|
||||
await transport.send_json(
|
||||
{
|
||||
"type": "pong",
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
|
||||
}
|
||||
)
|
||||
return True
|
||||
@@ -0,0 +1,301 @@
|
||||
import asyncio
|
||||
import time
|
||||
import queue
|
||||
import threading
|
||||
import json
|
||||
from typing import Any, List
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.manage_api_client import ManageApiClient, report as manage_report
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ReportProcessor(MessageProcessor):
|
||||
"""上报处理器：完整迁移reportHandle.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""这个处理器不直接处理消息，而是被其他处理器调用"""
|
||||
return False
|
||||
|
||||
def enqueue_asr_report(self, context: SessionContext, text: str, audio_data: List[bytes]):
|
||||
"""ASR上报队列 - 完整迁移自enqueue_asr_report"""
|
||||
if not self._should_report(context, context.report_asr_enable):
|
||||
return
|
||||
|
||||
report_time = int(time.time() * 1000)
|
||||
if context.chat_history_conf != 2:
|
||||
audio_data = None
|
||||
|
||||
# 将上报任务放入队列
|
||||
self._enqueue_report(context, {
|
||||
"type": 1, # 用户类型
|
||||
"text": text,
|
||||
"audio_data": audio_data,
|
||||
"report_time": report_time,
|
||||
"session_id": context.session_id,
|
||||
"device_id": context.device_id,
|
||||
})
|
||||
|
||||
# 确保上报线程已启动
|
||||
self._ensure_report_thread(context)
|
||||
|
||||
def enqueue_tts_report(self, context: SessionContext, text: str, opus_data: bytes):
|
||||
"""TTS上报队列 - 完整迁移自enqueue_tts_report"""
|
||||
if not self._should_report(context, context.report_tts_enable):
|
||||
return
|
||||
|
||||
report_time = int(time.time() * 1000)
|
||||
if context.chat_history_conf != 2:
|
||||
opus_data = None
|
||||
|
||||
# 将上报任务放入队列
|
||||
self._enqueue_report(context, {
|
||||
"type": 2, # 智能体类型
|
||||
"text": text,
|
||||
"audio_data": opus_data,
|
||||
"report_time": report_time,
|
||||
"session_id": context.session_id,
|
||||
"device_id": context.device_id,
|
||||
})
|
||||
|
||||
# 确保上报线程已启动
|
||||
self._ensure_report_thread(context)
|
||||
|
||||
def enqueue_tool_report(
|
||||
self,
|
||||
context: SessionContext,
|
||||
tool_name: str,
|
||||
tool_input: dict,
|
||||
tool_result: str = None,
|
||||
report_tool_call: bool = True,
|
||||
):
|
||||
if not self._should_report(context, True):
|
||||
return
|
||||
|
||||
timestamp = int(time.time() * 1000)
|
||||
entries = []
|
||||
if report_tool_call:
|
||||
entries.append(
|
||||
(
|
||||
json.dumps(
|
||||
[{"type": "tool", "text": f"{tool_name}({json.dumps(tool_input, ensure_ascii=False)})"}],
|
||||
ensure_ascii=False,
|
||||
),
|
||||
timestamp,
|
||||
)
|
||||
)
|
||||
if tool_result is not None:
|
||||
entries.append(
|
||||
(
|
||||
json.dumps(
|
||||
[{"type": "tool_result", "text": json.dumps({"result": str(tool_result)}, ensure_ascii=False)}],
|
||||
ensure_ascii=False,
|
||||
),
|
||||
timestamp + 1,
|
||||
)
|
||||
)
|
||||
|
||||
for text, report_time in entries:
|
||||
self._enqueue_report(
|
||||
context,
|
||||
{
|
||||
"type": 3,
|
||||
"text": text,
|
||||
"audio_data": None,
|
||||
"report_time": report_time,
|
||||
"session_id": context.session_id,
|
||||
"device_id": context.device_id,
|
||||
},
|
||||
)
|
||||
if entries:
|
||||
self._ensure_report_thread(context)
|
||||
|
||||
def _ensure_report_thread(self, context: SessionContext):
|
||||
"""确保上报线程已启动"""
|
||||
if context.report_thread is None or not context.report_thread.is_alive():
|
||||
if not getattr(context, "_report_cleanup_registered", False):
|
||||
context.register_cleanup(
|
||||
lambda: asyncio.to_thread(self.cleanup_session, context)
|
||||
)
|
||||
context._report_cleanup_registered = True
|
||||
context.report_thread = threading.Thread(
|
||||
target=self._report_worker,
|
||||
args=(context,),
|
||||
daemon=True
|
||||
)
|
||||
context.report_thread.start()
|
||||
logger.info(f"上报线程已启动: {context.session_id}")
|
||||
|
||||
def _enqueue_report(self, context: SessionContext, report_task: dict) -> None:
|
||||
try:
|
||||
context.report_queue.put_nowait(report_task)
|
||||
except queue.Full:
|
||||
try:
|
||||
context.report_queue.get_nowait()
|
||||
context.report_queue.put_nowait(report_task)
|
||||
logger.warning("上报队列已满，丢弃最旧任务: {}", context.session_id)
|
||||
except (queue.Empty, queue.Full):
|
||||
logger.warning("上报队列持续拥塞，丢弃当前任务: {}", context.session_id)
|
||||
|
||||
def _should_report(self, context: SessionContext, enabled: bool) -> bool:
|
||||
return bool(
|
||||
enabled
|
||||
and context.read_config_from_api
|
||||
and not context.need_bind
|
||||
and context.chat_history_conf != 0
|
||||
)
|
||||
|
||||
def _report_worker(self, context: SessionContext):
|
||||
"""上报工作线程 - 完整迁移自ConnectionHandler中的上报逻辑"""
|
||||
logger.info(f"上报工作线程启动: {context.session_id}")
|
||||
event_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(event_loop)
|
||||
|
||||
try:
|
||||
while not context.stop_event.is_set():
|
||||
try:
|
||||
# 从队列获取上报任务
|
||||
report_task = context.report_queue.get(timeout=1)
|
||||
if report_task is None:
|
||||
break
|
||||
|
||||
# 执行上报
|
||||
self._execute_report(context, report_task, event_loop)
|
||||
|
||||
except queue.Empty:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f"上报工作线程异常: {e}")
|
||||
finally:
|
||||
try:
|
||||
event_loop.run_until_complete(
|
||||
ManageApiClient.close_current_loop_client()
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("关闭上报HTTP客户端失败: {}", exc)
|
||||
event_loop.close()
|
||||
logger.info(f"上报工作线程退出: {context.session_id}")
|
||||
|
||||
def _execute_report(
|
||||
self, context: SessionContext, report_task: dict,
|
||||
event_loop=None
|
||||
):
|
||||
"""执行聊天记录上报操作 - 完整迁移自report函数"""
|
||||
try:
|
||||
report_type = report_task["type"]
|
||||
text = report_task["text"]
|
||||
audio_data = report_task["audio_data"]
|
||||
report_time = report_task["report_time"]
|
||||
|
||||
# 处理音频数据
|
||||
processed_audio = None
|
||||
if audio_data:
|
||||
if isinstance(audio_data, list):
|
||||
# ASR音频数据（多个音频片段）
|
||||
processed_audio = self._process_asr_audio(audio_data)
|
||||
elif isinstance(audio_data, bytes):
|
||||
# TTS音频数据（opus格式）
|
||||
processed_audio = self._opus_to_wav(audio_data)
|
||||
|
||||
# 执行上报
|
||||
coroutine = manage_report(
|
||||
mac_address=report_task["device_id"],
|
||||
session_id=report_task["session_id"],
|
||||
chat_type=report_type,
|
||||
content=text,
|
||||
audio=processed_audio,
|
||||
report_time=report_time,
|
||||
)
|
||||
if event_loop is None:
|
||||
asyncio.run(coroutine)
|
||||
else:
|
||||
event_loop.run_until_complete(coroutine)
|
||||
|
||||
logger.debug(f"上报成功: type={report_type}, text={text[:50]}...")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"聊天记录上报失败: {e}")
|
||||
|
||||
def _process_asr_audio(self, audio_data_list: List[bytes]) -> bytes:
|
||||
"""处理ASR音频数据"""
|
||||
try:
|
||||
# 将多个音频片段合并
|
||||
combined_audio = b''.join(audio_data_list)
|
||||
return self._pcm_to_wav(combined_audio)
|
||||
except Exception as e:
|
||||
logger.error(f"处理ASR音频数据失败: {e}")
|
||||
return b''
|
||||
|
||||
def _pcm_to_wav(self, pcm_data: bytes) -> bytes:
|
||||
import io
|
||||
import wave
|
||||
|
||||
wav_buffer = io.BytesIO()
|
||||
with wave.open(wav_buffer, 'wb') as wav_file:
|
||||
wav_file.setnchannels(1)
|
||||
wav_file.setsampwidth(2)
|
||||
wav_file.setframerate(16000)
|
||||
wav_file.writeframes(pcm_data)
|
||||
return wav_buffer.getvalue()
|
||||
|
||||
def _opus_to_wav(self, opus_data: bytes) -> bytes:
|
||||
"""将Opus数据转换为WAV格式的字节流 - 完整迁移自opus_to_wav"""
|
||||
try:
|
||||
import opuslib_next
|
||||
import io
|
||||
import wave
|
||||
|
||||
# Opus解码器配置
|
||||
sample_rate = 16000
|
||||
channels = 1
|
||||
|
||||
# 创建Opus解码器
|
||||
decoder = opuslib_next.Decoder(sample_rate, channels)
|
||||
|
||||
# 解码Opus数据
|
||||
pcm_data = decoder.decode(opus_data, frame_size=960)
|
||||
|
||||
# 创建WAV文件
|
||||
wav_buffer = io.BytesIO()
|
||||
with wave.open(wav_buffer, 'wb') as wav_file:
|
||||
wav_file.setnchannels(channels)
|
||||
wav_file.setsampwidth(2) # 16-bit
|
||||
wav_file.setframerate(sample_rate)
|
||||
wav_file.writeframes(pcm_data)
|
||||
|
||||
return wav_buffer.getvalue()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Opus转WAV失败: {e}")
|
||||
return b''
|
||||
|
||||
def cleanup_session(self, context: SessionContext):
|
||||
"""清理会话上报资源"""
|
||||
# 停止上报线程
|
||||
if context.report_thread and context.report_thread.is_alive():
|
||||
context.stop_event.set()
|
||||
try:
|
||||
context.report_queue.put_nowait(None)
|
||||
except queue.Full:
|
||||
try:
|
||||
context.report_queue.get_nowait()
|
||||
context.report_queue.put_nowait(None)
|
||||
except (queue.Empty, queue.Full):
|
||||
pass
|
||||
context.report_thread.join(timeout=5)
|
||||
if context.report_thread.is_alive():
|
||||
logger.warning(f"上报线程未能按时退出: {context.session_id}")
|
||||
else:
|
||||
context.report_thread = None
|
||||
|
||||
# 清理上报队列
|
||||
try:
|
||||
while not context.report_queue.empty():
|
||||
context.report_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
pass
|
||||
|
||||
logger.info(f"上报资源清理完成: {context.session_id}")
|
||||
@@ -0,0 +1,177 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
from core.utils.restart import restart_server
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ServerProcessor(MessageProcessor):
|
||||
"""服务器消息处理器：完整迁移serverMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理server类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "server":
|
||||
await self.handle_server_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_server_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理server消息 - 完整迁移自serverMessageHandler.py"""
|
||||
# 如果配置是从API读取的，则需要验证secret
|
||||
if not context.read_config_from_api:
|
||||
return
|
||||
|
||||
# 获取post请求的secret
|
||||
post_secret = msg_json.get("content", {}).get("secret", "")
|
||||
secret = context.config.get("manager-api", {}).get("secret", "")
|
||||
|
||||
# 如果secret不匹配，则返回
|
||||
if post_secret != secret:
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"服务器密钥验证失败"
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
return
|
||||
|
||||
# 处理不同的action
|
||||
action = msg_json.get("action")
|
||||
|
||||
if action == "update_config":
|
||||
await self._handle_update_config(context, transport, msg_json)
|
||||
elif action == "restart":
|
||||
await self._handle_restart(context, transport, msg_json)
|
||||
else:
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"未知的服务器操作: {action}"
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
|
||||
async def _handle_update_config(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理配置更新 - 完整迁移自update_config逻辑"""
|
||||
try:
|
||||
# 检查是否有服务器实例
|
||||
if not context.server:
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"无法获取服务器实例",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
return
|
||||
|
||||
# 更新WebSocketServer的配置
|
||||
if not await context.server.update_config():
|
||||
error_msg = ""
|
||||
if hasattr(context.server, "get_last_update_error"):
|
||||
error_msg = context.server.get_last_update_error()
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
error_msg or "更新服务器配置失败",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
return
|
||||
|
||||
# 发送成功响应
|
||||
await self._send_success_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"配置更新成功",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"更新配置失败: {str(e)}")
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"更新配置失败: {str(e)}",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
|
||||
async def _handle_restart(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理服务器重启 - 完整迁移自handle_restart逻辑"""
|
||||
try:
|
||||
# 发送确认响应
|
||||
await self._send_success_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"服务器重启中...",
|
||||
{"action": "restart"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
|
||||
# 异步执行重启操作
|
||||
threading.Thread(target=lambda : restart_server(logger), daemon=True).start()
|
||||
except Exception as e:
|
||||
logger.error(f"处理重启请求失败: {str(e)}")
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"重启失败: {str(e)}",
|
||||
{"action": "restart"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
|
||||
async def _send_success_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, content: dict = None):
|
||||
"""发送成功响应"""
|
||||
response = {
|
||||
"type": "server",
|
||||
"status": "success",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if content:
|
||||
response["content"] = content
|
||||
|
||||
await transport.send(json.dumps(response))
|
||||
logger.info(f"服务器操作成功: {message}")
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, content: dict = None):
|
||||
"""发送错误响应"""
|
||||
response = {
|
||||
"type": "server",
|
||||
"status": "fail",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if content:
|
||||
response["content"] = content
|
||||
|
||||
await transport.send(json.dumps(response))
|
||||
logger.error(f"服务器操作失败: {message}")
|
||||
|
||||
async def _close_transport(self, transport: TransportInterface):
|
||||
try:
|
||||
await transport.close()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,54 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TextProcessor(MessageProcessor):
|
||||
"""
|
||||
纯文本消息处理器：处理非JSON格式的文本消息
|
||||
这是新架构中缺失的重要组件，用于处理直接发送的文本聊天内容
|
||||
"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理纯文本消息"""
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
# 尝试解析为JSON，如果成功则不是纯文本消息
|
||||
json.loads(message)
|
||||
return False # JSON消息由其他processor处理
|
||||
except json.JSONDecodeError:
|
||||
# 确实是纯文本消息，进行聊天处理
|
||||
await self.handle_text_message(context, transport, message)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_text_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""处理纯文本消息 - 直接调用ChatProcessor进行聊天"""
|
||||
try:
|
||||
# 记录收到纯文本消息
|
||||
logger.info(f"收到纯文本消息: {text[:100]}...")
|
||||
|
||||
# 使用ChatProcessor处理聊天
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
await chat_processor.handle_chat(context, transport, text)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理纯文本消息失败: {e}")
|
||||
# 发送错误响应
|
||||
await self._send_error_response(transport, "文本处理失败，请重试")
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, error_message: str):
|
||||
"""发送错误响应"""
|
||||
try:
|
||||
await transport.send(json.dumps({
|
||||
"type": "error",
|
||||
"message": error_message
|
||||
}))
|
||||
except Exception as e:
|
||||
logger.error(f"发送错误响应失败: {e}")
|
||||
@@ -0,0 +1,48 @@
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TimeoutProcessor(MessageProcessor):
|
||||
"""超时检查处理器：检查会话是否超时"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""检查会话超时"""
|
||||
return await self.handle_timeout(context, transport)
|
||||
|
||||
async def handle_timeout(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
) -> bool:
|
||||
"""End an expired logical conversation without closing a long connection."""
|
||||
if not getattr(context, "conversation_active", False):
|
||||
return False
|
||||
|
||||
timeout_seconds = context.config.get("close_connection_no_voice_time", 120)
|
||||
if not context.is_timeout(timeout_seconds):
|
||||
return False
|
||||
|
||||
try:
|
||||
if transport.keeps_connection_between_sessions:
|
||||
logger.info(f"会话超时，结束长连接逻辑会话: {context.session_id}")
|
||||
from core.processors.audio_receive_processor import (
|
||||
AudioReceiveProcessor,
|
||||
)
|
||||
|
||||
await AudioReceiveProcessor()._no_voice_close_connect(
|
||||
context,
|
||||
transport,
|
||||
have_voice=False,
|
||||
)
|
||||
else:
|
||||
logger.info(f"会话超时，准备关闭连接: {context.session_id}")
|
||||
await transport.close()
|
||||
except Exception as e:
|
||||
logger.error(f"处理超时连接失败: {e}")
|
||||
|
||||
return True
|
||||
@@ -109,7 +109,7 @@ class ASRProvider(ASRProviderBase):
|
||||
self.token, expire_time_str = AccessToken.create_token(self.access_key_id, self.access_key_secret)
|
||||
if not self.token:
|
||||
raise ValueError("无法获取有效的访问Token")
|
||||
|
||||
|
||||
try:
|
||||
expire_str = str(expire_time_str).strip()
|
||||
if expire_str.isdigit():
|
||||
@@ -151,7 +151,7 @@ class ASRProvider(ASRProviderBase):
|
||||
"""开始识别会话"""
|
||||
if self._is_token_expired():
|
||||
self._refresh_token()
|
||||
|
||||
|
||||
# 建立连接
|
||||
headers = {"X-NLS-Token": self.token}
|
||||
self.asr_ws = await websockets.connect(
|
||||
@@ -169,7 +169,10 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
self.is_processing = True
|
||||
self.server_ready = False # 重置服务器准备状态
|
||||
self.forward_task = asyncio.create_task(self._forward_results(conn))
|
||||
session_id = getattr(conn, "session_id", None)
|
||||
self.forward_task = self._create_session_task(
|
||||
conn, self._forward_results(conn, session_id)
|
||||
)
|
||||
|
||||
# 发送开始请求
|
||||
start_request = {
|
||||
@@ -193,10 +196,13 @@ class ASRProvider(ASRProviderBase):
|
||||
await self.asr_ws.send(json.dumps(start_request, ensure_ascii=False))
|
||||
logger.bind(tag=TAG).debug("已发送开始请求，等待服务器准备...")
|
||||
|
||||
async def _forward_results(self, conn: "ConnectionHandler"):
|
||||
async def _forward_results(self, conn: "ConnectionHandler", session_id=None):
|
||||
"""转发识别结果"""
|
||||
try:
|
||||
while not conn.stop_event.is_set():
|
||||
while (
|
||||
not conn.stop_event.is_set()
|
||||
and self._session_is_current(conn, session_id)
|
||||
):
|
||||
# 获取当前连接的音频数据
|
||||
audio_data = conn.asr_audio
|
||||
try:
|
||||
@@ -276,7 +282,7 @@ class ASRProvider(ASRProviderBase):
|
||||
finally:
|
||||
# 清理连接的音频缓存
|
||||
await self._cleanup()
|
||||
conn.reset_audio_states()
|
||||
self._reset_audio_if_current(conn, session_id)
|
||||
|
||||
async def _send_stop_request(self):
|
||||
"""发送停止识别请求（不关闭连接）"""
|
||||
@@ -308,6 +314,21 @@ class ASRProvider(ASRProviderBase):
|
||||
self.server_ready = False
|
||||
logger.bind(tag=TAG).debug("ASR状态已重置")
|
||||
|
||||
forward_task = self.forward_task
|
||||
current_task = asyncio.current_task()
|
||||
if (
|
||||
forward_task
|
||||
and forward_task is not current_task
|
||||
and not forward_task.done()
|
||||
):
|
||||
forward_task.cancel()
|
||||
try:
|
||||
await forward_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"等待ASR转发任务退出失败: {e}")
|
||||
|
||||
# 关闭连接
|
||||
if self.asr_ws:
|
||||
try:
|
||||
@@ -319,8 +340,10 @@ class ASRProvider(ASRProviderBase):
|
||||
finally:
|
||||
self.asr_ws = None
|
||||
|
||||
# 清理任务引用
|
||||
self.forward_task = None
|
||||
# Never discard a live task reference. The current forward task reaches
|
||||
# this branch from its own finally block and is already completing.
|
||||
if self.forward_task is forward_task:
|
||||
self.forward_task = None
|
||||
|
||||
logger.bind(tag=TAG).debug("ASR会话清理完成")
|
||||
|
||||
|
||||
@@ -103,7 +103,10 @@ class ASRProvider(ASRProviderBase):
|
||||
logger.bind(tag=TAG).debug("WebSocket连接建立成功")
|
||||
|
||||
self.server_ready = False
|
||||
self.forward_task = asyncio.create_task(self._forward_results(conn))
|
||||
session_id = getattr(conn, "session_id", None)
|
||||
self.forward_task = self._create_session_task(
|
||||
conn, self._forward_results(conn, session_id)
|
||||
)
|
||||
|
||||
# 发送run-task指令
|
||||
run_task_msg = self._build_run_task_message()
|
||||
@@ -154,10 +157,13 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
return message
|
||||
|
||||
async def _forward_results(self, conn: "ConnectionHandler"):
|
||||
async def _forward_results(self, conn: "ConnectionHandler", session_id=None):
|
||||
"""转发识别结果"""
|
||||
try:
|
||||
while not conn.stop_event.is_set():
|
||||
while (
|
||||
not conn.stop_event.is_set()
|
||||
and self._session_is_current(conn, session_id)
|
||||
):
|
||||
# 获取当前连接的音频数据
|
||||
audio_data = conn.asr_audio
|
||||
try:
|
||||
@@ -243,7 +249,7 @@ class ASRProvider(ASRProviderBase):
|
||||
finally:
|
||||
# 清理连接的音频缓存
|
||||
await self._cleanup()
|
||||
conn.reset_audio_states()
|
||||
self._reset_audio_if_current(conn, session_id)
|
||||
|
||||
async def _send_stop_request(self):
|
||||
"""发送停止请求(用于手动模式停止录音)"""
|
||||
@@ -285,6 +291,21 @@ class ASRProvider(ASRProviderBase):
|
||||
self.server_ready = False
|
||||
logger.bind(tag=TAG).debug("ASR状态已重置")
|
||||
|
||||
forward_task = self.forward_task
|
||||
current_task = asyncio.current_task()
|
||||
if (
|
||||
forward_task
|
||||
and forward_task is not current_task
|
||||
and not forward_task.done()
|
||||
):
|
||||
forward_task.cancel()
|
||||
try:
|
||||
await forward_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"等待ASR转发任务退出失败: {e}")
|
||||
|
||||
# 关闭连接
|
||||
if self.asr_ws:
|
||||
try:
|
||||
@@ -301,8 +322,8 @@ class ASRProvider(ASRProviderBase):
|
||||
finally:
|
||||
self.asr_ws = None
|
||||
|
||||
# 清理任务引用
|
||||
self.forward_task = None
|
||||
if self.forward_task is forward_task:
|
||||
self.forward_task = None
|
||||
self.task_id = None
|
||||
|
||||
logger.bind(tag=TAG).debug("ASR会话清理完成")
|
||||
@@ -315,4 +336,4 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
async def close(self):
|
||||
"""关闭资源"""
|
||||
await self._cleanup()
|
||||
await self._cleanup()
|
||||
|
||||
@@ -32,8 +32,37 @@ class ASRProviderBase(ABC):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _session_is_current(conn: "ConnectionHandler", session_id: str) -> bool:
|
||||
"""Return whether an async ASR result still belongs to this session."""
|
||||
return session_id is None or getattr(conn, "session_id", None) == session_id
|
||||
|
||||
def _create_session_task(self, conn: "ConnectionHandler", coroutine):
|
||||
"""Bind streaming ASR work to the logical session when supported."""
|
||||
creator = getattr(conn, "create_background_task", None)
|
||||
if callable(creator):
|
||||
try:
|
||||
return creator(coroutine, turn_scoped=True)
|
||||
except TypeError:
|
||||
return creator(coroutine)
|
||||
return asyncio.create_task(coroutine)
|
||||
|
||||
def _reset_audio_if_current(
|
||||
self, conn: "ConnectionHandler", session_id: str
|
||||
) -> None:
|
||||
if not self._session_is_current(conn, session_id):
|
||||
return
|
||||
reset_audio_states = getattr(conn, "reset_audio_states", None)
|
||||
if callable(reset_audio_states):
|
||||
reset_audio_states()
|
||||
|
||||
# 打开音频通道
|
||||
async def open_audio_channels(self, conn: "ConnectionHandler"):
|
||||
# The pipeline runtime feeds PCM directly and does not use the legacy
|
||||
# priority queue. Starting that worker would route frames back through
|
||||
# ConnectionHandler-only functions.
|
||||
if getattr(conn, "uses_pipeline_runtime", False):
|
||||
return
|
||||
conn.asr_priority_thread = threading.Thread(
|
||||
target=self.asr_text_priority_thread, args=(conn,), daemon=True
|
||||
)
|
||||
@@ -72,18 +101,26 @@ class ASRProviderBase(ABC):
|
||||
return
|
||||
|
||||
# 自动模式下通过VAD检测到语音停止时触发识别
|
||||
if conn.asr.interface_type != InterfaceType.STREAM and conn.client_voice_stop:
|
||||
interface_type = getattr(
|
||||
self,
|
||||
"interface_type",
|
||||
getattr(getattr(conn, "asr", None), "interface_type", None),
|
||||
)
|
||||
if interface_type != InterfaceType.STREAM and conn.client_voice_stop:
|
||||
# 直接使用asr_audio中的PCM数据
|
||||
pcm_bytes = b"".join(conn.asr_audio)
|
||||
# 检查是否有足够的音频数据（每帧1920字节，15帧约28800字节）
|
||||
if len(pcm_bytes) > 1920 * 15:
|
||||
await self.handle_voice_stop(conn, [pcm_bytes])
|
||||
conn.reset_audio_states()
|
||||
reset_audio_states = getattr(conn, "reset_audio_states", None)
|
||||
if callable(reset_audio_states):
|
||||
reset_audio_states()
|
||||
|
||||
# 处理语音停止
|
||||
async def handle_voice_stop(self, conn: "ConnectionHandler", asr_audio_task: List[bytes]):
|
||||
"""并行处理ASR和声纹识别"""
|
||||
try:
|
||||
session_id = getattr(conn, "session_id", None)
|
||||
total_start_time = time.monotonic()
|
||||
|
||||
# 数据已经是PCM直接使用
|
||||
@@ -96,13 +133,11 @@ class ASRProviderBase(ABC):
|
||||
wav_data = self._pcm_to_wav(combined_pcm_data)
|
||||
|
||||
# 定义ASR任务
|
||||
asr_task = self.speech_to_text_wrapper(
|
||||
asr_audio_task, conn.session_id
|
||||
)
|
||||
asr_task = self.speech_to_text_wrapper(asr_audio_task, session_id)
|
||||
|
||||
if conn.voiceprint_provider and wav_data:
|
||||
voiceprint_task = conn.voiceprint_provider.identify_speaker(
|
||||
wav_data, conn.session_id
|
||||
wav_data, session_id
|
||||
)
|
||||
# 并发等待两个结果
|
||||
asr_result, voiceprint_result = await asyncio.gather(
|
||||
@@ -112,6 +147,12 @@ class ASRProviderBase(ABC):
|
||||
asr_result = await asr_task
|
||||
voiceprint_result = None
|
||||
|
||||
if not self._session_is_current(conn, session_id):
|
||||
logger.bind(tag=TAG).info(
|
||||
f"丢弃旧会话ASR结果: session_id={session_id}"
|
||||
)
|
||||
return
|
||||
|
||||
# 记录识别结果 - 检查是否为异常
|
||||
if isinstance(asr_result, Exception):
|
||||
logger.bind(tag=TAG).error(f"ASR识别失败: {asr_result}")
|
||||
@@ -165,9 +206,13 @@ class ASRProviderBase(ABC):
|
||||
|
||||
if text_len > 0:
|
||||
audio_snapshot = asr_audio_task.copy()
|
||||
enqueue_asr_report(conn, enhanced_text, audio_snapshot)
|
||||
# 使用自定义模块进行上报
|
||||
await startToChat(conn, enhanced_text)
|
||||
result_handler = getattr(conn, "asr_result_handler", None)
|
||||
if callable(result_handler):
|
||||
await result_handler(enhanced_text, audio_snapshot)
|
||||
else:
|
||||
enqueue_asr_report(conn, enhanced_text, audio_snapshot)
|
||||
# Legacy ConnectionHandler path.
|
||||
await startToChat(conn, enhanced_text)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
|
||||
import traceback
|
||||
|
||||
@@ -117,7 +117,10 @@ class ASRProvider(ASRProviderBase):
|
||||
raise e
|
||||
|
||||
# 启动接收ASR结果的异步任务
|
||||
self.forward_task = asyncio.create_task(self._forward_asr_results(conn))
|
||||
session_id = getattr(conn, "session_id", None)
|
||||
self.forward_task = self._create_session_task(
|
||||
conn, self._forward_asr_results(conn, session_id)
|
||||
)
|
||||
|
||||
# 发送缓存的音频数据
|
||||
if conn.asr_audio and len(conn.asr_audio) > 0:
|
||||
@@ -156,9 +159,13 @@ class ASRProvider(ASRProviderBase):
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).info(f"发送音频数据时发生错误: {e}")
|
||||
|
||||
async def _forward_asr_results(self, conn: "ConnectionHandler"):
|
||||
async def _forward_asr_results(self, conn: "ConnectionHandler", session_id=None):
|
||||
try:
|
||||
while self.asr_ws and not conn.stop_event.is_set():
|
||||
while (
|
||||
self.asr_ws
|
||||
and not conn.stop_event.is_set()
|
||||
and self._session_is_current(conn, session_id)
|
||||
):
|
||||
# 获取当前连接的音频数据
|
||||
audio_data = conn.asr_audio
|
||||
try:
|
||||
@@ -249,21 +256,47 @@ class ASRProvider(ASRProviderBase):
|
||||
if hasattr(e, "__cause__") and e.__cause__:
|
||||
logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
|
||||
finally:
|
||||
if self.asr_ws:
|
||||
await self.asr_ws.close()
|
||||
self.asr_ws = None
|
||||
self.is_processing = False
|
||||
self._is_stopping = False
|
||||
await self._cleanup()
|
||||
# 重置所有音频相关状态
|
||||
conn.reset_audio_states()
|
||||
self._reset_audio_if_current(conn, session_id)
|
||||
|
||||
def stop_ws_connection(self):
|
||||
if self.asr_ws:
|
||||
asyncio.create_task(self.asr_ws.close())
|
||||
self.asr_ws = None
|
||||
# The forward task owns the WebSocket and closes it from _cleanup().
|
||||
# Scheduling an untracked close here races with that cleanup path.
|
||||
self.is_processing = False
|
||||
self._is_stopping = False
|
||||
|
||||
async def _cleanup(self):
|
||||
"""取消转发任务并关闭流式 ASR 连接。"""
|
||||
self.is_processing = False
|
||||
self._is_stopping = False
|
||||
|
||||
forward_task = self.forward_task
|
||||
current_task = asyncio.current_task()
|
||||
if (
|
||||
forward_task
|
||||
and forward_task is not current_task
|
||||
and not forward_task.done()
|
||||
):
|
||||
forward_task.cancel()
|
||||
try:
|
||||
await forward_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"等待ASR转发任务退出失败: {e}")
|
||||
|
||||
if self.asr_ws:
|
||||
try:
|
||||
await asyncio.wait_for(self.asr_ws.close(), timeout=2.0)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"关闭ASR WebSocket连接失败: {e}")
|
||||
finally:
|
||||
self.asr_ws = None
|
||||
|
||||
if self.forward_task is forward_task:
|
||||
self.forward_task = None
|
||||
|
||||
async def _send_stop_request(self):
|
||||
"""发送最后一个音频帧以通知服务器结束"""
|
||||
self._is_stopping = True # 先标记为停止状态，阻止后续音频发送
|
||||
@@ -417,14 +450,4 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
async def close(self):
|
||||
"""资源清理方法"""
|
||||
if self.asr_ws:
|
||||
await self.asr_ws.close()
|
||||
self.asr_ws = None
|
||||
if self.forward_task:
|
||||
self.forward_task.cancel()
|
||||
try:
|
||||
await self.forward_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self.forward_task = None
|
||||
self.is_processing = False
|
||||
await self._cleanup()
|
||||
|
||||
@@ -90,8 +90,11 @@ class ASRProvider(ASRProviderBase):
|
||||
batch_size_s=60,
|
||||
)
|
||||
text = lang_tag_filter(result[0]["text"])
|
||||
recognized_content = (
|
||||
text.get("content", "") if isinstance(text, dict) else text
|
||||
)
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}"
|
||||
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {recognized_content}"
|
||||
)
|
||||
|
||||
return text, artifacts.file_path
|
||||
|
||||
@@ -0,0 +1,522 @@
|
||||
"""
|
||||
SharedASRManager: 全局 ASR 管理器
|
||||
实现单例模型 + 单推理执行器 + 队列限流。
|
||||
单例的原因是：推理是 CPU/GPU-bound，不是 I/O-bound，多实例不仅会占用内存，还会降低吞吐能力
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Dict, Any, Optional, Tuple, List
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
TAG = __name__
|
||||
|
||||
|
||||
class SharedASRManager:
|
||||
"""
|
||||
全局共享 ASR 管理器
|
||||
"""
|
||||
|
||||
# 支持预加载的本地模型类型
|
||||
LOCAL_MODEL_TYPES = [
|
||||
"fun_local", # FunASR 本地
|
||||
"sherpa_onnx_local", # Sherpa ONNX
|
||||
"sense_voice" # SenseVoice
|
||||
]
|
||||
|
||||
def __init__(self, config: Dict[str, Any], asr_type: str = None):
|
||||
"""
|
||||
初始化 ASR 管理器
|
||||
Args:
|
||||
config: 服务器配置
|
||||
asr_type: ASR 类型（Optional，用于显式指定）
|
||||
"""
|
||||
self.config = config
|
||||
self.asr_type = asr_type
|
||||
|
||||
# 模型实例（全局单例）
|
||||
self.model_instance = None
|
||||
|
||||
# 任务队列（限流）
|
||||
queue_max_size = self._get_queue_max_size()
|
||||
self.task_queue: asyncio.Queue = asyncio.Queue(maxsize=queue_max_size)
|
||||
|
||||
# 推理锁（使得推理串行化）
|
||||
self.inference_lock = asyncio.Lock()
|
||||
# 线程池执行器，用于阻塞调用
|
||||
self.executor: Optional[ThreadPoolExecutor] = None
|
||||
|
||||
# 运行状态
|
||||
self.running = False
|
||||
self._inference_task: Optional[asyncio.Task] = None
|
||||
self.is_local_model = self._check_local_model()
|
||||
self._variant_lock = asyncio.Lock()
|
||||
self._variant_managers = {}
|
||||
self._max_shared_models = max(
|
||||
1, int(config.get("shared_asr_max_models", 3) or 3)
|
||||
)
|
||||
|
||||
logger.bind(tag=TAG).info(
|
||||
f"SharedASRManager 初始化完成, "
|
||||
f"类型: {self.asr_type}, "
|
||||
f"本地模型: {self.is_local_model}, "
|
||||
f"队列大小: {queue_max_size}"
|
||||
)
|
||||
|
||||
def _get_queue_max_size(self) -> int:
|
||||
"""获取队列最大大小"""
|
||||
# 尝试从配置获取
|
||||
selected_asr = self.config.get("selected_module", {}).get("ASR")
|
||||
if selected_asr:
|
||||
asr_config = self.config.get("ASR", {}).get(selected_asr, {})
|
||||
return asr_config.get("queue_max_size", 100)
|
||||
return 100
|
||||
|
||||
def _check_local_model(self) -> bool:
|
||||
"""检查是否为本地模型"""
|
||||
if self.asr_type:
|
||||
return self.asr_type in self.LOCAL_MODEL_TYPES
|
||||
|
||||
# 从配置推断
|
||||
selected_asr = self.config.get("selected_module", {}).get("ASR")
|
||||
if not selected_asr:
|
||||
return False
|
||||
|
||||
asr_config = self.config.get("ASR", {}).get(selected_asr, {})
|
||||
asr_type = asr_config.get("type", selected_asr)
|
||||
self.asr_type = asr_type
|
||||
|
||||
return asr_type in self.LOCAL_MODEL_TYPES
|
||||
|
||||
async def initialize(self):
|
||||
"""
|
||||
初始化管理器
|
||||
- 预加载模型
|
||||
- 启动推理执行器
|
||||
"""
|
||||
if not self.is_local_model:
|
||||
logger.bind(tag=TAG).info("非本地模型，跳过预加载")
|
||||
return
|
||||
|
||||
if self.running:
|
||||
logger.bind(tag=TAG).warning("管理器已在运行中")
|
||||
return
|
||||
|
||||
try:
|
||||
logger.bind(tag=TAG).info(f"开始预加载 ASR 模型: {self.asr_type}")
|
||||
|
||||
# 预加载模型
|
||||
await self._preload_model()
|
||||
|
||||
# 启动推理执行器
|
||||
self.running = True
|
||||
self._inference_task = asyncio.create_task(self._inference_loop())
|
||||
|
||||
logger.bind(tag=TAG).info("ASR 模型预加载完成，推理执行器已启动")
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR 模型预加载失败: {e}")
|
||||
raise
|
||||
|
||||
async def _preload_model(self):
|
||||
"""在线程池中预加载模型"""
|
||||
# 创建线程池
|
||||
self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="asr_worker")
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
self.model_instance = await loop.run_in_executor(
|
||||
self.executor,
|
||||
self._create_model_instance
|
||||
)
|
||||
|
||||
logger.bind(tag=TAG).info("模型实例创建完成")
|
||||
|
||||
def _create_model_instance(self):
|
||||
"""
|
||||
实际创建模型实例（在线程中执行）
|
||||
|
||||
Returns:
|
||||
ASR Provider 实例
|
||||
"""
|
||||
from core.utils.modules_initialize import initialize_asr
|
||||
|
||||
logger.bind(tag=TAG).info("正在创建 ASR 模型实例...")
|
||||
instance = initialize_asr(self.config)
|
||||
logger.bind(tag=TAG).info("ASR 模型实例创建成功")
|
||||
|
||||
return instance
|
||||
|
||||
async def submit_task(
|
||||
self,
|
||||
opus_data: List[bytes],
|
||||
session_id: str,
|
||||
audio_format: str = "opus"
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
提交推理任务
|
||||
|
||||
Args:
|
||||
opus_data: 音频数据
|
||||
session_id: 会话 ID
|
||||
audio_format: 音频格式
|
||||
|
||||
Returns:
|
||||
(识别文本, 文件路径)
|
||||
|
||||
Raises:
|
||||
RuntimeError: 队列满或服务未运行
|
||||
"""
|
||||
if not self.running:
|
||||
raise RuntimeError("ASR 服务未运行")
|
||||
|
||||
# 检查队列是否满（限流）
|
||||
if self.task_queue.full():
|
||||
queue_status = self.get_queue_status()
|
||||
logger.bind(tag=TAG).warning(
|
||||
f"ASR 队列已满: {queue_status}"
|
||||
)
|
||||
raise RuntimeError("ASR 服务繁忙，请稍后重试")
|
||||
|
||||
# 创建 Future 用于返回结果
|
||||
result_future: asyncio.Future = asyncio.Future()
|
||||
|
||||
# 构造任务
|
||||
task = {
|
||||
'opus_data': opus_data,
|
||||
'session_id': session_id,
|
||||
'audio_format': audio_format,
|
||||
'future': result_future
|
||||
}
|
||||
|
||||
# 放入队列
|
||||
await self.task_queue.put(task)
|
||||
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"任务已提交, session: {session_id}, "
|
||||
f"队列大小: {self.task_queue.qsize()}"
|
||||
)
|
||||
|
||||
# 等待结果
|
||||
return await result_future
|
||||
|
||||
async def _inference_loop(self):
|
||||
"""
|
||||
单个推理执行器循环
|
||||
|
||||
核心原则：
|
||||
- 只有一个执行器
|
||||
- 串行处理任务
|
||||
- 带超时的队列获取
|
||||
"""
|
||||
logger.bind(tag=TAG).info("推理执行器启动")
|
||||
|
||||
while self.running:
|
||||
task = None
|
||||
try:
|
||||
# 带超时的队列获取，避免关闭时卡住
|
||||
try:
|
||||
task = await asyncio.wait_for(
|
||||
self.task_queue.get(),
|
||||
timeout=1.0
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
# 超时后检查 running 状态，继续循环
|
||||
continue
|
||||
|
||||
if task['future'].done():
|
||||
continue
|
||||
|
||||
# 执行推理（加锁保证串行）
|
||||
async with self.inference_lock:
|
||||
if task['future'].done():
|
||||
continue
|
||||
result = await self._run_inference(
|
||||
task['opus_data'],
|
||||
task['session_id'],
|
||||
task['audio_format']
|
||||
)
|
||||
|
||||
# 返回结果
|
||||
if not task['future'].done():
|
||||
task['future'].set_result(result)
|
||||
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"推理完成, session: {task['session_id']}"
|
||||
)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
logger.bind(tag=TAG).info("推理执行器被取消")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"推理执行失败: {e}")
|
||||
if task and 'future' in task and not task['future'].done():
|
||||
task['future'].set_exception(e)
|
||||
finally:
|
||||
if task is not None:
|
||||
self.task_queue.task_done()
|
||||
|
||||
logger.bind(tag=TAG).info("推理执行器已停止")
|
||||
|
||||
async def _run_inference(
|
||||
self,
|
||||
opus_data: List[bytes],
|
||||
session_id: str,
|
||||
audio_format: str
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
执行实际推理（在线程池中）
|
||||
|
||||
Args:
|
||||
opus_data: 音频数据
|
||||
session_id: 会话 ID
|
||||
audio_format: 音频格式
|
||||
|
||||
Returns:
|
||||
(识别文本, 文件路径)
|
||||
"""
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
# 在线程池中执行推理
|
||||
result = await loop.run_in_executor(
|
||||
self.executor,
|
||||
lambda: self._sync_wrapper(opus_data, session_id)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def _sync_wrapper(
|
||||
self,
|
||||
opus_data: List[bytes],
|
||||
session_id: str,
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Run the provider's complete artifact wrapper in the worker thread."""
|
||||
import asyncio
|
||||
|
||||
async def _call():
|
||||
return await self.model_instance.speech_to_text_wrapper(
|
||||
opus_data, session_id
|
||||
)
|
||||
|
||||
# 创建新的事件循环执行
|
||||
loop = None
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
result = loop.run_until_complete(_call())
|
||||
return result
|
||||
finally:
|
||||
if loop:
|
||||
loop.close()
|
||||
|
||||
def matches_config(self, config: Dict[str, Any]) -> bool:
|
||||
"""Return whether this manager owns the exact selected ASR config."""
|
||||
selected = config.get("selected_module", {}).get("ASR")
|
||||
manager_selected = self.config.get("selected_module", {}).get("ASR")
|
||||
if not selected or selected != manager_selected:
|
||||
return False
|
||||
return (
|
||||
config.get("ASR", {}).get(selected)
|
||||
== self.config.get("ASR", {}).get(manager_selected)
|
||||
and bool(config.get("delete_audio", True))
|
||||
== bool(self.config.get("delete_audio", True))
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _config_fingerprint(cls, config: Dict[str, Any]) -> str:
|
||||
selected = config.get("selected_module", {}).get("ASR")
|
||||
payload = {
|
||||
"selected": selected,
|
||||
"config": config.get("ASR", {}).get(selected),
|
||||
"delete_audio": bool(config.get("delete_audio", True)),
|
||||
}
|
||||
return json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
@classmethod
|
||||
def _config_is_local(cls, config: Dict[str, Any]) -> bool:
|
||||
selected = config.get("selected_module", {}).get("ASR")
|
||||
asr_config = config.get("ASR", {}).get(selected, {})
|
||||
asr_type = asr_config.get("type", selected)
|
||||
return asr_type in cls.LOCAL_MODEL_TYPES
|
||||
|
||||
async def acquire_for_config(
|
||||
self, config: Dict[str, Any]
|
||||
) -> Optional["SharedASRManager"]:
|
||||
"""Acquire a bounded shared local model matching one Agent config."""
|
||||
if self.matches_config(config):
|
||||
return self if self.is_ready() else None
|
||||
if not self._config_is_local(config):
|
||||
return None
|
||||
|
||||
fingerprint = self._config_fingerprint(config)
|
||||
async with self._variant_lock:
|
||||
entry = self._variant_managers.get(fingerprint)
|
||||
if entry is not None:
|
||||
entry["references"] += 1
|
||||
return entry["manager"]
|
||||
|
||||
if len(self._variant_managers) >= self._max_shared_models - 1:
|
||||
idle_fingerprint = next(
|
||||
(
|
||||
key
|
||||
for key, candidate in self._variant_managers.items()
|
||||
if candidate["references"] == 0
|
||||
),
|
||||
None,
|
||||
)
|
||||
if idle_fingerprint is None:
|
||||
raise RuntimeError(
|
||||
"ASR共享模型容量已满，请提高shared_asr_max_models或统一Agent ASR配置"
|
||||
)
|
||||
idle_manager = self._variant_managers.pop(
|
||||
idle_fingerprint
|
||||
)["manager"]
|
||||
# Keep the lock until shutdown completes so a replacement can
|
||||
# never overlap the retiring model and exceed the hard limit.
|
||||
await idle_manager.shutdown()
|
||||
|
||||
manager = SharedASRManager(copy.deepcopy(config))
|
||||
try:
|
||||
await manager.initialize()
|
||||
except Exception:
|
||||
await manager.shutdown()
|
||||
raise
|
||||
self._variant_managers[fingerprint] = {
|
||||
"manager": manager,
|
||||
"references": 1,
|
||||
}
|
||||
logger.bind(tag=TAG).info(
|
||||
"已加载Agent专用共享ASR模型，当前模型数: {}",
|
||||
len(self._variant_managers) + 1,
|
||||
)
|
||||
return manager
|
||||
|
||||
async def release_for_config(self, manager: "SharedASRManager") -> None:
|
||||
"""Release an Agent model; idle variants stay cached for safe reuse."""
|
||||
if manager is self:
|
||||
return
|
||||
async with self._variant_lock:
|
||||
for entry in self._variant_managers.values():
|
||||
if entry["manager"] is not manager:
|
||||
continue
|
||||
entry["references"] = max(0, entry["references"] - 1)
|
||||
break
|
||||
|
||||
async def shutdown(self):
|
||||
"""
|
||||
优雅停机
|
||||
|
||||
步骤：
|
||||
1. 停止接收新任务
|
||||
2. 等待当前任务完成（带超时）
|
||||
3. 取消未完成的任务
|
||||
4. 关闭线程池
|
||||
"""
|
||||
async with self._variant_lock:
|
||||
variants = [
|
||||
entry["manager"] for entry in self._variant_managers.values()
|
||||
]
|
||||
self._variant_managers.clear()
|
||||
if variants:
|
||||
await asyncio.gather(
|
||||
*(manager.shutdown() for manager in variants),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
if (
|
||||
not self.running
|
||||
and self.executor is None
|
||||
and self.model_instance is None
|
||||
):
|
||||
return
|
||||
|
||||
logger.bind(tag=TAG).info("开始关闭 ASR 管理器...")
|
||||
|
||||
# 停止接收新任务
|
||||
self.running = False
|
||||
|
||||
# 等待推理任务完成
|
||||
if self._inference_task and not self._inference_task.done():
|
||||
try:
|
||||
# 最多等待 5 秒
|
||||
await asyncio.wait_for(
|
||||
self._inference_task,
|
||||
timeout=5.0
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.bind(tag=TAG).warning("推理任务超时，强制取消")
|
||||
self._inference_task.cancel()
|
||||
try:
|
||||
await self._inference_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
# 取消所有队列中未完成的任务
|
||||
cancelled_count = 0
|
||||
while not self.task_queue.empty():
|
||||
try:
|
||||
task = self.task_queue.get_nowait()
|
||||
if not task['future'].done():
|
||||
task['future'].set_exception(
|
||||
RuntimeError("ASR 服务正在关闭")
|
||||
)
|
||||
cancelled_count += 1
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
|
||||
if cancelled_count > 0:
|
||||
logger.bind(tag=TAG).info(f"已取消 {cancelled_count} 个待处理任务")
|
||||
|
||||
# 关闭线程池
|
||||
if self.executor:
|
||||
self.executor.shutdown(wait=False)
|
||||
self.executor = None
|
||||
logger.bind(tag=TAG).info("线程池已关闭")
|
||||
|
||||
# 清理模型实例
|
||||
self.model_instance = None
|
||||
|
||||
logger.bind(tag=TAG).info("ASR 管理器已关闭")
|
||||
|
||||
def get_queue_status(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取队列状态（用于监控）
|
||||
|
||||
Returns:
|
||||
队列状态字典
|
||||
"""
|
||||
queue_size = self.task_queue.qsize()
|
||||
queue_max = self.task_queue.maxsize
|
||||
|
||||
return {
|
||||
'queue_size': queue_size,
|
||||
'queue_max': queue_max,
|
||||
'is_busy': queue_size > queue_max * 0.8,
|
||||
'utilization': queue_size / queue_max if queue_max > 0 else 0,
|
||||
'running': self.running
|
||||
}
|
||||
|
||||
def is_ready(self) -> bool:
|
||||
"""检查管理器是否就绪"""
|
||||
return (
|
||||
self.running and
|
||||
self.model_instance is not None and
|
||||
self.executor is not None
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def is_local_model_type(cls, asr_type: str) -> bool:
|
||||
"""
|
||||
检查 ASR 类型是否为本地模型
|
||||
|
||||
Args:
|
||||
asr_type: ASR 类型
|
||||
|
||||
Returns:
|
||||
是否为本地模型
|
||||
"""
|
||||
return asr_type in cls.LOCAL_MODEL_TYPES
|
||||
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
SharedASRProxy: 共享 ASR 管理器的代理类
|
||||
|
||||
功能：
|
||||
- 包装 SharedASRManager
|
||||
- 提供与原 ASR Provider 相同的接口
|
||||
- 处理队列满等异常情况
|
||||
"""
|
||||
|
||||
from typing import List, Tuple, Optional, Dict, Any
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
TAG = __name__
|
||||
|
||||
|
||||
class SharedASRProxy(ASRProviderBase):
|
||||
"""
|
||||
共享 ASR 管理器的代理类
|
||||
|
||||
该类提供与原 ASR Provider 相同的接口，
|
||||
但实际推理工作由 SharedASRManager 完成。
|
||||
"""
|
||||
|
||||
def __init__(self, manager):
|
||||
"""
|
||||
初始化代理
|
||||
|
||||
Args:
|
||||
manager: SharedASRManager 实例
|
||||
"""
|
||||
super().__init__()
|
||||
self.manager = manager
|
||||
|
||||
# 从共享管理器获取接口类型
|
||||
if manager.model_instance and hasattr(manager.model_instance, 'interface_type'):
|
||||
self.interface_type = manager.model_instance.interface_type
|
||||
else:
|
||||
self.interface_type = InterfaceType.LOCAL
|
||||
|
||||
logger.bind(tag=TAG).info("SharedASRProxy 初始化完成")
|
||||
|
||||
async def speech_to_text(
|
||||
self,
|
||||
opus_data: List[bytes],
|
||||
session_id: str,
|
||||
audio_format: str = "opus"
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
语音转文本（通过共享管理器）
|
||||
|
||||
Args:
|
||||
opus_data: 音频数据（Opus 编码的字节列表）
|
||||
session_id: 会话 ID
|
||||
audio_format: 音频格式，默认 "opus"
|
||||
|
||||
Returns:
|
||||
Tuple[str, str]: (识别的文本, 音频文件路径)
|
||||
"""
|
||||
try:
|
||||
# 检查管理器状态
|
||||
if not self.manager.is_ready():
|
||||
logger.bind(tag=TAG).error("ASR 管理器未就绪")
|
||||
return "", None
|
||||
|
||||
# 提交任务到共享管理器
|
||||
result = await self.manager.submit_task(
|
||||
opus_data,
|
||||
session_id,
|
||||
audio_format
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except RuntimeError as e:
|
||||
# 队列满或服务未运行
|
||||
logger.bind(tag=TAG).warning(f"ASR 服务繁忙: {e}")
|
||||
# 返回友好提示，而不是空字符串
|
||||
return "服务繁忙，请稍后重试", None
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR 推理失败: {e}")
|
||||
return "", None
|
||||
|
||||
async def speech_to_text_wrapper(
|
||||
self, pcm_data: List[bytes], session_id: str
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Delegate PCM inference without using uninitialized proxy file state."""
|
||||
return await self.manager.submit_task(pcm_data, session_id, "pcm")
|
||||
|
||||
def get_queue_status(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取队列状态
|
||||
|
||||
Returns:
|
||||
队列状态字典
|
||||
"""
|
||||
return self.manager.get_queue_status()
|
||||
|
||||
def is_ready(self) -> bool:
|
||||
"""
|
||||
检查代理是否就绪
|
||||
|
||||
Returns:
|
||||
是否就绪
|
||||
"""
|
||||
return self.manager.is_ready()
|
||||
|
||||
async def close(self):
|
||||
"""
|
||||
关闭代理
|
||||
|
||||
注意：不关闭共享管理器，由服务器统一管理
|
||||
"""
|
||||
logger.bind(tag=TAG).debug("SharedASRProxy 关闭")
|
||||
# 代理不负责关闭共享管理器
|
||||
pass
|
||||
@@ -141,7 +141,10 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
logger.bind(tag=TAG).info("ASR WebSocket连接已建立")
|
||||
self.server_ready = False
|
||||
self.forward_task = asyncio.create_task(self._forward_results(conn))
|
||||
session_id = getattr(conn, "session_id", None)
|
||||
self.forward_task = self._create_session_task(
|
||||
conn, self._forward_results(conn, session_id)
|
||||
)
|
||||
|
||||
# 发送首帧音频
|
||||
if conn.asr_audio and len(conn.asr_audio) > 0:
|
||||
@@ -185,10 +188,13 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
await self.asr_ws.send(json.dumps(frame_data, ensure_ascii=False))
|
||||
|
||||
async def _forward_results(self, conn: "ConnectionHandler"):
|
||||
async def _forward_results(self, conn: "ConnectionHandler", session_id=None):
|
||||
"""转发识别结果"""
|
||||
try:
|
||||
while not conn.stop_event.is_set():
|
||||
while (
|
||||
not conn.stop_event.is_set()
|
||||
and self._session_is_current(conn, session_id)
|
||||
):
|
||||
try:
|
||||
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=60)
|
||||
result = json.loads(response)
|
||||
@@ -247,7 +253,7 @@ class ASRProvider(ASRProviderBase):
|
||||
finally:
|
||||
# 清理连接资源
|
||||
await self._cleanup()
|
||||
conn.reset_audio_states()
|
||||
self._reset_audio_if_current(conn, session_id)
|
||||
|
||||
async def handle_voice_stop(
|
||||
self, conn: "ConnectionHandler", asr_audio_task: List[bytes]
|
||||
@@ -272,9 +278,8 @@ class ASRProvider(ASRProviderBase):
|
||||
logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
|
||||
|
||||
def stop_ws_connection(self):
|
||||
if self.asr_ws:
|
||||
asyncio.create_task(self.asr_ws.close())
|
||||
self.asr_ws = None
|
||||
# The forward task owns the WebSocket and closes it from _cleanup().
|
||||
# Scheduling an untracked close here races with that cleanup path.
|
||||
self.is_processing = False
|
||||
|
||||
async def _send_stop_request(self):
|
||||
@@ -299,6 +304,21 @@ class ASRProvider(ASRProviderBase):
|
||||
self.server_ready = False
|
||||
logger.bind(tag=TAG).debug("ASR状态已重置")
|
||||
|
||||
forward_task = self.forward_task
|
||||
current_task = asyncio.current_task()
|
||||
if (
|
||||
forward_task
|
||||
and forward_task is not current_task
|
||||
and not forward_task.done()
|
||||
):
|
||||
forward_task.cancel()
|
||||
try:
|
||||
await forward_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"等待ASR转发任务退出失败: {e}")
|
||||
|
||||
# 关闭连接
|
||||
if self.asr_ws:
|
||||
try:
|
||||
@@ -310,8 +330,8 @@ class ASRProvider(ASRProviderBase):
|
||||
finally:
|
||||
self.asr_ws = None
|
||||
|
||||
# 清理任务引用
|
||||
self.forward_task = None
|
||||
if self.forward_task is forward_task:
|
||||
self.forward_task = None
|
||||
|
||||
logger.bind(tag=TAG).debug("ASR会话清理完成")
|
||||
|
||||
@@ -323,15 +343,4 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
async def close(self):
|
||||
"""资源清理方法"""
|
||||
if self.asr_ws:
|
||||
await self.asr_ws.close()
|
||||
self.asr_ws = None
|
||||
if self.forward_task:
|
||||
self.forward_task.cancel()
|
||||
try:
|
||||
await self.forward_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self.forward_task = None
|
||||
self.is_processing = False
|
||||
|
||||
await self._cleanup()
|
||||
|
||||
@@ -127,7 +127,16 @@ class DeviceIoTExecutor(ToolExecutor):
|
||||
send_message = json.dumps(
|
||||
{"type": "iot", "commands": [command]}
|
||||
)
|
||||
await self.conn.websocket.send(send_message)
|
||||
|
||||
# 使用transport接口发送消息
|
||||
if hasattr(self.conn, 'transport') and self.conn.transport:
|
||||
await self.conn.transport.send(send_message)
|
||||
elif hasattr(self.conn, 'websocket') and self.conn.websocket:
|
||||
# 兼容旧版本
|
||||
logger.warning("未找到SessionContext的传输层接口, 回退使用旧版conn.websocket发送消息")
|
||||
await self.conn.websocket.send(send_message)
|
||||
else:
|
||||
raise AttributeError("无法找到可用的传输层接口")
|
||||
return
|
||||
|
||||
raise Exception(f"未找到设备{device_name}的方法{method_name}")
|
||||
|
||||
@@ -17,7 +17,7 @@ class MCPClient:
|
||||
self.name_mapping = {}
|
||||
self.ready = False
|
||||
self.call_results = {} # To store Futures for tool call responses
|
||||
self.next_id = 1
|
||||
self.next_id = 10000
|
||||
self.lock = asyncio.Lock()
|
||||
self._cached_available_tools = None # Cache for get_available_tools
|
||||
|
||||
@@ -91,3 +91,12 @@ class MCPClient:
|
||||
async with self.lock:
|
||||
if id in self.call_results:
|
||||
self.call_results.pop(id)
|
||||
|
||||
async def close(self):
|
||||
async with self.lock:
|
||||
pending = list(self.call_results.values())
|
||||
self.call_results.clear()
|
||||
self.ready = False
|
||||
for future in pending:
|
||||
if not future.done():
|
||||
future.set_exception(ConnectionError("MCP会话已关闭"))
|
||||
|
||||
@@ -3,11 +3,11 @@
|
||||
import json
|
||||
import asyncio
|
||||
import re
|
||||
from concurrent.futures import Future
|
||||
from core.utils.util import get_vision_url, sanitize_tool_name
|
||||
from core.utils.util import get_vision_url
|
||||
from core.utils.auth import AuthToken
|
||||
from config.logger import setup_logging
|
||||
from typing import TYPE_CHECKING
|
||||
from .mcp_client import MCPClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from core.connection import ConnectionHandler
|
||||
@@ -15,108 +15,45 @@ if TYPE_CHECKING:
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class MCPClient:
|
||||
"""设备端MCP客户端，用于管理MCP状态和工具"""
|
||||
|
||||
def __init__(self):
|
||||
self.tools = {} # sanitized_name -> tool_data
|
||||
self.name_mapping = {}
|
||||
self.ready = False
|
||||
self.call_results = {} # To store Futures for tool call responses
|
||||
self.next_id = 1
|
||||
self.lock = asyncio.Lock()
|
||||
self._cached_available_tools = None # Cache for get_available_tools
|
||||
|
||||
def has_tool(self, name: str) -> bool:
|
||||
return name in self.tools
|
||||
|
||||
def get_available_tools(self) -> list:
|
||||
# Check if the cache is valid
|
||||
if self._cached_available_tools is not None:
|
||||
return self._cached_available_tools
|
||||
|
||||
# If cache is not valid, regenerate the list
|
||||
result = []
|
||||
for tool_name, tool_data in self.tools.items():
|
||||
function_def = {
|
||||
"name": tool_name,
|
||||
"description": tool_data["description"],
|
||||
"parameters": {
|
||||
"type": tool_data["inputSchema"].get("type", "object"),
|
||||
"properties": tool_data["inputSchema"].get("properties", {}),
|
||||
"required": tool_data["inputSchema"].get("required", []),
|
||||
},
|
||||
}
|
||||
result.append({"type": "function", "function": function_def})
|
||||
|
||||
self._cached_available_tools = result # Store the generated list in cache
|
||||
return result
|
||||
|
||||
async def is_ready(self) -> bool:
|
||||
async with self.lock:
|
||||
return self.ready
|
||||
|
||||
async def set_ready(self, status: bool):
|
||||
async with self.lock:
|
||||
self.ready = status
|
||||
|
||||
async def add_tool(self, tool_data: dict):
|
||||
async with self.lock:
|
||||
sanitized_name = sanitize_tool_name(tool_data["name"])
|
||||
self.tools[sanitized_name] = tool_data
|
||||
self.name_mapping[sanitized_name] = tool_data["name"]
|
||||
self._cached_available_tools = (
|
||||
None # Invalidate the cache when a tool is added
|
||||
)
|
||||
|
||||
async def get_next_id(self) -> int:
|
||||
async with self.lock:
|
||||
current_id = self.next_id
|
||||
self.next_id += 1
|
||||
return current_id
|
||||
|
||||
async def register_call_result_future(self, id: int, future: Future):
|
||||
async with self.lock:
|
||||
self.call_results[id] = future
|
||||
|
||||
async def resolve_call_result(self, id: int, result: any):
|
||||
async with self.lock:
|
||||
if id in self.call_results:
|
||||
future = self.call_results.pop(id)
|
||||
if not future.done():
|
||||
future.set_result(result)
|
||||
|
||||
async def reject_call_result(self, id: int, exception: Exception):
|
||||
async with self.lock:
|
||||
if id in self.call_results:
|
||||
future = self.call_results.pop(id)
|
||||
if not future.done():
|
||||
future.set_exception(exception)
|
||||
|
||||
async def cleanup_call_result(self, id: int):
|
||||
async with self.lock:
|
||||
if id in self.call_results:
|
||||
self.call_results.pop(id)
|
||||
|
||||
|
||||
async def send_mcp_message(conn: "ConnectionHandler", payload: dict):
|
||||
async def send_mcp_message(
|
||||
conn: "ConnectionHandler",
|
||||
payload: dict,
|
||||
transport=None,
|
||||
*,
|
||||
raise_on_error: bool = False,
|
||||
):
|
||||
"""Helper to send MCP messages, encapsulating common logic."""
|
||||
if not conn.features.get("mcp"):
|
||||
features = getattr(conn, "features", {}) or {}
|
||||
if not features.get("mcp"):
|
||||
logger.bind(tag=TAG).warning("客户端不支持MCP，无法发送MCP消息")
|
||||
return
|
||||
|
||||
message = json.dumps({"type": "mcp", "payload": payload})
|
||||
|
||||
try:
|
||||
await conn.websocket.send(message)
|
||||
# 优先使用传入的transport，否则尝试从conn获取
|
||||
if transport:
|
||||
await transport.send(message)
|
||||
elif getattr(conn, "transport", None):
|
||||
# 新架构
|
||||
await conn.transport.send(message)
|
||||
elif getattr(conn, "websocket", None):
|
||||
# 兼容旧版本
|
||||
await conn.websocket.send(message)
|
||||
else:
|
||||
raise AttributeError("无法找到可用的传输层接口")
|
||||
logger.bind(tag=TAG).debug(f"成功发送MCP消息: {message}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送MCP消息失败: {e}")
|
||||
if raise_on_error:
|
||||
raise
|
||||
|
||||
|
||||
async def handle_mcp_message(
|
||||
conn: "ConnectionHandler", mcp_client: MCPClient, payload: dict
|
||||
conn: "ConnectionHandler",
|
||||
mcp_client: MCPClient,
|
||||
payload: dict,
|
||||
transport=None,
|
||||
):
|
||||
"""处理MCP消息,包括初始化、工具列表和工具调用响应等"""
|
||||
logger.bind(tag=TAG).debug(f"处理MCP消息: {str(payload)[:100]}")
|
||||
@@ -150,7 +87,7 @@ async def handle_mcp_message(
|
||||
|
||||
await asyncio.sleep(1)
|
||||
logger.bind(tag=TAG).debug("初始化完成，开始请求MCP工具列表")
|
||||
await send_mcp_tools_list_request(conn)
|
||||
await send_mcp_tools_list_request(conn, transport)
|
||||
|
||||
return
|
||||
|
||||
@@ -207,7 +144,7 @@ async def handle_mcp_message(
|
||||
next_cursor = result.get("nextCursor", "")
|
||||
if next_cursor:
|
||||
logger.bind(tag=TAG).debug(f"有更多工具，nextCursor: {next_cursor}")
|
||||
await send_mcp_tools_list_continue_request(conn, next_cursor)
|
||||
await send_mcp_tools_list_continue_request(conn, next_cursor, transport)
|
||||
else:
|
||||
await mcp_client.set_ready(True)
|
||||
logger.bind(tag=TAG).debug("所有工具已获取，MCP客户端准备就绪")
|
||||
@@ -235,7 +172,9 @@ async def handle_mcp_message(
|
||||
)
|
||||
|
||||
|
||||
async def send_mcp_initialize_message(conn: "ConnectionHandler"):
|
||||
async def send_mcp_initialize_message(
|
||||
conn: "ConnectionHandler", transport=None
|
||||
):
|
||||
"""发送MCP初始化消息"""
|
||||
|
||||
vision_url = get_vision_url(conn.config)
|
||||
@@ -267,10 +206,12 @@ async def send_mcp_initialize_message(conn: "ConnectionHandler"):
|
||||
},
|
||||
}
|
||||
logger.bind(tag=TAG).debug("发送MCP初始化消息")
|
||||
await send_mcp_message(conn, payload)
|
||||
await send_mcp_message(conn, payload, transport)
|
||||
|
||||
|
||||
async def send_mcp_tools_list_request(conn: "ConnectionHandler"):
|
||||
async def send_mcp_tools_list_request(
|
||||
conn: "ConnectionHandler", transport=None
|
||||
):
|
||||
"""发送MCP工具列表请求"""
|
||||
payload = {
|
||||
"jsonrpc": "2.0",
|
||||
@@ -278,10 +219,12 @@ async def send_mcp_tools_list_request(conn: "ConnectionHandler"):
|
||||
"method": "tools/list",
|
||||
}
|
||||
logger.bind(tag=TAG).debug("发送MCP工具列表请求")
|
||||
await send_mcp_message(conn, payload)
|
||||
await send_mcp_message(conn, payload, transport)
|
||||
|
||||
|
||||
async def send_mcp_tools_list_continue_request(conn: "ConnectionHandler", cursor: str):
|
||||
async def send_mcp_tools_list_continue_request(
|
||||
conn: "ConnectionHandler", cursor: str, transport=None
|
||||
):
|
||||
"""发送带有cursor的MCP工具列表请求"""
|
||||
payload = {
|
||||
"jsonrpc": "2.0",
|
||||
@@ -290,7 +233,7 @@ async def send_mcp_tools_list_continue_request(conn: "ConnectionHandler", cursor
|
||||
"params": {"cursor": cursor},
|
||||
}
|
||||
logger.bind(tag=TAG).info(f"发送带cursor的MCP工具列表请求: {cursor}")
|
||||
await send_mcp_message(conn, payload)
|
||||
await send_mcp_message(conn, payload, transport)
|
||||
|
||||
|
||||
async def call_mcp_tool(
|
||||
@@ -299,6 +242,7 @@ async def call_mcp_tool(
|
||||
tool_name: str,
|
||||
args: str = "{}",
|
||||
timeout: int = 30,
|
||||
return_raw: bool = False,
|
||||
):
|
||||
"""
|
||||
调用指定的工具，并等待响应
|
||||
@@ -359,6 +303,7 @@ async def call_mcp_tool(
|
||||
raise ValueError(f"参数必须是字典类型，实际类型: {type(arguments)}")
|
||||
|
||||
except Exception as e:
|
||||
await mcp_client.cleanup_call_result(tool_call_id)
|
||||
if not isinstance(e, ValueError):
|
||||
raise ValueError(f"参数处理失败: {str(e)}")
|
||||
raise e
|
||||
@@ -372,7 +317,11 @@ async def call_mcp_tool(
|
||||
}
|
||||
|
||||
logger.bind(tag=TAG).info(f"发送客户端mcp工具调用请求: {actual_name}，参数: {args}")
|
||||
await send_mcp_message(conn, payload)
|
||||
try:
|
||||
await send_mcp_message(conn, payload, raise_on_error=True)
|
||||
except Exception:
|
||||
await mcp_client.cleanup_call_result(tool_call_id)
|
||||
raise
|
||||
|
||||
try:
|
||||
# Wait for response or timeout
|
||||
@@ -388,13 +337,21 @@ async def call_mcp_tool(
|
||||
)
|
||||
raise RuntimeError(f"工具调用错误: {error_msg}")
|
||||
|
||||
if return_raw:
|
||||
return raw_result
|
||||
|
||||
content = raw_result.get("content")
|
||||
if isinstance(content, list) and len(content) > 0:
|
||||
if isinstance(content[0], dict) and "text" in content[0]:
|
||||
# 直接返回文本内容，不进行JSON解析
|
||||
return content[0]["text"]
|
||||
# 如果结果不是预期的格式，将其转换为字符串
|
||||
if return_raw:
|
||||
return raw_result
|
||||
return str(raw_result)
|
||||
except asyncio.CancelledError:
|
||||
await mcp_client.cleanup_call_result(tool_call_id)
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
await mcp_client.cleanup_call_result(tool_call_id)
|
||||
raise TimeoutError("工具调用请求超时")
|
||||
|
||||
@@ -22,6 +22,7 @@ class MCPEndpointClient:
|
||||
self.lock = asyncio.Lock()
|
||||
self._cached_available_tools = None # Cache for get_available_tools
|
||||
self.websocket = None # WebSocket连接
|
||||
self.listener_task = None
|
||||
|
||||
def has_tool(self, name: str) -> bool:
|
||||
return name in self.tools
|
||||
@@ -107,6 +108,18 @@ class MCPEndpointClient:
|
||||
|
||||
async def close(self):
|
||||
"""关闭WebSocket连接"""
|
||||
current_task = asyncio.current_task()
|
||||
if (
|
||||
self.listener_task
|
||||
and self.listener_task is not current_task
|
||||
and not self.listener_task.done()
|
||||
):
|
||||
self.listener_task.cancel()
|
||||
try:
|
||||
await self.listener_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self.listener_task = None
|
||||
if self.websocket:
|
||||
await self.websocket.close()
|
||||
self.websocket = None
|
||||
|
||||
@@ -16,6 +16,8 @@ async def connect_mcp_endpoint(mcp_endpoint_url: str, conn=None) -> MCPEndpointC
|
||||
if not mcp_endpoint_url or "你的" in mcp_endpoint_url or mcp_endpoint_url == "null":
|
||||
return None
|
||||
|
||||
websocket = None
|
||||
mcp_client = None
|
||||
try:
|
||||
websocket = await websockets.connect(mcp_endpoint_url)
|
||||
|
||||
@@ -23,7 +25,9 @@ async def connect_mcp_endpoint(mcp_endpoint_url: str, conn=None) -> MCPEndpointC
|
||||
mcp_client.set_websocket(websocket)
|
||||
|
||||
# 启动消息监听器
|
||||
asyncio.create_task(_message_listener(mcp_client))
|
||||
mcp_client.listener_task = asyncio.create_task(
|
||||
_message_listener(mcp_client), name="xiaozhi-mcp-endpoint-listener"
|
||||
)
|
||||
|
||||
# 发送初始化消息
|
||||
await send_mcp_endpoint_initialize(mcp_client)
|
||||
@@ -39,6 +43,10 @@ async def connect_mcp_endpoint(mcp_endpoint_url: str, conn=None) -> MCPEndpointC
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"连接MCP接入点失败: {e}")
|
||||
if mcp_client is not None:
|
||||
await mcp_client.close()
|
||||
elif websocket is not None:
|
||||
await websocket.close()
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -72,16 +72,25 @@ class ServerMCPClient:
|
||||
|
||||
async def cleanup(self):
|
||||
"""清理MCP客户端资源"""
|
||||
if not self._worker_task:
|
||||
task = self._worker_task
|
||||
if not task:
|
||||
return
|
||||
|
||||
self._shutdown_evt.set()
|
||||
try:
|
||||
await asyncio.wait_for(self._worker_task, timeout=20)
|
||||
except (asyncio.TimeoutError, Exception) as e:
|
||||
await asyncio.wait_for(asyncio.shield(task), timeout=20)
|
||||
except asyncio.TimeoutError:
|
||||
self.logger.bind(tag=TAG).warning("服务端MCP关闭超时，取消工作任务")
|
||||
task.cancel()
|
||||
done, _ = await asyncio.wait({task}, timeout=5)
|
||||
if task not in done:
|
||||
self.logger.bind(tag=TAG).error("服务端MCP工作任务取消超时")
|
||||
return
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"服务端MCP客户端关闭错误: {e}")
|
||||
finally:
|
||||
self._worker_task = None
|
||||
if task.done():
|
||||
self._worker_task = None
|
||||
|
||||
def has_tool(self, name: str) -> bool:
|
||||
"""检查是否包含指定工具
|
||||
@@ -197,7 +206,7 @@ class ServerMCPClient:
|
||||
if "API_ACCESS_TOKEN" in self.config:
|
||||
headers["Authorization"] = f"Bearer {self.config['API_ACCESS_TOKEN']}"
|
||||
self.logger.bind(tag=TAG).warning(f"你正在使用旧过时的配置 API_ACCESS_TOKEN ，请在.mcp_server_settings.json中将API_ACCESS_TOKEN直接设置在headers中，例如 'Authorization': 'Bearer API_ACCESS_TOKEN'")
|
||||
|
||||
|
||||
# 根据transport类型选择不同的客户端，默认为SSE
|
||||
transport_type = self.config.get("transport", "sse")
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
import queue
|
||||
@@ -16,8 +17,6 @@ from config.logger import setup_logging
|
||||
from core.utils import opus_encoder_utils
|
||||
from core.utils.tts import MarkdownCleaner, convert_percentage_to_range
|
||||
from core.utils.output_counter import add_device_output
|
||||
from core.handle.reportHandle import enqueue_tts_report
|
||||
from core.handle.sendAudioHandle import sendAudioMessage
|
||||
from core.utils.util import audio_bytes_to_data_stream, audio_to_data_stream
|
||||
from core.providers.tts.dto.dto import (
|
||||
TTSMessageDTO,
|
||||
@@ -30,6 +29,58 @@ TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def sendAudioMessage(conn, sentenceType, audios, text, sentence_id=None):
|
||||
"""兼容函数：使用新的processor发送音频消息"""
|
||||
try:
|
||||
# 获取transport接口
|
||||
transport = getattr(conn, 'transport', None)
|
||||
if not transport:
|
||||
logger.error("SessionContext中没有transport接口")
|
||||
return
|
||||
|
||||
# 使用AudioSendProcessor发送音频
|
||||
from core.processors.audio_send_processor import AudioSendProcessor
|
||||
processor = AudioSendProcessor()
|
||||
|
||||
await processor.send_audio_message(
|
||||
conn,
|
||||
transport,
|
||||
sentenceType,
|
||||
audios,
|
||||
text,
|
||||
sentence_id=sentence_id,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送音频消息失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def enqueue_tts_report(conn, text, audio_data):
|
||||
"""兼容函数：使用新的processor处理TTS报告"""
|
||||
try:
|
||||
# 获取transport接口
|
||||
transport = getattr(conn, 'transport', None)
|
||||
if not transport:
|
||||
logger.error("SessionContext中没有transport接口")
|
||||
return
|
||||
|
||||
# 使用ReportProcessor处理报告
|
||||
from core.processors.report_processor import ReportProcessor
|
||||
processor = ReportProcessor()
|
||||
|
||||
# 异步执行报告
|
||||
if hasattr(conn, 'loop') and conn.loop:
|
||||
# 直接调用同步方法
|
||||
processor.enqueue_tts_report(conn, text, audio_data)
|
||||
else:
|
||||
logger.warning("SessionContext中没有事件循环，跳过TTS报告")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"TTS报告处理失败: {e}")
|
||||
|
||||
|
||||
class TTSProviderBase(ABC):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
self.interface_type = InterfaceType.NON_STREAM
|
||||
@@ -189,7 +240,7 @@ class TTSProviderBase(ABC):
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def to_tts(self, text):
|
||||
# 保留原始文本用于日志/显示
|
||||
original_text = text
|
||||
@@ -312,13 +363,17 @@ class TTSProviderBase(ABC):
|
||||
|
||||
# tts 消化线程
|
||||
self.tts_priority_thread = threading.Thread(
|
||||
target=self.tts_text_priority_thread, daemon=True
|
||||
target=self.tts_text_priority_thread,
|
||||
name=f"xiaozhi-tts-text-{id(self)}",
|
||||
daemon=True,
|
||||
)
|
||||
self.tts_priority_thread.start()
|
||||
|
||||
# 音频播放 消化线程
|
||||
self.audio_play_priority_thread = threading.Thread(
|
||||
target=self._audio_play_priority_thread, daemon=True
|
||||
target=self._audio_play_priority_thread,
|
||||
name=f"xiaozhi-tts-audio-{id(self)}",
|
||||
daemon=True,
|
||||
)
|
||||
self.audio_play_priority_thread.start()
|
||||
|
||||
@@ -369,6 +424,8 @@ class TTSProviderBase(ABC):
|
||||
while not self.conn.stop_event.is_set():
|
||||
try:
|
||||
message = self.tts_text_queue.get(timeout=1)
|
||||
if message is None:
|
||||
break
|
||||
if self.conn.client_abort:
|
||||
logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
|
||||
continue
|
||||
@@ -417,11 +474,15 @@ class TTSProviderBase(ABC):
|
||||
try:
|
||||
try:
|
||||
item = self.tts_audio_queue.get(timeout=0.1)
|
||||
if item is None:
|
||||
break
|
||||
if len(item) == 4:
|
||||
sentence_type, audio_datas, text, sentence_id = item
|
||||
else:
|
||||
sentence_type, audio_datas, text = item
|
||||
sentence_id = None
|
||||
sentence_id = getattr(
|
||||
self, "current_sentence_id", None
|
||||
) or getattr(self.conn, "sentence_id", None)
|
||||
except queue.Empty:
|
||||
if self.conn.stop_event.is_set():
|
||||
break
|
||||
@@ -459,7 +520,19 @@ class TTSProviderBase(ABC):
|
||||
sendAudioMessage(self.conn, sentence_type, audio_datas, text, sentence_id),
|
||||
self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
self._pending_audio_future = future
|
||||
try:
|
||||
future.result(timeout=max(1, self.tts_timeout))
|
||||
except concurrent.futures.CancelledError:
|
||||
break
|
||||
except concurrent.futures.TimeoutError:
|
||||
future.cancel()
|
||||
logger.bind(tag=TAG).warning(
|
||||
"TTS音频发送超时，取消当前发送任务"
|
||||
)
|
||||
finally:
|
||||
if getattr(self, "_pending_audio_future", None) is future:
|
||||
self._pending_audio_future = None
|
||||
|
||||
# 记录输出和报告
|
||||
if self.conn.max_output_size > 0 and text:
|
||||
@@ -476,6 +549,22 @@ class TTSProviderBase(ABC):
|
||||
|
||||
async def close(self):
|
||||
"""资源清理方法"""
|
||||
self.tts_stop_request = True
|
||||
pending_future = getattr(self, "_pending_audio_future", None)
|
||||
if pending_future and not pending_future.done():
|
||||
pending_future.cancel()
|
||||
# Wake workers immediately; relying on queue timeouts leaves Provider
|
||||
# threads alive after the component has released its ownership.
|
||||
self.tts_text_queue.put(None)
|
||||
self.tts_audio_queue.put(None)
|
||||
for thread_name in ("tts_priority_thread", "audio_play_priority_thread"):
|
||||
thread = getattr(self, thread_name, None)
|
||||
if thread and thread.is_alive() and thread is not threading.current_thread():
|
||||
await asyncio.to_thread(thread.join, 2)
|
||||
if thread.is_alive():
|
||||
logger.bind(tag=TAG).warning(
|
||||
"TTS工作线程未能按时退出: {}", thread.name
|
||||
)
|
||||
self._sentence_text_map.clear()
|
||||
if hasattr(self, "ws") and self.ws:
|
||||
await self.ws.close()
|
||||
|
||||
@@ -0,0 +1,450 @@
|
||||
import asyncio
|
||||
from typing import Dict, Any, List, Optional
|
||||
from config.logger import setup_logging
|
||||
from core.websocket_server_new import NewWebSocketServer
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class MultiProtocolServer:
|
||||
"""
|
||||
协议服务器管理器
|
||||
提供统一的启动、停止和状态监控接口
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
self.config = config
|
||||
self.logger = setup_logging()
|
||||
|
||||
# 服务器实例
|
||||
self.servers: Dict[str, Any] = {}
|
||||
self.server_tasks: Dict[str, asyncio.Task] = {}
|
||||
self.monitor_task: Optional[asyncio.Task] = None
|
||||
self._lifecycle_lock = asyncio.Lock()
|
||||
self.management_owner = None
|
||||
|
||||
# 服务器状态
|
||||
self.is_running = False
|
||||
self.startup_complete = False
|
||||
|
||||
# 初始化服务器
|
||||
self._initialize_servers()
|
||||
|
||||
def _initialize_servers(self):
|
||||
"""初始化所有协议服务器"""
|
||||
try:
|
||||
self.servers.clear()
|
||||
# 检查配置中启用的协议
|
||||
enabled_protocols = self.config.get('enabled_protocols', ['websocket'])
|
||||
|
||||
# 初始化WebSocket服务器
|
||||
if 'websocket' in enabled_protocols:
|
||||
self.servers['websocket'] = NewWebSocketServer(self.config)
|
||||
logger.info("WebSocket服务器已初始化")
|
||||
|
||||
if not self.servers:
|
||||
logger.warning("没有启用任何协议服务器")
|
||||
|
||||
for server in self.servers.values():
|
||||
self._bind_management_owner(server)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"初始化服务器失败: {e}")
|
||||
raise
|
||||
|
||||
def set_management_owner(self, owner: Any) -> None:
|
||||
"""Route management commands through the facade that owns all protocols."""
|
||||
self.management_owner = owner
|
||||
for server in self.servers.values():
|
||||
self._bind_management_owner(server)
|
||||
|
||||
def _bind_management_owner(self, server: Any) -> None:
|
||||
owner = self.management_owner or server
|
||||
server.management_owner = owner
|
||||
connection_service = getattr(server, "connection_service", None)
|
||||
if connection_service is not None:
|
||||
connection_service.server = owner
|
||||
|
||||
async def start(self):
|
||||
"""启动所有服务器，并由管理器持有协议任务。"""
|
||||
async with self._lifecycle_lock:
|
||||
await self._start_unlocked()
|
||||
|
||||
async def _start_unlocked(self):
|
||||
if self.is_running:
|
||||
logger.warning("服务器已经在运行中")
|
||||
return
|
||||
if self.server_tasks:
|
||||
raise RuntimeError("上次协议停止尚未完成，请先重试 stop 清理残留资源")
|
||||
|
||||
try:
|
||||
logger.info("开始启动多协议服务器...")
|
||||
self.is_running = True
|
||||
self.startup_complete = False
|
||||
|
||||
# 启动所有服务器
|
||||
for protocol, server in self.servers.items():
|
||||
try:
|
||||
logger.info(f"启动{protocol}服务器...")
|
||||
task = asyncio.create_task(
|
||||
server.start(), name=f"xiaozhi-{protocol}-server"
|
||||
)
|
||||
self.server_tasks[protocol] = task
|
||||
await self._wait_for_server_started(protocol, server, task)
|
||||
|
||||
logger.info(f"{protocol}服务器启动成功")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"启动{protocol}服务器失败: {e}")
|
||||
raise
|
||||
|
||||
self.startup_complete = True
|
||||
logger.info("多协议服务器启动完成")
|
||||
|
||||
# 启动监控任务；必须保存引用，以便停止和重启时回收。
|
||||
self.monitor_task = asyncio.create_task(
|
||||
self._monitor_servers(), name="xiaozhi-protocol-monitor"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"启动多协议服务器失败: {e}")
|
||||
try:
|
||||
await self._stop_unlocked()
|
||||
except Exception as cleanup_error:
|
||||
logger.error(f"启动失败后的协议清理失败: {cleanup_error}")
|
||||
raise
|
||||
|
||||
async def _wait_for_server_started(self, protocol, server, task):
|
||||
"""等待协议监听器真正就绪，而不是依赖固定 sleep。"""
|
||||
started_event = getattr(server, "_started_event", None)
|
||||
if not isinstance(started_event, asyncio.Event):
|
||||
await asyncio.sleep(0)
|
||||
if task.done():
|
||||
await task
|
||||
return
|
||||
|
||||
timeout = float(self.config.get("server_startup_timeout", 10))
|
||||
event_waiter = asyncio.create_task(started_event.wait())
|
||||
try:
|
||||
done, _ = await asyncio.wait(
|
||||
{task, event_waiter},
|
||||
timeout=timeout,
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
if started_event.is_set():
|
||||
if task.done():
|
||||
await task
|
||||
return
|
||||
if task in done:
|
||||
await task
|
||||
raise RuntimeError(f"{protocol}服务器在就绪前退出")
|
||||
raise TimeoutError(f"等待{protocol}服务器启动超时")
|
||||
finally:
|
||||
if not event_waiter.done():
|
||||
event_waiter.cancel()
|
||||
try:
|
||||
await event_waiter
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def stop(self):
|
||||
"""停止所有服务器"""
|
||||
async with self._lifecycle_lock:
|
||||
await self._stop_unlocked()
|
||||
|
||||
async def _stop_unlocked(self):
|
||||
if not self.is_running and not self.server_tasks and not self.monitor_task:
|
||||
return
|
||||
|
||||
logger.info("开始停止多协议服务器...")
|
||||
self.is_running = False
|
||||
self.startup_complete = False
|
||||
|
||||
if self.monitor_task and not self.monitor_task.done():
|
||||
self.monitor_task.cancel()
|
||||
try:
|
||||
await self.monitor_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self.monitor_task = None
|
||||
|
||||
# 停止所有服务器
|
||||
errors = []
|
||||
stopped_protocols = []
|
||||
for protocol, server in self.servers.items():
|
||||
try:
|
||||
logger.info(f"停止{protocol}服务器...")
|
||||
stopped = await server.stop()
|
||||
if stopped is False:
|
||||
raise RuntimeError("协议仍有取消不响应的后台任务")
|
||||
logger.info(f"{protocol}服务器已停止")
|
||||
stopped_protocols.append(protocol)
|
||||
except Exception as e:
|
||||
logger.error(f"停止{protocol}服务器失败: {e}")
|
||||
errors.append((protocol, e))
|
||||
|
||||
# 只释放已确认停止的任务。失败协议保留所有权，允许再次 stop。
|
||||
for protocol in stopped_protocols:
|
||||
task = self.server_tasks.get(protocol)
|
||||
if task is None:
|
||||
continue
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
errors.append((f"{protocol}任务", e))
|
||||
logger.error(f"回收{protocol}服务器任务失败: {e}")
|
||||
continue
|
||||
self.server_tasks.pop(protocol, None)
|
||||
|
||||
if errors:
|
||||
details = ", ".join(
|
||||
f"{protocol}: {error}" for protocol, error in errors
|
||||
)
|
||||
raise RuntimeError(f"停止协议服务器时发生错误: {details}")
|
||||
logger.info("多协议服务器已停止")
|
||||
|
||||
async def restart(self):
|
||||
"""重启所有服务器"""
|
||||
async with self._lifecycle_lock:
|
||||
logger.info("重启多协议服务器...")
|
||||
await self._stop_unlocked()
|
||||
await self._start_unlocked()
|
||||
|
||||
async def restart_server(self, protocol: str):
|
||||
"""重启指定协议的服务器"""
|
||||
async with self._lifecycle_lock:
|
||||
return await self._restart_server_unlocked(protocol)
|
||||
|
||||
async def _restart_server_unlocked(self, protocol: str):
|
||||
if protocol not in self.servers:
|
||||
logger.error(f"未找到协议服务器: {protocol}")
|
||||
return False
|
||||
|
||||
try:
|
||||
logger.info(f"重启{protocol}服务器...")
|
||||
|
||||
# 停止指定服务器
|
||||
server = self.servers[protocol]
|
||||
stopped = await server.stop()
|
||||
if stopped is False:
|
||||
raise RuntimeError("协议仍有取消不响应的后台任务")
|
||||
|
||||
# 取消任务
|
||||
if protocol in self.server_tasks:
|
||||
task = self.server_tasks[protocol]
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
# 重新启动
|
||||
task = asyncio.create_task(
|
||||
server.start(), name=f"xiaozhi-{protocol}-server"
|
||||
)
|
||||
self.server_tasks[protocol] = task
|
||||
await self._wait_for_server_started(protocol, server, task)
|
||||
|
||||
logger.info(f"{protocol}服务器重启成功")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"重启{protocol}服务器失败: {e}")
|
||||
return False
|
||||
|
||||
async def update_config(self, new_config: Dict[str, Any]):
|
||||
"""更新配置"""
|
||||
async with self._lifecycle_lock:
|
||||
return await self._update_config_unlocked(new_config)
|
||||
|
||||
async def _update_config_unlocked(self, new_config: Dict[str, Any]):
|
||||
old_config = self.config
|
||||
was_running = self.is_running
|
||||
config_changed = False
|
||||
old_runtime_stopped = False
|
||||
try:
|
||||
logger.info("更新多协议服务器配置...")
|
||||
# 检查配置变化
|
||||
config_changed = self._check_config_changes(self.config, new_config)
|
||||
|
||||
# 如果配置有重大变化，重新初始化服务器
|
||||
if config_changed:
|
||||
logger.info("配置有重大变化，重新初始化服务器...")
|
||||
await self._stop_unlocked()
|
||||
old_runtime_stopped = True
|
||||
self.config = new_config
|
||||
self._initialize_servers()
|
||||
if was_running:
|
||||
await self._start_unlocked()
|
||||
else:
|
||||
# 更新各个服务器的配置
|
||||
for protocol, server in self.servers.items():
|
||||
updated = True
|
||||
if hasattr(server, 'apply_config'):
|
||||
updated = await server.apply_config(new_config)
|
||||
elif hasattr(server, 'update_config'):
|
||||
updated = await server.update_config(new_config)
|
||||
if updated is False:
|
||||
raise RuntimeError(f"{protocol}配置更新失败")
|
||||
self._bind_management_owner(server)
|
||||
self.config = new_config
|
||||
|
||||
logger.info("配置更新完成")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"更新配置失败: {e}")
|
||||
if config_changed and not old_runtime_stopped:
|
||||
# A listener still owns resources. Preserve the old server
|
||||
# objects/tasks so stop can be retried safely.
|
||||
self.config = old_config
|
||||
return False
|
||||
try:
|
||||
await self._stop_unlocked()
|
||||
self.config = old_config
|
||||
self._initialize_servers()
|
||||
if was_running:
|
||||
await self._start_unlocked()
|
||||
except Exception:
|
||||
logger.exception("回滚多协议服务器配置失败")
|
||||
return False
|
||||
|
||||
def _check_config_changes(self, old_config: Dict[str, Any], new_config: Dict[str, Any]) -> bool:
|
||||
"""检查配置是否有重大变化"""
|
||||
# 检查启用的协议是否变化
|
||||
old_protocols = set(old_config.get('enabled_protocols', ['websocket']))
|
||||
new_protocols = set(new_config.get('enabled_protocols', ['websocket']))
|
||||
|
||||
if old_protocols != new_protocols:
|
||||
logger.info(f"启用协议发生变化: {old_protocols} -> {new_protocols}")
|
||||
return True
|
||||
|
||||
if old_config.get("_shared_asr_manager") is not new_config.get(
|
||||
"_shared_asr_manager"
|
||||
):
|
||||
logger.info("共享ASR管理器发生变化")
|
||||
return True
|
||||
|
||||
# 检查服务器端口配置
|
||||
server_configs = {
|
||||
'server': ('port', 'host', 'ip'),
|
||||
}
|
||||
for config_key, listener_keys in server_configs.items():
|
||||
old_server_config = old_config.get(config_key, {})
|
||||
new_server_config = new_config.get(config_key, {})
|
||||
|
||||
# 检查端口和主机配置
|
||||
for key in listener_keys:
|
||||
if old_server_config.get(key) != new_server_config.get(key):
|
||||
logger.info(f"服务器配置{config_key}.{key}发生变化")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
async def _monitor_servers(self):
|
||||
"""监控服务器状态"""
|
||||
try:
|
||||
while self.is_running:
|
||||
await asyncio.sleep(30) # 每30秒检查一次
|
||||
|
||||
# 检查服务器任务状态
|
||||
for protocol, task in list(self.server_tasks.items()):
|
||||
if task.done():
|
||||
exception = None if task.cancelled() else task.exception()
|
||||
if exception:
|
||||
logger.error(f"{protocol}服务器异常退出: {exception}")
|
||||
else:
|
||||
logger.warning(f"{protocol}服务器任务意外退出")
|
||||
# 监控任务本身不能重入 lifecycle lock。
|
||||
async with self._lifecycle_lock:
|
||||
if self.is_running:
|
||||
await self._restart_server_unlocked(protocol)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"服务器监控任务出错: {e}")
|
||||
|
||||
def get_server_status(self) -> Dict[str, Any]:
|
||||
"""获取所有服务器状态"""
|
||||
status = {
|
||||
'is_running': self.is_running,
|
||||
'startup_complete': self.startup_complete,
|
||||
'enabled_protocols': list(self.servers.keys()),
|
||||
'servers': {}
|
||||
}
|
||||
|
||||
# 获取各个服务器的状态
|
||||
for protocol, server in self.servers.items():
|
||||
try:
|
||||
if hasattr(server, 'get_server_status'):
|
||||
server_status = server.get_server_status()
|
||||
else:
|
||||
server_status = {'type': protocol, 'status': 'unknown'}
|
||||
|
||||
# 添加任务状态
|
||||
task = self.server_tasks.get(protocol)
|
||||
if task:
|
||||
server_status['task_status'] = 'running' if not task.done() else 'stopped'
|
||||
if task.done() and not task.cancelled() and task.exception():
|
||||
server_status['task_error'] = str(task.exception())
|
||||
|
||||
status['servers'][protocol] = server_status
|
||||
|
||||
except Exception as e:
|
||||
status['servers'][protocol] = {
|
||||
'type': protocol,
|
||||
'status': 'error',
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
return status
|
||||
|
||||
def get_active_connections_count(self) -> Dict[str, int]:
|
||||
"""获取各协议的活跃连接数"""
|
||||
connections = {}
|
||||
|
||||
for protocol, server in self.servers.items():
|
||||
try:
|
||||
if hasattr(server, 'get_active_connections_count'):
|
||||
connections[protocol] = server.get_active_connections_count()
|
||||
elif hasattr(server, 'connections'):
|
||||
connections[protocol] = len(server.connections)
|
||||
else:
|
||||
connections[protocol] = 0
|
||||
except Exception as e:
|
||||
logger.error(f"获取{protocol}连接数失败: {e}")
|
||||
connections[protocol] = -1
|
||||
|
||||
return connections
|
||||
|
||||
async def broadcast_message(self, message: Dict[str, Any], protocol: Optional[str] = None):
|
||||
"""向所有连接广播消息"""
|
||||
try:
|
||||
if protocol:
|
||||
# 向指定协议广播
|
||||
if protocol in self.servers:
|
||||
server = self.servers[protocol]
|
||||
if hasattr(server, 'broadcast_message'):
|
||||
await server.broadcast_message(message)
|
||||
else:
|
||||
# 向所有协议广播
|
||||
for server in self.servers.values():
|
||||
if hasattr(server, 'broadcast_message'):
|
||||
await server.broadcast_message(message)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"广播消息失败: {e}")
|
||||
|
||||
def get_supported_protocols(self) -> List[str]:
|
||||
"""获取支持的协议列表"""
|
||||
return ['websocket']
|
||||
|
||||
def is_protocol_enabled(self, protocol: str) -> bool:
|
||||
"""检查协议是否启用"""
|
||||
return protocol in self.servers
|
||||
@@ -0,0 +1,229 @@
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
import numpy as np
|
||||
import opuslib_next
|
||||
|
||||
from config.logger import setup_logging
|
||||
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AudioIngressService:
|
||||
"""Decode transport audio once and apply optional server-side AEC."""
|
||||
|
||||
def __init__(self, decoder_factory=None):
|
||||
self._decoder_factory = decoder_factory or opuslib_next.Decoder
|
||||
|
||||
def process(self, context: Any, opus_packet: bytes, timestamp: int = 0) -> Optional[bytes]:
|
||||
pcm_frame = self.decode(context, opus_packet)
|
||||
if pcm_frame and timestamp > 0 and getattr(context, "client_aec", False):
|
||||
return self.apply_aec(context, timestamp, pcm_frame)
|
||||
return pcm_frame
|
||||
|
||||
def decode(self, context: Any, opus_packet: bytes) -> Optional[bytes]:
|
||||
if not opus_packet:
|
||||
return None
|
||||
|
||||
sample_rate = int(getattr(context, "input_sample_rate", 16000) or 16000)
|
||||
channels = int(getattr(context, "input_channels", 1) or 1)
|
||||
frame_duration = int(
|
||||
getattr(context, "input_frame_duration", 60) or 60
|
||||
)
|
||||
decoder_config = (sample_rate, channels)
|
||||
|
||||
try:
|
||||
if getattr(context, "_audio_ingress_decoder_config", None) != decoder_config:
|
||||
context._audio_ingress_decoder = self._decoder_factory(sample_rate, channels)
|
||||
context._audio_ingress_decoder_config = decoder_config
|
||||
self._register_cleanup(context)
|
||||
|
||||
frame_size = max(1, sample_rate * frame_duration // 1000)
|
||||
pcm_frame = context._audio_ingress_decoder.decode(bytes(opus_packet), frame_size)
|
||||
return bytes(pcm_frame)
|
||||
except Exception as exc:
|
||||
logger.debug("Opus decode failed: {}", exc)
|
||||
return None
|
||||
|
||||
def apply_aec(self, context: Any, timestamp: int, pcm_frame: bytes) -> bytes:
|
||||
"""Apply the AEC algorithm used by the upstream gateway path."""
|
||||
try:
|
||||
self._expire_aec_references(context)
|
||||
audio_cache = getattr(context, "aec_audio_cache", None)
|
||||
if not pcm_frame or not audio_cache:
|
||||
return pcm_frame
|
||||
|
||||
mic_audio = np.frombuffer(pcm_frame, dtype=np.int16).astype(np.float32)
|
||||
mic_rms = np.sqrt(np.mean(mic_audio**2))
|
||||
if mic_rms < 100:
|
||||
return pcm_frame
|
||||
|
||||
sorted_timestamps = sorted(audio_cache.keys())
|
||||
if len(sorted_timestamps) < 2:
|
||||
return pcm_frame
|
||||
|
||||
sample_count = len(mic_audio)
|
||||
closest_idx = min(
|
||||
range(len(sorted_timestamps)),
|
||||
key=lambda index: abs(sorted_timestamps[index] - timestamp),
|
||||
)
|
||||
|
||||
mic_window = np.hanning(sample_count)
|
||||
mic_fft = np.fft.rfft(mic_audio * mic_window)
|
||||
mic_psd = np.abs(mic_fft) ** 2
|
||||
mic_log_psd = 10 * np.log10(mic_psd + 1e-8)
|
||||
mic_power = np.dot(mic_log_psd, mic_log_psd)
|
||||
|
||||
best_corr = -1
|
||||
best_ref_idx = closest_idx
|
||||
best_ref_rms = 0.0
|
||||
for offset in range(-2, 3):
|
||||
test_idx = closest_idx + offset
|
||||
if test_idx < 0 or test_idx >= len(sorted_timestamps):
|
||||
continue
|
||||
|
||||
test_ref = np.frombuffer(
|
||||
audio_cache[sorted_timestamps[test_idx]], dtype=np.int16
|
||||
).astype(np.float32)
|
||||
test_ref_rms = np.sqrt(np.mean(test_ref**2))
|
||||
if test_ref_rms < 50:
|
||||
continue
|
||||
|
||||
test_fft = np.fft.rfft(test_ref * np.hanning(len(test_ref)))
|
||||
test_log_psd = 10 * np.log10(np.abs(test_fft) ** 2 + 1e-8)
|
||||
cross_power = np.dot(mic_log_psd, test_log_psd)
|
||||
reference_power = np.dot(test_log_psd, test_log_psd)
|
||||
corr = abs(cross_power) / (
|
||||
np.sqrt(mic_power) * np.sqrt(reference_power) + 1e-8
|
||||
)
|
||||
if corr > best_corr:
|
||||
best_corr = corr
|
||||
best_ref_idx = test_idx
|
||||
best_ref_rms = test_ref_rms
|
||||
|
||||
best_ref = np.frombuffer(
|
||||
audio_cache[sorted_timestamps[best_ref_idx]], dtype=np.int16
|
||||
).astype(np.float32)
|
||||
if best_ref_rms < 50:
|
||||
return pcm_frame
|
||||
|
||||
aligned_ref = best_ref[:sample_count]
|
||||
if len(aligned_ref) < sample_count:
|
||||
aligned_ref = np.pad(aligned_ref, (0, sample_count - len(aligned_ref)))
|
||||
|
||||
mic_mag = np.abs(mic_fft)
|
||||
mic_phase = np.angle(mic_fft)
|
||||
ref_mag = np.abs(np.fft.rfft(aligned_ref * np.hanning(sample_count)))
|
||||
scale = np.sum(mic_mag * ref_mag) / (np.dot(ref_mag, ref_mag) + 1e-8)
|
||||
coefficient = max(
|
||||
0.5, min(3.0, 1.0 + scale * 3 + (best_corr - 0.97) * 30)
|
||||
)
|
||||
result_mag = np.maximum(
|
||||
mic_mag - ref_mag * scale * coefficient * 1.5, mic_mag * 0.1
|
||||
)
|
||||
output = np.fft.irfft(result_mag * np.exp(1j * mic_phase), sample_count)
|
||||
if best_corr >= 0.97 and best_ref_rms > 500:
|
||||
output *= 0.3
|
||||
|
||||
return np.clip(output, -32768, 32767).astype(np.int16).tobytes()
|
||||
except Exception as exc:
|
||||
logger.warning("AEC processing failed: {}", exc)
|
||||
return pcm_frame
|
||||
|
||||
def cache_output_reference(
|
||||
self, context: Any, opus_packet: bytes, timestamp: int
|
||||
) -> None:
|
||||
if not getattr(context, "client_aec", False) or timestamp <= 0:
|
||||
return
|
||||
|
||||
try:
|
||||
sample_rate = int(
|
||||
getattr(
|
||||
context,
|
||||
"output_sample_rate",
|
||||
getattr(context, "sample_rate", 24000),
|
||||
)
|
||||
or 24000
|
||||
)
|
||||
channels = int(getattr(context, "output_channels", 1) or 1)
|
||||
frame_duration = int(
|
||||
getattr(context, "output_frame_duration", 60) or 60
|
||||
)
|
||||
decoder_config = (sample_rate, channels)
|
||||
decoder = getattr(context, "_audio_reference_decoder", None)
|
||||
if (
|
||||
decoder is None
|
||||
or getattr(context, "_audio_reference_decoder_config", None)
|
||||
!= decoder_config
|
||||
):
|
||||
decoder = self._decoder_factory(sample_rate, channels)
|
||||
context._audio_reference_decoder = decoder
|
||||
context._audio_reference_decoder_config = decoder_config
|
||||
self._register_cleanup(context)
|
||||
frame_size = max(1, sample_rate * frame_duration // 1000)
|
||||
pcm_data = bytes(decoder.decode(bytes(opus_packet), frame_size))
|
||||
input_rate = int(getattr(context, "input_sample_rate", 16000) or 16000)
|
||||
if sample_rate != input_rate:
|
||||
pcm_data = self._resample_pcm(pcm_data, sample_rate, input_rate)
|
||||
self._expire_aec_references(context)
|
||||
context.aec_audio_cache[timestamp] = pcm_data
|
||||
context.aec_audio_cache_time[timestamp] = time.time()
|
||||
except Exception as exc:
|
||||
logger.debug("AEC reference decode failed: {}", exc)
|
||||
|
||||
@staticmethod
|
||||
def _expire_aec_references(context: Any) -> None:
|
||||
cache = getattr(context, "aec_audio_cache", {})
|
||||
cache_times = getattr(context, "aec_audio_cache_time", {})
|
||||
config = getattr(context, "config", {}) or {}
|
||||
max_age = max(1, int(config.get("aec_reference_max_age_seconds", 120)))
|
||||
max_frames = max(2, int(config.get("aec_reference_max_frames", 256)))
|
||||
now = time.time()
|
||||
|
||||
expired = [
|
||||
timestamp
|
||||
for timestamp, cached_at in list(cache_times.items())
|
||||
if now - cached_at > max_age
|
||||
]
|
||||
overflow = max(0, len(cache) - max_frames + 1)
|
||||
if overflow:
|
||||
expired.extend(sorted(cache_times, key=cache_times.get)[:overflow])
|
||||
for timestamp in set(expired):
|
||||
cache.pop(timestamp, None)
|
||||
cache_times.pop(timestamp, None)
|
||||
|
||||
def cleanup(self, context: Any) -> None:
|
||||
for name in (
|
||||
"_audio_ingress_decoder",
|
||||
"_audio_ingress_decoder_config",
|
||||
"_audio_reference_decoder",
|
||||
"_audio_reference_decoder_config",
|
||||
):
|
||||
if hasattr(context, name):
|
||||
delattr(context, name)
|
||||
if hasattr(context, "aec_audio_cache"):
|
||||
context.aec_audio_cache.clear()
|
||||
if hasattr(context, "aec_audio_cache_time"):
|
||||
context.aec_audio_cache_time.clear()
|
||||
|
||||
def _register_cleanup(self, context: Any) -> None:
|
||||
if getattr(context, "_audio_ingress_cleanup_registered", False):
|
||||
return
|
||||
register_cleanup = getattr(context, "register_cleanup", None)
|
||||
if callable(register_cleanup):
|
||||
register_cleanup(lambda: self.cleanup(context))
|
||||
context._audio_ingress_cleanup_registered = True
|
||||
|
||||
@staticmethod
|
||||
def _resample_pcm(pcm_data: bytes, source_rate: int, target_rate: int) -> bytes:
|
||||
if not pcm_data or source_rate == target_rate:
|
||||
return pcm_data
|
||||
source = np.frombuffer(pcm_data, dtype=np.int16)
|
||||
if len(source) < 2:
|
||||
return pcm_data
|
||||
target_size = max(1, round(len(source) * target_rate / source_rate))
|
||||
source_positions = np.linspace(0.0, 1.0, len(source), endpoint=False)
|
||||
target_positions = np.linspace(0.0, 1.0, target_size, endpoint=False)
|
||||
target = np.interp(target_positions, source_positions, source)
|
||||
return np.clip(target, -32768, 32767).astype(np.int16).tobytes()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,90 @@
|
||||
from abc import ABC, abstractmethod
|
||||
import json
|
||||
from typing import Any, AsyncGenerator
|
||||
|
||||
|
||||
class TransportInterface(ABC):
|
||||
"""
|
||||
传输层抽象接口。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def send(self, data: Any) -> None:
|
||||
"""发送一条消息。"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def send_json(self, message: Any) -> None:
|
||||
"""Send a control message without exposing transport framing to callers."""
|
||||
payload = message if isinstance(message, str) else json.dumps(message)
|
||||
await self.send(payload)
|
||||
|
||||
async def send_audio(self, audio: bytes, timestamp: int = 0) -> None:
|
||||
"""Send one encoded audio frame over the transport's audio channel."""
|
||||
await self.send(audio)
|
||||
|
||||
async def prepare_audio_channel(self, audio_params=None, version: int = 3) -> None:
|
||||
"""Prepare transport-specific audio negotiation when required."""
|
||||
|
||||
async def wait_audio_ready(self, timeout: float = 0) -> bool:
|
||||
"""Wait until encoded audio can be delivered to the peer."""
|
||||
return True
|
||||
|
||||
async def mark_business_ready(self) -> None:
|
||||
"""Allow a transport handshake to proceed after runtime initialization."""
|
||||
|
||||
async def mark_session_ready(self, session_id: str = None) -> None:
|
||||
"""Release a logical-session handshake after its runtime is ready."""
|
||||
|
||||
async def end_session(self, session_id: str) -> None:
|
||||
"""Tell the device to return to Idle without closing the connection."""
|
||||
await self.send_json({"type": "goodbye", "session_id": session_id})
|
||||
|
||||
@abstractmethod
|
||||
async def receive(self) -> AsyncGenerator[Any, None]:
|
||||
"""异步消息流。"""
|
||||
yield # pragma: no cover
|
||||
|
||||
@abstractmethod
|
||||
async def close(self) -> None:
|
||||
"""关闭底层连接。"""
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def is_connected(self) -> bool:
|
||||
"""连接是否存活。"""
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def transport_type(self) -> str:
|
||||
"""Stable transport identifier used by shared connection logic."""
|
||||
return "unknown"
|
||||
|
||||
@property
|
||||
def has_datagram_audio(self) -> bool:
|
||||
"""Whether audio is carried by a channel separate from control messages."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def requires_audio_tail_grace(self) -> bool:
|
||||
"""Whether control can overtake the last audio frames of a turn."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def keeps_connection_between_sessions(self) -> bool:
|
||||
"""Whether ending a conversation should leave the transport connected."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def is_protocol_authenticated(self) -> bool:
|
||||
"""Whether the transport already authenticated the peer during handshake."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def raw_connection(self):
|
||||
"""Underlying connection for temporary compatibility with legacy code."""
|
||||
return None
|
||||
|
||||
@property
|
||||
def session_id(self):
|
||||
return None
|
||||
@@ -0,0 +1,154 @@
|
||||
import struct
|
||||
import time
|
||||
from typing import Any, AsyncGenerator
|
||||
from .transport_interface import TransportInterface
|
||||
|
||||
|
||||
class WebSocketTransport(TransportInterface):
|
||||
"""
|
||||
WebSocket 传输实现：包装 websockets 库的协议对象，
|
||||
提供统一的 send/receive/close 接口。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
websocket,
|
||||
from_mqtt_gateway: bool = False,
|
||||
protocol_version: int = 1,
|
||||
):
|
||||
self._ws = websocket
|
||||
self._from_mqtt_gateway = from_mqtt_gateway
|
||||
self._protocol_version = protocol_version if protocol_version in (1, 2, 3) else 1
|
||||
self._gateway_sequence = 0
|
||||
|
||||
async def send(self, data: Any) -> None:
|
||||
if isinstance(data, bytes):
|
||||
await self.send_audio(data)
|
||||
return
|
||||
|
||||
if isinstance(data, (str, bytes)):
|
||||
await self._ws.send(data)
|
||||
else:
|
||||
await self._ws.send(str(data))
|
||||
|
||||
async def send_audio(self, audio: bytes, timestamp: int = 0) -> None:
|
||||
if self._from_mqtt_gateway:
|
||||
await self._ws.send(self._frame_gateway_audio(audio, timestamp))
|
||||
return
|
||||
await self._ws.send(self._frame_websocket_audio(audio, timestamp))
|
||||
|
||||
def _frame_websocket_audio(self, audio: bytes, timestamp: int = 0) -> bytes:
|
||||
if self._protocol_version == 2:
|
||||
return struct.pack("!HHIII", 2, 0, 0, timestamp, len(audio)) + audio
|
||||
if self._protocol_version == 3:
|
||||
return struct.pack("!BBH", 0, 0, len(audio)) + audio
|
||||
return audio
|
||||
|
||||
def _parse_websocket_audio(self, message: bytes):
|
||||
if self._protocol_version == 2:
|
||||
if len(message) < 16:
|
||||
return None
|
||||
version, message_type, _, timestamp, audio_length = struct.unpack(
|
||||
"!HHIII", message[:16]
|
||||
)
|
||||
if (
|
||||
version != 2
|
||||
or message_type != 0
|
||||
or audio_length <= 0
|
||||
or len(message) != 16 + audio_length
|
||||
):
|
||||
return None
|
||||
return {
|
||||
"type": "audio",
|
||||
"data": message[16:],
|
||||
"timestamp": timestamp,
|
||||
"_transport_type": "websocket",
|
||||
}
|
||||
if self._protocol_version == 3:
|
||||
if len(message) < 4:
|
||||
return None
|
||||
message_type, _, audio_length = struct.unpack("!BBH", message[:4])
|
||||
if (
|
||||
message_type != 0
|
||||
or audio_length <= 0
|
||||
or len(message) != 4 + audio_length
|
||||
):
|
||||
return None
|
||||
return {
|
||||
"type": "audio",
|
||||
"data": message[4:],
|
||||
"timestamp": 0,
|
||||
"_transport_type": "websocket",
|
||||
}
|
||||
return message
|
||||
|
||||
def _frame_gateway_audio(self, audio: bytes, timestamp: int = 0) -> bytes:
|
||||
self._gateway_sequence += 1
|
||||
if timestamp <= 0:
|
||||
timestamp = int(time.time() * 1000) % (2 ** 32)
|
||||
header = bytearray(16)
|
||||
header[0] = 1
|
||||
header[2:4] = len(audio).to_bytes(2, "big")
|
||||
header[4:8] = self._gateway_sequence.to_bytes(4, "big")
|
||||
header[8:12] = timestamp.to_bytes(4, "big")
|
||||
header[12:16] = len(audio).to_bytes(4, "big")
|
||||
return bytes(header) + audio
|
||||
|
||||
async def receive(self) -> AsyncGenerator[Any, None]:
|
||||
async for message in self._ws:
|
||||
if self._from_mqtt_gateway and isinstance(message, bytes):
|
||||
if len(message) < 16:
|
||||
continue
|
||||
if message[:8] != b"\x00" * 8:
|
||||
continue
|
||||
timestamp = int.from_bytes(message[8:12], "big")
|
||||
audio_length = int.from_bytes(message[12:16], "big")
|
||||
if audio_length <= 0 or len(message) != 16 + audio_length:
|
||||
continue
|
||||
message = {
|
||||
"type": "audio",
|
||||
"data": message[16:],
|
||||
"timestamp": timestamp,
|
||||
"_transport_type": "gateway",
|
||||
}
|
||||
elif isinstance(message, bytes):
|
||||
message = self._parse_websocket_audio(message)
|
||||
if message is None:
|
||||
continue
|
||||
yield message
|
||||
|
||||
async def close(self) -> None:
|
||||
try:
|
||||
if hasattr(self._ws, "closed") and not self._ws.closed:
|
||||
await self._ws.close()
|
||||
elif hasattr(self._ws, "state") and self._ws.state.name != "CLOSED":
|
||||
await self._ws.close()
|
||||
else:
|
||||
await self._ws.close()
|
||||
except Exception:
|
||||
raise RuntimeError("WebSocket close failed")
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
try:
|
||||
if hasattr(self._ws, "closed"):
|
||||
return not self._ws.closed
|
||||
if hasattr(self._ws, "state"):
|
||||
return getattr(self._ws.state, "name", "CLOSED") != "CLOSED"
|
||||
except Exception:
|
||||
raise RuntimeError("WebSocket connection check failed")
|
||||
return False
|
||||
|
||||
@property
|
||||
def transport_type(self) -> str:
|
||||
return "gateway" if self._from_mqtt_gateway else "websocket"
|
||||
|
||||
@property
|
||||
def requires_audio_tail_grace(self) -> bool:
|
||||
# The gateway receives MQTT control and UDP audio independently before
|
||||
# serializing both streams onto this WebSocket.
|
||||
return self._from_mqtt_gateway
|
||||
|
||||
@property
|
||||
def raw_connection(self):
|
||||
return self._ws
|
||||
@@ -0,0 +1,106 @@
|
||||
import asyncio
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
from config.logger import setup_logging
|
||||
from core.utils.modules_initialize import initialize_modules
|
||||
from core.providers.asr.shared_asr_manager import SharedASRManager
|
||||
|
||||
|
||||
def _validate_selected_modules(config: Dict[str, Any]) -> Tuple[bool, str]:
|
||||
selected = config.get("selected_module", {})
|
||||
required_sections = {
|
||||
"VAD": "VAD",
|
||||
"ASR": "ASR",
|
||||
"LLM": "LLM",
|
||||
"TTS": "TTS",
|
||||
"Memory": "Memory",
|
||||
"Intent": "Intent",
|
||||
}
|
||||
for module_key, section_key in required_sections.items():
|
||||
module_name = selected.get(module_key)
|
||||
if not module_name:
|
||||
continue
|
||||
section = config.get(section_key, {})
|
||||
if module_name not in section:
|
||||
return False, f"{section_key}配置缺失: {module_name}"
|
||||
return True, ""
|
||||
|
||||
|
||||
async def _cleanup_instance(instance: Any) -> None:
|
||||
if instance is None:
|
||||
return
|
||||
try:
|
||||
if hasattr(instance, "close"):
|
||||
result = instance.close()
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
if hasattr(instance, "cleanup"):
|
||||
result = instance.cleanup()
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
if hasattr(instance, "cleanup_audio_files"):
|
||||
instance.cleanup_audio_files()
|
||||
except Exception:
|
||||
# 校验阶段的清理异常不影响结果
|
||||
pass
|
||||
|
||||
|
||||
async def _cleanup_modules(modules: Dict[str, Any]) -> None:
|
||||
for instance in modules.values():
|
||||
await _cleanup_instance(instance)
|
||||
|
||||
|
||||
async def validate_config_components(
|
||||
config: Dict[str, Any], logger=None
|
||||
) -> Tuple[bool, str]:
|
||||
"""
|
||||
预初始化选中组件用于校验配置，捕获配置/密钥类错误。
|
||||
返回 (是否通过, 错误信息)。
|
||||
"""
|
||||
logger = logger or setup_logging()
|
||||
ok, msg = _validate_selected_modules(config)
|
||||
if not ok:
|
||||
return False, msg
|
||||
|
||||
selected = config.get("selected_module", {})
|
||||
init_vad = bool(selected.get("VAD"))
|
||||
init_asr = bool(selected.get("ASR"))
|
||||
asr_type = None
|
||||
selected_asr = selected.get("ASR")
|
||||
if selected_asr:
|
||||
asr_config = config.get("ASR", {}).get(selected_asr, {})
|
||||
asr_type = asr_config.get("type", selected_asr)
|
||||
if SharedASRManager.is_local_model_type(asr_type):
|
||||
init_asr = False
|
||||
init_llm = bool(selected.get("LLM"))
|
||||
init_tts = bool(selected.get("TTS"))
|
||||
init_memory = bool(selected.get("Memory"))
|
||||
init_intent = bool(selected.get("Intent"))
|
||||
|
||||
modules: Dict[str, Any] = {}
|
||||
manager = None
|
||||
try:
|
||||
modules = initialize_modules(
|
||||
logger,
|
||||
config,
|
||||
init_vad,
|
||||
init_asr,
|
||||
init_llm,
|
||||
init_tts,
|
||||
init_memory,
|
||||
init_intent,
|
||||
)
|
||||
if selected_asr and asr_type and SharedASRManager.is_local_model_type(asr_type):
|
||||
manager = SharedASRManager(config, asr_type)
|
||||
await manager.initialize()
|
||||
return True, ""
|
||||
except Exception as e:
|
||||
logger.error(f"配置校验失败: {e}")
|
||||
return False, str(e)
|
||||
finally:
|
||||
if manager:
|
||||
try:
|
||||
await manager.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
await _cleanup_modules(modules)
|
||||
@@ -0,0 +1,18 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def restart_server(logger):
|
||||
"""实际执行重启的方法"""
|
||||
time.sleep(1)
|
||||
logger.info("执行服务器重启...")
|
||||
subprocess.Popen(
|
||||
[sys.executable, "app.py"],
|
||||
stdin=sys.stdin,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
start_new_session=True,
|
||||
)
|
||||
os._exit(0)
|
||||
@@ -1,5 +1,6 @@
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
from config.logger import setup_logging
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from core.connection import ConnectionHandler
|
||||
@@ -91,18 +92,27 @@ async def get_emotion(conn: "ConnectionHandler", text):
|
||||
emotion = EMOJI_MAP[char]
|
||||
break
|
||||
try:
|
||||
await conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "llm",
|
||||
"text": emoji,
|
||||
"emotion": emotion,
|
||||
"session_id": conn.session_id,
|
||||
}
|
||||
)
|
||||
message = json.dumps(
|
||||
{
|
||||
"type": "llm",
|
||||
"text": emoji,
|
||||
"emotion": emotion,
|
||||
"session_id": conn.session_id,
|
||||
}
|
||||
)
|
||||
|
||||
# 使用transport接口发送消息
|
||||
if hasattr(conn, 'transport') and conn.transport:
|
||||
await conn.transport.send(message)
|
||||
elif hasattr(conn, 'websocket') and conn.websocket:
|
||||
# 兼容旧版本
|
||||
await conn.websocket.send(message)
|
||||
else:
|
||||
raise AttributeError("无法找到可用的传输层接口")
|
||||
|
||||
except Exception as e:
|
||||
conn.logger.bind(tag=TAG).warning(f"发送情绪表情失败，错误:{e}")
|
||||
logger = setup_logging()
|
||||
logger.warning(f"发送情绪表情失败，错误:{e}")
|
||||
return
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import json
|
||||
import websockets
|
||||
from typing import Dict, Any, Optional
|
||||
from config.logger import setup_logging
|
||||
from core.services.connection_service import ConnectionService
|
||||
from core.transport.websocket_transport import WebSocketTransport
|
||||
from config.config_loader import get_config_from_api_async
|
||||
from core.utils.util import check_vad_update, check_asr_update
|
||||
from core.auth import AuthMiddleware, AuthenticationError
|
||||
from core.utils.config_validation import validate_config_components
|
||||
from core.providers.asr.shared_asr_manager import SharedASRManager
|
||||
|
||||
|
||||
class SuppressInvalidHandshakeFilter(logging.Filter):
|
||||
"""过滤掉无效握手错误日志（如HTTPS访问WS端口）"""
|
||||
|
||||
def filter(self, record):
|
||||
msg = record.getMessage()
|
||||
suppress_keywords = [
|
||||
"opening handshake failed",
|
||||
"did not receive a valid HTTP request",
|
||||
"connection closed while reading HTTP request",
|
||||
"line without CRLF",
|
||||
]
|
||||
return not any(keyword in msg for keyword in suppress_keywords)
|
||||
|
||||
|
||||
def _setup_websockets_logger():
|
||||
"""配置 websockets 相关的所有 logger，过滤无效握手错误"""
|
||||
filter_instance = SuppressInvalidHandshakeFilter()
|
||||
for logger_name in ["websockets", "websockets.server", "websockets.client"]:
|
||||
ws_logger = logging.getLogger(logger_name)
|
||||
ws_logger.addFilter(filter_instance)
|
||||
|
||||
|
||||
_setup_websockets_logger()
|
||||
|
||||
|
||||
logger = setup_logging()
|
||||
TAG = __name__
|
||||
|
||||
|
||||
class NewWebSocketServer:
|
||||
"""
|
||||
新的WebSocket服务器：使用新架构替代旧的ConnectionHandler
|
||||
集成ConnectionService、MessageRouter和新的Processor架构
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
self.config = config
|
||||
self.logger = setup_logging()
|
||||
self.config_lock = asyncio.Lock()
|
||||
self.last_update_error = None
|
||||
|
||||
# 创建连接服务
|
||||
self.connection_service = ConnectionService(config)
|
||||
self.connection_service.server = self
|
||||
|
||||
# 活跃连接管理
|
||||
self.active_connections = set()
|
||||
|
||||
# 认证中间件
|
||||
self.auth_middleware = AuthMiddleware(config)
|
||||
|
||||
# 服务器实例和控制
|
||||
self._server = None
|
||||
self._stop_event = asyncio.Event()
|
||||
self._started_event = asyncio.Event()
|
||||
self._is_running = False
|
||||
|
||||
async def start(self):
|
||||
"""启动WebSocket服务器"""
|
||||
server_config = self.config["server"]
|
||||
host = server_config.get("ip", "0.0.0.0")
|
||||
port = int(server_config.get("port", 8000))
|
||||
|
||||
logger.bind(tag=TAG).info(f"启动新架构WebSocket服务器: {host}:{port}")
|
||||
|
||||
self._stop_event.clear()
|
||||
self._started_event.clear()
|
||||
|
||||
try:
|
||||
self._server = await websockets.serve(
|
||||
self._handle_connection,
|
||||
host,
|
||||
port,
|
||||
process_request=self._http_response
|
||||
)
|
||||
self._is_running = True
|
||||
self._started_event.set()
|
||||
logger.bind(tag=TAG).info("WebSocket服务器启动成功")
|
||||
|
||||
# 等待停止信号
|
||||
await self._stop_event.wait()
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"WebSocket服务器启动失败: {e}")
|
||||
raise
|
||||
finally:
|
||||
self._is_running = False
|
||||
self._started_event.clear()
|
||||
|
||||
async def stop(self):
|
||||
"""停止WebSocket服务器"""
|
||||
if not self._is_running:
|
||||
logger.bind(tag=TAG).debug("WebSocket服务器未运行，无需停止")
|
||||
return
|
||||
|
||||
logger.bind(tag=TAG).info("正在停止WebSocket服务器...")
|
||||
|
||||
# 关闭所有活跃连接
|
||||
for transport in list(self.active_connections):
|
||||
try:
|
||||
await transport.close()
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"关闭连接失败: {e}")
|
||||
self.active_connections.clear()
|
||||
|
||||
# 关闭服务器
|
||||
if self._server:
|
||||
self._server.close()
|
||||
try:
|
||||
await asyncio.wait_for(self._server.wait_closed(), timeout=5.0)
|
||||
except asyncio.TimeoutError:
|
||||
logger.bind(tag=TAG).warning("等待服务器关闭超时")
|
||||
self._server = None
|
||||
|
||||
# 发送停止信号
|
||||
self._stop_event.set()
|
||||
self._is_running = False
|
||||
self._started_event.clear()
|
||||
|
||||
logger.bind(tag=TAG).info("WebSocket服务器已停止")
|
||||
|
||||
async def _handle_connection(self, websocket):
|
||||
"""处理新连接 - 使用新架构"""
|
||||
# 提取连接头信息
|
||||
headers = self._extract_headers(websocket)
|
||||
device_id = headers.get('device-id')
|
||||
|
||||
# 如果没有 device-id，提示并关闭连接
|
||||
if not device_id:
|
||||
await websocket.send("端口正常，如需测试连接，请使用test_page.html")
|
||||
await websocket.close()
|
||||
return
|
||||
|
||||
# 连接时认证
|
||||
try:
|
||||
await self._handle_auth(headers)
|
||||
except AuthenticationError as e:
|
||||
logger.bind(tag=TAG).warning(f"认证失败: {e}")
|
||||
await websocket.send("认证失败")
|
||||
await websocket.close()
|
||||
return
|
||||
|
||||
# 创建WebSocket传输层
|
||||
try:
|
||||
protocol_version = int(headers.get('protocol-version', 1) or 1)
|
||||
except (TypeError, ValueError):
|
||||
protocol_version = 1
|
||||
transport = WebSocketTransport(
|
||||
websocket,
|
||||
from_mqtt_gateway=headers.get('from_mqtt_gateway') == 'true',
|
||||
protocol_version=protocol_version,
|
||||
)
|
||||
|
||||
# 记录活跃连接
|
||||
self.active_connections.add(transport)
|
||||
|
||||
try:
|
||||
logger.bind(tag=TAG).info(
|
||||
f"新连接建立: {device_id} from {headers.get('x-real-ip', 'unknown')}"
|
||||
)
|
||||
|
||||
# 使用ConnectionService处理连接
|
||||
await self.connection_service.handle_connection(transport, headers)
|
||||
|
||||
except websockets.exceptions.ConnectionClosed:
|
||||
logger.bind(tag=TAG).info("WebSocket连接正常关闭")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理WebSocket连接时出错: {e}", exc_info=True)
|
||||
# 将错误反馈给管理端，避免长时间等待
|
||||
try:
|
||||
if hasattr(websocket, "closed") and not websocket.closed:
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "server",
|
||||
"status": "error",
|
||||
"message": f"Server error: {e}",
|
||||
"content": {"action": "unknown"},
|
||||
}
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
# 确保从活动连接集合中移除
|
||||
self.active_connections.discard(transport)
|
||||
|
||||
# 强制关闭连接（如果还没有关闭的话）
|
||||
try:
|
||||
if hasattr(websocket, "closed") and not websocket.closed:
|
||||
await websocket.close()
|
||||
elif hasattr(websocket, "state") and websocket.state.name != "CLOSED":
|
||||
await websocket.close()
|
||||
except Exception as close_error:
|
||||
logger.bind(tag=TAG).error(f"强制关闭WebSocket连接时出错: {close_error}")
|
||||
|
||||
async def _handle_auth(self, headers: Dict[str, str]):
|
||||
"""
|
||||
连接时认证
|
||||
|
||||
Args:
|
||||
headers: HTTP 请求头
|
||||
|
||||
Raises:
|
||||
AuthenticationError: 认证失败时抛出
|
||||
"""
|
||||
await self.auth_middleware.authenticate_async(headers)
|
||||
|
||||
def _extract_headers(self, websocket) -> Dict[str, str]:
|
||||
"""
|
||||
从WebSocket请求中提取头信息
|
||||
|
||||
支持从以下来源提取信息：
|
||||
1. HTTP 请求头
|
||||
2. URL 查询参数（device-id, client-id, authorization）
|
||||
3. 路径参数（如 ?from=mqtt_gateway）
|
||||
"""
|
||||
headers = {}
|
||||
|
||||
# 1. 提取 HTTP 请求头
|
||||
if hasattr(websocket, 'request') and hasattr(websocket.request, 'headers'):
|
||||
for name, value in websocket.request.headers.items():
|
||||
headers[name.lower()] = value
|
||||
elif hasattr(websocket, 'request_headers'):
|
||||
for name, value in websocket.request_headers.items():
|
||||
headers[name.lower()] = value
|
||||
|
||||
# 2. 提取路径参数（如果有的话）
|
||||
request_path = None
|
||||
if hasattr(websocket, 'request') and hasattr(websocket.request, 'path'):
|
||||
request_path = websocket.request.path
|
||||
elif hasattr(websocket, 'path'):
|
||||
request_path = websocket.path
|
||||
|
||||
if request_path:
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
parsed = urlparse(request_path)
|
||||
query_params = parse_qs(parsed.query)
|
||||
|
||||
# 处理关键参数：device-id, client-id, authorization
|
||||
key_params = ['device-id', 'client-id', 'authorization']
|
||||
for key in key_params:
|
||||
if key in query_params and query_params[key]:
|
||||
# URL 参数优先级低于 header
|
||||
if key not in headers or not headers[key]:
|
||||
headers[key] = query_params[key][0]
|
||||
|
||||
# 处理其他参数
|
||||
for key, values in query_params.items():
|
||||
if values and key not in headers:
|
||||
headers[key] = values[0]
|
||||
|
||||
# 检查是否来自 MQTT 网关
|
||||
if request_path.endswith("?from=mqtt_gateway") or "from=mqtt_gateway" in request_path:
|
||||
headers['from_mqtt_gateway'] = 'true'
|
||||
|
||||
# 3. 提取远程地址
|
||||
if hasattr(websocket, 'remote_address'):
|
||||
# 如果 headers 中没有 x-real-ip，使用 remote_address
|
||||
if 'x-real-ip' not in headers:
|
||||
headers['x-real-ip'] = websocket.remote_address[0]
|
||||
|
||||
return headers
|
||||
|
||||
async def _http_response(self, websocket, request_headers):
|
||||
"""处理HTTP请求"""
|
||||
# 检查是否为 WebSocket 升级请求
|
||||
if request_headers.headers.get("connection", "").lower() == "upgrade":
|
||||
# 如果是 WebSocket 请求，返回 None 允许握手继续
|
||||
return None
|
||||
else:
|
||||
# 如果是普通 HTTP 请求，返回服务器状态
|
||||
return websocket.respond(200, "New Architecture WebSocket Server is running\n")
|
||||
|
||||
async def apply_config(self, new_config: Dict[str, Any]) -> bool:
|
||||
"""Apply a facade-validated config to future WebSocket connections."""
|
||||
self.config = new_config
|
||||
self.connection_service = ConnectionService(new_config)
|
||||
self.connection_service.server = getattr(self, "management_owner", self)
|
||||
self.auth_middleware = AuthMiddleware(new_config)
|
||||
return True
|
||||
|
||||
async def update_config(
|
||||
self, new_config: Optional[Dict[str, Any]] = None
|
||||
) -> bool:
|
||||
"""
|
||||
更新服务器配置并重新初始化组件
|
||||
|
||||
Returns:
|
||||
bool: 更新是否成功
|
||||
"""
|
||||
try:
|
||||
async with self.config_lock:
|
||||
logger.bind(tag=TAG).info("开始更新服务器配置")
|
||||
self.last_update_error = None
|
||||
old_config = self.config
|
||||
old_connection_service = self.connection_service
|
||||
old_auth_middleware = self.auth_middleware
|
||||
|
||||
# 管理命令可自行拉取配置；多协议管理器则直接传入已合并配置。
|
||||
if new_config is None:
|
||||
new_config = await get_config_from_api_async(self.config)
|
||||
if new_config is None:
|
||||
logger.bind(tag=TAG).error("获取新配置失败")
|
||||
self.last_update_error = "获取新配置失败"
|
||||
return False
|
||||
|
||||
logger.bind(tag=TAG).info("获取新配置成功")
|
||||
|
||||
new_shared_manager = None
|
||||
reuse_manager = False
|
||||
|
||||
# 校验新配置（预初始化组件以发现配置错误）
|
||||
ok, error_msg = await validate_config_components(new_config, logger)
|
||||
if not ok:
|
||||
logger.bind(tag=TAG).error(f"配置校验失败: {error_msg}")
|
||||
self.last_update_error = f"配置校验失败: {error_msg}"
|
||||
return False
|
||||
|
||||
# 准备共享 ASR 管理器（本地模型走共享预加载）
|
||||
old_shared_manager = old_config.get("_shared_asr_manager")
|
||||
selected_asr = new_config.get("selected_module", {}).get("ASR")
|
||||
if selected_asr:
|
||||
asr_config = new_config.get("ASR", {}).get(selected_asr, {})
|
||||
asr_type = asr_config.get("type", selected_asr)
|
||||
if SharedASRManager.is_local_model_type(asr_type):
|
||||
if (
|
||||
old_shared_manager
|
||||
and getattr(old_shared_manager, "asr_type", None) == asr_type
|
||||
and old_shared_manager.is_ready()
|
||||
):
|
||||
new_shared_manager = old_shared_manager
|
||||
reuse_manager = True
|
||||
else:
|
||||
new_shared_manager = SharedASRManager(new_config, asr_type)
|
||||
await new_shared_manager.initialize()
|
||||
# 非本地模型时不立即关闭旧共享管理器，待更新成功后统一处理
|
||||
|
||||
# 检查 VAD 和 ASR 类型是否需要更新
|
||||
update_vad = check_vad_update(self.config, new_config)
|
||||
update_asr = check_asr_update(self.config, new_config)
|
||||
logger.bind(tag=TAG).info(
|
||||
f"检查VAD和ASR类型是否需要更新: VAD={update_vad}, ASR={update_asr}"
|
||||
)
|
||||
|
||||
# 检查配置是否有重大变化
|
||||
changed_configs = self._get_changed_configs(self.config, new_config)
|
||||
|
||||
# 更新配置
|
||||
self.config = new_config
|
||||
if new_shared_manager:
|
||||
self.config["_shared_asr_manager"] = new_shared_manager
|
||||
elif "_shared_asr_manager" in self.config:
|
||||
del self.config["_shared_asr_manager"]
|
||||
|
||||
# 根据变化类型进行更新
|
||||
if changed_configs:
|
||||
logger.bind(tag=TAG).info(f"配置项变化: {', '.join(changed_configs)}")
|
||||
|
||||
# 重新创建连接服务，使用新配置
|
||||
# 注意：已建立的连接会继续使用旧配置，只有新连接使用新配置
|
||||
self.connection_service = ConnectionService(new_config)
|
||||
self.connection_service.server = self
|
||||
|
||||
# 如果 ASR 配置变化且复用旧共享管理器，提示重启
|
||||
if update_asr and reuse_manager:
|
||||
logger.bind(tag=TAG).warning(
|
||||
"ASR 配置已变化，但仍复用已有共享 ASR 管理器，建议重启服务"
|
||||
)
|
||||
else:
|
||||
# 即使没有重大变化，也更新 ConnectionService 的配置引用
|
||||
self.connection_service.config = new_config
|
||||
self.connection_service.server = self
|
||||
|
||||
# 更新认证中间件
|
||||
self.auth_middleware = AuthMiddleware(new_config)
|
||||
|
||||
# 更新成功后再关闭旧共享管理器（避免失败回滚时不可用）
|
||||
if old_shared_manager and old_shared_manager is not new_shared_manager:
|
||||
await old_shared_manager.shutdown()
|
||||
|
||||
logger.bind(tag=TAG).info("配置更新任务执行完毕")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"更新服务器配置失败: {str(e)}", exc_info=True)
|
||||
self.last_update_error = f"更新服务器配置失败: {str(e)}"
|
||||
try:
|
||||
if new_shared_manager and not reuse_manager:
|
||||
await new_shared_manager.shutdown()
|
||||
self.config = old_config
|
||||
self.connection_service = old_connection_service
|
||||
self.auth_middleware = old_auth_middleware
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
def get_last_update_error(self) -> str:
|
||||
return self.last_update_error or ""
|
||||
|
||||
def _get_changed_configs(self, old_config: Dict[str, Any], new_config: Dict[str, Any]) -> list:
|
||||
"""
|
||||
获取变化的配置项列表
|
||||
|
||||
Returns:
|
||||
list: 变化的配置项名称列表
|
||||
"""
|
||||
changed = []
|
||||
key_configs = [
|
||||
"selected_module",
|
||||
"VAD",
|
||||
"ASR",
|
||||
"LLM",
|
||||
"TTS",
|
||||
"Memory",
|
||||
"Intent"
|
||||
]
|
||||
|
||||
for key in key_configs:
|
||||
old_value = old_config.get(key)
|
||||
new_value = new_config.get(key)
|
||||
if old_value != new_value:
|
||||
changed.append(key)
|
||||
|
||||
return changed
|
||||
|
||||
def get_active_connections_count(self) -> int:
|
||||
"""获取活跃连接数"""
|
||||
return len(self.active_connections)
|
||||
|
||||
def get_server_status(self) -> Dict[str, Any]:
|
||||
"""获取服务器状态"""
|
||||
return {
|
||||
"active_connections": self.get_active_connections_count(),
|
||||
"server_type": "new_architecture",
|
||||
"processors": self.connection_service.message_router.list_processors()
|
||||
}
|
||||
@@ -0,0 +1,437 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
小智服务器门面类
|
||||
统一管理所有协议服务器的启动和停止
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Dict, Any, Optional
|
||||
from config.logger import setup_logging
|
||||
from core.servers.multi_protocol_server import MultiProtocolServer
|
||||
|
||||
logger = setup_logging()
|
||||
TAG = __name__
|
||||
|
||||
|
||||
class XiaozhiServerFacade:
|
||||
"""
|
||||
小智服务器门面类
|
||||
提供统一的服务器管理接口，屏蔽内部协议复杂性
|
||||
|
||||
功能：
|
||||
- 协议管理
|
||||
- 本地 ASR 模型预加载
|
||||
- 优雅启动和停止
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
"""
|
||||
初始化服务器门面
|
||||
|
||||
Args:
|
||||
config: 服务器配置字典
|
||||
"""
|
||||
self.config = config
|
||||
self.multi_protocol_server: Optional[MultiProtocolServer] = None
|
||||
self.shared_asr_manager = None # 共享 ASR 管理器
|
||||
self._retired_shared_asr_managers = []
|
||||
self.is_initialized = False
|
||||
self.is_running = False
|
||||
self.last_update_error = None
|
||||
self._cleanup_pending = False
|
||||
|
||||
# 处理协议配置
|
||||
self._setup_protocol_config()
|
||||
|
||||
def _setup_protocol_config(self):
|
||||
"""设置协议配置"""
|
||||
try:
|
||||
protocols = self.config.get("protocols", {})
|
||||
if not isinstance(protocols, dict):
|
||||
protocols = {}
|
||||
requested = protocols.get("enabled_protocols")
|
||||
requested = requested if isinstance(requested, list) else []
|
||||
websocket_enabled = protocols.get("websocket_enabled")
|
||||
if websocket_enabled is None:
|
||||
websocket_enabled = not protocols or "websocket" in requested
|
||||
|
||||
enabled_protocols = []
|
||||
if websocket_enabled:
|
||||
enabled_protocols.append("websocket")
|
||||
|
||||
self.config["enabled_protocols"] = enabled_protocols
|
||||
logger.info(f"启用的协议: {enabled_protocols}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"设置协议配置失败: {e}")
|
||||
# 使用最基本的配置
|
||||
self.config["enabled_protocols"] = ["websocket"]
|
||||
|
||||
async def initialize(self):
|
||||
"""初始化服务器"""
|
||||
if self.is_initialized:
|
||||
logger.bind(tag=TAG).warning("服务器已经初始化")
|
||||
return
|
||||
|
||||
try:
|
||||
logger.bind(tag=TAG).info("正在初始化小智服务器...")
|
||||
|
||||
# 检查并预加载本地 ASR 模型（关键步骤）
|
||||
await self._preload_asr_if_needed()
|
||||
|
||||
# 创建多协议服务器
|
||||
self.multi_protocol_server = MultiProtocolServer(self.config)
|
||||
self.multi_protocol_server.set_management_owner(self)
|
||||
|
||||
self.is_initialized = True
|
||||
logger.bind(tag=TAG).info("小智服务器初始化完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"初始化服务器失败: {e}")
|
||||
raise
|
||||
|
||||
async def _preload_asr_if_needed(self):
|
||||
"""
|
||||
检查并预加载本地 ASR 模型
|
||||
|
||||
如果配置使用本地 ASR 模型（如 FunASR），则在服务器启动时预加载，
|
||||
避免首次语音识别时的延迟导致客户端超时。
|
||||
"""
|
||||
try:
|
||||
# 获取 ASR 配置
|
||||
selected_asr = self.config.get("selected_module", {}).get("ASR")
|
||||
if not selected_asr:
|
||||
logger.bind(tag=TAG).info("未配置 ASR 模块，跳过预加载")
|
||||
return
|
||||
|
||||
# 获取 ASR 类型
|
||||
asr_config = self.config.get("ASR", {}).get(selected_asr, {})
|
||||
asr_type = asr_config.get("type", selected_asr)
|
||||
|
||||
# 导入 SharedASRManager 检查是否为本地模型
|
||||
from core.providers.asr.shared_asr_manager import SharedASRManager
|
||||
|
||||
if SharedASRManager.is_local_model_type(asr_type):
|
||||
logger.bind(tag=TAG).info(
|
||||
f"检测到本地 ASR 模型: {asr_type}，开始预加载..."
|
||||
)
|
||||
|
||||
# 创建全局 ASR 管理器
|
||||
self.shared_asr_manager = SharedASRManager(self.config, asr_type)
|
||||
|
||||
# 预加载模型
|
||||
await self.shared_asr_manager.initialize()
|
||||
|
||||
# 将管理器放入配置中供后续使用
|
||||
self.config['_shared_asr_manager'] = self.shared_asr_manager
|
||||
|
||||
logger.bind(tag=TAG).info(
|
||||
f"ASR 模型预加载完成，类型: {asr_type}"
|
||||
)
|
||||
else:
|
||||
logger.bind(tag=TAG).info(
|
||||
f"ASR 类型为远程服务: {asr_type}，无需预加载"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR 预加载失败: {e}")
|
||||
# 预加载失败不影响服务器启动，继续使用懒加载模式
|
||||
logger.bind(tag=TAG).warning("将回退到懒加载模式")
|
||||
|
||||
async def start(self):
|
||||
"""启动服务器"""
|
||||
if self._cleanup_pending:
|
||||
raise RuntimeError("上次停止尚未完成，请先重试 stop 清理残留资源")
|
||||
if not self.is_initialized:
|
||||
await self.initialize()
|
||||
|
||||
if self.is_running:
|
||||
logger.warning("服务器已经在运行中")
|
||||
return
|
||||
|
||||
try:
|
||||
logger.info("正在启动小智服务器...")
|
||||
|
||||
# 启动多协议服务器
|
||||
await self.multi_protocol_server.start()
|
||||
|
||||
self.is_running = True
|
||||
logger.info("小智服务器启动成功")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"启动服务器失败: {e}")
|
||||
self.is_running = False
|
||||
# initialize() may already own a shared ASR manager and partially
|
||||
# started listeners. Release both before propagating startup failure.
|
||||
try:
|
||||
await self.stop()
|
||||
except Exception as cleanup_error:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"启动失败后的资源清理失败: {cleanup_error}"
|
||||
)
|
||||
raise
|
||||
|
||||
async def stop(self):
|
||||
"""停止服务器"""
|
||||
if (
|
||||
not self.is_running
|
||||
and self.multi_protocol_server is None
|
||||
and self.shared_asr_manager is None
|
||||
and not self._retired_shared_asr_managers
|
||||
):
|
||||
logger.bind(tag=TAG).info("服务器未在运行")
|
||||
return
|
||||
|
||||
logger.bind(tag=TAG).info("正在停止小智服务器...")
|
||||
errors = []
|
||||
protocols_stopped = self.multi_protocol_server is None
|
||||
if self.multi_protocol_server:
|
||||
try:
|
||||
await self.multi_protocol_server.stop()
|
||||
except Exception as e:
|
||||
errors.append(("多协议服务器", e))
|
||||
logger.bind(tag=TAG).error(f"停止多协议服务器失败: {e}")
|
||||
else:
|
||||
self.multi_protocol_server = None
|
||||
protocols_stopped = True
|
||||
|
||||
if self.shared_asr_manager and protocols_stopped:
|
||||
logger.bind(tag=TAG).info("正在关闭共享 ASR 管理器...")
|
||||
try:
|
||||
await self.shared_asr_manager.shutdown()
|
||||
except Exception as e:
|
||||
errors.append(("共享 ASR", e))
|
||||
logger.bind(tag=TAG).error(f"关闭共享 ASR 管理器失败: {e}")
|
||||
else:
|
||||
self.shared_asr_manager = None
|
||||
self.config.pop('_shared_asr_manager', None)
|
||||
elif self.shared_asr_manager:
|
||||
logger.bind(tag=TAG).warning(
|
||||
"协议服务器仍持有连接，延后关闭共享 ASR 管理器"
|
||||
)
|
||||
|
||||
if protocols_stopped and self._retired_shared_asr_managers:
|
||||
remaining_retired = []
|
||||
for manager in self._retired_shared_asr_managers:
|
||||
try:
|
||||
await manager.shutdown()
|
||||
except Exception as e:
|
||||
remaining_retired.append(manager)
|
||||
errors.append(("旧共享 ASR", e))
|
||||
logger.bind(tag=TAG).error(
|
||||
f"关闭旧共享ASR管理器失败: {e}"
|
||||
)
|
||||
self._retired_shared_asr_managers = remaining_retired
|
||||
|
||||
self.is_running = False
|
||||
self.is_initialized = bool(
|
||||
self.multi_protocol_server
|
||||
or self.shared_asr_manager
|
||||
or self._retired_shared_asr_managers
|
||||
)
|
||||
if not self.is_initialized:
|
||||
self.shared_asr_manager = None
|
||||
self.config.pop('_shared_asr_manager', None)
|
||||
logger.bind(tag=TAG).info("小智服务器已停止")
|
||||
else:
|
||||
logger.bind(tag=TAG).warning(
|
||||
"服务器部分资源停止失败，已保留所有权供重试清理"
|
||||
)
|
||||
|
||||
if errors:
|
||||
self._cleanup_pending = True
|
||||
details = ", ".join(f"{owner}: {error}" for owner, error in errors)
|
||||
raise RuntimeError(f"停止服务器时发生错误: {details}")
|
||||
self._cleanup_pending = False
|
||||
|
||||
async def restart(self):
|
||||
"""重启服务器"""
|
||||
logger.info("重启小智服务器...")
|
||||
await self.stop()
|
||||
await asyncio.sleep(1) # 等待清理完成
|
||||
await self.start()
|
||||
|
||||
async def update_config(
|
||||
self, new_config: Optional[Dict[str, Any]] = None
|
||||
) -> bool:
|
||||
"""
|
||||
更新服务器配置
|
||||
|
||||
Args:
|
||||
new_config: 新的配置字典
|
||||
|
||||
Returns:
|
||||
bool: 更新是否成功
|
||||
"""
|
||||
old_config = self.config
|
||||
old_shared_manager = self.shared_asr_manager
|
||||
prepared_shared_manager = old_shared_manager
|
||||
owns_prepared_manager = False
|
||||
try:
|
||||
logger.info("更新服务器配置...")
|
||||
self.last_update_error = None
|
||||
|
||||
if new_config is None:
|
||||
from config.config_loader import get_config_from_api_async
|
||||
|
||||
new_config = await get_config_from_api_async(self.config)
|
||||
if new_config is None:
|
||||
raise RuntimeError("获取新配置失败")
|
||||
|
||||
# 使用新顶层对象，避免 MultiProtocolServer 的旧配置引用被原地改写，
|
||||
# 从而导致协议/端口变化无法被检测。
|
||||
merged_config = dict(self.config)
|
||||
merged_config.update(new_config)
|
||||
self.config = merged_config
|
||||
self._setup_protocol_config()
|
||||
|
||||
from core.utils.config_validation import validate_config_components
|
||||
from core.utils.util import check_asr_update
|
||||
from core.providers.asr.shared_asr_manager import SharedASRManager
|
||||
|
||||
ok, error_msg = await validate_config_components(self.config, logger)
|
||||
if not ok:
|
||||
raise RuntimeError(f"配置校验失败: {error_msg}")
|
||||
|
||||
selected_asr = self.config.get("selected_module", {}).get("ASR")
|
||||
asr_config = self.config.get("ASR", {}).get(selected_asr, {})
|
||||
asr_type = asr_config.get("type", selected_asr) if selected_asr else None
|
||||
needs_new_asr = check_asr_update(old_config, self.config)
|
||||
|
||||
if asr_type and SharedASRManager.is_local_model_type(asr_type):
|
||||
if not (
|
||||
old_shared_manager
|
||||
and not needs_new_asr
|
||||
and old_shared_manager.is_ready()
|
||||
):
|
||||
prepared_shared_manager = SharedASRManager(self.config, asr_type)
|
||||
await prepared_shared_manager.initialize()
|
||||
owns_prepared_manager = True
|
||||
self.config["_shared_asr_manager"] = prepared_shared_manager
|
||||
else:
|
||||
prepared_shared_manager = None
|
||||
self.config.pop("_shared_asr_manager", None)
|
||||
|
||||
# 已初始化时即更新实例集合；运行中会完成监听器切换。
|
||||
if self.multi_protocol_server:
|
||||
success = await self.multi_protocol_server.update_config(self.config)
|
||||
if success:
|
||||
logger.info("服务器配置更新成功")
|
||||
else:
|
||||
raise RuntimeError("多协议服务器配置更新失败")
|
||||
|
||||
self.shared_asr_manager = prepared_shared_manager
|
||||
if (
|
||||
old_shared_manager
|
||||
and old_shared_manager is not prepared_shared_manager
|
||||
):
|
||||
try:
|
||||
await old_shared_manager.shutdown()
|
||||
except Exception as e:
|
||||
# 新配置已经提交，保留旧资源所有权供 stop 重试。
|
||||
self._retired_shared_asr_managers.append(
|
||||
old_shared_manager
|
||||
)
|
||||
logger.bind(tag=TAG).error(f"关闭旧共享ASR管理器失败: {e}")
|
||||
|
||||
logger.info("配置更新完成")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"更新配置失败: {e}")
|
||||
self.last_update_error = str(e)
|
||||
if owns_prepared_manager and prepared_shared_manager:
|
||||
try:
|
||||
await prepared_shared_manager.shutdown()
|
||||
except Exception as cleanup_error:
|
||||
self._retired_shared_asr_managers.append(
|
||||
prepared_shared_manager
|
||||
)
|
||||
logger.bind(tag=TAG).error(
|
||||
f"回滚新共享ASR管理器失败: {cleanup_error}"
|
||||
)
|
||||
self.config = old_config
|
||||
self.shared_asr_manager = old_shared_manager
|
||||
if self.multi_protocol_server:
|
||||
self.is_running = self.multi_protocol_server.is_running
|
||||
if (
|
||||
self.multi_protocol_server.server_tasks
|
||||
and not self.multi_protocol_server.is_running
|
||||
):
|
||||
self._cleanup_pending = True
|
||||
return False
|
||||
|
||||
def get_last_update_error(self) -> str:
|
||||
return self.last_update_error or ""
|
||||
|
||||
def get_server_status(self) -> Dict[str, Any]:
|
||||
"""获取服务器状态"""
|
||||
base_status = {
|
||||
'is_initialized': self.is_initialized,
|
||||
'is_running': self.is_running,
|
||||
'enabled_protocols': self.config.get('enabled_protocols', [])
|
||||
}
|
||||
|
||||
if self.multi_protocol_server:
|
||||
server_status = self.multi_protocol_server.get_server_status()
|
||||
base_status.update(server_status)
|
||||
|
||||
# 添加 ASR 状态
|
||||
if self.shared_asr_manager:
|
||||
base_status['asr'] = {
|
||||
'mode': 'shared',
|
||||
'ready': self.shared_asr_manager.is_ready(),
|
||||
'queue_status': self.shared_asr_manager.get_queue_status()
|
||||
}
|
||||
else:
|
||||
base_status['asr'] = {'mode': 'lazy_load'}
|
||||
|
||||
return base_status
|
||||
|
||||
def get_active_connections_count(self) -> Dict[str, int]:
|
||||
"""获取各协议的活跃连接数"""
|
||||
if self.multi_protocol_server:
|
||||
return self.multi_protocol_server.get_active_connections_count()
|
||||
return {}
|
||||
|
||||
def get_supported_protocols(self) -> list:
|
||||
"""获取支持的协议列表"""
|
||||
if self.multi_protocol_server:
|
||||
return self.multi_protocol_server.get_supported_protocols()
|
||||
return ['websocket']
|
||||
|
||||
def is_protocol_enabled(self, protocol: str) -> bool:
|
||||
"""检查协议是否启用"""
|
||||
enabled_protocols = self.config.get('enabled_protocols', [])
|
||||
return protocol in enabled_protocols
|
||||
|
||||
async def broadcast_message(self, message: Dict[str, Any], protocol: Optional[str] = None):
|
||||
"""
|
||||
向所有连接广播消息
|
||||
|
||||
Args:
|
||||
message: 要广播的消息
|
||||
protocol: 指定协议，None表示向所有协议广播
|
||||
"""
|
||||
if self.multi_protocol_server:
|
||||
await self.multi_protocol_server.broadcast_message(message, protocol)
|
||||
|
||||
def get_websocket_info(self) -> Dict[str, Any]:
|
||||
"""获取WebSocket连接信息"""
|
||||
if not self.is_protocol_enabled('websocket'):
|
||||
return {'enabled': False}
|
||||
|
||||
server_config = self.config.get('server', {})
|
||||
return {
|
||||
'enabled': True,
|
||||
'host': server_config.get('ip', '0.0.0.0'),
|
||||
'port': server_config.get('port', 8000),
|
||||
'path': '/xiaozhi/v1/'
|
||||
}
|
||||
|
||||
def get_connection_info(self) -> Dict[str, Any]:
|
||||
"""获取所有协议的连接信息"""
|
||||
return {
|
||||
'websocket': self.get_websocket_info(),
|
||||
'active_connections': self.get_active_connections_count()
|
||||
}
|
||||
Reference in New Issue
Block a user