mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-29 00:03:52 +08:00
refactor: introduce transport-neutral connection runtime
This commit is contained in:
caixypromise
@@ -0,0 +1,132 @@
|
||||
import json
|
||||
import uuid
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.components.component_manager import ComponentType
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AbortProcessor(MessageProcessor):
|
||||
"""中断消息处理器：完整迁移abortMessageHandler.py和abortHandle.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理abort类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "abort":
|
||||
await self.handle_abort_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_abort_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理中断消息 - 完整迁移自abortHandle.py的handleAbortMessage"""
|
||||
msg_session_id = msg_json.get("session_id")
|
||||
if msg_session_id and msg_session_id != context.session_id:
|
||||
logger.warning(
|
||||
f"忽略非当前会话的abort消息: "
|
||||
f"msg={msg_session_id}, current={context.session_id}"
|
||||
)
|
||||
return
|
||||
logger.info("Abort message received")
|
||||
|
||||
# 设置成打断状态，会自动打断llm、tts任务 - 完整迁移原逻辑
|
||||
context.abort_requested = True
|
||||
context.close_after_chat = False
|
||||
|
||||
# Stop the old LLM/tool producers before a subsequent listen/start can
|
||||
# clear abort_requested and begin a new turn.
|
||||
cancel_tasks = getattr(context, "cancel_turn_tasks", None)
|
||||
if callable(cancel_tasks):
|
||||
await cancel_tasks()
|
||||
|
||||
# 清理队列 - 完整迁移原逻辑
|
||||
await self._clear_queues(context)
|
||||
|
||||
# 打断客户端说话状态 - 完整迁移原逻辑
|
||||
await transport.send_json({
|
||||
"type": "tts",
|
||||
"state": "stop",
|
||||
"session_id": context.session_id
|
||||
})
|
||||
|
||||
# 清理说话状态 - 完整迁移原逻辑
|
||||
self._clear_speak_status(context)
|
||||
|
||||
logger.info("Abort message received-end")
|
||||
|
||||
async def _clear_queues(self, context: SessionContext):
|
||||
"""清理所有队列 - 完整迁移原clear_queues逻辑"""
|
||||
try:
|
||||
# 清理TTS音频队列
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
for queue_name in ('tts_text_queue', 'tts_audio_queue'):
|
||||
pending_queue = getattr(tts_instance, queue_name, None)
|
||||
if pending_queue is None:
|
||||
continue
|
||||
try:
|
||||
while not pending_queue.empty():
|
||||
pending_queue.get_nowait()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Rotate the turn token so late text/audio from the aborted turn is stale.
|
||||
context.sentence_id = uuid.uuid4().hex
|
||||
|
||||
# 清理ASR音频队列
|
||||
context.clear_audio_buffer()
|
||||
|
||||
# 清理其他可能的队列
|
||||
if hasattr(context, 'clear_queues'):
|
||||
context.clear_queues()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"清理队列时出错: {e}")
|
||||
|
||||
def _clear_speak_status(self, context: SessionContext):
|
||||
"""清理说话状态 - 完整迁移原clearSpeakStatus逻辑"""
|
||||
try:
|
||||
# 清理说话状态
|
||||
context.is_speaking = False
|
||||
|
||||
# 如果有其他说话状态相关的属性，也一并清理
|
||||
if hasattr(context, 'clearSpeakStatus'):
|
||||
context.clearSpeakStatus()
|
||||
|
||||
# 重置相关状态
|
||||
context.client_have_voice = False
|
||||
context.client_voice_stop = True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"清理说话状态时出错: {e}")
|
||||
|
||||
async def _send_abort_confirmation(self, transport: TransportInterface, session_id: str):
|
||||
"""发送中断确认响应（可选）"""
|
||||
response = {
|
||||
"type": "abort",
|
||||
"status": "success",
|
||||
"message": "中断操作已完成",
|
||||
"session_id": session_id
|
||||
}
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送中断确认响应失败: {e}")
|
||||
|
||||
async def _get_component(self, context: SessionContext, component_type: ComponentType):
|
||||
if not context.component_manager:
|
||||
return None
|
||||
return await context.component_manager.get_component(component_type, context)
|
||||
@@ -0,0 +1,439 @@
|
||||
import time
|
||||
import json
|
||||
import asyncio
|
||||
import uuid
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.components.component_manager import ComponentType
|
||||
from core.services.audio_ingress_service import AudioIngressService
|
||||
from core.utils.util import audio_to_data
|
||||
from core.utils.output_counter import check_device_output_limit
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
def arm_wake_audio_suppression(context: SessionContext) -> None:
|
||||
"""Ignore reordered wake-word audio for a bounded interval."""
|
||||
context.just_woken_up = True
|
||||
previous = getattr(context, "wake_audio_suppression_task", None)
|
||||
if previous and not previous.done():
|
||||
previous.cancel()
|
||||
|
||||
delay_ms = max(
|
||||
0,
|
||||
int(context.config.get("wake_audio_suppression_ms", 2000)),
|
||||
)
|
||||
if delay_ms == 0:
|
||||
context.just_woken_up = False
|
||||
context.wake_audio_suppression_task = None
|
||||
return
|
||||
|
||||
session_id = context.session_id
|
||||
context.wake_audio_suppression_task = context.create_background_task(
|
||||
_release_wake_audio_suppression(
|
||||
context,
|
||||
session_id,
|
||||
delay_ms / 1000,
|
||||
),
|
||||
conversation_scoped=True,
|
||||
)
|
||||
|
||||
|
||||
async def _release_wake_audio_suppression(
|
||||
context: SessionContext,
|
||||
session_id: str,
|
||||
delay_seconds: float,
|
||||
) -> None:
|
||||
current_task = asyncio.current_task()
|
||||
try:
|
||||
await asyncio.sleep(delay_seconds)
|
||||
if context.session_id == session_id:
|
||||
context.just_woken_up = False
|
||||
finally:
|
||||
if getattr(context, "wake_audio_suppression_task", None) is current_task:
|
||||
context.wake_audio_suppression_task = None
|
||||
|
||||
|
||||
class AudioReceiveProcessor(MessageProcessor):
|
||||
"""音频接收处理器：完整迁移receiveAudioHandle.py的所有功能"""
|
||||
|
||||
def __init__(self, audio_ingress_service=None):
|
||||
self.audio_ingress_service = audio_ingress_service or AudioIngressService()
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理音频消息"""
|
||||
if isinstance(message, bytes):
|
||||
await self.handle_audio_message(context, transport, message, 0)
|
||||
return True
|
||||
if isinstance(message, dict) and message.get("type") == "audio":
|
||||
audio_data = message.get("data")
|
||||
if isinstance(audio_data, (bytes, bytearray)):
|
||||
await self.handle_audio_message(
|
||||
context,
|
||||
transport,
|
||||
bytes(audio_data),
|
||||
int(message.get("timestamp", 0) or 0),
|
||||
)
|
||||
return True
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_audio_message(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
audio: bytes,
|
||||
timestamp: int = 0,
|
||||
):
|
||||
"""处理音频消息 - 完整迁移自handleAudioMessage"""
|
||||
if (
|
||||
getattr(transport, "requires_audio_tail_grace", False)
|
||||
and not getattr(context, "accepting_input_audio", False)
|
||||
):
|
||||
return
|
||||
|
||||
pcm_frame = self.audio_ingress_service.process(context, audio, timestamp)
|
||||
if not pcm_frame:
|
||||
return
|
||||
|
||||
# Do not feed encoded wake-word history into stateful VAD. Native MQTT
|
||||
# gates the usual pre-start history above; this bounded suppression only
|
||||
# covers frames that crossed the independent control/audio channels.
|
||||
if context.just_woken_up:
|
||||
if not getattr(context, "wake_audio_suppression_task", None):
|
||||
arm_wake_audio_suppression(context)
|
||||
context.asr_audio.clear()
|
||||
context.reset_vad_states()
|
||||
return
|
||||
|
||||
# 获取VAD组件
|
||||
vad_component = await self._get_component(context, ComponentType.VAD)
|
||||
if not vad_component or not hasattr(vad_component, 'vad_instance'):
|
||||
now = time.time()
|
||||
last_log = getattr(context, "_vad_missing_last_log", 0.0)
|
||||
if now - last_log > 10:
|
||||
logger.warning("VAD组件未初始化")
|
||||
context._vad_missing_last_log = now
|
||||
return
|
||||
|
||||
vad_instance = vad_component.vad_instance
|
||||
|
||||
# 当前片段是否有人说话
|
||||
have_voice = vad_instance.is_vad(context, pcm_frame)
|
||||
|
||||
if have_voice:
|
||||
if (
|
||||
getattr(context, "client_aec", False)
|
||||
and context.is_speaking
|
||||
and context.listen_mode != "manual"
|
||||
):
|
||||
await self._handle_abort_message(context, transport)
|
||||
|
||||
# 设备长时间空闲检测，用于say goodbye
|
||||
await self._no_voice_close_connect(context, transport, have_voice)
|
||||
|
||||
# 接收音频
|
||||
asr_component = await self._get_component(context, ComponentType.ASR)
|
||||
asr_instance = asr_component.asr_instance if asr_component and hasattr(asr_component, 'asr_instance') else None
|
||||
|
||||
# 自动模式兜底：没有收到 listen stop 时，基于VAD触发一次停止
|
||||
if (
|
||||
not have_voice
|
||||
and context.client_have_voice
|
||||
and not context.client_voice_stop
|
||||
and not getattr(context, "listen_stop_pending", False)
|
||||
and context.listen_mode != "manual"
|
||||
and asr_instance is not None
|
||||
):
|
||||
context.client_voice_stop = True
|
||||
try:
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
if hasattr(asr_instance, "interface_type") and asr_instance.interface_type == InterfaceType.STREAM:
|
||||
if hasattr(asr_instance, "_send_stop_request"):
|
||||
context.create_background_task(
|
||||
asr_instance._send_stop_request(),
|
||||
turn_scoped=True,
|
||||
)
|
||||
else:
|
||||
if len(context.asr_audio) > 0:
|
||||
asr_audio_task = context.asr_audio.copy()
|
||||
context.asr_audio.clear()
|
||||
context.reset_vad_states()
|
||||
await asr_instance.handle_voice_stop(context, asr_audio_task)
|
||||
except Exception as e:
|
||||
logger.error(f"自动VAD停止处理失败: {e}")
|
||||
|
||||
if asr_instance and hasattr(asr_instance, 'receive_audio'):
|
||||
await asr_instance.receive_audio(context, pcm_frame, have_voice)
|
||||
|
||||
async def start_to_chat(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
text: str,
|
||||
*,
|
||||
preserve_close_after_chat: bool = False,
|
||||
):
|
||||
"""开始聊天 - 完整迁移自startToChat"""
|
||||
# 检查输入是否是JSON格式（包含说话人信息）
|
||||
speaker_name = None
|
||||
actual_text = text
|
||||
|
||||
try:
|
||||
# 尝试解析JSON格式的输入
|
||||
if text.strip().startswith('{') and text.strip().endswith('}'):
|
||||
data = json.loads(text)
|
||||
if 'speaker' in data and 'content' in data:
|
||||
speaker_name = data['speaker']
|
||||
actual_content = data['content']
|
||||
logger.info(f"解析到说话人信息: {speaker_name}")
|
||||
|
||||
if speaker_name not in context.introduced_speakers:
|
||||
context.introduced_speakers.add(speaker_name)
|
||||
actual_text = text
|
||||
else:
|
||||
actual_text = actual_content
|
||||
except (json.JSONDecodeError, KeyError):
|
||||
# 如果解析失败，继续使用原始文本
|
||||
pass
|
||||
|
||||
# 保存说话人信息到上下文
|
||||
if speaker_name:
|
||||
context.current_speaker = speaker_name
|
||||
else:
|
||||
context.current_speaker = None
|
||||
|
||||
# 检查设备绑定
|
||||
if context.need_bind:
|
||||
await self.prompt_bind_device(context, transport)
|
||||
return
|
||||
|
||||
# 如果当日的输出字数大于限定的字数
|
||||
if context.max_output_size > 0:
|
||||
if check_device_output_limit(
|
||||
context.headers.get("device-id"), context.max_output_size
|
||||
):
|
||||
await self._max_out_size(context, transport)
|
||||
return
|
||||
|
||||
if context.is_speaking and getattr(context, "listen_mode", "auto") != "manual":
|
||||
await self._handle_abort_message(context, transport)
|
||||
|
||||
context.abort_requested = False
|
||||
if not preserve_close_after_chat:
|
||||
context.close_after_chat = False
|
||||
|
||||
# 首先进行意图分析，使用实际文本内容
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
intent_handled = await chat_processor.handle_user_intent(context, transport, actual_text)
|
||||
|
||||
if intent_handled:
|
||||
# 如果意图已被处理，不再进行聊天
|
||||
return
|
||||
|
||||
# 意图未被处理，继续常规聊天流程，使用实际文本内容
|
||||
await self._send_stt_message(context, transport, actual_text)
|
||||
|
||||
# 与旧架构一致：将聊天处理放入线程池，避免阻塞事件循环
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
|
||||
if hasattr(context, "create_background_task"):
|
||||
context.create_background_task(
|
||||
chat_processor.handle_chat(
|
||||
context,
|
||||
transport,
|
||||
actual_text,
|
||||
skip_intent=True,
|
||||
),
|
||||
turn_scoped=True,
|
||||
)
|
||||
else:
|
||||
await chat_processor.handle_chat(
|
||||
context, transport, actual_text, skip_intent=True
|
||||
)
|
||||
|
||||
async def _no_voice_close_connect(self, context: SessionContext, transport: TransportInterface, have_voice: bool):
|
||||
"""无声音时关闭连接检测 - 完整迁移自no_voice_close_connect"""
|
||||
if have_voice:
|
||||
context.update_activity()
|
||||
return
|
||||
|
||||
# 只有在已经初始化过时间戳的情况下才进行超时检查
|
||||
if context.last_activity_time_ms > 0.0:
|
||||
no_voice_time = time.time() * 1000 - context.last_activity_time_ms
|
||||
close_connection_no_voice_time = int(
|
||||
context.config.get("close_connection_no_voice_time", 120)
|
||||
)
|
||||
|
||||
if (
|
||||
not context.close_after_chat
|
||||
and no_voice_time > 1000 * close_connection_no_voice_time
|
||||
):
|
||||
context.close_after_chat = True
|
||||
context.abort_requested = False
|
||||
|
||||
end_prompt = context.config.get("end_prompt", {})
|
||||
if end_prompt and end_prompt.get("enable", True) is False:
|
||||
logger.info("结束对话，无需发送结束提示语")
|
||||
if transport.keeps_connection_between_sessions:
|
||||
session_id = context.session_id
|
||||
end_conversation = getattr(context, "end_conversation", None)
|
||||
if callable(end_conversation):
|
||||
await end_conversation(session_id)
|
||||
await transport.end_session(session_id)
|
||||
context.close_after_chat = False
|
||||
context.reset_vad_states()
|
||||
else:
|
||||
await transport.close()
|
||||
return
|
||||
|
||||
prompt = end_prompt.get("prompt")
|
||||
if not prompt:
|
||||
prompt = "请你以```时间过得真快```未来头，用富有感情、依依不舍的话来结束这场对话吧。！"
|
||||
await self.start_to_chat(
|
||||
context,
|
||||
transport,
|
||||
prompt,
|
||||
preserve_close_after_chat=True,
|
||||
)
|
||||
|
||||
async def _max_out_size(self, context: SessionContext, transport: TransportInterface):
|
||||
"""超出最大输出字数处理 - 完整迁移自max_out_size"""
|
||||
# 播放超出最大输出字数的提示
|
||||
context.abort_requested = False
|
||||
text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
file_path = "config/assets/max_output_size.wav"
|
||||
opus_packets = await audio_to_data(file_path)
|
||||
|
||||
# 获取TTS组件并添加到队列
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
if hasattr(tts_instance, 'tts_audio_queue'):
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
tts_instance.tts_audio_queue.put(
|
||||
(SentenceType.LAST, opus_packets, text, context.sentence_id)
|
||||
)
|
||||
|
||||
context.close_after_chat = True
|
||||
|
||||
async def prompt_bind_device(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
):
|
||||
"""检查设备绑定 - 完整迁移自check_bind_device"""
|
||||
bind_code = context.bind_code
|
||||
|
||||
if bind_code:
|
||||
# 确保bind_code是6位数字
|
||||
if len(bind_code) != 6:
|
||||
logger.error(f"无效的绑定码格式: {bind_code}")
|
||||
text = "绑定码格式错误，请检查配置。"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
return
|
||||
|
||||
text = f"请登录控制面板，输入{bind_code}，绑定设备。"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
# 获取TTS组件
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if not tts_component or not hasattr(tts_component, 'tts_instance'):
|
||||
return
|
||||
|
||||
tts_instance = tts_component.tts_instance
|
||||
if not hasattr(tts_instance, 'tts_audio_queue'):
|
||||
return
|
||||
|
||||
# 播放提示音
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
music_path = "config/assets/bind_code.wav"
|
||||
opus_packets = await audio_to_data(music_path)
|
||||
tts_instance.tts_audio_queue.put(
|
||||
(SentenceType.FIRST, opus_packets, text, context.sentence_id)
|
||||
)
|
||||
|
||||
# 逐个播放数字
|
||||
for i in range(6): # 确保只播放6位数字
|
||||
try:
|
||||
digit = bind_code[i]
|
||||
num_path = f"config/assets/bind_code/{digit}.wav"
|
||||
num_packets = await audio_to_data(num_path)
|
||||
tts_instance.tts_audio_queue.put(
|
||||
(SentenceType.MIDDLE, num_packets, None, context.sentence_id)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"播放数字音频失败: {e}")
|
||||
continue
|
||||
tts_instance.tts_audio_queue.put(
|
||||
(SentenceType.LAST, [], None, context.sentence_id)
|
||||
)
|
||||
else:
|
||||
# 播放未绑定提示
|
||||
context.abort_requested = False
|
||||
text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
# 获取TTS组件
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
if hasattr(tts_instance, 'tts_audio_queue'):
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
music_path = "config/assets/bind_not_found.wav"
|
||||
opus_packets = await audio_to_data(music_path)
|
||||
tts_instance.tts_audio_queue.put(
|
||||
(SentenceType.LAST, opus_packets, text, context.sentence_id)
|
||||
)
|
||||
|
||||
async def _handle_abort_message(self, context: SessionContext, transport: TransportInterface):
|
||||
"""处理中断消息"""
|
||||
from core.processors.abort_processor import AbortProcessor
|
||||
|
||||
await AbortProcessor().handle_abort_message(
|
||||
context,
|
||||
transport,
|
||||
{"type": "abort", "session_id": context.session_id},
|
||||
)
|
||||
|
||||
async def _clear_queues(self, context: SessionContext):
|
||||
"""清理所有队列"""
|
||||
# 清理TTS音频队列
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
for queue_name in ('tts_text_queue', 'tts_audio_queue'):
|
||||
pending_queue = getattr(tts_instance, queue_name, None)
|
||||
if pending_queue is None:
|
||||
continue
|
||||
try:
|
||||
while not pending_queue.empty():
|
||||
pending_queue.get_nowait()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Invalidate late TTS text/audio generated by the interrupted turn.
|
||||
context.sentence_id = uuid.uuid4().hex
|
||||
|
||||
# 清理ASR音频队列
|
||||
context.clear_audio_buffer()
|
||||
|
||||
async def _get_component(self, context: SessionContext, component_type: ComponentType):
|
||||
if not context.component_manager:
|
||||
return None
|
||||
return await context.component_manager.get_component(component_type, context)
|
||||
|
||||
async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息"""
|
||||
from core.processors.audio_send_processor import AudioSendProcessor
|
||||
|
||||
await AudioSendProcessor().send_stt_message(
|
||||
context, transport, text
|
||||
)
|
||||
@@ -0,0 +1,578 @@
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from typing import Any, List
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
from core.utils import textUtils
|
||||
from core.components.component_manager import ComponentType
|
||||
from core.services.audio_ingress_service import AudioIngressService
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AudioSendProcessor(MessageProcessor):
|
||||
"""音频发送处理器：完整迁移sendAudioHandle.py的所有功能"""
|
||||
|
||||
def __init__(self, audio_ingress_service=None):
|
||||
self.audio_ingress_service = audio_ingress_service or AudioIngressService()
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""这个处理器不直接处理消息，而是被其他处理器调用"""
|
||||
return False
|
||||
|
||||
async def send_audio_message(self, context: SessionContext, transport: TransportInterface,
|
||||
sentence_type: SentenceType, audios: bytes, text: str,
|
||||
sentence_id: str = None):
|
||||
"""发送音频消息 - 完整迁移自sendAudioMessage"""
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
return
|
||||
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
if not tts_component or not hasattr(tts_component, 'tts_instance'):
|
||||
return
|
||||
|
||||
tts_instance = tts_component.tts_instance
|
||||
|
||||
if hasattr(tts_instance, 'tts_audio_first_sentence') and tts_instance.tts_audio_first_sentence:
|
||||
logger.info(f"发送第一段语音: {text}")
|
||||
tts_instance.tts_audio_first_sentence = False
|
||||
await self.start_tts_stream(context, transport)
|
||||
|
||||
if sentence_type == SentenceType.FIRST:
|
||||
await self.send_tts_message(
|
||||
context, transport, "sentence_start", text, sentence_id=sentence_id
|
||||
)
|
||||
|
||||
await self.send_audio(
|
||||
context, transport, audios, sentence_id=sentence_id
|
||||
)
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
return
|
||||
|
||||
# 发送句子开始消息
|
||||
if sentence_type is not SentenceType.MIDDLE:
|
||||
logger.info(f"发送音频消息: {sentence_type}, {text}")
|
||||
|
||||
# 发送结束消息（如果是最后一个文本）
|
||||
if (
|
||||
not getattr(context, "calling", False)
|
||||
and context.llm_finish_task
|
||||
and sentence_type == SentenceType.LAST
|
||||
):
|
||||
# Latch the terminal action before sending tts:stop. The device can
|
||||
# immediately answer with listen:start, whose handler resets the
|
||||
# mutable turn flags while this coroutine is still awaiting I/O.
|
||||
close_after_chat = bool(context.close_after_chat)
|
||||
closing_session_id = (
|
||||
getattr(transport, "session_id", None) or context.session_id
|
||||
)
|
||||
if close_after_chat:
|
||||
context.close_after_chat = False
|
||||
await self.send_tts_message(
|
||||
context, transport, "stop", None, sentence_id=sentence_id
|
||||
)
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
return
|
||||
context.is_speaking = False
|
||||
if close_after_chat:
|
||||
# MQTT/UDP：结束后回到Idle，不关闭连接
|
||||
if transport.keeps_connection_between_sessions:
|
||||
end_conversation = getattr(context, "end_conversation", None)
|
||||
if callable(end_conversation):
|
||||
await end_conversation(closing_session_id)
|
||||
await transport.end_session(closing_session_id)
|
||||
else:
|
||||
await transport.close()
|
||||
|
||||
async def send_audio(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
audios: bytes,
|
||||
frame_duration: int = 60,
|
||||
sentence_id: str = None,
|
||||
):
|
||||
"""发送单个opus包，支持流控 - 完整迁移自sendAudio"""
|
||||
if audios is None or len(audios) == 0:
|
||||
return
|
||||
if getattr(context, "audio_flow_control", {}).get("send_failed"):
|
||||
return
|
||||
|
||||
# MQTT/UDP：等待UDP远端地址就绪，避免首包丢失
|
||||
if transport.has_datagram_audio:
|
||||
if not await transport.wait_audio_ready(timeout=2):
|
||||
logger.warning("UDP远端地址未就绪，跳过音频发送")
|
||||
return
|
||||
|
||||
audio_list = [audios] if isinstance(audios, bytes) else audios
|
||||
if not isinstance(audio_list, list):
|
||||
return
|
||||
|
||||
flow = context.audio_flow_control
|
||||
pre_buffer_count = max(
|
||||
0, int(context.config.get("tts_pre_buffer_count", 5))
|
||||
)
|
||||
for audio in audio_list:
|
||||
if context.abort_requested or not self._is_current_turn(context, sentence_id):
|
||||
break
|
||||
|
||||
# Match the legacy AudioRateController contract: send only the
|
||||
# initial pre-buffer inline, then enqueue the remaining frames so
|
||||
# the TTS consumer can prefetch later sentence chunks.
|
||||
packet_count = int(flow.get("packet_count", 0))
|
||||
if packet_count < pre_buffer_count and "_send_queue" not in flow:
|
||||
sent = await self._send_audio_packet(
|
||||
context,
|
||||
transport,
|
||||
flow,
|
||||
audio,
|
||||
frame_duration,
|
||||
sentence_id,
|
||||
paced=False,
|
||||
)
|
||||
if not sent:
|
||||
break
|
||||
continue
|
||||
|
||||
queue = self._ensure_audio_sender(
|
||||
context, transport, flow, sentence_id
|
||||
)
|
||||
await queue.put(("audio", audio, frame_duration, sentence_id))
|
||||
|
||||
@staticmethod
|
||||
async def _pace_audio_send(
|
||||
context: SessionContext,
|
||||
frame_duration: int,
|
||||
flow: dict = None,
|
||||
) -> None:
|
||||
flow = flow if flow is not None else context.audio_flow_control
|
||||
packet_count = int(flow.get("packet_count", 0))
|
||||
pre_buffer_count = max(0, int(context.config.get("tts_pre_buffer_count", 5)))
|
||||
if packet_count < pre_buffer_count:
|
||||
return
|
||||
|
||||
configured_delay = int(context.config.get("tts_audio_send_delay", -1))
|
||||
if configured_delay > 0:
|
||||
await asyncio.sleep(configured_delay / 1000.0)
|
||||
return
|
||||
|
||||
delay_ms = max(0, int(frame_duration))
|
||||
if delay_ms <= 0:
|
||||
return
|
||||
|
||||
now = time.monotonic()
|
||||
# packet_count is the number already sent. The first packet after the
|
||||
# pre-buffer must therefore wait one complete frame, not become an
|
||||
# extra immediate packet.
|
||||
paced_packet_index = packet_count - pre_buffer_count + 1
|
||||
pacing_started_at = flow.get("pacing_started_at")
|
||||
pacing_delay_ms = flow.get("pacing_delay_ms")
|
||||
if pacing_started_at is None or pacing_delay_ms != delay_ms:
|
||||
pacing_started_at = now
|
||||
flow["pacing_started_at"] = pacing_started_at
|
||||
flow["pacing_delay_ms"] = delay_ms
|
||||
|
||||
delay_seconds = delay_ms / 1000.0
|
||||
target = pacing_started_at + paced_packet_index * delay_seconds
|
||||
|
||||
# TTS producers can pause between sentence chunks. Rebase after a
|
||||
# full-frame gap instead of bursting stale deadlines to catch up.
|
||||
if now - target >= delay_seconds:
|
||||
pacing_started_at = now - paced_packet_index * delay_seconds
|
||||
flow["pacing_started_at"] = pacing_started_at
|
||||
target = now
|
||||
|
||||
wait_seconds = target - now
|
||||
if wait_seconds > 0:
|
||||
await asyncio.sleep(wait_seconds)
|
||||
|
||||
def _ensure_audio_sender(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
flow: dict,
|
||||
sentence_id: str = None,
|
||||
) -> asyncio.Queue:
|
||||
queue = flow.get("_send_queue")
|
||||
task = flow.get("_send_task")
|
||||
if queue is not None and task is not None and not task.done():
|
||||
return queue
|
||||
|
||||
queue_size = max(
|
||||
1, int(context.config.get("tts_audio_queue_size", 256))
|
||||
)
|
||||
queue = asyncio.Queue(maxsize=queue_size)
|
||||
sender = self._audio_send_loop(
|
||||
context, transport, flow, queue, sentence_id
|
||||
)
|
||||
create_task = getattr(context, "create_background_task", None)
|
||||
if callable(create_task):
|
||||
task = create_task(sender, turn_scoped=True)
|
||||
else:
|
||||
task = asyncio.create_task(sender)
|
||||
flow["_send_queue"] = queue
|
||||
flow["_send_task"] = task
|
||||
return queue
|
||||
|
||||
async def _audio_send_loop(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
flow: dict,
|
||||
queue: asyncio.Queue,
|
||||
sentence_id: str = None,
|
||||
) -> None:
|
||||
try:
|
||||
while True:
|
||||
item = await queue.get()
|
||||
try:
|
||||
item_type = item[0]
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
continue
|
||||
if item_type == "audio":
|
||||
_, audio, frame_duration, item_sentence_id = item
|
||||
await self._send_audio_packet(
|
||||
context,
|
||||
transport,
|
||||
flow,
|
||||
audio,
|
||||
frame_duration,
|
||||
item_sentence_id,
|
||||
paced=True,
|
||||
)
|
||||
elif item_type == "json":
|
||||
_, message, item_sentence_id = item
|
||||
if self._is_current_turn(context, item_sentence_id):
|
||||
await transport.send_json(message)
|
||||
finally:
|
||||
queue.task_done()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as error:
|
||||
flow["send_failed"] = True
|
||||
logger.error("后台音频发送循环失败: {}", error)
|
||||
finally:
|
||||
while True:
|
||||
try:
|
||||
queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
else:
|
||||
queue.task_done()
|
||||
|
||||
async def _send_audio_packet(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
flow: dict,
|
||||
audio: bytes,
|
||||
frame_duration: int,
|
||||
sentence_id: str = None,
|
||||
*,
|
||||
paced: bool,
|
||||
) -> bool:
|
||||
if paced:
|
||||
await self._pace_audio_send(context, frame_duration, flow)
|
||||
if context.abort_requested or not self._is_current_turn(context, sentence_id):
|
||||
return False
|
||||
|
||||
context.update_activity()
|
||||
timestamp = self._next_audio_timestamp(
|
||||
context, frame_duration, flow
|
||||
)
|
||||
sent = await self._send_audio_with_retry(
|
||||
context,
|
||||
transport,
|
||||
audio,
|
||||
timestamp,
|
||||
sentence_id,
|
||||
flow,
|
||||
)
|
||||
if not sent or not self._is_current_turn(context, sentence_id):
|
||||
return False
|
||||
self.audio_ingress_service.cache_output_reference(
|
||||
context, audio, timestamp
|
||||
)
|
||||
flow["packet_count"] = int(flow.get("packet_count", 0)) + 1
|
||||
return True
|
||||
|
||||
async def _send_audio_with_retry(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
audio: bytes,
|
||||
timestamp: int,
|
||||
sentence_id: str = None,
|
||||
flow: dict = None,
|
||||
) -> bool:
|
||||
flow = flow if flow is not None else context.audio_flow_control
|
||||
retries = max(0, int(context.config.get("audio_send_retries", 2)))
|
||||
retry_delay = max(
|
||||
0, int(context.config.get("audio_send_retry_delay_ms", 20))
|
||||
) / 1000.0
|
||||
for attempt in range(retries + 1):
|
||||
if context.abort_requested or not self._is_current_turn(context, sentence_id):
|
||||
return False
|
||||
try:
|
||||
await transport.send_audio(audio, timestamp)
|
||||
return self._is_current_turn(context, sentence_id)
|
||||
except Exception as error:
|
||||
if attempt >= retries:
|
||||
flow["send_failed"] = True
|
||||
logger.error(
|
||||
"Audio send failed after {} attempts: {}",
|
||||
attempt + 1,
|
||||
error,
|
||||
)
|
||||
raise
|
||||
await asyncio.sleep(retry_delay)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _next_audio_timestamp(
|
||||
context: SessionContext,
|
||||
frame_duration: int,
|
||||
flow: dict = None,
|
||||
) -> int:
|
||||
flow = flow if flow is not None else getattr(
|
||||
context, "audio_flow_control", None
|
||||
)
|
||||
if flow is None:
|
||||
flow = {}
|
||||
context.audio_flow_control = flow
|
||||
timestamp_step = int(frame_duration) or int(
|
||||
getattr(context, "output_frame_duration", 60) or 60
|
||||
)
|
||||
timestamp = int(flow.get("output_timestamp", 0)) + timestamp_step
|
||||
timestamp %= 2 ** 32
|
||||
flow["output_timestamp"] = timestamp
|
||||
return timestamp
|
||||
|
||||
async def send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息 - 完整迁移自send_stt_message"""
|
||||
end_prompt = getattr(context, "config", {}).get("end_prompt", {}).get("prompt")
|
||||
if end_prompt and text == end_prompt:
|
||||
await self.start_tts_stream(context, transport)
|
||||
return
|
||||
|
||||
display_text = text
|
||||
parsed_data = None
|
||||
if isinstance(text, dict):
|
||||
parsed_data = text
|
||||
elif isinstance(text, str):
|
||||
stripped = text.strip()
|
||||
if stripped.startswith("{") and stripped.endswith("}"):
|
||||
try:
|
||||
parsed_data = json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
if isinstance(parsed_data, dict) and "content" in parsed_data:
|
||||
display_text = parsed_data["content"]
|
||||
if "speaker" in parsed_data:
|
||||
context.current_speaker = parsed_data["speaker"]
|
||||
|
||||
stt_text = textUtils.get_string_no_punctuation_or_emoji(
|
||||
str(display_text)
|
||||
)
|
||||
await self._send_json(transport, {
|
||||
"type": "stt",
|
||||
"text": stt_text,
|
||||
"session_id": context.session_id
|
||||
})
|
||||
logger.info(f"发送STT消息: {stt_text}")
|
||||
# Legacy WS sends TTS start together with STT, before synthesis begins.
|
||||
# This lead time is required by MQTT/UDP clients because the device
|
||||
# changes to Speaking asynchronously and drops UDP audio beforehand.
|
||||
await self.start_tts_stream(context, transport)
|
||||
|
||||
async def start_tts_stream(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
) -> bool:
|
||||
"""Open one device-side TTS stream without sending duplicate starts."""
|
||||
if getattr(context, "is_speaking", False):
|
||||
return False
|
||||
|
||||
context.is_speaking = True
|
||||
flow_control = getattr(context, "audio_flow_control", None) or {}
|
||||
await self._stop_audio_sender(flow_control)
|
||||
context.audio_flow_control = {}
|
||||
try:
|
||||
await self._send_json(transport, {
|
||||
"type": "tts",
|
||||
"state": "start",
|
||||
"session_id": context.session_id,
|
||||
})
|
||||
except Exception:
|
||||
context.is_speaking = False
|
||||
raise
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
async def _send_json(
|
||||
transport: TransportInterface,
|
||||
message: dict,
|
||||
) -> None:
|
||||
send_json = getattr(transport, "send_json", None)
|
||||
if callable(send_json):
|
||||
await send_json(message)
|
||||
return
|
||||
await transport.send(json.dumps(message))
|
||||
|
||||
async def send_tts_message(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
state: str,
|
||||
text: str = None,
|
||||
sentence_id: str = None,
|
||||
):
|
||||
"""发送TTS消息 - 完整迁移自send_tts_message"""
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
return False
|
||||
if state == "sentence_start":
|
||||
flow = getattr(context, "audio_flow_control", {})
|
||||
queue = flow.get("_send_queue")
|
||||
task = flow.get("_send_task")
|
||||
if queue is not None and task is not None and not task.done():
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": context.session_id,
|
||||
}
|
||||
if text:
|
||||
message["text"] = text
|
||||
await queue.put(("json", message, sentence_id))
|
||||
return True
|
||||
if state == "stop":
|
||||
if context.config.get("enable_stop_tts_notify", False):
|
||||
from core.utils.util import audio_to_data
|
||||
|
||||
notify_path = context.config.get(
|
||||
"stop_tts_notify_voice", "config/assets/tts_notify.mp3"
|
||||
)
|
||||
notify_audio = await audio_to_data(notify_path, is_opus=True)
|
||||
if notify_audio:
|
||||
await self.send_audio(
|
||||
context,
|
||||
transport,
|
||||
notify_audio,
|
||||
sentence_id=sentence_id,
|
||||
)
|
||||
if not await self._wait_for_audio_completion(context, sentence_id):
|
||||
return False
|
||||
|
||||
if not self._is_current_turn(context, sentence_id):
|
||||
return False
|
||||
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": context.session_id
|
||||
}
|
||||
if text:
|
||||
message["text"] = text
|
||||
|
||||
await transport.send_json(message)
|
||||
logger.debug(f"发送TTS消息: state={state}, text={text}")
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
async def _wait_for_audio_completion(
|
||||
context: SessionContext, sentence_id: str = None
|
||||
) -> bool:
|
||||
flow = context.audio_flow_control
|
||||
queue = flow.get("_send_queue")
|
||||
sender_task = flow.get("_send_task")
|
||||
if queue is not None and sender_task is not None:
|
||||
join_task = asyncio.create_task(queue.join())
|
||||
done, _ = await asyncio.wait(
|
||||
{join_task, sender_task},
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
if sender_task in done and not join_task.done():
|
||||
join_task.cancel()
|
||||
await asyncio.gather(join_task, return_exceptions=True)
|
||||
return False
|
||||
await join_task
|
||||
await AudioSendProcessor._stop_audio_sender(flow)
|
||||
if flow.get("send_failed"):
|
||||
return False
|
||||
|
||||
packet_count = int(flow.get("packet_count", 0))
|
||||
if packet_count <= 0:
|
||||
return AudioSendProcessor._is_current_turn(context, sentence_id)
|
||||
frame_duration = max(1, int(getattr(context, "output_frame_duration", 60)))
|
||||
pre_buffer_count = max(0, int(context.config.get("tts_pre_buffer_count", 5)))
|
||||
tail_frames = max(
|
||||
0,
|
||||
int(context.config.get("tts_stop_buffer_frames", pre_buffer_count + 2)),
|
||||
)
|
||||
if tail_frames:
|
||||
await asyncio.sleep(tail_frames * frame_duration / 1000.0)
|
||||
return AudioSendProcessor._is_current_turn(context, sentence_id)
|
||||
|
||||
@staticmethod
|
||||
async def _stop_audio_sender(flow: dict) -> None:
|
||||
task = flow.pop("_send_task", None)
|
||||
flow.pop("_send_queue", None)
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
@staticmethod
|
||||
def _is_current_turn(context: SessionContext, sentence_id: str = None) -> bool:
|
||||
if sentence_id is None:
|
||||
return bool(
|
||||
getattr(context, "conversation_active", True)
|
||||
and getattr(context, "sentence_id", None) is None
|
||||
)
|
||||
return sentence_id == context.sentence_id
|
||||
|
||||
async def send_music_message(self, context: SessionContext, transport: TransportInterface,
|
||||
music_path: str, text: str):
|
||||
"""发送音乐消息 - 完整迁移自send_music_message"""
|
||||
from core.utils.util import audio_to_data
|
||||
|
||||
try:
|
||||
# 获取音频数据
|
||||
opus_packets = await audio_to_data(music_path)
|
||||
if opus_packets:
|
||||
# 发送音乐开始消息
|
||||
await self.send_tts_message(context, transport, "start", text)
|
||||
|
||||
# 发送音频数据
|
||||
await self.send_audio(context, transport, opus_packets)
|
||||
|
||||
# 发送音乐结束消息
|
||||
await self.send_tts_message(context, transport, "stop", None)
|
||||
|
||||
logger.info(f"发送音乐: {music_path}")
|
||||
else:
|
||||
logger.warning(f"无法加载音乐文件: {music_path}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送音乐失败: {e}")
|
||||
|
||||
async def _get_component(self, context: SessionContext, component_type: ComponentType):
|
||||
if not context.component_manager:
|
||||
return None
|
||||
return await context.component_manager.get_component(component_type, context)
|
||||
|
||||
async def send_welcome_audio(self, context: SessionContext, transport: TransportInterface):
|
||||
"""发送欢迎音频"""
|
||||
welcome_audio_path = context.config.get("welcome_audio_path")
|
||||
if welcome_audio_path:
|
||||
await self.send_music_message(context, transport, welcome_audio_path, "欢迎使用小智助手")
|
||||
|
||||
async def send_goodbye_audio(self, context: SessionContext, transport: TransportInterface):
|
||||
"""发送告别音频"""
|
||||
goodbye_audio_path = context.config.get("goodbye_audio_path")
|
||||
if goodbye_audio_path:
|
||||
await self.send_music_message(context, transport, goodbye_audio_path, "再见，期待下次相遇")
|
||||
@@ -0,0 +1,55 @@
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.auth import AuthMiddleware, AuthenticationError
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AuthProcessor(MessageProcessor):
|
||||
"""认证处理器：处理连接认证逻辑"""
|
||||
|
||||
def __init__(self):
|
||||
self.auth_middleware = None
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理认证相关逻辑"""
|
||||
# 服务器管理连接走server消息处理（基于secret校验），跳过认证
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
import json
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "server":
|
||||
return False
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
elif isinstance(message, dict) and message.get("type") == "server":
|
||||
return False
|
||||
|
||||
# 如果已经认证，跳过
|
||||
if context.is_authenticated:
|
||||
return False
|
||||
|
||||
# 初始化认证中间件（延迟初始化）
|
||||
if self.auth_middleware is None:
|
||||
self.auth_middleware = AuthMiddleware(context.config)
|
||||
|
||||
# 检查是否为认证消息（通过headers进行认证）
|
||||
if context.headers:
|
||||
try:
|
||||
await self.auth_middleware.authenticate_async(context.headers)
|
||||
context.is_authenticated = True
|
||||
logger.info(f"设备认证成功: {context.device_id}")
|
||||
return False # 认证成功，继续处理其他消息
|
||||
except AuthenticationError as e:
|
||||
logger.error(f"设备认证失败: {e}")
|
||||
# 发送认证失败消息
|
||||
await transport.send("Authentication failed")
|
||||
await transport.close()
|
||||
return True # 认证失败，停止处理
|
||||
|
||||
# 如果没有认证信息，要求认证
|
||||
await transport.send("Authentication required")
|
||||
return True # 停止后续处理
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,34 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class GoodbyeProcessor(MessageProcessor):
|
||||
"""Goodbye消息处理器：处理会话结束"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "goodbye":
|
||||
logger.info(f"收到goodbye: session_id={msg_json.get('session_id')}")
|
||||
# 长连接传输仅结束逻辑会话，短连接传输关闭连接。
|
||||
if transport.keeps_connection_between_sessions:
|
||||
end_conversation = getattr(context, "end_conversation", None)
|
||||
if callable(end_conversation):
|
||||
await end_conversation(msg_json.get("session_id"))
|
||||
else:
|
||||
await transport.close()
|
||||
return True
|
||||
return False
|
||||
@@ -0,0 +1,351 @@
|
||||
import time
|
||||
import json
|
||||
import random
|
||||
import asyncio
|
||||
import uuid
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.utils.dialogue import Message
|
||||
from core.utils.util import audio_to_data, remove_punctuation_and_length, opus_datas_to_wav_bytes
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
from core.processors.audio_receive_processor import arm_wake_audio_suppression
|
||||
from core.utils.wakeup_word import WakeupWordsConfig
|
||||
from core.components.component_manager import ComponentType
|
||||
from core.providers.tools.device_mcp import (
|
||||
MCPClient,
|
||||
send_mcp_initialize_message,
|
||||
)
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
# 唤醒词配置
|
||||
WAKEUP_CONFIG = {
|
||||
"refresh_time": 5,
|
||||
"words": ["你好", "你好啊", "嘿，你好", "嗨"],
|
||||
}
|
||||
|
||||
# 创建全局的唤醒词配置管理器
|
||||
wakeup_words_config = WakeupWordsConfig()
|
||||
|
||||
# 用于防止并发调用wakeupWordsResponse的锁
|
||||
_wakeup_response_lock = asyncio.Lock()
|
||||
|
||||
|
||||
class HelloProcessor(MessageProcessor):
|
||||
"""Hello消息处理器：完整迁移helloHandle.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理hello类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "hello":
|
||||
await self.handle_hello_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_hello_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理hello消息 - 完整迁移自handleHelloMessage"""
|
||||
pending_tasks = []
|
||||
for task_name in (
|
||||
"listen_stop_task",
|
||||
"listen_start_task",
|
||||
):
|
||||
pending_task = getattr(context, task_name, None)
|
||||
if pending_task and not pending_task.done():
|
||||
pending_task.cancel()
|
||||
pending_tasks.append(pending_task)
|
||||
setattr(context, task_name, None)
|
||||
if pending_tasks:
|
||||
await asyncio.gather(*pending_tasks, return_exceptions=True)
|
||||
|
||||
transport_session_id = getattr(transport, "session_id", None)
|
||||
if transport_session_id:
|
||||
if (
|
||||
context.session_id != transport_session_id
|
||||
and callable(getattr(context, "end_conversation", None))
|
||||
):
|
||||
# A previous finalizer may already have published
|
||||
# conversation_active=False while persistence is still in
|
||||
# progress. Always join it before exposing the new session.
|
||||
await context.end_conversation(context.session_id)
|
||||
context.session_id = transport_session_id
|
||||
if context.welcome_msg:
|
||||
context.welcome_msg["session_id"] = context.session_id
|
||||
dialogue = getattr(context, "dialogue", None)
|
||||
if getattr(context, "prompt", None) and dialogue and not dialogue.dialogue:
|
||||
dialogue.update_system_message(context.prompt)
|
||||
inject_fewshot = getattr(context, "inject_tool_call_fewshot", None)
|
||||
if callable(inject_fewshot):
|
||||
inject_fewshot()
|
||||
context.conversation_active = True
|
||||
context.listen_stop_pending = False
|
||||
context.listen_stop_deadline = 0.0
|
||||
if transport.requires_audio_tail_grace:
|
||||
context.accepting_input_audio = False
|
||||
|
||||
# 处理音频参数
|
||||
audio_params = msg_json.get("audio_params")
|
||||
if audio_params:
|
||||
format = audio_params.get("format")
|
||||
logger.info(f"客户端音频格式: {format}")
|
||||
context.audio_format = format
|
||||
context.input_sample_rate = int(
|
||||
audio_params.get("sample_rate", getattr(context, "input_sample_rate", 16000))
|
||||
)
|
||||
context.input_channels = int(
|
||||
audio_params.get("channels", getattr(context, "input_channels", 1))
|
||||
)
|
||||
context.input_frame_duration = int(
|
||||
audio_params.get(
|
||||
"frame_duration", getattr(context, "input_frame_duration", 60)
|
||||
)
|
||||
)
|
||||
|
||||
# 处理客户端特性
|
||||
features = msg_json.get("features") or {}
|
||||
context.features = dict(features)
|
||||
context.client_aec = bool(features.get("aec"))
|
||||
mcp_enabled = bool(features.get("mcp"))
|
||||
previous_mcp_client = getattr(context, "mcp_client", None)
|
||||
if not mcp_enabled and previous_mcp_client:
|
||||
initialize_task = getattr(
|
||||
context, "mcp_initialize_task", None
|
||||
)
|
||||
if initialize_task and not initialize_task.done():
|
||||
initialize_task.cancel()
|
||||
await asyncio.gather(
|
||||
initialize_task, return_exceptions=True
|
||||
)
|
||||
context.mcp_initialize_task = None
|
||||
previous_mcp_cleanup = getattr(
|
||||
context, "_mcp_cleanup_callback", None
|
||||
)
|
||||
if previous_mcp_cleanup:
|
||||
context.unregister_cleanup(previous_mcp_cleanup)
|
||||
context._mcp_cleanup_callback = None
|
||||
if hasattr(previous_mcp_client, "close"):
|
||||
await previous_mcp_client.close()
|
||||
context.mcp_client = None
|
||||
if features:
|
||||
logger.info(f"客户端特性: {features}")
|
||||
if mcp_enabled:
|
||||
logger.info("客户端支持MCP")
|
||||
if context.mcp_client is None:
|
||||
context.mcp_client = MCPClient()
|
||||
context._mcp_cleanup_callback = (
|
||||
context.mcp_client.close
|
||||
)
|
||||
context.register_cleanup(
|
||||
context._mcp_cleanup_callback
|
||||
)
|
||||
initialize_task = getattr(
|
||||
context, "mcp_initialize_task", None
|
||||
)
|
||||
if (
|
||||
not await context.mcp_client.is_ready()
|
||||
and (
|
||||
initialize_task is None
|
||||
or initialize_task.done()
|
||||
)
|
||||
):
|
||||
context.mcp_initialize_task = (
|
||||
context.create_background_task(
|
||||
send_mcp_initialize_message(
|
||||
context, transport
|
||||
),
|
||||
conversation_scoped=False,
|
||||
)
|
||||
)
|
||||
if features.get("aec"):
|
||||
logger.info("客户端启用了服务端AEC")
|
||||
|
||||
# MQTT/UDP 场景已由MQTT连接层发送hello回复，避免重复发送websocket格式
|
||||
if transport.has_datagram_audio:
|
||||
return
|
||||
|
||||
# 发送欢迎消息
|
||||
if context.welcome_msg:
|
||||
await transport.send_json(context.welcome_msg)
|
||||
else:
|
||||
# 默认欢迎消息
|
||||
welcome_msg = {
|
||||
"type": "hello",
|
||||
"session_id": context.session_id,
|
||||
"version": 1,
|
||||
"transport": "websocket"
|
||||
}
|
||||
await transport.send_json(welcome_msg)
|
||||
|
||||
async def check_wakeup_words(self, context: SessionContext, transport: TransportInterface, text: str) -> bool:
|
||||
"""检查唤醒词 - 完整迁移自checkWakeupWords"""
|
||||
enable_wakeup_words_response_cache = context.config.get("enable_wakeup_words_response_cache", False)
|
||||
|
||||
# 等待tts初始化，最多等待3秒
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < 3:
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
else:
|
||||
return False
|
||||
|
||||
if not enable_wakeup_words_response_cache:
|
||||
return False
|
||||
|
||||
_, filtered_text = remove_punctuation_and_length(text)
|
||||
if filtered_text not in context.config.get("wakeup_words", []):
|
||||
return False
|
||||
|
||||
sentence_id = uuid.uuid4().hex
|
||||
context.sentence_id = sentence_id
|
||||
context.just_woken_up = True
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
# 获取当前音色
|
||||
tts_instance = getattr(tts_component, 'tts_instance', None) if tts_component else None
|
||||
voice = getattr(tts_instance, "voice", "default") if tts_instance else "default"
|
||||
if not voice:
|
||||
voice = "default"
|
||||
|
||||
# 获取唤醒词回复配置
|
||||
response = wakeup_words_config.get_wakeup_response(voice)
|
||||
if not response or not response.get("file_path"):
|
||||
response = {
|
||||
"voice": "default",
|
||||
"file_path": "config/assets/wakeup_words.wav",
|
||||
"time": 0,
|
||||
"text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",
|
||||
}
|
||||
|
||||
# 获取音频数据
|
||||
opus_packets = await audio_to_data(response.get("file_path"))
|
||||
# 播放唤醒词回复
|
||||
context.abort_requested = False
|
||||
context.llm_finish_task = True
|
||||
if tts_instance is not None:
|
||||
# A prior turn may have consumed the one-shot start marker. Cached
|
||||
# playback is a new TTS stream and must always emit start first.
|
||||
tts_instance.tts_audio_first_sentence = True
|
||||
|
||||
logger.info(f"播放唤醒词回复: {response.get('text')}")
|
||||
try:
|
||||
await self._send_audio_message(
|
||||
context,
|
||||
transport,
|
||||
SentenceType.FIRST,
|
||||
opus_packets,
|
||||
response.get("text"),
|
||||
sentence_id,
|
||||
)
|
||||
await self._send_audio_message(
|
||||
context,
|
||||
transport,
|
||||
SentenceType.LAST,
|
||||
[],
|
||||
None,
|
||||
sentence_id,
|
||||
)
|
||||
finally:
|
||||
# Cached wake playback bypasses ListenProcessor, so it must arm
|
||||
# the same bounded release used by the regular wake-word path.
|
||||
arm_wake_audio_suppression(context)
|
||||
|
||||
# 补充对话
|
||||
if context.dialogue:
|
||||
context.dialogue.put(Message(role="assistant", content=response.get("text")))
|
||||
|
||||
# 检查是否需要更新唤醒词回复
|
||||
if time.time() - response.get("time", 0) > WAKEUP_CONFIG["refresh_time"]:
|
||||
if not _wakeup_response_lock.locked():
|
||||
context.create_background_task(
|
||||
self._wakeup_words_response(context, transport)
|
||||
)
|
||||
return True
|
||||
|
||||
async def _wakeup_words_response(self, context: SessionContext, transport: TransportInterface):
|
||||
"""生成唤醒词回复 - 完整迁移自wakeupWordsResponse"""
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
llm_component = await self._get_component(context, ComponentType.LLM)
|
||||
|
||||
tts_instance = getattr(tts_component, 'tts_instance', None) if tts_component else None
|
||||
llm_instance = getattr(llm_component, 'llm_instance', None) if llm_component else None
|
||||
|
||||
if not tts_instance or not llm_instance or not hasattr(llm_instance, 'response_no_stream'):
|
||||
return
|
||||
|
||||
try:
|
||||
# 尝试获取锁，如果获取不到就返回
|
||||
async with _wakeup_response_lock:
|
||||
# 生成唤醒词回复
|
||||
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
|
||||
question = (
|
||||
"此刻用户正在和你说```"
|
||||
+ wakeup_word
|
||||
+ "```。\n请你根据以上用户的内容进行20-30字回复。要符合系统设置的角色情感和态度，不要像机器人一样说话。\n"
|
||||
+ "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。"
|
||||
)
|
||||
|
||||
result = await asyncio.to_thread(
|
||||
llm_instance.response_no_stream,
|
||||
context.config.get("prompt", ""),
|
||||
question,
|
||||
)
|
||||
if not result or len(result) == 0:
|
||||
return
|
||||
|
||||
# 生成TTS音频
|
||||
tts_result = await asyncio.to_thread(tts_instance.to_tts, result)
|
||||
if not tts_result:
|
||||
return
|
||||
|
||||
# 获取当前音色
|
||||
voice = getattr(tts_instance, "voice", "default")
|
||||
|
||||
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
|
||||
file_path = wakeup_words_config.generate_file_path(voice)
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(wav_bytes)
|
||||
# 更新配置
|
||||
wakeup_words_config.update_wakeup_response(voice, file_path, result)
|
||||
except Exception as e:
|
||||
logger.error(f"生成唤醒词回复失败: {e}")
|
||||
|
||||
async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息"""
|
||||
from core.processors.audio_send_processor import AudioSendProcessor
|
||||
|
||||
await AudioSendProcessor().send_stt_message(
|
||||
context, transport, text
|
||||
)
|
||||
|
||||
async def _send_audio_message(self, context: SessionContext, transport: TransportInterface,
|
||||
sentence_type: SentenceType, audios: bytes, text: str,
|
||||
sentence_id: str = None):
|
||||
"""发送音频消息"""
|
||||
# 这里应该调用AudioSendProcessor
|
||||
from core.processors.audio_send_processor import AudioSendProcessor
|
||||
audio_send_processor = AudioSendProcessor()
|
||||
await audio_send_processor.send_audio_message(
|
||||
context,
|
||||
transport,
|
||||
sentence_type,
|
||||
audios,
|
||||
text,
|
||||
sentence_id=sentence_id,
|
||||
)
|
||||
|
||||
async def _get_component(self, context: SessionContext, component_type: ComponentType):
|
||||
if not context.component_manager:
|
||||
return None
|
||||
return await context.component_manager.get_component(component_type, context)
|
||||
@@ -0,0 +1,124 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.providers.tools.device_iot import handleIotStatus, handleIotDescriptors
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class IotProcessor(MessageProcessor):
|
||||
"""IoT消息处理器：完整迁移iotMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理iot类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "iot":
|
||||
await self.handle_iot_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_iot_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理IoT消息 - 完整迁移自iotMessageHandler.py"""
|
||||
tasks = []
|
||||
|
||||
# 处理设备描述符 - 完整迁移原逻辑
|
||||
if "descriptors" in msg_json:
|
||||
logger.debug("处理IoT设备描述符")
|
||||
task = context.create_background_task(
|
||||
self._handle_iot_descriptors(context, transport, msg_json["descriptors"])
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
# 处理设备状态 - 完整迁移原逻辑
|
||||
if "states" in msg_json:
|
||||
logger.debug("处理IoT设备状态")
|
||||
task = context.create_background_task(
|
||||
self._handle_iot_status(context, transport, msg_json["states"])
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
# 如果没有有效的IoT数据
|
||||
if not tasks:
|
||||
logger.warning("IoT消息缺少descriptors或states字段")
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"IoT消息格式错误：缺少descriptors或states字段"
|
||||
)
|
||||
return
|
||||
|
||||
# 任务由 SessionContext 持有，结束会话时统一取消。
|
||||
|
||||
async def _handle_iot_descriptors(self, context: SessionContext, transport: TransportInterface, descriptors: Any):
|
||||
"""处理IoT设备描述符 - 包装原handleIotDescriptors函数"""
|
||||
try:
|
||||
# 调用原有的handleIotDescriptors函数
|
||||
# 注意：这里需要传入context而不是conn，因为handleIotDescriptors可能需要适配
|
||||
await handleIotDescriptors(context, descriptors)
|
||||
logger.debug("IoT设备描述符处理完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理IoT设备描述符失败: {e}", exc_info=True)
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"IoT设备描述符处理失败: {str(e)}"
|
||||
)
|
||||
|
||||
async def _handle_iot_status(self, context: SessionContext, transport: TransportInterface, states: Any):
|
||||
"""处理IoT设备状态 - 包装原handleIotStatus函数"""
|
||||
try:
|
||||
# 调用原有的handleIotStatus函数
|
||||
# 注意：这里需要传入context而不是conn，因为handleIotStatus可能需要适配
|
||||
await handleIotStatus(context, states)
|
||||
logger.debug("IoT设备状态处理完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理IoT设备状态失败: {e}", exc_info=True)
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"IoT设备状态处理失败: {str(e)}"
|
||||
)
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, session_id: str, message: str):
|
||||
"""发送IoT错误响应"""
|
||||
response = {
|
||||
"type": "iot",
|
||||
"status": "error",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送IoT错误响应失败: {e}")
|
||||
|
||||
async def _send_success_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, data: dict = None):
|
||||
"""发送IoT成功响应"""
|
||||
response = {
|
||||
"type": "iot",
|
||||
"status": "success",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if data:
|
||||
response["data"] = data
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送IoT成功响应失败: {e}")
|
||||
@@ -0,0 +1,433 @@
|
||||
import time
|
||||
import json
|
||||
import asyncio
|
||||
import uuid
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from core.utils.dialogue import Message
|
||||
from core.components.component_manager import ComponentType
|
||||
from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType
|
||||
from core.processors.audio_receive_processor import arm_wake_audio_suppression
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ListenProcessor(MessageProcessor):
|
||||
"""Listen消息处理器：完整迁移listenMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理listen类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "listen":
|
||||
await self.handle_listen_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_listen_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理listen消息 - 完整迁移自listenMessageHandler.py"""
|
||||
msg_session_id = msg_json.get("session_id")
|
||||
if msg_session_id and msg_session_id != context.session_id:
|
||||
logger.warning(
|
||||
f"忽略非当前会话的listen消息: "
|
||||
f"msg={msg_session_id}, current={context.session_id}"
|
||||
)
|
||||
return
|
||||
|
||||
state = msg_json.get("state")
|
||||
has_datagram_audio = getattr(
|
||||
transport, "has_datagram_audio", False
|
||||
)
|
||||
if (
|
||||
has_datagram_audio
|
||||
and state in {"start", "detect"}
|
||||
and not context.conversation_active
|
||||
):
|
||||
if transport.keeps_connection_between_sessions:
|
||||
await transport.send_json(
|
||||
{
|
||||
"type": "goodbye",
|
||||
"session_id": msg_session_id or context.session_id,
|
||||
}
|
||||
)
|
||||
logger.info(
|
||||
"忽略已结束会话的listen {}: session_id={}",
|
||||
state,
|
||||
context.session_id,
|
||||
)
|
||||
return
|
||||
|
||||
# 设置拾音模式
|
||||
if "mode" in msg_json:
|
||||
context.listen_mode = msg_json["mode"]
|
||||
logger.debug(f"客户端拾音模式：{context.listen_mode}")
|
||||
|
||||
# 处理不同的状态
|
||||
if state == "start":
|
||||
pending_start = getattr(context, "listen_start_task", None)
|
||||
if pending_start and not pending_start.done():
|
||||
logger.debug("忽略尾包隔离期内的重复listen start")
|
||||
return
|
||||
tail_quarantine = await self._flush_pending_listen_stop(
|
||||
context, transport
|
||||
)
|
||||
# 开始监听语音
|
||||
logger.info(f"listen start: session_id={context.session_id}, mode={context.listen_mode}")
|
||||
context.reset_audio_states()
|
||||
if (
|
||||
getattr(transport, "requires_audio_tail_grace", False)
|
||||
and tail_quarantine > 0
|
||||
):
|
||||
context.accepting_input_audio = False
|
||||
context.listen_start_task = context.create_background_task(
|
||||
self._open_input_after_tail_quarantine(
|
||||
context,
|
||||
context.session_id,
|
||||
tail_quarantine,
|
||||
),
|
||||
turn_scoped=True,
|
||||
)
|
||||
else:
|
||||
context.accepting_input_audio = True
|
||||
context.abort_requested = False
|
||||
context.close_after_chat = False
|
||||
logger.debug("开始语音监听")
|
||||
|
||||
elif state == "stop":
|
||||
# 停止监听语音
|
||||
logger.info(f"listen stop: session_id={context.session_id}")
|
||||
context.client_have_voice = True
|
||||
context.listen_stop_pending = True
|
||||
pending_start = getattr(context, "listen_start_task", None)
|
||||
if pending_start and not pending_start.done():
|
||||
pending_start.cancel()
|
||||
context.listen_start_task = None
|
||||
|
||||
if getattr(transport, "requires_audio_tail_grace", False):
|
||||
pending = getattr(context, "listen_stop_task", None)
|
||||
if pending and not pending.done():
|
||||
logger.debug("忽略重复的listen stop")
|
||||
return
|
||||
delay_ms = max(
|
||||
0,
|
||||
min(
|
||||
1000,
|
||||
int(
|
||||
context.config.get(
|
||||
"mqtt_udp_tail_grace_ms", 180
|
||||
)
|
||||
),
|
||||
),
|
||||
)
|
||||
context.listen_stop_deadline = (
|
||||
time.monotonic() + delay_ms / 1000
|
||||
)
|
||||
context.listen_stop_task = context.create_background_task(
|
||||
self._finalize_listen_stop(
|
||||
context,
|
||||
transport,
|
||||
context.session_id,
|
||||
delay_ms / 1000,
|
||||
),
|
||||
turn_scoped=True,
|
||||
)
|
||||
else:
|
||||
context.listen_stop_deadline = 0.0
|
||||
await self._finalize_listen_stop(
|
||||
context,
|
||||
transport,
|
||||
context.session_id,
|
||||
0,
|
||||
)
|
||||
logger.debug("停止语音监听")
|
||||
|
||||
elif state == "detect":
|
||||
# 检测到文本输入
|
||||
logger.info(f"listen detect: session_id={context.session_id}, text={msg_json.get('text')}")
|
||||
context.client_have_voice = False
|
||||
context.asr_audio.clear()
|
||||
|
||||
if "text" in msg_json:
|
||||
context.update_activity()
|
||||
original_text = msg_json["text"] # 保留原始文本
|
||||
filtered_len, filtered_text = remove_punctuation_and_length(original_text)
|
||||
|
||||
if original_text.startswith("[device_call]"):
|
||||
await self._handle_device_call(
|
||||
context,
|
||||
transport,
|
||||
original_text[len("[device_call]"):].strip(),
|
||||
)
|
||||
return
|
||||
|
||||
# 识别是否是唤醒词
|
||||
is_wakeup_words = filtered_text in context.config.get("wakeup_words", [])
|
||||
# 是否开启唤醒词回复
|
||||
enable_greeting = context.config.get("enable_greeting", True)
|
||||
|
||||
if is_wakeup_words and not enable_greeting:
|
||||
# 如果是唤醒词，且关闭了唤醒词回复，就不用回答
|
||||
# Native already drops wake history until listen/start.
|
||||
# Keeping the greeting suppression window armed here would
|
||||
# discard the beginning of the user's first real utterance.
|
||||
await self._send_stt_message(context, transport, original_text)
|
||||
await self._send_tts_message(context, transport, "stop", None)
|
||||
context.is_speaking = False
|
||||
|
||||
elif is_wakeup_words:
|
||||
# 处理唤醒词
|
||||
self._arm_wake_audio_suppression(context)
|
||||
# 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
|
||||
await self._enqueue_asr_report(context, "嘿，你好呀", [])
|
||||
await self._start_to_chat(context, transport, "嘿，你好呀", skip_intent=True)
|
||||
|
||||
else:
|
||||
# 处理普通文本
|
||||
# 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
|
||||
await self._enqueue_asr_report(context, original_text, [])
|
||||
# 否则需要LLM对文字内容进行答复
|
||||
await self._start_to_chat(context, transport, original_text)
|
||||
|
||||
async def _flush_pending_listen_stop(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
) -> float:
|
||||
"""Finalize buffered speech before a new listen/start resets the turn."""
|
||||
pending = getattr(context, "listen_stop_task", None)
|
||||
had_pending_stop = bool(getattr(context, "listen_stop_pending", False))
|
||||
tail_quarantine = max(
|
||||
0.0,
|
||||
float(getattr(context, "listen_stop_deadline", 0.0) or 0.0)
|
||||
- time.monotonic(),
|
||||
)
|
||||
if pending and not pending.done():
|
||||
pending.cancel()
|
||||
try:
|
||||
await pending
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
context.listen_stop_task = None
|
||||
if had_pending_stop:
|
||||
await self._finalize_listen_stop(
|
||||
context,
|
||||
transport,
|
||||
context.session_id,
|
||||
0,
|
||||
)
|
||||
else:
|
||||
context.listen_stop_pending = False
|
||||
if tail_quarantine <= 0:
|
||||
context.listen_stop_deadline = 0.0
|
||||
return tail_quarantine
|
||||
|
||||
@staticmethod
|
||||
async def _open_input_after_tail_quarantine(
|
||||
context: SessionContext,
|
||||
session_id: str,
|
||||
delay_seconds: float,
|
||||
) -> None:
|
||||
current_task = asyncio.current_task()
|
||||
try:
|
||||
await asyncio.sleep(delay_seconds)
|
||||
if (
|
||||
context.session_id == session_id
|
||||
and context.conversation_active
|
||||
and not context.listen_stop_pending
|
||||
):
|
||||
context.accepting_input_audio = True
|
||||
finally:
|
||||
if getattr(context, "listen_start_task", None) is current_task:
|
||||
context.listen_start_task = None
|
||||
context.listen_stop_deadline = 0.0
|
||||
|
||||
async def _finalize_listen_stop(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
session_id: str,
|
||||
delay_seconds: float,
|
||||
) -> None:
|
||||
"""Finalize ASR after datagram tail frames have crossed the router."""
|
||||
current_task = asyncio.current_task()
|
||||
try:
|
||||
if delay_seconds > 0:
|
||||
await asyncio.sleep(delay_seconds)
|
||||
if context.session_id != session_id:
|
||||
return
|
||||
if (
|
||||
getattr(transport, "has_datagram_audio", False)
|
||||
and not context.conversation_active
|
||||
):
|
||||
return
|
||||
|
||||
# Close the tail-grace ingress gate before resolving ASR.
|
||||
# Component lookup may fail; leaving it open would leak subsequent
|
||||
# packets into a turn that has already stopped.
|
||||
context.listen_stop_pending = False
|
||||
context.client_voice_stop = True
|
||||
if getattr(transport, "requires_audio_tail_grace", False):
|
||||
context.accepting_input_audio = False
|
||||
|
||||
asr_component = await self._get_component(
|
||||
context, ComponentType.ASR
|
||||
)
|
||||
if context.session_id != session_id:
|
||||
return
|
||||
asr_instance = (
|
||||
getattr(asr_component, "asr_instance", None)
|
||||
if asr_component
|
||||
else None
|
||||
)
|
||||
|
||||
if not asr_instance or not hasattr(
|
||||
asr_instance, "interface_type"
|
||||
):
|
||||
return
|
||||
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
|
||||
if asr_instance.interface_type == InterfaceType.STREAM:
|
||||
if hasattr(asr_instance, "_send_stop_request"):
|
||||
await asr_instance._send_stop_request()
|
||||
return
|
||||
|
||||
if context.asr_audio:
|
||||
asr_audio_task = context.asr_audio.copy()
|
||||
context.asr_audio.clear()
|
||||
context.reset_vad_states()
|
||||
await asr_instance.handle_voice_stop(
|
||||
context, asr_audio_task
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as error:
|
||||
logger.error(
|
||||
"listen stop收尾失败: session_id={}, error={}",
|
||||
session_id,
|
||||
error,
|
||||
)
|
||||
finally:
|
||||
if getattr(context, "listen_stop_task", None) is current_task:
|
||||
context.listen_stop_task = None
|
||||
context.listen_stop_deadline = 0.0
|
||||
if context.session_id == session_id:
|
||||
context.listen_stop_pending = False
|
||||
if getattr(transport, "requires_audio_tail_grace", False):
|
||||
context.accepting_input_audio = False
|
||||
|
||||
def _arm_wake_audio_suppression(self, context: SessionContext) -> None:
|
||||
"""Bound the UDP/TCP reorder window used by the legacy gateway flow."""
|
||||
arm_wake_audio_suppression(context)
|
||||
|
||||
async def _handle_device_call(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
call_text: str,
|
||||
) -> None:
|
||||
"""Restore the legacy device-call announcement and call-state handoff."""
|
||||
logger.info(f"收到设备呼叫指令: {call_text}")
|
||||
context.incoming_call = True
|
||||
context.sentence_id = uuid.uuid4().hex
|
||||
await self._send_stt_message(context, transport, call_text)
|
||||
|
||||
tts_component = await self._get_component(context, ComponentType.TTS)
|
||||
tts_instance = (
|
||||
getattr(tts_component, "tts_instance", None)
|
||||
if tts_component
|
||||
else None
|
||||
)
|
||||
if tts_instance:
|
||||
tts_instance.store_tts_text(context.sentence_id, call_text)
|
||||
tts_instance.tts_text_queue.put(
|
||||
TTSMessageDTO(
|
||||
sentence_id=context.sentence_id,
|
||||
sentence_type=SentenceType.FIRST,
|
||||
content_type=ContentType.ACTION,
|
||||
)
|
||||
)
|
||||
tts_instance.tts_one_sentence(
|
||||
context,
|
||||
ContentType.TEXT,
|
||||
content_detail=call_text,
|
||||
)
|
||||
tts_instance.tts_text_queue.put(
|
||||
TTSMessageDTO(
|
||||
sentence_id=context.sentence_id,
|
||||
sentence_type=SentenceType.LAST,
|
||||
content_type=ContentType.ACTION,
|
||||
)
|
||||
)
|
||||
|
||||
context.dialogue.put(Message(role="assistant", content=call_text))
|
||||
|
||||
async def _handle_audio_message(self, context: SessionContext, transport: TransportInterface, audio: bytes):
|
||||
"""处理音频消息 - 调用AudioReceiveProcessor"""
|
||||
# 这里应该调用AudioReceiveProcessor来处理音频
|
||||
from core.processors.audio_receive_processor import AudioReceiveProcessor
|
||||
audio_processor = AudioReceiveProcessor()
|
||||
await audio_processor.handle_audio_message(context, transport, audio)
|
||||
|
||||
async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息"""
|
||||
from core.processors.audio_send_processor import AudioSendProcessor
|
||||
|
||||
await AudioSendProcessor().send_stt_message(
|
||||
context, transport, text
|
||||
)
|
||||
|
||||
async def _send_tts_message(self, context: SessionContext, transport: TransportInterface, state: str, text: str = None):
|
||||
"""发送TTS消息"""
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": context.session_id
|
||||
}
|
||||
if text:
|
||||
message["text"] = text
|
||||
|
||||
await transport.send(json.dumps(message))
|
||||
logger.debug(f"发送TTS消息: state={state}, text={text}")
|
||||
|
||||
async def _get_component(self, context: SessionContext, component_type: ComponentType):
|
||||
if not context.component_manager:
|
||||
return None
|
||||
return await context.component_manager.get_component(component_type, context)
|
||||
|
||||
async def _enqueue_asr_report(self, context: SessionContext, text: str, audio_data: list):
|
||||
"""ASR上报队列"""
|
||||
if context.report_asr_enable:
|
||||
from core.processors.report_processor import ReportProcessor
|
||||
report_processor = ReportProcessor()
|
||||
report_processor.enqueue_asr_report(context, text, audio_data)
|
||||
|
||||
async def _start_to_chat(self, context: SessionContext, transport: TransportInterface, text: str, skip_intent: bool = False):
|
||||
"""开始聊天 - 调用ChatProcessor"""
|
||||
# 与旧架构一致：先发送STT，再异步触发聊天，避免阻塞事件循环
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
|
||||
if hasattr(context, "create_background_task"):
|
||||
context.create_background_task(
|
||||
chat_processor.handle_chat(
|
||||
context, transport, text, skip_intent=skip_intent
|
||||
),
|
||||
turn_scoped=True,
|
||||
)
|
||||
logger.info(f"chat任务已提交: session_id={context.session_id}")
|
||||
else:
|
||||
await chat_processor.handle_chat(context, transport, text, skip_intent=skip_intent)
|
||||
@@ -0,0 +1,96 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.providers.tools.device_mcp import handle_mcp_message
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class McpProcessor(MessageProcessor):
|
||||
"""MCP消息处理器：完整迁移mcpMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理mcp类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "mcp":
|
||||
await self.handle_mcp_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_mcp_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理MCP消息 - 完整迁移自mcpMessageHandler.py"""
|
||||
if "payload" in msg_json:
|
||||
# 检查MCP客户端是否存在
|
||||
if not context.mcp_client:
|
||||
logger.warning("MCP客户端未初始化，无法处理MCP消息")
|
||||
await self._send_error_response(transport, context.session_id, "MCP客户端未初始化")
|
||||
return
|
||||
|
||||
# 创建异步任务处理MCP消息 - 完整迁移原逻辑
|
||||
context.create_background_task(
|
||||
self._handle_mcp_payload(
|
||||
context, transport, msg_json["payload"]
|
||||
),
|
||||
conversation_scoped=False,
|
||||
)
|
||||
else:
|
||||
logger.warning("MCP消息缺少payload字段")
|
||||
await self._send_error_response(transport, context.session_id, "MCP消息格式错误：缺少payload")
|
||||
|
||||
async def _handle_mcp_payload(self, context: SessionContext, transport: TransportInterface, payload: dict):
|
||||
"""处理MCP payload - 包装原handle_mcp_message函数"""
|
||||
try:
|
||||
# 调用原有的handle_mcp_message函数
|
||||
# 注意：这里需要传入context而不是conn，因为handle_mcp_message可能需要适配
|
||||
await handle_mcp_message(context, context.mcp_client, payload, transport)
|
||||
logger.debug("MCP消息处理完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理MCP消息失败: {e}", exc_info=True)
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"MCP消息处理失败: {str(e)}"
|
||||
)
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, session_id: str, message: str):
|
||||
"""发送MCP错误响应"""
|
||||
response = {
|
||||
"type": "mcp",
|
||||
"status": "error",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送MCP错误响应失败: {e}")
|
||||
|
||||
async def _send_success_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, data: dict = None):
|
||||
"""发送MCP成功响应"""
|
||||
response = {
|
||||
"type": "mcp",
|
||||
"status": "success",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if data:
|
||||
response["data"] = data
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送MCP成功响应失败: {e}")
|
||||
@@ -0,0 +1,124 @@
|
||||
import json
|
||||
from typing import Any, List
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.processors.hello_processor import HelloProcessor
|
||||
from core.processors.listen_processor import ListenProcessor
|
||||
from core.processors.audio_receive_processor import AudioReceiveProcessor
|
||||
from core.processors.auth_processor import AuthProcessor
|
||||
from core.processors.timeout_processor import TimeoutProcessor
|
||||
from core.processors.server_processor import ServerProcessor
|
||||
from core.processors.mcp_processor import McpProcessor
|
||||
from core.processors.iot_processor import IotProcessor
|
||||
from core.processors.abort_processor import AbortProcessor
|
||||
from core.processors.goodbye_processor import GoodbyeProcessor
|
||||
from core.processors.text_processor import TextProcessor
|
||||
from core.processors.ping_processor import PingProcessor
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class MessageRouter(MessageProcessor):
|
||||
"""
|
||||
消息路由器：协调所有独立的processor
|
||||
按功能职责分离，避免耦合，每个processor专注单一职责
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# 初始化所有独立的processor
|
||||
self.auth_processor = AuthProcessor()
|
||||
self.timeout_processor = TimeoutProcessor()
|
||||
self.abort_processor = AbortProcessor()
|
||||
self.goodbye_processor = GoodbyeProcessor()
|
||||
self.hello_processor = HelloProcessor()
|
||||
self.listen_processor = ListenProcessor()
|
||||
self.server_processor = ServerProcessor()
|
||||
self.mcp_processor = McpProcessor()
|
||||
self.iot_processor = IotProcessor()
|
||||
self.audio_receive_processor = AudioReceiveProcessor()
|
||||
self.text_processor = TextProcessor()
|
||||
self.ping_processor = PingProcessor()
|
||||
|
||||
# 按优先级排序的processor列表
|
||||
self.processors: List[MessageProcessor] = [
|
||||
self.timeout_processor, # 首先检查超时
|
||||
self.auth_processor, # 然后检查认证
|
||||
self.server_processor, # 服务器消息（管理端下发动作）
|
||||
self.abort_processor, # 中断消息
|
||||
self.goodbye_processor, # goodbye消息
|
||||
self.hello_processor, # hello消息
|
||||
self.ping_processor, # 可选JSON ping/pong心跳
|
||||
self.listen_processor, # listen消息
|
||||
self.mcp_processor, # MCP消息
|
||||
self.iot_processor, # IoT消息
|
||||
self.audio_receive_processor, # 音频消息
|
||||
self.text_processor, # 纯文本消息（放在最后，作为兜底处理）
|
||||
]
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""
|
||||
路由消息到合适的processor
|
||||
每个processor专注处理自己的消息类型，避免耦合
|
||||
"""
|
||||
# Audio activity is updated only after VAD confirms voice. Treating
|
||||
# every silent frame as activity prevents logical-session timeouts.
|
||||
is_audio = isinstance(message, bytes) or (
|
||||
isinstance(message, dict) and message.get("type") == "audio"
|
||||
)
|
||||
if not is_audio:
|
||||
context.update_activity()
|
||||
|
||||
# 按优先级顺序尝试每个processor
|
||||
for processor in self.processors:
|
||||
try:
|
||||
if await processor.process(context, transport, message):
|
||||
# 消息已被处理，记录日志并返回
|
||||
logger.debug(f"消息被 {processor.__class__.__name__} 处理")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"{processor.__class__.__name__} 处理消息时出错: {e}", exc_info=True)
|
||||
continue
|
||||
|
||||
# 如果没有processor处理该消息，记录警告
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
msg_type = msg_json.get("type", "unknown") if isinstance(msg_json, dict) else "non-dict"
|
||||
logger.warning(f"未处理的消息类型: {msg_type}, 内容: {message[:100]}...")
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(f"未处理的非JSON消息: {message[:100]}...")
|
||||
elif isinstance(message, bytes):
|
||||
logger.warning(f"未处理的二进制消息，大小: {len(message)} bytes")
|
||||
else:
|
||||
logger.warning(f"未处理的消息类型: {type(message)}, {message}")
|
||||
|
||||
return False
|
||||
|
||||
def add_processor(self, processor: MessageProcessor, priority: int = None):
|
||||
"""
|
||||
添加新的processor
|
||||
priority: 优先级，数字越小优先级越高，None表示添加到末尾
|
||||
"""
|
||||
if priority is None:
|
||||
self.processors.append(processor)
|
||||
else:
|
||||
self.processors.insert(priority, processor)
|
||||
logger.info(f"添加processor: {processor.__class__.__name__}")
|
||||
|
||||
def remove_processor(self, processor_class):
|
||||
"""移除指定类型的processor"""
|
||||
self.processors = [p for p in self.processors if not isinstance(p, processor_class)]
|
||||
logger.info(f"移除processor: {processor_class.__name__}")
|
||||
|
||||
def get_processor(self, processor_class):
|
||||
"""获取指定类型的processor"""
|
||||
for processor in self.processors:
|
||||
if isinstance(processor, processor_class):
|
||||
return processor
|
||||
return None
|
||||
|
||||
def list_processors(self) -> List[str]:
|
||||
"""列出所有processor的名称"""
|
||||
return [processor.__class__.__name__ for processor in self.processors]
|
||||
@@ -0,0 +1,40 @@
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from core.context.session_context import SessionContext
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class PingProcessor(MessageProcessor):
|
||||
"""Handle the optional legacy JSON ping/pong control contract."""
|
||||
|
||||
async def process(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
message: Any,
|
||||
) -> bool:
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
message = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
if not isinstance(message, dict) or message.get("type") != "ping":
|
||||
return False
|
||||
|
||||
if not context.config.get("enable_websocket_ping", False):
|
||||
logger.debug("WebSocket心跳功能未启用，忽略PING消息")
|
||||
return True
|
||||
|
||||
await transport.send_json(
|
||||
{
|
||||
"type": "pong",
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
|
||||
}
|
||||
)
|
||||
return True
|
||||
@@ -0,0 +1,301 @@
|
||||
import asyncio
|
||||
import time
|
||||
import queue
|
||||
import threading
|
||||
import json
|
||||
from typing import Any, List
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.manage_api_client import ManageApiClient, report as manage_report
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ReportProcessor(MessageProcessor):
|
||||
"""上报处理器：完整迁移reportHandle.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""这个处理器不直接处理消息，而是被其他处理器调用"""
|
||||
return False
|
||||
|
||||
def enqueue_asr_report(self, context: SessionContext, text: str, audio_data: List[bytes]):
|
||||
"""ASR上报队列 - 完整迁移自enqueue_asr_report"""
|
||||
if not self._should_report(context, context.report_asr_enable):
|
||||
return
|
||||
|
||||
report_time = int(time.time() * 1000)
|
||||
if context.chat_history_conf != 2:
|
||||
audio_data = None
|
||||
|
||||
# 将上报任务放入队列
|
||||
self._enqueue_report(context, {
|
||||
"type": 1, # 用户类型
|
||||
"text": text,
|
||||
"audio_data": audio_data,
|
||||
"report_time": report_time,
|
||||
"session_id": context.session_id,
|
||||
"device_id": context.device_id,
|
||||
})
|
||||
|
||||
# 确保上报线程已启动
|
||||
self._ensure_report_thread(context)
|
||||
|
||||
def enqueue_tts_report(self, context: SessionContext, text: str, opus_data: bytes):
|
||||
"""TTS上报队列 - 完整迁移自enqueue_tts_report"""
|
||||
if not self._should_report(context, context.report_tts_enable):
|
||||
return
|
||||
|
||||
report_time = int(time.time() * 1000)
|
||||
if context.chat_history_conf != 2:
|
||||
opus_data = None
|
||||
|
||||
# 将上报任务放入队列
|
||||
self._enqueue_report(context, {
|
||||
"type": 2, # 智能体类型
|
||||
"text": text,
|
||||
"audio_data": opus_data,
|
||||
"report_time": report_time,
|
||||
"session_id": context.session_id,
|
||||
"device_id": context.device_id,
|
||||
})
|
||||
|
||||
# 确保上报线程已启动
|
||||
self._ensure_report_thread(context)
|
||||
|
||||
def enqueue_tool_report(
|
||||
self,
|
||||
context: SessionContext,
|
||||
tool_name: str,
|
||||
tool_input: dict,
|
||||
tool_result: str = None,
|
||||
report_tool_call: bool = True,
|
||||
):
|
||||
if not self._should_report(context, True):
|
||||
return
|
||||
|
||||
timestamp = int(time.time() * 1000)
|
||||
entries = []
|
||||
if report_tool_call:
|
||||
entries.append(
|
||||
(
|
||||
json.dumps(
|
||||
[{"type": "tool", "text": f"{tool_name}({json.dumps(tool_input, ensure_ascii=False)})"}],
|
||||
ensure_ascii=False,
|
||||
),
|
||||
timestamp,
|
||||
)
|
||||
)
|
||||
if tool_result is not None:
|
||||
entries.append(
|
||||
(
|
||||
json.dumps(
|
||||
[{"type": "tool_result", "text": json.dumps({"result": str(tool_result)}, ensure_ascii=False)}],
|
||||
ensure_ascii=False,
|
||||
),
|
||||
timestamp + 1,
|
||||
)
|
||||
)
|
||||
|
||||
for text, report_time in entries:
|
||||
self._enqueue_report(
|
||||
context,
|
||||
{
|
||||
"type": 3,
|
||||
"text": text,
|
||||
"audio_data": None,
|
||||
"report_time": report_time,
|
||||
"session_id": context.session_id,
|
||||
"device_id": context.device_id,
|
||||
},
|
||||
)
|
||||
if entries:
|
||||
self._ensure_report_thread(context)
|
||||
|
||||
def _ensure_report_thread(self, context: SessionContext):
|
||||
"""确保上报线程已启动"""
|
||||
if context.report_thread is None or not context.report_thread.is_alive():
|
||||
if not getattr(context, "_report_cleanup_registered", False):
|
||||
context.register_cleanup(
|
||||
lambda: asyncio.to_thread(self.cleanup_session, context)
|
||||
)
|
||||
context._report_cleanup_registered = True
|
||||
context.report_thread = threading.Thread(
|
||||
target=self._report_worker,
|
||||
args=(context,),
|
||||
daemon=True
|
||||
)
|
||||
context.report_thread.start()
|
||||
logger.info(f"上报线程已启动: {context.session_id}")
|
||||
|
||||
def _enqueue_report(self, context: SessionContext, report_task: dict) -> None:
|
||||
try:
|
||||
context.report_queue.put_nowait(report_task)
|
||||
except queue.Full:
|
||||
try:
|
||||
context.report_queue.get_nowait()
|
||||
context.report_queue.put_nowait(report_task)
|
||||
logger.warning("上报队列已满，丢弃最旧任务: {}", context.session_id)
|
||||
except (queue.Empty, queue.Full):
|
||||
logger.warning("上报队列持续拥塞，丢弃当前任务: {}", context.session_id)
|
||||
|
||||
def _should_report(self, context: SessionContext, enabled: bool) -> bool:
|
||||
return bool(
|
||||
enabled
|
||||
and context.read_config_from_api
|
||||
and not context.need_bind
|
||||
and context.chat_history_conf != 0
|
||||
)
|
||||
|
||||
def _report_worker(self, context: SessionContext):
|
||||
"""上报工作线程 - 完整迁移自ConnectionHandler中的上报逻辑"""
|
||||
logger.info(f"上报工作线程启动: {context.session_id}")
|
||||
event_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(event_loop)
|
||||
|
||||
try:
|
||||
while not context.stop_event.is_set():
|
||||
try:
|
||||
# 从队列获取上报任务
|
||||
report_task = context.report_queue.get(timeout=1)
|
||||
if report_task is None:
|
||||
break
|
||||
|
||||
# 执行上报
|
||||
self._execute_report(context, report_task, event_loop)
|
||||
|
||||
except queue.Empty:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f"上报工作线程异常: {e}")
|
||||
finally:
|
||||
try:
|
||||
event_loop.run_until_complete(
|
||||
ManageApiClient.close_current_loop_client()
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("关闭上报HTTP客户端失败: {}", exc)
|
||||
event_loop.close()
|
||||
logger.info(f"上报工作线程退出: {context.session_id}")
|
||||
|
||||
def _execute_report(
|
||||
self, context: SessionContext, report_task: dict,
|
||||
event_loop=None
|
||||
):
|
||||
"""执行聊天记录上报操作 - 完整迁移自report函数"""
|
||||
try:
|
||||
report_type = report_task["type"]
|
||||
text = report_task["text"]
|
||||
audio_data = report_task["audio_data"]
|
||||
report_time = report_task["report_time"]
|
||||
|
||||
# 处理音频数据
|
||||
processed_audio = None
|
||||
if audio_data:
|
||||
if isinstance(audio_data, list):
|
||||
# ASR音频数据（多个音频片段）
|
||||
processed_audio = self._process_asr_audio(audio_data)
|
||||
elif isinstance(audio_data, bytes):
|
||||
# TTS音频数据（opus格式）
|
||||
processed_audio = self._opus_to_wav(audio_data)
|
||||
|
||||
# 执行上报
|
||||
coroutine = manage_report(
|
||||
mac_address=report_task["device_id"],
|
||||
session_id=report_task["session_id"],
|
||||
chat_type=report_type,
|
||||
content=text,
|
||||
audio=processed_audio,
|
||||
report_time=report_time,
|
||||
)
|
||||
if event_loop is None:
|
||||
asyncio.run(coroutine)
|
||||
else:
|
||||
event_loop.run_until_complete(coroutine)
|
||||
|
||||
logger.debug(f"上报成功: type={report_type}, text={text[:50]}...")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"聊天记录上报失败: {e}")
|
||||
|
||||
def _process_asr_audio(self, audio_data_list: List[bytes]) -> bytes:
|
||||
"""处理ASR音频数据"""
|
||||
try:
|
||||
# 将多个音频片段合并
|
||||
combined_audio = b''.join(audio_data_list)
|
||||
return self._pcm_to_wav(combined_audio)
|
||||
except Exception as e:
|
||||
logger.error(f"处理ASR音频数据失败: {e}")
|
||||
return b''
|
||||
|
||||
def _pcm_to_wav(self, pcm_data: bytes) -> bytes:
|
||||
import io
|
||||
import wave
|
||||
|
||||
wav_buffer = io.BytesIO()
|
||||
with wave.open(wav_buffer, 'wb') as wav_file:
|
||||
wav_file.setnchannels(1)
|
||||
wav_file.setsampwidth(2)
|
||||
wav_file.setframerate(16000)
|
||||
wav_file.writeframes(pcm_data)
|
||||
return wav_buffer.getvalue()
|
||||
|
||||
def _opus_to_wav(self, opus_data: bytes) -> bytes:
|
||||
"""将Opus数据转换为WAV格式的字节流 - 完整迁移自opus_to_wav"""
|
||||
try:
|
||||
import opuslib_next
|
||||
import io
|
||||
import wave
|
||||
|
||||
# Opus解码器配置
|
||||
sample_rate = 16000
|
||||
channels = 1
|
||||
|
||||
# 创建Opus解码器
|
||||
decoder = opuslib_next.Decoder(sample_rate, channels)
|
||||
|
||||
# 解码Opus数据
|
||||
pcm_data = decoder.decode(opus_data, frame_size=960)
|
||||
|
||||
# 创建WAV文件
|
||||
wav_buffer = io.BytesIO()
|
||||
with wave.open(wav_buffer, 'wb') as wav_file:
|
||||
wav_file.setnchannels(channels)
|
||||
wav_file.setsampwidth(2) # 16-bit
|
||||
wav_file.setframerate(sample_rate)
|
||||
wav_file.writeframes(pcm_data)
|
||||
|
||||
return wav_buffer.getvalue()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Opus转WAV失败: {e}")
|
||||
return b''
|
||||
|
||||
def cleanup_session(self, context: SessionContext):
|
||||
"""清理会话上报资源"""
|
||||
# 停止上报线程
|
||||
if context.report_thread and context.report_thread.is_alive():
|
||||
context.stop_event.set()
|
||||
try:
|
||||
context.report_queue.put_nowait(None)
|
||||
except queue.Full:
|
||||
try:
|
||||
context.report_queue.get_nowait()
|
||||
context.report_queue.put_nowait(None)
|
||||
except (queue.Empty, queue.Full):
|
||||
pass
|
||||
context.report_thread.join(timeout=5)
|
||||
if context.report_thread.is_alive():
|
||||
logger.warning(f"上报线程未能按时退出: {context.session_id}")
|
||||
else:
|
||||
context.report_thread = None
|
||||
|
||||
# 清理上报队列
|
||||
try:
|
||||
while not context.report_queue.empty():
|
||||
context.report_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
pass
|
||||
|
||||
logger.info(f"上报资源清理完成: {context.session_id}")
|
||||
@@ -0,0 +1,177 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
from core.utils.restart import restart_server
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ServerProcessor(MessageProcessor):
|
||||
"""服务器消息处理器：完整迁移serverMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理server类型的消息"""
|
||||
msg_json = None
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
except json.JSONDecodeError:
|
||||
msg_json = None
|
||||
elif isinstance(message, dict):
|
||||
msg_json = message
|
||||
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "server":
|
||||
await self.handle_server_message(context, transport, msg_json)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_server_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理server消息 - 完整迁移自serverMessageHandler.py"""
|
||||
# 如果配置是从API读取的，则需要验证secret
|
||||
if not context.read_config_from_api:
|
||||
return
|
||||
|
||||
# 获取post请求的secret
|
||||
post_secret = msg_json.get("content", {}).get("secret", "")
|
||||
secret = context.config.get("manager-api", {}).get("secret", "")
|
||||
|
||||
# 如果secret不匹配，则返回
|
||||
if post_secret != secret:
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"服务器密钥验证失败"
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
return
|
||||
|
||||
# 处理不同的action
|
||||
action = msg_json.get("action")
|
||||
|
||||
if action == "update_config":
|
||||
await self._handle_update_config(context, transport, msg_json)
|
||||
elif action == "restart":
|
||||
await self._handle_restart(context, transport, msg_json)
|
||||
else:
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"未知的服务器操作: {action}"
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
|
||||
async def _handle_update_config(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理配置更新 - 完整迁移自update_config逻辑"""
|
||||
try:
|
||||
# 检查是否有服务器实例
|
||||
if not context.server:
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"无法获取服务器实例",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
return
|
||||
|
||||
# 更新WebSocketServer的配置
|
||||
if not await context.server.update_config():
|
||||
error_msg = ""
|
||||
if hasattr(context.server, "get_last_update_error"):
|
||||
error_msg = context.server.get_last_update_error()
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
error_msg or "更新服务器配置失败",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
return
|
||||
|
||||
# 发送成功响应
|
||||
await self._send_success_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"配置更新成功",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"更新配置失败: {str(e)}")
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"更新配置失败: {str(e)}",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
|
||||
async def _handle_restart(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理服务器重启 - 完整迁移自handle_restart逻辑"""
|
||||
try:
|
||||
# 发送确认响应
|
||||
await self._send_success_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"服务器重启中...",
|
||||
{"action": "restart"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
|
||||
# 异步执行重启操作
|
||||
threading.Thread(target=lambda : restart_server(logger), daemon=True).start()
|
||||
except Exception as e:
|
||||
logger.error(f"处理重启请求失败: {str(e)}")
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"重启失败: {str(e)}",
|
||||
{"action": "restart"}
|
||||
)
|
||||
await self._close_transport(transport)
|
||||
|
||||
async def _send_success_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, content: dict = None):
|
||||
"""发送成功响应"""
|
||||
response = {
|
||||
"type": "server",
|
||||
"status": "success",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if content:
|
||||
response["content"] = content
|
||||
|
||||
await transport.send(json.dumps(response))
|
||||
logger.info(f"服务器操作成功: {message}")
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, content: dict = None):
|
||||
"""发送错误响应"""
|
||||
response = {
|
||||
"type": "server",
|
||||
"status": "fail",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if content:
|
||||
response["content"] = content
|
||||
|
||||
await transport.send(json.dumps(response))
|
||||
logger.error(f"服务器操作失败: {message}")
|
||||
|
||||
async def _close_transport(self, transport: TransportInterface):
|
||||
try:
|
||||
await transport.close()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,54 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TextProcessor(MessageProcessor):
|
||||
"""
|
||||
纯文本消息处理器：处理非JSON格式的文本消息
|
||||
这是新架构中缺失的重要组件，用于处理直接发送的文本聊天内容
|
||||
"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理纯文本消息"""
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
# 尝试解析为JSON，如果成功则不是纯文本消息
|
||||
json.loads(message)
|
||||
return False # JSON消息由其他processor处理
|
||||
except json.JSONDecodeError:
|
||||
# 确实是纯文本消息，进行聊天处理
|
||||
await self.handle_text_message(context, transport, message)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_text_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""处理纯文本消息 - 直接调用ChatProcessor进行聊天"""
|
||||
try:
|
||||
# 记录收到纯文本消息
|
||||
logger.info(f"收到纯文本消息: {text[:100]}...")
|
||||
|
||||
# 使用ChatProcessor处理聊天
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
await chat_processor.handle_chat(context, transport, text)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理纯文本消息失败: {e}")
|
||||
# 发送错误响应
|
||||
await self._send_error_response(transport, "文本处理失败，请重试")
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, error_message: str):
|
||||
"""发送错误响应"""
|
||||
try:
|
||||
await transport.send(json.dumps({
|
||||
"type": "error",
|
||||
"message": error_message
|
||||
}))
|
||||
except Exception as e:
|
||||
logger.error(f"发送错误响应失败: {e}")
|
||||
@@ -0,0 +1,48 @@
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TimeoutProcessor(MessageProcessor):
|
||||
"""超时检查处理器：检查会话是否超时"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""检查会话超时"""
|
||||
return await self.handle_timeout(context, transport)
|
||||
|
||||
async def handle_timeout(
|
||||
self,
|
||||
context: SessionContext,
|
||||
transport: TransportInterface,
|
||||
) -> bool:
|
||||
"""End an expired logical conversation without closing a long connection."""
|
||||
if not getattr(context, "conversation_active", False):
|
||||
return False
|
||||
|
||||
timeout_seconds = context.config.get("close_connection_no_voice_time", 120)
|
||||
if not context.is_timeout(timeout_seconds):
|
||||
return False
|
||||
|
||||
try:
|
||||
if transport.keeps_connection_between_sessions:
|
||||
logger.info(f"会话超时，结束长连接逻辑会话: {context.session_id}")
|
||||
from core.processors.audio_receive_processor import (
|
||||
AudioReceiveProcessor,
|
||||
)
|
||||
|
||||
await AudioReceiveProcessor()._no_voice_close_connect(
|
||||
context,
|
||||
transport,
|
||||
have_voice=False,
|
||||
)
|
||||
else:
|
||||
logger.info(f"会话超时，准备关闭连接: {context.session_id}")
|
||||
await transport.close()
|
||||
except Exception as e:
|
||||
logger.error(f"处理超时连接失败: {e}")
|
||||
|
||||
return True
|
||||
Reference in New Issue
Block a user