mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-29 10:03:56 +08:00
refactor: 重构底层代码，抽离conn，调整消息处理器并创建传输层接口。
feature: 支持mqtt非桥接版本。
This commit is contained in:
caixypromise
@@ -0,0 +1,102 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AbortProcessor(MessageProcessor):
|
||||
"""中断消息处理器：完整迁移abortMessageHandler.py和abortHandle.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理abort类型的消息"""
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "abort":
|
||||
await self.handle_abort_message(context, transport, msg_json)
|
||||
return True
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def handle_abort_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理中断消息 - 完整迁移自abortHandle.py的handleAbortMessage"""
|
||||
logger.info("Abort message received")
|
||||
|
||||
# 设置成打断状态，会自动打断llm、tts任务 - 完整迁移原逻辑
|
||||
context.abort_requested = True
|
||||
|
||||
# 清理队列 - 完整迁移原逻辑
|
||||
await self._clear_queues(context)
|
||||
|
||||
# 打断客户端说话状态 - 完整迁移原逻辑
|
||||
await transport.send(json.dumps({
|
||||
"type": "tts",
|
||||
"state": "stop",
|
||||
"session_id": context.session_id
|
||||
}))
|
||||
|
||||
# 清理说话状态 - 完整迁移原逻辑
|
||||
self._clear_speak_status(context)
|
||||
|
||||
logger.info("Abort message received-end")
|
||||
|
||||
async def _clear_queues(self, context: SessionContext):
|
||||
"""清理所有队列 - 完整迁移原clear_queues逻辑"""
|
||||
try:
|
||||
# 清理TTS音频队列
|
||||
tts_component = context.components.get('tts')
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
if hasattr(tts_instance, 'tts_audio_queue'):
|
||||
try:
|
||||
while not tts_instance.tts_audio_queue.empty():
|
||||
tts_instance.tts_audio_queue.get_nowait()
|
||||
except:
|
||||
pass
|
||||
|
||||
# 清理ASR音频队列
|
||||
context.clear_audio_buffer()
|
||||
|
||||
# 清理其他可能的队列
|
||||
if hasattr(context, 'clear_queues'):
|
||||
context.clear_queues()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"清理队列时出错: {e}")
|
||||
|
||||
def _clear_speak_status(self, context: SessionContext):
|
||||
"""清理说话状态 - 完整迁移原clearSpeakStatus逻辑"""
|
||||
try:
|
||||
# 清理说话状态
|
||||
context.is_speaking = False
|
||||
|
||||
# 如果有其他说话状态相关的属性，也一并清理
|
||||
if hasattr(context, 'clearSpeakStatus'):
|
||||
context.clearSpeakStatus()
|
||||
|
||||
# 重置相关状态
|
||||
context.client_have_voice = False
|
||||
context.client_voice_stop = True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"清理说话状态时出错: {e}")
|
||||
|
||||
async def _send_abort_confirmation(self, transport: TransportInterface, session_id: str):
|
||||
"""发送中断确认响应（可选）"""
|
||||
response = {
|
||||
"type": "abort",
|
||||
"status": "success",
|
||||
"message": "中断操作已完成",
|
||||
"session_id": session_id
|
||||
}
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送中断确认响应失败: {e}")
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
import time
|
||||
import json
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.utils.util import audio_to_data
|
||||
from core.utils.output_counter import check_device_output_limit
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AudioReceiveProcessor(MessageProcessor):
|
||||
"""音频接收处理器：完整迁移receiveAudioHandle.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理音频消息"""
|
||||
if isinstance(message, bytes):
|
||||
await self.handle_audio_message(context, transport, message)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_audio_message(self, context: SessionContext, transport: TransportInterface, audio: bytes):
|
||||
"""处理音频消息 - 完整迁移自handleAudioMessage"""
|
||||
# 获取VAD组件
|
||||
vad_component = context.components.get('vad')
|
||||
if not vad_component or not hasattr(vad_component, 'vad_instance'):
|
||||
logger.warning("VAD组件未初始化")
|
||||
return
|
||||
|
||||
vad_instance = vad_component.vad_instance
|
||||
|
||||
# 当前片段是否有人说话
|
||||
have_voice = vad_instance.is_vad(context, audio)
|
||||
|
||||
# 如果设备刚刚被唤醒，短暂忽略VAD检测
|
||||
if have_voice and context.just_woken_up:
|
||||
have_voice = False
|
||||
# 设置一个短暂延迟后恢复VAD检测
|
||||
context.asr_audio.clear()
|
||||
if not hasattr(context, "vad_resume_task") or context.vad_resume_task.done():
|
||||
context.vad_resume_task = asyncio.create_task(self._resume_vad_detection(context))
|
||||
return
|
||||
|
||||
if have_voice:
|
||||
if context.is_speaking:
|
||||
await self._handle_abort_message(context, transport)
|
||||
|
||||
# 设备长时间空闲检测，用于say goodbye
|
||||
await self._no_voice_close_connect(context, transport, have_voice)
|
||||
|
||||
# 接收音频
|
||||
asr_component = context.components.get('asr')
|
||||
if asr_component and hasattr(asr_component, 'asr_instance'):
|
||||
asr_instance = asr_component.asr_instance
|
||||
if hasattr(asr_instance, 'receive_audio'):
|
||||
await asr_instance.receive_audio(context, audio, have_voice)
|
||||
|
||||
async def _resume_vad_detection(self, context: SessionContext):
|
||||
"""恢复VAD检测 - 完整迁移自resume_vad_detection"""
|
||||
# 等待1秒后恢复VAD检测
|
||||
await asyncio.sleep(1)
|
||||
context.just_woken_up = False
|
||||
|
||||
async def start_to_chat(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""开始聊天 - 完整迁移自startToChat"""
|
||||
# 检查输入是否是JSON格式（包含说话人信息）
|
||||
speaker_name = None
|
||||
actual_text = text
|
||||
|
||||
try:
|
||||
# 尝试解析JSON格式的输入
|
||||
if text.strip().startswith('{') and text.strip().endswith('}'):
|
||||
data = json.loads(text)
|
||||
if 'speaker' in data and 'content' in data:
|
||||
speaker_name = data['speaker']
|
||||
actual_text = data['content']
|
||||
logger.info(f"解析到说话人信息: {speaker_name}")
|
||||
|
||||
# 直接使用JSON格式的文本，不解析
|
||||
actual_text = text
|
||||
except (json.JSONDecodeError, KeyError):
|
||||
# 如果解析失败，继续使用原始文本
|
||||
pass
|
||||
|
||||
# 保存说话人信息到上下文
|
||||
if speaker_name:
|
||||
context.current_speaker = speaker_name
|
||||
else:
|
||||
context.current_speaker = None
|
||||
|
||||
# 检查设备绑定
|
||||
if context.need_bind:
|
||||
await self._check_bind_device(context, transport)
|
||||
return
|
||||
|
||||
# 如果当日的输出字数大于限定的字数
|
||||
if context.max_output_size > 0:
|
||||
if check_device_output_limit(
|
||||
context.headers.get("device-id"), context.max_output_size
|
||||
):
|
||||
await self._max_out_size(context, transport)
|
||||
return
|
||||
|
||||
if context.is_speaking:
|
||||
await self._handle_abort_message(context, transport)
|
||||
|
||||
# 首先进行意图分析，使用实际文本内容
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
intent_handled = await chat_processor.handle_user_intent(context, transport, actual_text)
|
||||
|
||||
if intent_handled:
|
||||
# 如果意图已被处理，不再进行聊天
|
||||
return
|
||||
|
||||
# 意图未被处理，继续常规聊天流程，使用实际文本内容
|
||||
await self._send_stt_message(context, transport, actual_text)
|
||||
|
||||
# 使用ChatProcessor处理聊天
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
await chat_processor.handle_chat(context, transport, actual_text)
|
||||
|
||||
async def _no_voice_close_connect(self, context: SessionContext, transport: TransportInterface, have_voice: bool):
|
||||
"""无声音时关闭连接检测 - 完整迁移自no_voice_close_connect"""
|
||||
if have_voice:
|
||||
context.update_activity()
|
||||
return
|
||||
|
||||
# 只有在已经初始化过时间戳的情况下才进行超时检查
|
||||
if context.last_activity_time_ms > 0.0:
|
||||
no_voice_time = time.time() * 1000 - context.last_activity_time_ms
|
||||
close_connection_no_voice_time = int(
|
||||
context.config.get("close_connection_no_voice_time", 120)
|
||||
)
|
||||
|
||||
if (
|
||||
not context.close_after_chat
|
||||
and no_voice_time > 1000 * close_connection_no_voice_time
|
||||
):
|
||||
context.close_after_chat = True
|
||||
context.abort_requested = False
|
||||
|
||||
end_prompt = context.config.get("end_prompt", {})
|
||||
if end_prompt and end_prompt.get("enable", True) is False:
|
||||
logger.info("结束对话，无需发送结束提示语")
|
||||
await transport.close()
|
||||
return
|
||||
|
||||
prompt = end_prompt.get("prompt")
|
||||
if not prompt:
|
||||
prompt = "请你以```时间过得真快```未来头，用富有感情、依依不舍的话来结束这场对话吧。！"
|
||||
await self.start_to_chat(context, transport, prompt)
|
||||
|
||||
async def _max_out_size(self, context: SessionContext, transport: TransportInterface):
|
||||
"""超出最大输出字数处理 - 完整迁移自max_out_size"""
|
||||
# 播放超出最大输出字数的提示
|
||||
context.abort_requested = False
|
||||
text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
file_path = "config/assets/max_output_size.wav"
|
||||
opus_packets = audio_to_data(file_path)
|
||||
|
||||
# 获取TTS组件并添加到队列
|
||||
tts_component = context.components.get('tts')
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
if hasattr(tts_instance, 'tts_audio_queue'):
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
tts_instance.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))
|
||||
|
||||
context.close_after_chat = True
|
||||
|
||||
async def _check_bind_device(self, context: SessionContext, transport: TransportInterface):
|
||||
"""检查设备绑定 - 完整迁移自check_bind_device"""
|
||||
bind_code = context.bind_code
|
||||
|
||||
if bind_code:
|
||||
# 确保bind_code是6位数字
|
||||
if len(bind_code) != 6:
|
||||
logger.error(f"无效的绑定码格式: {bind_code}")
|
||||
text = "绑定码格式错误，请检查配置。"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
return
|
||||
|
||||
text = f"请登录控制面板，输入{bind_code}，绑定设备。"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
# 获取TTS组件
|
||||
tts_component = context.components.get('tts')
|
||||
if not tts_component or not hasattr(tts_component, 'tts_instance'):
|
||||
return
|
||||
|
||||
tts_instance = tts_component.tts_instance
|
||||
if not hasattr(tts_instance, 'tts_audio_queue'):
|
||||
return
|
||||
|
||||
# 播放提示音
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
music_path = "config/assets/bind_code.wav"
|
||||
opus_packets = audio_to_data(music_path)
|
||||
tts_instance.tts_audio_queue.put((SentenceType.FIRST, opus_packets, text))
|
||||
|
||||
# 逐个播放数字
|
||||
for i in range(6): # 确保只播放6位数字
|
||||
try:
|
||||
digit = bind_code[i]
|
||||
num_path = f"config/assets/bind_code/{digit}.wav"
|
||||
num_packets = audio_to_data(num_path)
|
||||
tts_instance.tts_audio_queue.put((SentenceType.MIDDLE, num_packets, None))
|
||||
except Exception as e:
|
||||
logger.error(f"播放数字音频失败: {e}")
|
||||
continue
|
||||
tts_instance.tts_audio_queue.put((SentenceType.LAST, [], None))
|
||||
else:
|
||||
# 播放未绑定提示
|
||||
context.abort_requested = False
|
||||
text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
# 获取TTS组件
|
||||
tts_component = context.components.get('tts')
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
if hasattr(tts_instance, 'tts_audio_queue'):
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
music_path = "config/assets/bind_not_found.wav"
|
||||
opus_packets = audio_to_data(music_path)
|
||||
tts_instance.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))
|
||||
|
||||
async def _handle_abort_message(self, context: SessionContext, transport: TransportInterface):
|
||||
"""处理中断消息"""
|
||||
logger.info("Audio processor: Abort message received")
|
||||
context.abort_requested = True
|
||||
|
||||
# 清理队列
|
||||
await self._clear_queues(context)
|
||||
|
||||
# 打断客户端说话状态
|
||||
await transport.send(json.dumps({
|
||||
"type": "tts",
|
||||
"state": "stop",
|
||||
"session_id": context.session_id
|
||||
}))
|
||||
|
||||
# 清理说话状态
|
||||
context.is_speaking = False
|
||||
|
||||
async def _clear_queues(self, context: SessionContext):
|
||||
"""清理所有队列"""
|
||||
# 清理TTS音频队列
|
||||
tts_component = context.components.get('tts')
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
tts_instance = tts_component.tts_instance
|
||||
if hasattr(tts_instance, 'tts_audio_queue'):
|
||||
try:
|
||||
while not tts_instance.tts_audio_queue.empty():
|
||||
tts_instance.tts_audio_queue.get_nowait()
|
||||
except:
|
||||
pass
|
||||
|
||||
# 清理ASR音频队列
|
||||
context.clear_audio_buffer()
|
||||
|
||||
async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息"""
|
||||
await transport.send(json.dumps({
|
||||
"type": "stt",
|
||||
"text": text,
|
||||
"session_id": context.session_id
|
||||
}))
|
||||
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
import json
|
||||
import time
|
||||
import asyncio
|
||||
from typing import Any, List
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
from core.utils import textUtils
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AudioSendProcessor(MessageProcessor):
|
||||
"""音频发送处理器：完整迁移sendAudioHandle.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""这个处理器不直接处理消息，而是被其他处理器调用"""
|
||||
return False
|
||||
|
||||
async def send_audio_message(self, context: SessionContext, transport: TransportInterface,
|
||||
sentence_type: SentenceType, audios: bytes, text: str):
|
||||
"""发送音频消息 - 完整迁移自sendAudioMessage"""
|
||||
tts_component = context.components.get('tts')
|
||||
if not tts_component or not hasattr(tts_component, 'tts_instance'):
|
||||
return
|
||||
|
||||
tts_instance = tts_component.tts_instance
|
||||
|
||||
if hasattr(tts_instance, 'tts_audio_first_sentence') and tts_instance.tts_audio_first_sentence:
|
||||
logger.info(f"发送第一段语音: {text}")
|
||||
tts_instance.tts_audio_first_sentence = False
|
||||
await self.send_tts_message(context, transport, "start", None)
|
||||
|
||||
if sentence_type == SentenceType.FIRST:
|
||||
await self.send_tts_message(context, transport, "sentence_start", text)
|
||||
|
||||
await self.send_audio(context, transport, audios)
|
||||
|
||||
# 发送句子开始消息
|
||||
if sentence_type is not SentenceType.MIDDLE:
|
||||
logger.info(f"发送音频消息: {sentence_type}, {text}")
|
||||
|
||||
# 发送结束消息（如果是最后一个文本）
|
||||
if context.llm_finish_task and sentence_type == SentenceType.LAST:
|
||||
await self.send_tts_message(context, transport, "stop", None)
|
||||
context.is_speaking = False
|
||||
if context.close_after_chat:
|
||||
await transport.close()
|
||||
|
||||
async def send_audio(self, context: SessionContext, transport: TransportInterface,
|
||||
audios: bytes, frame_duration: int = 60):
|
||||
"""发送单个opus包，支持流控 - 完整迁移自sendAudio"""
|
||||
if audios is None or len(audios) == 0:
|
||||
return
|
||||
|
||||
if isinstance(audios, bytes):
|
||||
if context.abort_requested:
|
||||
return
|
||||
|
||||
context.update_activity()
|
||||
await transport.send(audios)
|
||||
await asyncio.sleep(frame_duration / 1000.0)
|
||||
elif isinstance(audios, list):
|
||||
for audio in audios:
|
||||
if context.abort_requested:
|
||||
break
|
||||
context.update_activity()
|
||||
await transport.send(audio)
|
||||
await asyncio.sleep(frame_duration / 1000.0)
|
||||
|
||||
async def send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息 - 完整迁移自send_stt_message"""
|
||||
await transport.send(json.dumps({
|
||||
"type": "stt",
|
||||
"text": text,
|
||||
"session_id": context.session_id
|
||||
}))
|
||||
logger.info(f"发送STT消息: {text}")
|
||||
|
||||
async def send_tts_message(self, context: SessionContext, transport: TransportInterface,
|
||||
state: str, text: str = None):
|
||||
"""发送TTS消息 - 完整迁移自send_tts_message"""
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": context.session_id
|
||||
}
|
||||
if text:
|
||||
message["text"] = text
|
||||
|
||||
await transport.send(json.dumps(message))
|
||||
logger.debug(f"发送TTS消息: state={state}, text={text}")
|
||||
|
||||
async def send_music_message(self, context: SessionContext, transport: TransportInterface,
|
||||
music_path: str, text: str):
|
||||
"""发送音乐消息 - 完整迁移自send_music_message"""
|
||||
from core.utils.util import audio_to_data
|
||||
|
||||
try:
|
||||
# 获取音频数据
|
||||
opus_packets = audio_to_data(music_path)
|
||||
if opus_packets:
|
||||
# 发送音乐开始消息
|
||||
await self.send_tts_message(context, transport, "start", text)
|
||||
|
||||
# 发送音频数据
|
||||
await self.send_audio(context, transport, opus_packets)
|
||||
|
||||
# 发送音乐结束消息
|
||||
await self.send_tts_message(context, transport, "stop", None)
|
||||
|
||||
logger.info(f"发送音乐: {music_path}")
|
||||
else:
|
||||
logger.warning(f"无法加载音乐文件: {music_path}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送音乐失败: {e}")
|
||||
|
||||
async def send_welcome_audio(self, context: SessionContext, transport: TransportInterface):
|
||||
"""发送欢迎音频"""
|
||||
welcome_audio_path = context.config.get("welcome_audio_path")
|
||||
if welcome_audio_path:
|
||||
await self.send_music_message(context, transport, welcome_audio_path, "欢迎使用小智助手")
|
||||
|
||||
async def send_goodbye_audio(self, context: SessionContext, transport: TransportInterface):
|
||||
"""发送告别音频"""
|
||||
goodbye_audio_path = context.config.get("goodbye_audio_path")
|
||||
if goodbye_audio_path:
|
||||
await self.send_music_message(context, transport, goodbye_audio_path, "再见，期待下次相遇")
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.auth import AuthMiddleware, AuthenticationError
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AuthProcessor(MessageProcessor):
|
||||
"""认证处理器：处理连接认证逻辑"""
|
||||
|
||||
def __init__(self):
|
||||
self.auth_middleware = None
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理认证相关逻辑"""
|
||||
# 如果已经认证，跳过
|
||||
if context.is_authenticated:
|
||||
return False
|
||||
|
||||
# 初始化认证中间件（延迟初始化）
|
||||
if self.auth_middleware is None:
|
||||
self.auth_middleware = AuthMiddleware(context.config)
|
||||
|
||||
# 检查是否为认证消息（通过headers进行认证）
|
||||
if context.headers:
|
||||
try:
|
||||
await self.auth_middleware.authenticate(context.headers)
|
||||
context.is_authenticated = True
|
||||
logger.info(f"设备认证成功: {context.device_id}")
|
||||
return False # 认证成功，继续处理其他消息
|
||||
except AuthenticationError as e:
|
||||
logger.error(f"设备认证失败: {e}")
|
||||
# 发送认证失败消息
|
||||
await transport.send("Authentication failed")
|
||||
await transport.close()
|
||||
return True # 认证失败，停止处理
|
||||
|
||||
# 如果没有认证信息，要求认证
|
||||
await transport.send("Authentication required")
|
||||
return True # 停止后续处理
|
||||
|
||||
@@ -0,0 +1,516 @@
|
||||
import json
|
||||
import uuid
|
||||
import asyncio
|
||||
from typing import Any, Dict
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.utils.dialogue import Message, Dialogue
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType
|
||||
from plugins_func.register import Action, ActionResponse
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ChatProcessor(MessageProcessor):
|
||||
"""聊天处理器：完整迁移intentHandler.py的所有功能"""
|
||||
|
||||
def __init__(self):
|
||||
# 会话对话历史管理
|
||||
self._dialogues: Dict[str, Dialogue] = {}
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理聊天消息"""
|
||||
# 这个处理器不直接处理原始消息，而是被其他处理器调用
|
||||
return False
|
||||
|
||||
async def handle_chat(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""处理聊天请求 - 完整迁移自handle_user_intent"""
|
||||
try:
|
||||
# 首先进行意图处理
|
||||
intent_handled = await self.handle_user_intent(context, transport, text)
|
||||
if intent_handled:
|
||||
return
|
||||
|
||||
# 如果意图未处理，进行常规聊天
|
||||
await self._regular_chat(context, transport, text)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理聊天失败: {e}")
|
||||
await self._send_error(transport, "聊天处理失败，请重试")
|
||||
|
||||
async def handle_user_intent(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""处理用户意图 - 完整迁移自intentHandler.py"""
|
||||
# 预处理输入文本，处理可能的JSON格式
|
||||
try:
|
||||
if text.strip().startswith('{') and text.strip().endswith('}'):
|
||||
parsed_data = json.loads(text)
|
||||
if isinstance(parsed_data, dict) and "content" in parsed_data:
|
||||
text = parsed_data["content"] # 提取content用于意图分析
|
||||
context.current_speaker = parsed_data.get("speaker") # 保留说话人信息
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
# 检查是否有明确的退出命令
|
||||
_, filtered_text = remove_punctuation_and_length(text)
|
||||
if await self._check_direct_exit(context, transport, filtered_text):
|
||||
return True
|
||||
|
||||
# 检查是否是唤醒词
|
||||
if await self._check_wakeup_words(context, transport, filtered_text):
|
||||
return True
|
||||
|
||||
if context.intent_type == "function_call":
|
||||
# 使用支持function calling的聊天方法,不再进行意图分析
|
||||
return False
|
||||
|
||||
# 使用LLM进行意图分析
|
||||
intent_result = await self._analyze_intent_with_llm(context, text)
|
||||
if not intent_result:
|
||||
return False
|
||||
|
||||
# 会话开始时生成sentence_id
|
||||
context.sentence_id = str(uuid.uuid4().hex)
|
||||
|
||||
# 处理各种意图
|
||||
return await self._process_intent_result(context, transport, intent_result, text)
|
||||
|
||||
def _get_dialogue(self, session_id: str) -> Dialogue:
|
||||
"""获取或创建对话历史"""
|
||||
if session_id not in self._dialogues:
|
||||
self._dialogues[session_id] = Dialogue()
|
||||
return self._dialogues[session_id]
|
||||
|
||||
async def _get_memory_context(self, context: SessionContext, query: str) -> str:
|
||||
"""获取记忆上下文"""
|
||||
try:
|
||||
memory_component = context.components.get('memory')
|
||||
if memory_component and hasattr(memory_component, 'memory_instance'):
|
||||
memory_instance = memory_component.memory_instance
|
||||
if hasattr(memory_instance, 'query_memory'):
|
||||
return await memory_instance.query_memory(query)
|
||||
except Exception as e:
|
||||
logger.warning(f"获取记忆上下文失败: {e}")
|
||||
|
||||
return None
|
||||
|
||||
async def _generate_llm_response(self, context: SessionContext, transport: TransportInterface,
|
||||
llm_instance, dialogue_context: list, dialogue: Dialogue):
|
||||
"""生成LLM回复"""
|
||||
try:
|
||||
# 初始化sentence_id并发送TTS FIRST标记（模拟原connection.py第692-700行）
|
||||
if not context.sentence_id:
|
||||
context.sentence_id = str(uuid.uuid4().hex)
|
||||
|
||||
# 发送TTS开始标记
|
||||
await self._send_tts_first_marker(context)
|
||||
|
||||
# 检查是否支持流式响应
|
||||
if hasattr(llm_instance, 'response'):
|
||||
# 使用流式响应
|
||||
response_generator = llm_instance.response(context.session_id, dialogue_context)
|
||||
|
||||
response_parts = []
|
||||
async for response_part in self._async_generator_wrapper(response_generator):
|
||||
if context.abort_requested:
|
||||
break
|
||||
|
||||
if response_part and len(response_part) > 0:
|
||||
response_parts.append(response_part)
|
||||
|
||||
# 原架构不发送流式响应给前端，直接进行TTS处理
|
||||
# 将响应片段放入TTS队列进行语音合成
|
||||
await self._process_response_part_for_tts(context, response_part)
|
||||
|
||||
# 完整回复
|
||||
full_response = "".join(response_parts)
|
||||
if full_response:
|
||||
# 添加助手回复到对话历史
|
||||
dialogue.put(Message(role="assistant", content=full_response))
|
||||
|
||||
# 原架构不发送response_complete给前端，只进行TTS处理
|
||||
# 发送TTS结束标记
|
||||
await self._finalize_tts_response(context, full_response)
|
||||
|
||||
logger.info(f"LLM回复完成: {full_response[:100]}...")
|
||||
|
||||
else:
|
||||
logger.warning("LLM实例不支持流式响应")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"生成LLM回复失败: {e}")
|
||||
await self._send_error(transport, "生成回复失败")
|
||||
|
||||
async def _async_generator_wrapper(self, generator):
|
||||
"""将同步生成器包装为异步生成器"""
|
||||
try:
|
||||
for item in generator:
|
||||
yield item
|
||||
# 让出控制权，避免阻塞事件循环
|
||||
await asyncio.sleep(0)
|
||||
except Exception as e:
|
||||
logger.error(f"生成器包装失败: {e}")
|
||||
|
||||
async def _send_tts_first_marker(self, context: SessionContext):
|
||||
"""发送TTS开始标记"""
|
||||
try:
|
||||
tts_component = context.components.get('tts')
|
||||
if not tts_component or not hasattr(tts_component, 'tts_instance'):
|
||||
return
|
||||
|
||||
tts_instance = tts_component.tts_instance
|
||||
if not tts_instance or not hasattr(tts_instance, 'tts_text_queue'):
|
||||
return
|
||||
|
||||
# 发送TTS开始标记（模拟原connection.py第694-700行）
|
||||
tts_instance.tts_text_queue.put(TTSMessageDTO(
|
||||
sentence_id=context.sentence_id,
|
||||
sentence_type=SentenceType.FIRST,
|
||||
content_type=ContentType.ACTION
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送TTS开始标记失败: {e}")
|
||||
|
||||
async def _process_response_part_for_tts(self, context: SessionContext, response_part: str):
|
||||
"""处理响应片段进行TTS - 模拟原架构逻辑"""
|
||||
try:
|
||||
tts_component = context.components.get('tts')
|
||||
if not tts_component or not hasattr(tts_component, 'tts_instance'):
|
||||
return
|
||||
|
||||
tts_instance = tts_component.tts_instance
|
||||
if not tts_instance or not hasattr(tts_instance, 'tts_text_queue'):
|
||||
return
|
||||
|
||||
# 将响应片段放入TTS队列（模拟原connection.py第782-789行逻辑）
|
||||
tts_instance.tts_text_queue.put(TTSMessageDTO(
|
||||
sentence_id=context.sentence_id,
|
||||
sentence_type=SentenceType.MIDDLE,
|
||||
content_type=ContentType.TEXT,
|
||||
content_detail=response_part
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理TTS响应片段失败: {e}")
|
||||
|
||||
async def _finalize_tts_response(self, context: SessionContext, full_response: str):
|
||||
"""完成TTS响应 - 发送结束标记"""
|
||||
try:
|
||||
tts_component = context.components.get('tts')
|
||||
if not tts_component or not hasattr(tts_component, 'tts_instance'):
|
||||
return
|
||||
|
||||
tts_instance = tts_component.tts_instance
|
||||
if not tts_instance or not hasattr(tts_instance, 'tts_text_queue'):
|
||||
return
|
||||
|
||||
# 发送TTS结束标记（模拟原speak_txt函数逻辑）
|
||||
tts_instance.tts_text_queue.put(TTSMessageDTO(
|
||||
sentence_id=context.sentence_id,
|
||||
sentence_type=SentenceType.LAST,
|
||||
content_type=ContentType.ACTION
|
||||
))
|
||||
|
||||
# 设置LLM完成标记
|
||||
context.llm_finish_task = True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"完成TTS响应失败: {e}")
|
||||
|
||||
async def _trigger_tts(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""触发TTS语音合成 - 完整迁移自原chat方法的TTS处理"""
|
||||
try:
|
||||
tts_component = context.components.get('tts')
|
||||
if not tts_component or not hasattr(tts_component, 'tts_instance'):
|
||||
logger.warning("TTS组件未初始化")
|
||||
return
|
||||
|
||||
tts_instance = tts_component.tts_instance
|
||||
|
||||
# 确保有sentence_id
|
||||
if not context.sentence_id:
|
||||
context.sentence_id = str(uuid.uuid4().hex)
|
||||
|
||||
logger.info(f"触发TTS合成: {text[:50]}...")
|
||||
|
||||
# 使用原来的TTS处理方式
|
||||
if hasattr(tts_instance, 'tts_text_queue') and hasattr(tts_instance, 'tts_one_sentence'):
|
||||
# 发送FIRST消息到TTS队列
|
||||
tts_instance.tts_text_queue.put(
|
||||
TTSMessageDTO(
|
||||
sentence_id=context.sentence_id,
|
||||
sentence_type=SentenceType.FIRST,
|
||||
content_type=ContentType.ACTION,
|
||||
)
|
||||
)
|
||||
|
||||
# 合成一句话
|
||||
tts_instance.tts_one_sentence(context, ContentType.TEXT, content_detail=text)
|
||||
|
||||
# 发送LAST消息到TTS队列
|
||||
tts_instance.tts_text_queue.put(
|
||||
TTSMessageDTO(
|
||||
sentence_id=context.sentence_id,
|
||||
sentence_type=SentenceType.LAST,
|
||||
content_type=ContentType.ACTION,
|
||||
)
|
||||
)
|
||||
|
||||
logger.info("TTS合成任务已提交到队列")
|
||||
else:
|
||||
logger.warning("TTS实例不支持队列处理")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"TTS合成失败: {e}")
|
||||
|
||||
async def _send_error(self, transport: TransportInterface, error_message: str):
|
||||
"""发送错误消息"""
|
||||
try:
|
||||
await transport.send(json.dumps({
|
||||
"type": "error",
|
||||
"message": error_message
|
||||
}))
|
||||
except Exception as e:
|
||||
logger.error(f"发送错误消息失败: {e}")
|
||||
|
||||
# === 意图处理相关方法：完整迁移自intentHandler.py ===
|
||||
|
||||
async def _check_direct_exit(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""检查是否有明确的退出命令 - 完整迁移自check_direct_exit"""
|
||||
_, text = remove_punctuation_and_length(text)
|
||||
cmd_exit = context.cmd_exit
|
||||
for cmd in cmd_exit:
|
||||
if text == cmd:
|
||||
logger.info(f"识别到明确的退出命令: {text}")
|
||||
await self._send_stt_message(context, transport, text)
|
||||
await transport.close()
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _check_wakeup_words(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""检查唤醒词 - 调用TextProcessor的方法"""
|
||||
# 这里需要调用TextProcessor的checkWakeupWords方法
|
||||
# 为了避免循环依赖，我们在这里实现简化版本
|
||||
_, filtered_text = remove_punctuation_and_length(text)
|
||||
if filtered_text in context.config.get("wakeup_words", []):
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _analyze_intent_with_llm(self, context: SessionContext, text: str):
|
||||
"""使用LLM分析用户意图 - 完整迁移自analyze_intent_with_llm"""
|
||||
intent_component = context.components.get('intent')
|
||||
if not intent_component or not hasattr(intent_component, 'intent_instance'):
|
||||
logger.warning("意图识别服务未初始化")
|
||||
return None
|
||||
|
||||
intent_instance = intent_component.intent_instance
|
||||
|
||||
# 对话历史记录
|
||||
dialogue = context.dialogue
|
||||
if not dialogue:
|
||||
return None
|
||||
|
||||
try:
|
||||
intent_result = await intent_instance.detect_intent(context, dialogue.dialogue, text)
|
||||
return intent_result
|
||||
except Exception as e:
|
||||
logger.error(f"意图识别失败: {str(e)}")
|
||||
|
||||
return None
|
||||
|
||||
async def _process_intent_result(self, context: SessionContext, transport: TransportInterface, intent_result: str, original_text: str):
|
||||
"""处理意图识别结果 - 完整迁移自process_intent_result"""
|
||||
try:
|
||||
# 尝试将结果解析为JSON
|
||||
intent_data = json.loads(intent_result)
|
||||
|
||||
# 检查是否有function_call
|
||||
if "function_call" in intent_data:
|
||||
# 直接从意图识别获取了function_call
|
||||
logger.debug(f"检测到function_call格式的意图结果: {intent_data['function_call']['name']}")
|
||||
function_name = intent_data["function_call"]["name"]
|
||||
if function_name == "continue_chat":
|
||||
return False
|
||||
|
||||
function_args = {}
|
||||
if "arguments" in intent_data["function_call"]:
|
||||
function_args = intent_data["function_call"]["arguments"]
|
||||
if function_args is None:
|
||||
function_args = {}
|
||||
|
||||
# 确保参数是字符串格式的JSON
|
||||
if isinstance(function_args, dict):
|
||||
function_args = json.dumps(function_args)
|
||||
|
||||
function_call_data = {
|
||||
"name": function_name,
|
||||
"id": str(uuid.uuid4().hex),
|
||||
"arguments": function_args,
|
||||
}
|
||||
|
||||
await self._send_stt_message(context, transport, original_text)
|
||||
context.abort_requested = False
|
||||
|
||||
# 使用executor执行函数调用和结果处理
|
||||
await self._process_function_call(context, transport, function_call_data, original_text)
|
||||
return True
|
||||
return False
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"处理意图结果时出错: {e}")
|
||||
return False
|
||||
|
||||
async def _process_function_call(self, context: SessionContext, transport: TransportInterface, function_call_data: dict, original_text: str):
|
||||
"""处理函数调用 - 完整迁移自process_function_call"""
|
||||
def process_function_call():
|
||||
# 添加用户消息到对话历史
|
||||
dialogue = context.dialogue
|
||||
if dialogue:
|
||||
dialogue.put(Message(role="user", content=original_text))
|
||||
|
||||
# 使用统一工具处理器处理所有工具调用
|
||||
try:
|
||||
func_handler = context.func_handler
|
||||
if not func_handler:
|
||||
raise Exception("函数处理器未初始化")
|
||||
|
||||
loop = context.loop
|
||||
result = asyncio.run_coroutine_threadsafe(
|
||||
func_handler.handle_llm_function_call(context, function_call_data),
|
||||
loop,
|
||||
).result()
|
||||
except Exception as e:
|
||||
logger.error(f"工具调用失败: {e}")
|
||||
result = ActionResponse(
|
||||
action=Action.ERROR, result=str(e), response=str(e)
|
||||
)
|
||||
|
||||
if result:
|
||||
function_name = function_call_data.get("name", "")
|
||||
if result.action == Action.RESPONSE: # 直接回复前端
|
||||
text = result.response
|
||||
if text is not None:
|
||||
self._speak_txt(context, text)
|
||||
elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
|
||||
text = result.result
|
||||
if dialogue:
|
||||
dialogue.put(Message(role="tool", content=text))
|
||||
intent_component = context.components.get('intent')
|
||||
if intent_component and hasattr(intent_component, 'intent_instance'):
|
||||
intent_instance = intent_component.intent_instance
|
||||
if hasattr(intent_instance, 'replyResult'):
|
||||
llm_result = intent_instance.replyResult(text, original_text)
|
||||
if llm_result is None:
|
||||
llm_result = text
|
||||
self._speak_txt(context, llm_result)
|
||||
elif (
|
||||
result.action == Action.NOTFOUND
|
||||
or result.action == Action.ERROR
|
||||
):
|
||||
text = result.result
|
||||
if text is not None:
|
||||
self._speak_txt(context, text)
|
||||
elif function_name != "play_music":
|
||||
# For backward compatibility with original code
|
||||
# 获取当前最新的文本索引
|
||||
text = result.response
|
||||
if text is None:
|
||||
text = result.result
|
||||
if text is not None:
|
||||
self._speak_txt(context, text)
|
||||
|
||||
# 将函数执行放在线程池中
|
||||
if context.executor:
|
||||
context.executor.submit(process_function_call)
|
||||
else:
|
||||
# 如果没有executor，直接执行
|
||||
process_function_call()
|
||||
|
||||
def _speak_txt(self, context: SessionContext, text: str):
|
||||
"""语音合成文本 - 完整迁移自speak_txt"""
|
||||
tts_component = context.components.get('tts')
|
||||
if not tts_component or not hasattr(tts_component, 'tts_instance'):
|
||||
return
|
||||
|
||||
tts_instance = tts_component.tts_instance
|
||||
sentence_id = context.sentence_id or str(uuid.uuid4().hex)
|
||||
|
||||
# 发送TTS消息队列
|
||||
if hasattr(tts_instance, 'tts_text_queue'):
|
||||
tts_instance.tts_text_queue.put(
|
||||
TTSMessageDTO(
|
||||
sentence_id=sentence_id,
|
||||
sentence_type=SentenceType.FIRST,
|
||||
content_type=ContentType.ACTION,
|
||||
)
|
||||
)
|
||||
|
||||
# 合成一句话
|
||||
if hasattr(tts_instance, 'tts_one_sentence'):
|
||||
tts_instance.tts_one_sentence(context, ContentType.TEXT, content_detail=text)
|
||||
|
||||
tts_instance.tts_text_queue.put(
|
||||
TTSMessageDTO(
|
||||
sentence_id=sentence_id,
|
||||
sentence_type=SentenceType.LAST,
|
||||
content_type=ContentType.ACTION,
|
||||
)
|
||||
)
|
||||
|
||||
# 添加到对话历史
|
||||
dialogue = context.dialogue
|
||||
if dialogue:
|
||||
dialogue.put(Message(role="assistant", content=text))
|
||||
|
||||
async def _regular_chat(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""常规聊天处理"""
|
||||
# 使用SessionContext的对话历史
|
||||
dialogue = context.dialogue
|
||||
if not dialogue:
|
||||
from core.utils.dialogue import Dialogue
|
||||
dialogue = Dialogue()
|
||||
context.dialogue = dialogue
|
||||
|
||||
# 获取LLM组件
|
||||
llm_component = context.components.get('llm')
|
||||
if not llm_component:
|
||||
await self._send_error(transport, "LLM组件未初始化")
|
||||
return
|
||||
|
||||
llm_instance = getattr(llm_component, 'llm_instance', None)
|
||||
if not llm_instance:
|
||||
await self._send_error(transport, "LLM实例未就绪")
|
||||
return
|
||||
|
||||
# 添加用户消息到对话历史
|
||||
dialogue.put(Message(role="user", content=text))
|
||||
|
||||
# 原架构不发送thinking状态给前端，直接开始处理
|
||||
|
||||
# 获取记忆上下文
|
||||
memory_context = await self._get_memory_context(context, text)
|
||||
|
||||
# 构建对话上下文
|
||||
dialogue_context = dialogue.get_llm_dialogue_with_memory(
|
||||
memory_context,
|
||||
context.config.get("voiceprint", {})
|
||||
)
|
||||
|
||||
# 调用LLM生成回复
|
||||
await self._generate_llm_response(context, transport, llm_instance, dialogue_context, dialogue)
|
||||
|
||||
async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息"""
|
||||
await transport.send(json.dumps({
|
||||
"type": "stt",
|
||||
"text": text,
|
||||
"session_id": context.session_id
|
||||
}))
|
||||
|
||||
def cleanup_session(self, session_id: str):
|
||||
"""清理会话对话历史"""
|
||||
if session_id in self._dialogues:
|
||||
del self._dialogues[session_id]
|
||||
logger.info(f"已清理会话对话历史: {session_id}")
|
||||
@@ -0,0 +1,206 @@
|
||||
import time
|
||||
import json
|
||||
import random
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.utils.dialogue import Message
|
||||
from core.utils.util import audio_to_data, remove_punctuation_and_length, opus_datas_to_wav_bytes
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
from core.utils.wakeup_word import WakeupWordsConfig
|
||||
from core.providers.tools.device_mcp import (
|
||||
MCPClient,
|
||||
send_mcp_initialize_message,
|
||||
send_mcp_tools_list_request,
|
||||
)
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
# 唤醒词配置
|
||||
WAKEUP_CONFIG = {
|
||||
"refresh_time": 5,
|
||||
"words": ["你好", "你好啊", "嘿，你好", "嗨"],
|
||||
}
|
||||
|
||||
# 创建全局的唤醒词配置管理器
|
||||
wakeup_words_config = WakeupWordsConfig()
|
||||
|
||||
# 用于防止并发调用wakeupWordsResponse的锁
|
||||
_wakeup_response_lock = asyncio.Lock()
|
||||
|
||||
|
||||
class HelloProcessor(MessageProcessor):
|
||||
"""Hello消息处理器：完整迁移helloHandle.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理hello类型的消息"""
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "hello":
|
||||
await self.handle_hello_message(context, transport, msg_json)
|
||||
return True
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def handle_hello_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理hello消息 - 完整迁移自handleHelloMessage"""
|
||||
# 处理音频参数
|
||||
audio_params = msg_json.get("audio_params")
|
||||
if audio_params:
|
||||
format = audio_params.get("format")
|
||||
logger.info(f"客户端音频格式: {format}")
|
||||
context.audio_format = format
|
||||
if not context.welcome_msg:
|
||||
context.welcome_msg = {}
|
||||
context.welcome_msg["audio_params"] = audio_params
|
||||
|
||||
# 处理客户端特性
|
||||
features = msg_json.get("features")
|
||||
if features:
|
||||
logger.info(f"客户端特性: {features}")
|
||||
context.features = features
|
||||
if features.get("mcp"):
|
||||
logger.info("客户端支持MCP")
|
||||
context.mcp_client = MCPClient()
|
||||
# 发送初始化 - 传递transport参数
|
||||
asyncio.create_task(send_mcp_initialize_message(context, transport))
|
||||
# 发送mcp消息，获取tools列表 - 传递transport参数
|
||||
asyncio.create_task(send_mcp_tools_list_request(context, transport))
|
||||
|
||||
# 发送欢迎消息
|
||||
if context.welcome_msg:
|
||||
await transport.send(json.dumps(context.welcome_msg))
|
||||
else:
|
||||
# 默认欢迎消息
|
||||
welcome_msg = {
|
||||
"type": "hello",
|
||||
"session_id": context.session_id,
|
||||
"version": 1,
|
||||
"transport": "websocket"
|
||||
}
|
||||
await transport.send(json.dumps(welcome_msg))
|
||||
|
||||
async def check_wakeup_words(self, context: SessionContext, transport: TransportInterface, text: str) -> bool:
|
||||
"""检查唤醒词 - 完整迁移自checkWakeupWords"""
|
||||
enable_wakeup_words_response_cache = context.config.get("enable_wakeup_words_response_cache", False)
|
||||
|
||||
# 等待tts初始化，最多等待3秒
|
||||
tts_component = context.components.get('tts')
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < 3:
|
||||
if tts_component and hasattr(tts_component, 'tts_instance'):
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
else:
|
||||
return False
|
||||
|
||||
if not enable_wakeup_words_response_cache:
|
||||
return False
|
||||
|
||||
_, filtered_text = remove_punctuation_and_length(text)
|
||||
if filtered_text not in context.config.get("wakeup_words", []):
|
||||
return False
|
||||
|
||||
context.just_woken_up = True
|
||||
await self._send_stt_message(context, transport, text)
|
||||
|
||||
# 获取当前音色
|
||||
tts_instance = getattr(tts_component, 'tts_instance', None) if tts_component else None
|
||||
voice = getattr(tts_instance, "voice", "default") if tts_instance else "default"
|
||||
if not voice:
|
||||
voice = "default"
|
||||
|
||||
# 获取唤醒词回复配置
|
||||
response = wakeup_words_config.get_wakeup_response(voice)
|
||||
if not response or not response.get("file_path"):
|
||||
response = {
|
||||
"voice": "default",
|
||||
"file_path": "config/assets/wakeup_words.wav",
|
||||
"time": 0,
|
||||
"text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",
|
||||
}
|
||||
|
||||
# 获取音频数据
|
||||
opus_packets = audio_to_data(response.get("file_path"))
|
||||
# 播放唤醒词回复
|
||||
context.abort_requested = False
|
||||
|
||||
logger.info(f"播放唤醒词回复: {response.get('text')}")
|
||||
await self._send_audio_message(context, transport, SentenceType.FIRST, opus_packets, response.get("text"))
|
||||
await self._send_audio_message(context, transport, SentenceType.LAST, [], None)
|
||||
|
||||
# 补充对话
|
||||
if context.dialogue:
|
||||
context.dialogue.put(Message(role="assistant", content=response.get("text")))
|
||||
|
||||
# 检查是否需要更新唤醒词回复
|
||||
if time.time() - response.get("time", 0) > WAKEUP_CONFIG["refresh_time"]:
|
||||
if not _wakeup_response_lock.locked():
|
||||
asyncio.create_task(self._wakeup_words_response(context, transport))
|
||||
return True
|
||||
|
||||
async def _wakeup_words_response(self, context: SessionContext, transport: TransportInterface):
|
||||
"""生成唤醒词回复 - 完整迁移自wakeupWordsResponse"""
|
||||
tts_component = context.components.get('tts')
|
||||
llm_component = context.components.get('llm')
|
||||
|
||||
tts_instance = getattr(tts_component, 'tts_instance', None) if tts_component else None
|
||||
llm_instance = getattr(llm_component, 'llm_instance', None) if llm_component else None
|
||||
|
||||
if not tts_instance or not llm_instance or not hasattr(llm_instance, 'response_no_stream'):
|
||||
return
|
||||
|
||||
try:
|
||||
# 尝试获取锁，如果获取不到就返回
|
||||
async with _wakeup_response_lock:
|
||||
# 生成唤醒词回复
|
||||
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
|
||||
question = (
|
||||
"此刻用户正在和你说```"
|
||||
+ wakeup_word
|
||||
+ "```。\n请你根据以上用户的内容进行20-30字回复。要符合系统设置的角色情感和态度，不要像机器人一样说话。\n"
|
||||
+ "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。"
|
||||
)
|
||||
|
||||
result = llm_instance.response_no_stream(context.config.get("prompt", ""), question)
|
||||
if not result or len(result) == 0:
|
||||
return
|
||||
|
||||
# 生成TTS音频
|
||||
tts_result = await asyncio.to_thread(tts_instance.to_tts, result)
|
||||
if not tts_result:
|
||||
return
|
||||
|
||||
# 获取当前音色
|
||||
voice = getattr(tts_instance, "voice", "default")
|
||||
|
||||
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
|
||||
file_path = wakeup_words_config.generate_file_path(voice)
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(wav_bytes)
|
||||
# 更新配置
|
||||
wakeup_words_config.update_wakeup_response(voice, file_path, result)
|
||||
except Exception as e:
|
||||
logger.error(f"生成唤醒词回复失败: {e}")
|
||||
|
||||
async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息"""
|
||||
await transport.send(json.dumps({
|
||||
"type": "stt",
|
||||
"text": text,
|
||||
"session_id": context.session_id
|
||||
}))
|
||||
|
||||
async def _send_audio_message(self, context: SessionContext, transport: TransportInterface,
|
||||
sentence_type: SentenceType, audios: bytes, text: str):
|
||||
"""发送音频消息"""
|
||||
# 这里应该调用AudioSendProcessor
|
||||
from core.processors.audio_send_processor import AudioSendProcessor
|
||||
audio_send_processor = AudioSendProcessor()
|
||||
await audio_send_processor.send_audio_message(context, transport, sentence_type, audios, text)
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.providers.tools.device_iot import handleIotStatus, handleIotDescriptors
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class IotProcessor(MessageProcessor):
|
||||
"""IoT消息处理器：完整迁移iotMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理iot类型的消息"""
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "iot":
|
||||
await self.handle_iot_message(context, transport, msg_json)
|
||||
return True
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def handle_iot_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理IoT消息 - 完整迁移自iotMessageHandler.py"""
|
||||
tasks = []
|
||||
|
||||
# 处理设备描述符 - 完整迁移原逻辑
|
||||
if "descriptors" in msg_json:
|
||||
logger.debug("处理IoT设备描述符")
|
||||
task = asyncio.create_task(
|
||||
self._handle_iot_descriptors(context, transport, msg_json["descriptors"])
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
# 处理设备状态 - 完整迁移原逻辑
|
||||
if "states" in msg_json:
|
||||
logger.debug("处理IoT设备状态")
|
||||
task = asyncio.create_task(
|
||||
self._handle_iot_status(context, transport, msg_json["states"])
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
# 如果没有有效的IoT数据
|
||||
if not tasks:
|
||||
logger.warning("IoT消息缺少descriptors或states字段")
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"IoT消息格式错误：缺少descriptors或states字段"
|
||||
)
|
||||
return
|
||||
|
||||
# 等待所有任务完成（可选，根据原逻辑决定）
|
||||
# await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
async def _handle_iot_descriptors(self, context: SessionContext, transport: TransportInterface, descriptors: Any):
|
||||
"""处理IoT设备描述符 - 包装原handleIotDescriptors函数"""
|
||||
try:
|
||||
# 调用原有的handleIotDescriptors函数
|
||||
# 注意：这里需要传入context而不是conn，因为handleIotDescriptors可能需要适配
|
||||
await handleIotDescriptors(context, descriptors)
|
||||
logger.debug("IoT设备描述符处理完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理IoT设备描述符失败: {e}", exc_info=True)
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"IoT设备描述符处理失败: {str(e)}"
|
||||
)
|
||||
|
||||
async def _handle_iot_status(self, context: SessionContext, transport: TransportInterface, states: Any):
|
||||
"""处理IoT设备状态 - 包装原handleIotStatus函数"""
|
||||
try:
|
||||
# 调用原有的handleIotStatus函数
|
||||
# 注意：这里需要传入context而不是conn，因为handleIotStatus可能需要适配
|
||||
await handleIotStatus(context, states)
|
||||
logger.debug("IoT设备状态处理完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理IoT设备状态失败: {e}", exc_info=True)
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"IoT设备状态处理失败: {str(e)}"
|
||||
)
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, session_id: str, message: str):
|
||||
"""发送IoT错误响应"""
|
||||
response = {
|
||||
"type": "iot",
|
||||
"status": "error",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送IoT错误响应失败: {e}")
|
||||
|
||||
async def _send_success_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, data: dict = None):
|
||||
"""发送IoT成功响应"""
|
||||
response = {
|
||||
"type": "iot",
|
||||
"status": "success",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if data:
|
||||
response["data"] = data
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送IoT成功响应失败: {e}")
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import time
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ListenProcessor(MessageProcessor):
|
||||
"""Listen消息处理器：完整迁移listenMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理listen类型的消息"""
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "listen":
|
||||
await self.handle_listen_message(context, transport, msg_json)
|
||||
return True
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def handle_listen_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理listen消息 - 完整迁移自listenMessageHandler.py"""
|
||||
# 设置拾音模式
|
||||
if "mode" in msg_json:
|
||||
context.listen_mode = msg_json["mode"]
|
||||
logger.debug(f"客户端拾音模式：{context.listen_mode}")
|
||||
|
||||
# 处理不同的状态
|
||||
state = msg_json.get("state")
|
||||
|
||||
if state == "start":
|
||||
# 开始监听语音
|
||||
context.client_have_voice = True
|
||||
context.client_voice_stop = False
|
||||
logger.debug("开始语音监听")
|
||||
|
||||
elif state == "stop":
|
||||
# 停止监听语音
|
||||
context.client_have_voice = True
|
||||
context.client_voice_stop = True
|
||||
# 如果有音频数据，处理最后的音频
|
||||
if len(context.asr_audio) > 0:
|
||||
await self._handle_audio_message(context, transport, b"")
|
||||
logger.debug("停止语音监听")
|
||||
|
||||
elif state == "detect":
|
||||
# 检测到文本输入
|
||||
context.client_have_voice = False
|
||||
context.asr_audio.clear()
|
||||
|
||||
if "text" in msg_json:
|
||||
context.update_activity()
|
||||
original_text = msg_json["text"] # 保留原始文本
|
||||
filtered_len, filtered_text = remove_punctuation_and_length(original_text)
|
||||
|
||||
# 识别是否是唤醒词
|
||||
is_wakeup_words = filtered_text in context.config.get("wakeup_words", [])
|
||||
# 是否开启唤醒词回复
|
||||
enable_greeting = context.config.get("enable_greeting", True)
|
||||
|
||||
if is_wakeup_words and not enable_greeting:
|
||||
# 如果是唤醒词，且关闭了唤醒词回复，就不用回答
|
||||
await self._send_stt_message(context, transport, original_text)
|
||||
await self._send_tts_message(context, transport, "stop", None)
|
||||
context.is_speaking = False
|
||||
|
||||
elif is_wakeup_words:
|
||||
# 处理唤醒词
|
||||
context.just_woken_up = True
|
||||
# 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
|
||||
await self._enqueue_asr_report(context, "嘿，你好呀", [])
|
||||
await self._start_to_chat(context, transport, "嘿，你好呀")
|
||||
|
||||
else:
|
||||
# 处理普通文本
|
||||
# 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
|
||||
await self._enqueue_asr_report(context, original_text, [])
|
||||
# 否则需要LLM对文字内容进行答复
|
||||
await self._start_to_chat(context, transport, original_text)
|
||||
|
||||
async def _handle_audio_message(self, context: SessionContext, transport: TransportInterface, audio: bytes):
|
||||
"""处理音频消息 - 调用AudioReceiveProcessor"""
|
||||
# 这里应该调用AudioReceiveProcessor来处理音频
|
||||
from core.processors.audio_receive_processor import AudioReceiveProcessor
|
||||
audio_processor = AudioReceiveProcessor()
|
||||
await audio_processor.handle_audio_message(context, transport, audio)
|
||||
|
||||
async def _send_stt_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""发送STT消息"""
|
||||
await transport.send(json.dumps({
|
||||
"type": "stt",
|
||||
"text": text,
|
||||
"session_id": context.session_id
|
||||
}))
|
||||
logger.info(f"发送STT消息: {text}")
|
||||
|
||||
async def _send_tts_message(self, context: SessionContext, transport: TransportInterface, state: str, text: str = None):
|
||||
"""发送TTS消息"""
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": context.session_id
|
||||
}
|
||||
if text:
|
||||
message["text"] = text
|
||||
|
||||
await transport.send(json.dumps(message))
|
||||
logger.debug(f"发送TTS消息: state={state}, text={text}")
|
||||
|
||||
async def _enqueue_asr_report(self, context: SessionContext, text: str, audio_data: list):
|
||||
"""ASR上报队列"""
|
||||
if context.report_asr_enable:
|
||||
from core.processors.report_processor import ReportProcessor
|
||||
report_processor = ReportProcessor()
|
||||
report_processor.enqueue_asr_report(context, text, audio_data)
|
||||
|
||||
async def _start_to_chat(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""开始聊天 - 调用ChatProcessor"""
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
await chat_processor.handle_chat(context, transport, text)
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.providers.tools.device_mcp import handle_mcp_message
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class McpProcessor(MessageProcessor):
|
||||
"""MCP消息处理器：完整迁移mcpMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理mcp类型的消息"""
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "mcp":
|
||||
await self.handle_mcp_message(context, transport, msg_json)
|
||||
return True
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def handle_mcp_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理MCP消息 - 完整迁移自mcpMessageHandler.py"""
|
||||
if "payload" in msg_json:
|
||||
# 检查MCP客户端是否存在
|
||||
if not context.mcp_client:
|
||||
logger.warning("MCP客户端未初始化，无法处理MCP消息")
|
||||
await self._send_error_response(transport, context.session_id, "MCP客户端未初始化")
|
||||
return
|
||||
|
||||
# 创建异步任务处理MCP消息 - 完整迁移原逻辑
|
||||
asyncio.create_task(
|
||||
self._handle_mcp_payload(context, transport, msg_json["payload"])
|
||||
)
|
||||
else:
|
||||
logger.warning("MCP消息缺少payload字段")
|
||||
await self._send_error_response(transport, context.session_id, "MCP消息格式错误：缺少payload")
|
||||
|
||||
async def _handle_mcp_payload(self, context: SessionContext, transport: TransportInterface, payload: dict):
|
||||
"""处理MCP payload - 包装原handle_mcp_message函数"""
|
||||
try:
|
||||
# 调用原有的handle_mcp_message函数
|
||||
# 注意：这里需要传入context而不是conn，因为handle_mcp_message可能需要适配
|
||||
await handle_mcp_message(context, context.mcp_client, payload, transport)
|
||||
logger.debug("MCP消息处理完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理MCP消息失败: {e}", exc_info=True)
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"MCP消息处理失败: {str(e)}"
|
||||
)
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, session_id: str, message: str):
|
||||
"""发送MCP错误响应"""
|
||||
response = {
|
||||
"type": "mcp",
|
||||
"status": "error",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送MCP错误响应失败: {e}")
|
||||
|
||||
async def _send_success_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, data: dict = None):
|
||||
"""发送MCP成功响应"""
|
||||
response = {
|
||||
"type": "mcp",
|
||||
"status": "success",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if data:
|
||||
response["data"] = data
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(response))
|
||||
except Exception as e:
|
||||
logger.error(f"发送MCP成功响应失败: {e}")
|
||||
@@ -0,0 +1,114 @@
|
||||
import json
|
||||
from typing import Any, List
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from core.processors.hello_processor import HelloProcessor
|
||||
from core.processors.listen_processor import ListenProcessor
|
||||
from core.processors.audio_receive_processor import AudioReceiveProcessor
|
||||
from core.processors.auth_processor import AuthProcessor
|
||||
from core.processors.timeout_processor import TimeoutProcessor
|
||||
from core.processors.server_processor import ServerProcessor
|
||||
from core.processors.mcp_processor import McpProcessor
|
||||
from core.processors.iot_processor import IotProcessor
|
||||
from core.processors.abort_processor import AbortProcessor
|
||||
from core.processors.text_processor import TextProcessor
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class MessageRouter(MessageProcessor):
|
||||
"""
|
||||
消息路由器：协调所有独立的processor
|
||||
按功能职责分离，避免耦合，每个processor专注单一职责
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# 初始化所有独立的processor
|
||||
self.auth_processor = AuthProcessor()
|
||||
self.timeout_processor = TimeoutProcessor()
|
||||
self.abort_processor = AbortProcessor()
|
||||
self.hello_processor = HelloProcessor()
|
||||
self.listen_processor = ListenProcessor()
|
||||
self.server_processor = ServerProcessor()
|
||||
self.mcp_processor = McpProcessor()
|
||||
self.iot_processor = IotProcessor()
|
||||
self.audio_receive_processor = AudioReceiveProcessor()
|
||||
self.text_processor = TextProcessor()
|
||||
|
||||
# 按优先级排序的processor列表
|
||||
self.processors: List[MessageProcessor] = [
|
||||
self.timeout_processor, # 首先检查超时
|
||||
self.auth_processor, # 然后检查认证
|
||||
self.abort_processor, # 中断消息
|
||||
self.hello_processor, # hello消息
|
||||
self.listen_processor, # listen消息
|
||||
self.server_processor, # 服务器消息
|
||||
self.mcp_processor, # MCP消息
|
||||
self.iot_processor, # IoT消息
|
||||
self.audio_receive_processor, # 音频消息
|
||||
self.text_processor, # 纯文本消息（放在最后，作为兜底处理）
|
||||
]
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""
|
||||
路由消息到合适的processor
|
||||
每个processor专注处理自己的消息类型，避免耦合
|
||||
"""
|
||||
# 更新活动时间
|
||||
context.update_activity()
|
||||
|
||||
# 按优先级顺序尝试每个processor
|
||||
for processor in self.processors:
|
||||
try:
|
||||
if await processor.process(context, transport, message):
|
||||
# 消息已被处理，记录日志并返回
|
||||
logger.debug(f"消息被 {processor.__class__.__name__} 处理")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"{processor.__class__.__name__} 处理消息时出错: {e}", exc_info=True)
|
||||
continue
|
||||
|
||||
# 如果没有processor处理该消息，记录警告
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
msg_type = msg_json.get("type", "unknown") if isinstance(msg_json, dict) else "non-dict"
|
||||
logger.warning(f"未处理的消息类型: {msg_type}, 内容: {message[:100]}...")
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(f"未处理的非JSON消息: {message[:100]}...")
|
||||
elif isinstance(message, bytes):
|
||||
logger.warning(f"未处理的二进制消息，大小: {len(message)} bytes")
|
||||
else:
|
||||
logger.warning(f"未处理的消息类型: {type(message)}")
|
||||
|
||||
return False
|
||||
|
||||
def add_processor(self, processor: MessageProcessor, priority: int = None):
|
||||
"""
|
||||
添加新的processor
|
||||
priority: 优先级，数字越小优先级越高，None表示添加到末尾
|
||||
"""
|
||||
if priority is None:
|
||||
self.processors.append(processor)
|
||||
else:
|
||||
self.processors.insert(priority, processor)
|
||||
logger.info(f"添加processor: {processor.__class__.__name__}")
|
||||
|
||||
def remove_processor(self, processor_class):
|
||||
"""移除指定类型的processor"""
|
||||
self.processors = [p for p in self.processors if not isinstance(p, processor_class)]
|
||||
logger.info(f"移除processor: {processor_class.__name__}")
|
||||
|
||||
def get_processor(self, processor_class):
|
||||
"""获取指定类型的processor"""
|
||||
for processor in self.processors:
|
||||
if isinstance(processor, processor_class):
|
||||
return processor
|
||||
return None
|
||||
|
||||
def list_processors(self) -> List[str]:
|
||||
"""列出所有processor的名称"""
|
||||
return [processor.__class__.__name__ for processor in self.processors]
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import time
|
||||
import queue
|
||||
import threading
|
||||
from typing import Any, List
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.manage_api_client import report as manage_report
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ReportProcessor(MessageProcessor):
|
||||
"""上报处理器：完整迁移reportHandle.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""这个处理器不直接处理消息，而是被其他处理器调用"""
|
||||
return False
|
||||
|
||||
def enqueue_asr_report(self, context: SessionContext, text: str, audio_data: List[bytes]):
|
||||
"""ASR上报队列 - 完整迁移自enqueue_asr_report"""
|
||||
if not context.report_asr_enable:
|
||||
return
|
||||
|
||||
report_time = int(time.time())
|
||||
|
||||
# 将上报任务放入队列
|
||||
context.report_queue.put({
|
||||
"type": 1, # 用户类型
|
||||
"text": text,
|
||||
"audio_data": audio_data,
|
||||
"report_time": report_time
|
||||
})
|
||||
|
||||
# 确保上报线程已启动
|
||||
self._ensure_report_thread(context)
|
||||
|
||||
def enqueue_tts_report(self, context: SessionContext, text: str, opus_data: bytes):
|
||||
"""TTS上报队列 - 完整迁移自enqueue_tts_report"""
|
||||
if not context.report_tts_enable:
|
||||
return
|
||||
|
||||
report_time = int(time.time())
|
||||
|
||||
# 将上报任务放入队列
|
||||
context.report_queue.put({
|
||||
"type": 2, # 智能体类型
|
||||
"text": text,
|
||||
"audio_data": opus_data,
|
||||
"report_time": report_time
|
||||
})
|
||||
|
||||
# 确保上报线程已启动
|
||||
self._ensure_report_thread(context)
|
||||
|
||||
def _ensure_report_thread(self, context: SessionContext):
|
||||
"""确保上报线程已启动"""
|
||||
if context.report_thread is None or not context.report_thread.is_alive():
|
||||
context.report_thread = threading.Thread(
|
||||
target=self._report_worker,
|
||||
args=(context,),
|
||||
daemon=True
|
||||
)
|
||||
context.report_thread.start()
|
||||
logger.info(f"上报线程已启动: {context.session_id}")
|
||||
|
||||
def _report_worker(self, context: SessionContext):
|
||||
"""上报工作线程 - 完整迁移自ConnectionHandler中的上报逻辑"""
|
||||
logger.info(f"上报工作线程启动: {context.session_id}")
|
||||
|
||||
while not context.stop_event.is_set():
|
||||
try:
|
||||
# 从队列获取上报任务
|
||||
report_task = context.report_queue.get(timeout=1)
|
||||
|
||||
# 执行上报
|
||||
self._execute_report(context, report_task)
|
||||
|
||||
except queue.Empty:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f"上报工作线程异常: {e}")
|
||||
|
||||
logger.info(f"上报工作线程退出: {context.session_id}")
|
||||
|
||||
def _execute_report(self, context: SessionContext, report_task: dict):
|
||||
"""执行聊天记录上报操作 - 完整迁移自report函数"""
|
||||
try:
|
||||
report_type = report_task["type"]
|
||||
text = report_task["text"]
|
||||
audio_data = report_task["audio_data"]
|
||||
report_time = report_task["report_time"]
|
||||
|
||||
# 处理音频数据
|
||||
processed_audio = None
|
||||
if audio_data:
|
||||
if isinstance(audio_data, list):
|
||||
# ASR音频数据（多个音频片段）
|
||||
processed_audio = self._process_asr_audio(audio_data)
|
||||
elif isinstance(audio_data, bytes):
|
||||
# TTS音频数据（opus格式）
|
||||
processed_audio = self._opus_to_wav(audio_data)
|
||||
|
||||
# 执行上报
|
||||
manage_report(
|
||||
mac_address=context.device_id,
|
||||
session_id=context.session_id,
|
||||
chat_type=report_type,
|
||||
content=text,
|
||||
audio=processed_audio,
|
||||
report_time=report_time,
|
||||
)
|
||||
|
||||
logger.debug(f"上报成功: type={report_type}, text={text[:50]}...")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"聊天记录上报失败: {e}")
|
||||
|
||||
def _process_asr_audio(self, audio_data_list: List[bytes]) -> bytes:
|
||||
"""处理ASR音频数据"""
|
||||
try:
|
||||
# 将多个音频片段合并
|
||||
combined_audio = b''.join(audio_data_list)
|
||||
return combined_audio
|
||||
except Exception as e:
|
||||
logger.error(f"处理ASR音频数据失败: {e}")
|
||||
return b''
|
||||
|
||||
def _opus_to_wav(self, opus_data: bytes) -> bytes:
|
||||
"""将Opus数据转换为WAV格式的字节流 - 完整迁移自opus_to_wav"""
|
||||
try:
|
||||
import opuslib_next
|
||||
import io
|
||||
import wave
|
||||
|
||||
# Opus解码器配置
|
||||
sample_rate = 16000
|
||||
channels = 1
|
||||
|
||||
# 创建Opus解码器
|
||||
decoder = opuslib_next.Decoder(sample_rate, channels)
|
||||
|
||||
# 解码Opus数据
|
||||
pcm_data = decoder.decode(opus_data, frame_size=960)
|
||||
|
||||
# 创建WAV文件
|
||||
wav_buffer = io.BytesIO()
|
||||
with wave.open(wav_buffer, 'wb') as wav_file:
|
||||
wav_file.setnchannels(channels)
|
||||
wav_file.setsampwidth(2) # 16-bit
|
||||
wav_file.setframerate(sample_rate)
|
||||
wav_file.writeframes(pcm_data)
|
||||
|
||||
return wav_buffer.getvalue()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Opus转WAV失败: {e}")
|
||||
return b''
|
||||
|
||||
def cleanup_session(self, context: SessionContext):
|
||||
"""清理会话上报资源"""
|
||||
# 停止上报线程
|
||||
if context.report_thread and context.report_thread.is_alive():
|
||||
context.stop_event.set()
|
||||
context.report_thread.join(timeout=5)
|
||||
|
||||
# 清理上报队列
|
||||
try:
|
||||
while not context.report_queue.empty():
|
||||
context.report_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
pass
|
||||
|
||||
logger.info(f"上报资源清理完成: {context.session_id}")
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ServerProcessor(MessageProcessor):
|
||||
"""服务器消息处理器：完整迁移serverMessageHandler.py的所有功能"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理server类型的消息"""
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, dict) and msg_json.get("type") == "server":
|
||||
await self.handle_server_message(context, transport, msg_json)
|
||||
return True
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return False
|
||||
|
||||
async def handle_server_message(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理server消息 - 完整迁移自serverMessageHandler.py"""
|
||||
# 如果配置是从API读取的，则需要验证secret
|
||||
if not context.read_config_from_api:
|
||||
return
|
||||
|
||||
# 获取post请求的secret
|
||||
post_secret = msg_json.get("content", {}).get("secret", "")
|
||||
secret = context.config.get("manager-api", {}).get("secret", "")
|
||||
|
||||
# 如果secret不匹配，则返回
|
||||
if post_secret != secret:
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"服务器密钥验证失败"
|
||||
)
|
||||
return
|
||||
|
||||
# 处理不同的action
|
||||
action = msg_json.get("action")
|
||||
|
||||
if action == "update_config":
|
||||
await self._handle_update_config(context, transport, msg_json)
|
||||
elif action == "restart":
|
||||
await self._handle_restart(context, transport, msg_json)
|
||||
else:
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"未知的服务器操作: {action}"
|
||||
)
|
||||
|
||||
async def _handle_update_config(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理配置更新 - 完整迁移自update_config逻辑"""
|
||||
try:
|
||||
# 检查是否有服务器实例
|
||||
if not context.server:
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"无法获取服务器实例",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
return
|
||||
|
||||
# 更新WebSocketServer的配置
|
||||
if not await context.server.update_config():
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"更新服务器配置失败",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
return
|
||||
|
||||
# 发送成功响应
|
||||
await self._send_success_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"配置更新成功",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"更新配置失败: {str(e)}")
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"更新配置失败: {str(e)}",
|
||||
{"action": "update_config"}
|
||||
)
|
||||
|
||||
async def _handle_restart(self, context: SessionContext, transport: TransportInterface, msg_json: dict):
|
||||
"""处理服务器重启 - 完整迁移自handle_restart逻辑"""
|
||||
try:
|
||||
# 这里应该调用context的handle_restart方法
|
||||
if hasattr(context, 'handle_restart'):
|
||||
await context.handle_restart(msg_json)
|
||||
else:
|
||||
logger.warning("SessionContext没有handle_restart方法")
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
"重启功能暂不可用",
|
||||
{"action": "restart"}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"处理重启请求失败: {str(e)}")
|
||||
await self._send_error_response(
|
||||
transport,
|
||||
context.session_id,
|
||||
f"重启失败: {str(e)}",
|
||||
{"action": "restart"}
|
||||
)
|
||||
|
||||
async def _send_success_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, content: dict = None):
|
||||
"""发送成功响应"""
|
||||
response = {
|
||||
"type": "server",
|
||||
"status": "success",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if content:
|
||||
response["content"] = content
|
||||
|
||||
await transport.send(json.dumps(response))
|
||||
logger.info(f"服务器操作成功: {message}")
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, session_id: str,
|
||||
message: str, content: dict = None):
|
||||
"""发送错误响应"""
|
||||
response = {
|
||||
"type": "server",
|
||||
"status": "error",
|
||||
"message": message,
|
||||
"session_id": session_id
|
||||
}
|
||||
if content:
|
||||
response["content"] = content
|
||||
|
||||
await transport.send(json.dumps(response))
|
||||
logger.error(f"服务器操作失败: {message}")
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TextProcessor(MessageProcessor):
|
||||
"""
|
||||
纯文本消息处理器：处理非JSON格式的文本消息
|
||||
这是新架构中缺失的重要组件，用于处理直接发送的文本聊天内容
|
||||
"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""处理纯文本消息"""
|
||||
if isinstance(message, str):
|
||||
try:
|
||||
# 尝试解析为JSON，如果成功则不是纯文本消息
|
||||
json.loads(message)
|
||||
return False # JSON消息由其他processor处理
|
||||
except json.JSONDecodeError:
|
||||
# 确实是纯文本消息，进行聊天处理
|
||||
await self.handle_text_message(context, transport, message)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def handle_text_message(self, context: SessionContext, transport: TransportInterface, text: str):
|
||||
"""处理纯文本消息 - 直接调用ChatProcessor进行聊天"""
|
||||
try:
|
||||
# 记录收到纯文本消息
|
||||
logger.info(f"收到纯文本消息: {text[:100]}...")
|
||||
|
||||
# 使用ChatProcessor处理聊天
|
||||
from core.processors.chat_processor import ChatProcessor
|
||||
chat_processor = ChatProcessor()
|
||||
await chat_processor.handle_chat(context, transport, text)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理纯文本消息失败: {e}")
|
||||
# 发送错误响应
|
||||
await self._send_error_response(transport, "文本处理失败，请重试")
|
||||
|
||||
async def _send_error_response(self, transport: TransportInterface, error_message: str):
|
||||
"""发送错误响应"""
|
||||
try:
|
||||
await transport.send(json.dumps({
|
||||
"type": "error",
|
||||
"message": error_message
|
||||
}))
|
||||
except Exception as e:
|
||||
logger.error(f"发送错误响应失败: {e}")
|
||||
@@ -0,0 +1,41 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from core.pipeline.message_pipeline import MessageProcessor
|
||||
from core.context.session_context import SessionContext
|
||||
from core.transport.transport_interface import TransportInterface
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TimeoutProcessor(MessageProcessor):
|
||||
"""超时检查处理器：检查会话是否超时"""
|
||||
|
||||
async def process(self, context: SessionContext, transport: TransportInterface, message: Any) -> bool:
|
||||
"""检查会话超时"""
|
||||
# 更新活动时间（在其他处理器中已更新，这里只检查）
|
||||
|
||||
# 获取超时配置
|
||||
timeout_seconds = context.config.get("close_connection_no_voice_time", 120)
|
||||
|
||||
# 检查是否超时
|
||||
if context.is_timeout(timeout_seconds):
|
||||
logger.info(f"会话超时，准备关闭连接: {context.session_id}")
|
||||
|
||||
# 发送超时通知
|
||||
timeout_msg = {
|
||||
"type": "timeout",
|
||||
"message": "连接超时，即将关闭",
|
||||
"session_id": context.session_id
|
||||
}
|
||||
|
||||
try:
|
||||
await transport.send(json.dumps(timeout_msg))
|
||||
await transport.close()
|
||||
except Exception as e:
|
||||
logger.error(f"发送超时消息失败: {e}")
|
||||
|
||||
return True # 消息已处理，停止后续处理
|
||||
|
||||
return False # 未超时，继续处理
|
||||
|
||||
Reference in New Issue
Block a user