update: 增加唤醒时声纹处理，1秒内发送至大模型【需优化唤醒锁机制，中途会遭受打断(偶发)，考虑忽略检测】

2026-07-26 00:53:54 +08:00 · 2025-08-28 17:58:23 +08:00
parent 91cd843cfe
commit 41b8fac3aa
10 changed files with 126 additions and 519 deletions
@@ -1,16 +1,5 @@
-import time
 import json
-import random
 import asyncio
-from core.utils.dialogue import Message
-from core.providers.tts.dto.dto import SentenceType
-from core.utils.wakeup_word import WakeupWordsConfig
-from core.handle.sendAudioHandle import sendAudioMessage, send_stt_message
-from core.utils.util import (
-    audio_to_data_stream,
-    remove_punctuation_and_length, 
-    opus_datas_to_wav_bytes
-)
 from core.providers.tools.device_mcp import (
    MCPClient,
    send_mcp_initialize_message,
@@ -19,18 +8,6 @@ from core.providers.tools.device_mcp import (

 TAG = __name__

-WAKEUP_CONFIG = {
-    "refresh_time": 5,
-    "words": ["你好", "你好啊", "嘿，你好", "嗨"],
-}
-
-# 创建全局的唤醒词配置管理器
-wakeup_words_config = WakeupWordsConfig()
-
-# 用于防止并发调用wakeupWordsResponse的锁
-_wakeup_response_lock = asyncio.Lock()
-
-
 async def handleHelloMessage(conn, msg_json):
    """处理hello消息"""
    audio_params = msg_json.get("audio_params")
@@ -51,100 +28,4 @@ async def handleHelloMessage(conn, msg_json):
            # 发送mcp消息，获取tools列表
            asyncio.create_task(send_mcp_tools_list_request(conn))

-    await conn.websocket.send(json.dumps(conn.welcome_msg))
-
-
-async def checkWakeupWords(conn, text):
-    enable_wakeup_words_response_cache = conn.config[
-        "enable_wakeup_words_response_cache"
-    ]
-
-    if not enable_wakeup_words_response_cache or not conn.tts:
-        return False
-
-    _, filtered_text = remove_punctuation_and_length(text)
-    if filtered_text not in conn.config.get("wakeup_words"):
-        return False
-
-    conn.just_woken_up = True
-    await send_stt_message(conn, text)
-
-    # 获取当前音色
-    voice = getattr(conn.tts, "voice", "default")
-    if not voice:
-        voice = "default"
-
-    # 获取唤醒词回复配置
-    response = wakeup_words_config.get_wakeup_response(voice)
-    if not response or not response.get("file_path"):
-        response = {
-            "voice": "default",
-            "file_path": "config/assets/wakeup_words.wav",
-            "time": 0,
-            "text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",
-        }
-    
-    # 获取音频数据
-    opus_packets = []
-    def handle_audio_frame(frame_data):
-        opus_packets.append(frame_data)
-    
-    audio_to_data_stream(response.get("file_path"), is_opus=True, callback=handle_audio_frame)
-
-    # 播放唤醒词回复
-    conn.client_abort = False
-
-    conn.logger.bind(tag=TAG).info(f"播放唤醒词回复: {response.get('text')}")
-    await sendAudioMessage(conn, SentenceType.FIRST, opus_packets, response.get("text"))
-    await sendAudioMessage(conn, SentenceType.LAST, [], None)
-
-    # 补充对话
-    conn.dialogue.put(Message(role="assistant", content=response.get("text")))
-
-    # 检查是否需要更新唤醒词回复
-    if time.time() - response.get("time", 0) > WAKEUP_CONFIG["refresh_time"]:
-        if not _wakeup_response_lock.locked():
-            asyncio.create_task(wakeupWordsResponse(conn))
-    return True
-
-
-async def wakeupWordsResponse(conn):
-    if not conn.tts or not conn.llm or not conn.llm.response_no_stream:
-        return
-
-    try:
-        # 尝试获取锁，如果获取不到就返回
-        if not await _wakeup_response_lock.acquire():
-            return
-
-        # 生成唤醒词回复
-        wakeup_word = random.choice(WAKEUP_CONFIG["words"])
-        question = (
-            "此刻用户正在和你说```"
-            + wakeup_word
-            + "```。\n请你根据以上用户的内容进行20-30字回复。要符合系统设置的角色情感和态度，不要像机器人一样说话。\n"
-            + "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。"
-        )
-
-        result = conn.llm.response_no_stream(conn.config["prompt"], question)
-        if not result or len(result) == 0:
-            return
-
-        # 生成TTS音频
-        tts_result = await asyncio.to_thread(conn.tts.to_tts, result)
-        if not tts_result:
-            return
-
-        # 获取当前音色
-        voice = getattr(conn.tts, "voice", "default")
-
-        wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
-        file_path = wakeup_words_config.generate_file_path(voice)
-        with open(file_path, "wb") as f:
-            f.write(wav_bytes)
-        # 更新配置
-        wakeup_words_config.update_wakeup_response(voice, file_path, result)
-    finally:
-        # 确保在任何情况下都释放锁
-        if _wakeup_response_lock.locked():
-            _wakeup_response_lock.release()
+    await conn.websocket.send(json.dumps(conn.welcome_msg))
@@ -1,12 +1,11 @@
 import json
-import asyncio
 import uuid
-from core.handle.sendAudioHandle import send_stt_message
-from core.handle.helloHandle import checkWakeupWords
-from core.utils.util import remove_punctuation_and_length
-from core.providers.tts.dto.dto import ContentType
+import asyncio
 from core.utils.dialogue import Message
+from core.providers.tts.dto.dto import ContentType
 from plugins_func.register import Action, ActionResponse
+from core.handle.sendAudioHandle import send_stt_message
+from core.utils.util import remove_punctuation_and_length
 from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType

 TAG = __name__
@@ -24,13 +23,9 @@ async def handle_user_intent(conn, text):
        pass

    # 检查是否有明确的退出命令
-    filtered_text = remove_punctuation_and_length(text)[1]
+    _, filtered_text = remove_punctuation_and_length(text)
    if await check_direct_exit(conn, filtered_text):
        return True
-    
-    # 检查是否是唤醒词
-    if await checkWakeupWords(conn, filtered_text):
-        return True

    if conn.intent_type == "function_call":
        # 使用支持function calling的聊天方法,不再进行意图分析
@@ -1,26 +1,21 @@
 import time
 import json
-import asyncio
-from core.utils.util import audio_to_data_stream
 from core.handle.abortHandle import handleAbortMessage
 from core.handle.intentHandler import handle_user_intent
 from core.utils.output_counter import check_device_output_limit
+from core.utils.util import play_audio_frames, play_audio_response
 from core.handle.sendAudioHandle import send_stt_message, SentenceType

 TAG = __name__


 async def handleAudioMessage(conn, audio):
+    # 检查是否在唤醒处理锁定期内
+    if getattr(conn, 'wakeup_processing_lock', 0) > time.monotonic():
+        return
+    
    # 当前片段是否有人说话
    have_voice = conn.vad.is_vad(conn, audio)
-    # 如果设备刚刚被唤醒，短暂忽略VAD检测
-    if have_voice and hasattr(conn, "just_woken_up") and conn.just_woken_up:
-        have_voice = False
-        # 设置一个短暂延迟后恢复VAD检测
-        conn.asr_audio.clear()
-        if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
-            conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
-        return
    if have_voice:
        if conn.client_is_speaking:
            await handleAbortMessage(conn)
@@ -29,13 +24,6 @@ async def handleAudioMessage(conn, audio):
    # 接收音频
    await conn.asr.receive_audio(conn, audio, have_voice)

-
-async def resume_vad_detection(conn):
-    # 等待2秒后恢复VAD检测
-    await asyncio.sleep(1)
-    conn.just_woken_up = False
-
-
 async def startToChat(conn, text):
    # 检查输入是否是JSON格式（包含说话人信息）
    speaker_name = None
@@ -119,9 +107,7 @@ async def max_out_size(conn):
    text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！"
    await send_stt_message(conn, text)
    file_path = "config/assets/max_output_size.wav"
-    conn.tts.tts_audio_queue.put((SentenceType.FIRST, [], text))
-    play_audio_frames(conn, file_path)
-    conn.tts.tts_audio_queue.put((SentenceType.LAST, [], None))
+    play_audio_response(conn, {"text": text, "file_path": file_path})
    conn.close_after_chat = True


@@ -156,17 +142,4 @@ async def check_bind_device(conn):
        text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
        await send_stt_message(conn, text)
        music_path = "config/assets/bind_not_found.wav"
-        conn.tts.tts_audio_queue.put((SentenceType.FIRST, [], text))
-        play_audio_frames(conn, music_path)
-        conn.tts.tts_audio_queue.put((SentenceType.LAST, [], None))
-
-def play_audio_frames(conn, file_path):
-    """播放音频文件并处理发送帧数据"""
-    def handle_audio_frame(frame_data):
-        conn.tts.tts_audio_queue.put((SentenceType.MIDDLE, frame_data, None))
-
-    audio_to_data_stream(
-        file_path,
-        is_opus=True,
-        callback=handle_audio_frame
-    )
+        play_audio_response(conn, {"text": text, "file_path": music_path})
@@ -30,7 +30,7 @@ async def sendAudioMessage(conn, sentenceType, audios, text):


 # 播放音频
-async def sendAudio(conn, audios, pre_buffer=False, frame_duration=60):
+async def sendAudio(conn, audios, pre_buffer=False):
    """
    发送单个opus包，支持流控
    Args:
@@ -38,7 +38,7 @@ async def sendAudio(conn, audios, pre_buffer=False, frame_duration=60):
        opus_packet: 单个opus数据包
        pre_buffer: 快速发送音频
    """
-    if audios is None:
+    if audios is None or len(audios) == 0:
        return

    if isinstance(audios, bytes):
@@ -60,6 +60,7 @@ async def sendAudio(conn, audios, pre_buffer=False, frame_duration=60):
                "start_time": time.perf_counter(),
            }

+        frame_duration=60
        flow_control = conn.audio_flow_control
        current_time = time.perf_counter()
        # 计算预期发送时间
@@ -76,38 +77,6 @@ async def sendAudio(conn, audios, pre_buffer=False, frame_duration=60):
        # 更新流控状态
        flow_control["packet_count"] += 1
        flow_control["last_send_time"] = time.perf_counter()
-    else:
-        if audios is None or len(audios) == 0:
-            return
-        # 流控参数优化
-        frame_duration = 60  # 帧时长（毫秒），匹配 Opus 编码
-        start_time = time.perf_counter()
-        play_position = 0
-
-        # 执行预缓冲
-        pre_buffer_frames = min(3, len(audios))
-        for i in range(pre_buffer_frames):
-            await conn.websocket.send(audios[i])
-        remaining_audios = audios[pre_buffer_frames:]
-
-        # 播放剩余音频帧
-        for opus_packet in remaining_audios:
-            if conn.client_abort:
-                break
-
-            # 重置没有声音的状态
-            conn.last_activity_time = time.time() * 1000
-
-            # 计算预期发送时间
-            expected_time = start_time + (play_position / 1000)
-            current_time = time.perf_counter()
-            delay = expected_time - current_time
-            if delay > 0:
-                await asyncio.sleep(delay)
-
-            await conn.websocket.send(opus_packet)
-
-            play_position += frame_duration


 async def send_tts_message(conn, state, text=None):
@@ -1,13 +1,11 @@
 import json
-import time
 import asyncio
+from core.utils.util import filter_sensitive_info
 from core.handle.abortHandle import handleAbortMessage
 from core.handle.helloHandle import handleHelloMessage
-from core.handle.reportHandle import enqueue_asr_report
+from core.handle.receiveAudioHandle import handleAudioMessage
 from core.providers.tools.device_mcp import handle_mcp_message
-from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
 from core.handle.sendAudioHandle import send_stt_message, send_tts_message
-from core.utils.util import remove_punctuation_and_length, filter_sensitive_info
 from core.providers.tools.device_iot import handleIotDescriptors, handleIotStatus

 TAG = __name__
@@ -46,14 +44,9 @@ async def handleTextMessage(conn, message):
                conn.client_have_voice = False
                conn.asr_audio.clear()
                if "text" in msg_json:
-                    conn.last_activity_time = time.time() * 1000
-                    original_text = msg_json["text"]  # 保留原始文本
-                    filtered_len, filtered_text = remove_punctuation_and_length(
-                        original_text
-                    )
-
+                    original_text = msg_json["text"]  # 保留设备上传的文本
                    # 识别是否是唤醒词
-                    is_wakeup_words = filtered_text in conn.config.get("wakeup_words")
+                    is_wakeup_words = original_text in conn.config.get("wakeup_words")
                    # 是否开启唤醒词回复
                    enable_greeting = conn.config.get("enable_greeting", True)

@@ -62,16 +55,10 @@ async def handleTextMessage(conn, message):
                        await send_stt_message(conn, original_text)
                        await send_tts_message(conn, "stop", None)
                        conn.client_is_speaking = False
-                    elif is_wakeup_words:
-                        conn.just_woken_up = True
-                        # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
-                        enqueue_asr_report(conn, "嘿，你好呀", [])
-                        await startToChat(conn, "嘿，你好呀")
                    else:
-                        # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
-                        enqueue_asr_report(conn, original_text, [])
-                        # 否则需要LLM对文字内容进行答复
-                        await startToChat(conn, original_text)
+                        # 检测到唤醒词，开始等待后续进行声纹识别
+                        conn.wakeup_mode = True
+                        conn.logger.bind(tag=TAG).info(f"检测到唤醒词~")
        elif msg_json["type"] == "iot":
            conn.logger.bind(tag=TAG).info(f"收到iot消息：{message}")
            if "descriptors" in msg_json: