update:流式优化暂不稳定，先回滚到早期代码

main/xiaozhi-server/core/handle/helloHandle.py

@@ -1,13 +1,33 @@
+import time
 import json
+import random
 import asyncio
+from core.utils.dialogue import Message
+from core.utils.util import audio_to_data
+from core.handle.sendAudioHandle import sendAudioMessage, send_stt_message
+from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes
+from core.providers.tts.dto.dto import ContentType, SentenceType
 from core.providers.tools.device_mcp import (
     MCPClient,
     send_mcp_initialize_message,
     send_mcp_tools_list_request,
 )
+from core.utils.wakeup_word import WakeupWordsConfig
 
 TAG = __name__
 
+WAKEUP_CONFIG = {
+    "refresh_time": 5,
+    "words": ["你好", "你好啊", "嘿，你好", "嗨"],
+}
+
+# 创建全局的唤醒词配置管理器
+wakeup_words_config = WakeupWordsConfig()
+
+# 用于防止并发调用wakeupWordsResponse的锁
+_wakeup_response_lock = asyncio.Lock()
+
+
 async def handleHelloMessage(conn, msg_json):
     """处理hello消息"""
     audio_params = msg_json.get("audio_params")
@@ -28,4 +48,94 @@ async def handleHelloMessage(conn, msg_json):
             # 发送mcp消息，获取tools列表
             asyncio.create_task(send_mcp_tools_list_request(conn))
 
-    await conn.websocket.send(json.dumps(conn.welcome_msg))
+    await conn.websocket.send(json.dumps(conn.welcome_msg))
+
+
+async def checkWakeupWords(conn, text):
+    enable_wakeup_words_response_cache = conn.config[
+        "enable_wakeup_words_response_cache"
+    ]
+
+    if not enable_wakeup_words_response_cache or not conn.tts:
+        return False
+
+    _, filtered_text = remove_punctuation_and_length(text)
+    if filtered_text not in conn.config.get("wakeup_words"):
+        return False
+
+    conn.just_woken_up = True
+    await send_stt_message(conn, text)
+
+    # 获取当前音色
+    voice = getattr(conn.tts, "voice", "default")
+    if not voice:
+        voice = "default"
+
+    # 获取唤醒词回复配置
+    response = wakeup_words_config.get_wakeup_response(voice)
+    if not response or not response.get("file_path"):
+        response = {
+            "voice": "default",
+            "file_path": "config/assets/wakeup_words.wav",
+            "time": 0,
+            "text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",
+        }
+
+    # 播放唤醒词回复
+    conn.client_abort = False
+    opus_packets, _ = audio_to_data(response.get("file_path"))
+
+    conn.logger.bind(tag=TAG).info(f"播放唤醒词回复: {response.get('text')}")
+    await sendAudioMessage(conn, SentenceType.FIRST, opus_packets, response.get("text"))
+    await sendAudioMessage(conn, SentenceType.LAST, [], None)
+
+    # 补充对话
+    conn.dialogue.put(Message(role="assistant", content=response.get("text")))
+
+    # 检查是否需要更新唤醒词回复
+    if time.time() - response.get("time", 0) > WAKEUP_CONFIG["refresh_time"]:
+        if not _wakeup_response_lock.locked():
+            asyncio.create_task(wakeupWordsResponse(conn))
+    return True
+
+
+async def wakeupWordsResponse(conn):
+    if not conn.tts or not conn.llm or not conn.llm.response_no_stream:
+        return
+
+    try:
+        # 尝试获取锁，如果获取不到就返回
+        if not await _wakeup_response_lock.acquire():
+            return
+
+        # 生成唤醒词回复
+        wakeup_word = random.choice(WAKEUP_CONFIG["words"])
+        question = (
+            "此刻用户正在和你说```"
+            + wakeup_word
+            + "```。\n请你根据以上用户的内容进行20-30字回复。要符合系统设置的角色情感和态度，不要像机器人一样说话。\n"
+            + "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。"
+        )
+
+        result = conn.llm.response_no_stream(conn.config["prompt"], question)
+        if not result or len(result) == 0:
+            return
+
+        # 生成TTS音频
+        tts_result = await asyncio.to_thread(conn.tts.to_tts, result)
+        if not tts_result:
+            return
+
+        # 获取当前音色
+        voice = getattr(conn.tts, "voice", "default")
+
+        wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
+        file_path = wakeup_words_config.generate_file_path(voice)
+        with open(file_path, "wb") as f:
+            f.write(wav_bytes)
+        # 更新配置
+        wakeup_words_config.update_wakeup_response(voice, file_path, result)
+    finally:
+        # 确保在任何情况下都释放锁
+        if _wakeup_response_lock.locked():
+            _wakeup_response_lock.release()

main/xiaozhi-server/core/handle/intentHandler.py

@@ -1,11 +1,12 @@
 import json
-import uuid
 import asyncio
-from core.utils.dialogue import Message
-from core.providers.tts.dto.dto import ContentType
-from plugins_func.register import Action, ActionResponse
+import uuid
 from core.handle.sendAudioHandle import send_stt_message
+from core.handle.helloHandle import checkWakeupWords
 from core.utils.util import remove_punctuation_and_length
+from core.providers.tts.dto.dto import ContentType
+from core.utils.dialogue import Message
+from plugins_func.register import Action, ActionResponse
 from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType
 
 TAG = __name__
@@ -23,9 +24,12 @@ async def handle_user_intent(conn, text):
         pass
 
     # 检查是否有明确的退出命令
-    _, filtered_text = remove_punctuation_and_length(text)
+    filtered_text = remove_punctuation_and_length(text)[1]
     if await check_direct_exit(conn, filtered_text):
         return True
+    # 检查是否是唤醒词
+    if await checkWakeupWords(conn, filtered_text):
+        return True
 
     if conn.intent_type == "function_call":
         # 使用支持function calling的聊天方法,不再进行意图分析

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -1,21 +1,28 @@
-import time
-import json
-from core.handle.abortHandle import handleAbortMessage
+from core.handle.sendAudioHandle import send_stt_message
 from core.handle.intentHandler import handle_user_intent
 from core.utils.output_counter import check_device_output_limit
-from core.utils.util import play_audio_frames, play_audio_response
-from core.handle.sendAudioHandle import send_stt_message, SentenceType
+from core.handle.abortHandle import handleAbortMessage
+import time
+import asyncio
+import json
+from core.handle.sendAudioHandle import SentenceType
+from core.utils.util import audio_to_data
 
 TAG = __name__
 
 
 async def handleAudioMessage(conn, audio):
-    # 检查是否在唤醒处理锁定期内
-    if getattr(conn, 'wakeup_processing_lock', 0) > time.monotonic():
-        return
-    
     # 当前片段是否有人说话
     have_voice = conn.vad.is_vad(conn, audio)
+    # 如果设备刚刚被唤醒，短暂忽略VAD检测
+    if have_voice and hasattr(conn, "just_woken_up") and conn.just_woken_up:
+        have_voice = False
+        # 设置一个短暂延迟后恢复VAD检测
+        conn.asr_audio.clear()
+        if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
+            conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
+        return
+
     if have_voice:
         if conn.client_is_speaking:
             await handleAbortMessage(conn)
@@ -24,11 +31,18 @@ async def handleAudioMessage(conn, audio):
     # 接收音频
     await conn.asr.receive_audio(conn, audio, have_voice)
 
+
+async def resume_vad_detection(conn):
+    # 等待2秒后恢复VAD检测
+    await asyncio.sleep(1)
+    conn.just_woken_up = False
+
+
 async def startToChat(conn, text):
     # 检查输入是否是JSON格式（包含说话人信息）
     speaker_name = None
     actual_text = text
-
+    
     try:
         # 尝试解析JSON格式的输入
         if text.strip().startswith('{') and text.strip().endswith('}'):
@@ -37,13 +51,13 @@ async def startToChat(conn, text):
                 speaker_name = data['speaker']
                 actual_text = data['content']
                 conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}")
-
+                
                 # 直接使用JSON格式的文本，不解析
                 actual_text = text
     except (json.JSONDecodeError, KeyError):
         # 如果解析失败，继续使用原始文本
         pass
-
+    
     # 保存说话人信息到连接对象
     if speaker_name:
         conn.current_speaker = speaker_name
@@ -107,7 +121,8 @@ async def max_out_size(conn):
     text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！"
     await send_stt_message(conn, text)
     file_path = "config/assets/max_output_size.wav"
-    play_audio_response(conn, {"text": text, "file_path": file_path})
+    opus_packets, _ = audio_to_data(file_path)
+    conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))
     conn.close_after_chat = True
 
 
@@ -125,15 +140,16 @@ async def check_bind_device(conn):
 
         # 播放提示音
         music_path = "config/assets/bind_code.wav"
-        conn.tts.tts_audio_queue.put((SentenceType.FIRST, [], text))
-        play_audio_frames(conn, music_path)
+        opus_packets, _ = audio_to_data(music_path)
+        conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, text))
 
         # 逐个播放数字
         for i in range(6):  # 确保只播放6位数字
             try:
                 digit = conn.bind_code[i]
                 num_path = f"config/assets/bind_code/{digit}.wav"
-                play_audio_frames(conn, num_path)
+                num_packets, _ = audio_to_data(num_path)
+                conn.tts.tts_audio_queue.put((SentenceType.MIDDLE, num_packets, None))
             except Exception as e:
                 conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
                 continue
@@ -142,4 +158,5 @@ async def check_bind_device(conn):
         text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
         await send_stt_message(conn, text)
         music_path = "config/assets/bind_not_found.wav"
-        play_audio_response(conn, {"text": text, "file_path": music_path})
+        opus_packets, _ = audio_to_data(music_path)
+        conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -1,25 +1,25 @@
 import json
-import time
 import asyncio
-from core.utils import textUtils
+import time
 from core.providers.tts.dto.dto import SentenceType
+from core.utils import textUtils
 
 TAG = __name__
 
 
 async def sendAudioMessage(conn, sentenceType, audios, text):
+    # 发送句子开始消息
+    conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}")
+
+    pre_buffer = False
     if conn.tts.tts_audio_first_sentence:
         conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
         conn.tts.tts_audio_first_sentence = False
-        await send_tts_message(conn, "start", None)
+        pre_buffer = True
 
-    if sentenceType == SentenceType.FIRST:
-        await send_tts_message(conn, "sentence_start", text)
+    await send_tts_message(conn, "sentence_start", text)
 
-    await sendAudio(conn, audios)
-    # 发送句子开始消息
-    if sentenceType is not SentenceType.MIDDLE:
-        conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}")
+    await sendAudio(conn, audios, pre_buffer)
 
     # 发送结束消息（如果是最后一个文本）
     if conn.llm_finish_task and sentenceType == SentenceType.LAST:
@@ -30,59 +30,45 @@ async def sendAudioMessage(conn, sentenceType, audios, text):
 
 
 # 播放音频
-async def sendAudio(conn, audios, pre_buffer=False):
-    """
-    发送单个opus包，支持流控
-    Args:
-        conn: 连接对象
-        opus_packet: 单个opus数据包
-        pre_buffer: 快速发送音频
-    """
+async def sendAudio(conn, audios, pre_buffer=True):
     if audios is None or len(audios) == 0:
         return
+    # 流控参数优化
+    frame_duration = 60  # 帧时长（毫秒），匹配 Opus 编码
+    start_time = time.perf_counter()
+    play_position = 0
 
-    if isinstance(audios, bytes):
+    # 仅当第一句话时执行预缓冲
+    if pre_buffer:
+        pre_buffer_frames = min(3, len(audios))
+        for i in range(pre_buffer_frames):
+            await conn.websocket.send(audios[i])
+        remaining_audios = audios[pre_buffer_frames:]
+    else:
+        remaining_audios = audios
+
+    # 播放剩余音频帧
+    for opus_packet in remaining_audios:
         if conn.client_abort:
-            return
-
-        # 短音频直接发送（例如：提示音）
-        if pre_buffer:
-            await conn.websocket.send(audios)
-            return
+            break
 
+        # 重置没有声音的状态
         conn.last_activity_time = time.time() * 1000
 
-        # 获取或初始化流控状态
-        if not hasattr(conn, "audio_flow_control"):
-            conn.audio_flow_control = {
-                "last_send_time": 0,
-                "packet_count": 0,
-                "start_time": time.perf_counter(),
-            }
-
-        frame_duration=60
-        flow_control = conn.audio_flow_control
-        current_time = time.perf_counter()
         # 计算预期发送时间
-        expected_time = flow_control["start_time"] + (
-            flow_control["packet_count"] * frame_duration / 1000
-        )
+        expected_time = start_time + (play_position / 1000)
+        current_time = time.perf_counter()
         delay = expected_time - current_time
         if delay > 0:
             await asyncio.sleep(delay)
 
-        # 发送数据包
-        await conn.websocket.send(audios)
+        await conn.websocket.send(opus_packet)
 
-        # 更新流控状态
-        flow_control["packet_count"] += 1
-        flow_control["last_send_time"] = time.perf_counter()
+        play_position += frame_duration
 
 
 async def send_tts_message(conn, state, text=None):
     """发送 TTS 状态消息"""
-    if text is None and state == "sentence_start":
-        return
     message = {"type": "tts", "state": state, "session_id": conn.session_id}
     if text is not None:
         message["text"] = textUtils.check_emoji(text)
@@ -95,12 +81,8 @@ async def send_tts_message(conn, state, text=None):
             stop_tts_notify_voice = conn.config.get(
                 "stop_tts_notify_voice", "config/assets/tts_notify.mp3"
             )
-            conn.tts.audio_to_opus_data_stream(
-                stop_tts_notify_voice,
-                callback=lambda audio_data: asyncio.run_coroutine_threadsafe(
-                    sendAudio(conn, audio_data, True), conn.loop
-                ),
-            )
+            audios, _ = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
+            await sendAudio(conn, audios)
         # 清除服务端讲话状态
         conn.clearSpeakStatus()
 
@@ -109,17 +91,18 @@ async def send_tts_message(conn, state, text=None):
 
 
 async def send_stt_message(conn, text):
-    """发送 STT 状态消息"""
     end_prompt_str = conn.config.get("end_prompt", {}).get("prompt")
     if end_prompt_str and end_prompt_str == text:
         await send_tts_message(conn, "start")
         return
 
+    """发送 STT 状态消息"""
+    
     # 解析JSON格式，提取实际的用户说话内容
     display_text = text
     try:
         # 尝试解析JSON格式
-        if text.strip().startswith("{") and text.strip().endswith("}"):
+        if text.strip().startswith('{') and text.strip().endswith('}'):
             parsed_data = json.loads(text)
             if isinstance(parsed_data, dict) and "content" in parsed_data:
                 # 如果是包含说话人信息的JSON格式，只显示content部分

main/xiaozhi-server/core/handle/textHandle.py

@@ -1,12 +1,14 @@
 import json
-import asyncio
-from core.utils.util import filter_sensitive_info
+import time
 from core.handle.abortHandle import handleAbortMessage
 from core.handle.helloHandle import handleHelloMessage
-from core.handle.receiveAudioHandle import handleAudioMessage
 from core.providers.tools.device_mcp import handle_mcp_message
+from core.utils.util import remove_punctuation_and_length, filter_sensitive_info
+from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
 from core.handle.sendAudioHandle import send_stt_message, send_tts_message
 from core.providers.tools.device_iot import handleIotDescriptors, handleIotStatus
+from core.handle.reportHandle import enqueue_asr_report
+import asyncio
 
 TAG = __name__
 
@@ -44,9 +46,14 @@ async def handleTextMessage(conn, message):
                 conn.client_have_voice = False
                 conn.asr_audio.clear()
                 if "text" in msg_json:
-                    original_text = msg_json["text"]  # 保留设备上传的文本
+                    conn.last_activity_time = time.time() * 1000
+                    original_text = msg_json["text"]  # 保留原始文本
+                    filtered_len, filtered_text = remove_punctuation_and_length(
+                        original_text
+                    )
+
                     # 识别是否是唤醒词
-                    is_wakeup_words = original_text in conn.config.get("wakeup_words")
+                    is_wakeup_words = filtered_text in conn.config.get("wakeup_words")
                     # 是否开启唤醒词回复
                     enable_greeting = conn.config.get("enable_greeting", True)
 
@@ -55,10 +62,16 @@ async def handleTextMessage(conn, message):
                         await send_stt_message(conn, original_text)
                         await send_tts_message(conn, "stop", None)
                         conn.client_is_speaking = False
+                    elif is_wakeup_words:
+                        conn.just_woken_up = True
+                        # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
+                        enqueue_asr_report(conn, "嘿，你好呀", [])
+                        await startToChat(conn, "嘿，你好呀")
                     else:
-                        # 检测到唤醒词，开始等待后续进行声纹识别
-                        conn.wakeup_mode = True
-                        conn.logger.bind(tag=TAG).info(f"检测到唤醒词~")
+                        # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
+                        enqueue_asr_report(conn, original_text, [])
+                        # 否则需要LLM对文字内容进行答复
+                        await startToChat(conn, original_text)
         elif msg_json["type"] == "iot":
             conn.logger.bind(tag=TAG).info(f"收到iot消息：{message}")
             if "descriptors" in msg_json: