update:优化

main/xiaozhi-server/core/handle/helloHandle.py

@@ -26,7 +26,8 @@ async def handleHelloMessage(conn, msg_json):
         format = audio_params.get("format")
         conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}")
         conn.audio_format = format
-        conn.asr.set_audio_format(format)
+        if conn.asr is not None:
+            conn.asr.set_audio_format(format)
         conn.welcome_msg["audio_params"] = audio_params
 
     await conn.websocket.send(json.dumps(conn.welcome_msg))
@@ -55,7 +56,7 @@ async def checkWakeupWords(conn, text):
         text_hello = WAKEUP_CONFIG["text"]
         if not text_hello:
             text_hello = text
-        conn.audio_play_queue.put((opus_packets, text_hello, 0))
+        conn.tts.audio_play_queue.put((opus_packets, text_hello, 0))
         if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
             asyncio.create_task(wakeupWordsResponse(conn))
         return True

main/xiaozhi-server/core/handle/intentHandler.py

@@ -2,6 +2,7 @@ from config.logger import setup_logging
 import json
 import uuid
 from core.handle.sendAudioHandle import send_stt_message
+from core.handle.helloHandle import checkWakeupWords
 from core.utils.util import remove_punctuation_and_length
 from core.utils.dialogue import Message
 from plugins_func.register import Action
@@ -15,10 +16,9 @@ async def handle_user_intent(conn, text):
     filtered_text = remove_punctuation_and_length(text)[1]
     if await check_direct_exit(conn, filtered_text):
         return True
-    # 4月4日因流式改造暂时关闭唤醒词加速功能
     # 检查是否是唤醒词
-    # if await checkWakeupWords(conn, filtered_text):
-    #     return True
+    if await checkWakeupWords(conn, filtered_text):
+        return True
 
     if conn.intent_type == "function_call":
         # 使用支持function calling的聊天方法,不再进行意图分析
@@ -109,21 +109,21 @@ async def process_intent_result(conn, intent_result, original_text):
                     if result.action == Action.RESPONSE:  # 直接回复前端
                         text = result.response
                         if text is not None:
-                            speak_and_play(conn, text)
+                            speak_txt(conn, text)
                     elif result.action == Action.REQLLM:  # 调用函数后再请求llm生成回复
                         text = result.result
                         conn.dialogue.put(Message(role="tool", content=text))
                         llm_result = conn.intent.replyResult(text, original_text)
                         if llm_result is None:
                             llm_result = text
-                        speak_and_play(conn, llm_result)
+                        speak_txt(conn, llm_result)
                     elif (
                         result.action == Action.NOTFOUND
                         or result.action == Action.ERROR
                     ):
                         text = result.result
                         if text is not None:
-                            speak_and_play(conn, text)
+                            speak_txt(conn, text)
                     elif function_name != "play_music":
                         # For backward compatibility with original code
                         # 获取当前最新的文本索引
@@ -131,7 +131,7 @@ async def process_intent_result(conn, intent_result, original_text):
                         if text is None:
                             text = result.result
                         if text is not None:
-                            speak_and_play(conn, text)
+                            speak_txt(conn, text)
 
             # 将函数执行放在线程池中
             conn.executor.submit(process_function_call)
@@ -142,6 +142,12 @@ async def process_intent_result(conn, intent_result, original_text):
         return False
 
 
-def speak_and_play(conn, text):
-    conn.tts.tts_one_sentence(conn, text)
+def speak_txt(conn, text):
+    text_index = (
+        conn.tts_last_text_index + 1 if hasattr(conn, "tts_last_text_index") else 0
+    )
+    conn.recode_first_last_text(text, text_index)
+    future = conn.executor.submit(conn.speak_and_play, None, text, text_index)
+    conn.llm_finish_task = True
+    conn.tts_queue.put((future, text_index))
     conn.dialogue.put(Message(role="assistant", content=text))

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -39,7 +39,9 @@ async def handleAudioMessage(conn, audio):
         if len(conn.asr_audio) < 15:
             conn.asr_server_receive = True
         else:
-            raw_text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)  # 确保ASR模块返回原始文本
+            raw_text, _ = await conn.asr.speech_to_text(
+                conn.asr_audio, conn.session_id
+            )  # 确保ASR模块返回原始文本
             conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
             text_len, _ = remove_punctuation_and_length(raw_text)
             if text_len > 0:
@@ -76,11 +78,7 @@ async def startToChat(conn, text):
 
     # 意图未被处理，继续常规聊天流程
     await send_stt_message(conn, text)
-    if conn.intent_type == "function_call":
-        # 使用支持function calling的聊天方法
-        conn.executor.submit(conn.chat_with_function_calling, text)
-    else:
-        conn.executor.submit(conn.chat, text)
+    conn.executor.submit(conn.chat, text)
 
 
 async def no_voice_close_connect(conn):
@@ -117,7 +115,7 @@ async def max_out_size(conn):
     conn.llm_finish_task = True
     file_path = "config/assets/max_output_size.wav"
     opus_packets, _ = audio_to_data(file_path)
-    conn.audio_play_queue.put((opus_packets, text, 0))
+    conn.tts.audio_play_queue.put((opus_packets, text, 0))
     conn.close_after_chat = True
 
 
@@ -139,7 +137,7 @@ async def check_bind_device(conn):
         # 播放提示音
         music_path = "config/assets/bind_code.wav"
         opus_packets, _ = audio_to_data(music_path)
-        conn.audio_play_queue.put((opus_packets, text, 0))
+        conn.tts.audio_play_queue.put((opus_packets, text, 0))
 
         # 逐个播放数字
         for i in range(6):  # 确保只播放6位数字
@@ -147,7 +145,7 @@ async def check_bind_device(conn):
                 digit = conn.bind_code[i]
                 num_path = f"config/assets/bind_code/{digit}.wav"
                 num_packets, _ = audio_to_data(num_path)
-                conn.audio_play_queue.put((num_packets, None, i + 1))
+                conn.tts.audio_play_queue.put((num_packets, None, i + 1))
             except Exception as e:
                 conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
                 continue
@@ -159,4 +157,4 @@ async def check_bind_device(conn):
         conn.llm_finish_task = True
         music_path = "config/assets/bind_not_found.wav"
         opus_packets, _ = audio_to_data(music_path)
-        conn.audio_play_queue.put((opus_packets, text, 0))
+        conn.tts.audio_play_queue.put((opus_packets, text, 0))

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -1,70 +1,104 @@
-import traceback
-
-from config.logger import setup_logging
 import json
 import asyncio
 import time
-
-from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType, MsgType
-from core.utils.util import (
-    remove_punctuation_and_length,
-    get_string_no_punctuation_or_emoji,
-)
+from core.utils.util import get_string_no_punctuation_or_emoji, analyze_emotion
 
 TAG = __name__
-logger = setup_logging()
+
+emoji_map = {
+    "neutral": "😶",
+    "happy": "🙂",
+    "laughing": "😆",
+    "funny": "😂",
+    "sad": "😔",
+    "angry": "😠",
+    "crying": "😭",
+    "loving": "😍",
+    "embarrassed": "😳",
+    "surprised": "😲",
+    "shocked": "😱",
+    "thinking": "🤔",
+    "winking": "😉",
+    "cool": "😎",
+    "relaxed": "😌",
+    "delicious": "🤤",
+    "kissy": "😘",
+    "confident": "😏",
+    "sleepy": "😴",
+    "silly": "😜",
+    "confused": "🙄",
+}
 
 
-async def sendAudioMessage(conn, ttsMessageDTO: TTSMessageDTO):
-    if ttsMessageDTO.u_id != conn.u_id:
-        logger.bind(tag=TAG).info(
-            f"msg id:{ttsMessageDTO.u_id},不是当前对话，当前对话id：{conn.u_id}"
-        )
-        return
+async def sendAudioMessage(conn, audios, text, text_index=0):
     # 发送句子开始消息
-    if SentenceType.SENTENCE_START == ttsMessageDTO.sentence_type:
-        logger.bind(tag=TAG).info(f"发送第一段语音: {ttsMessageDTO.tts_finish_text}")
-        await send_tts_message(conn, "sentence_start", ttsMessageDTO.tts_finish_text)
+    if text is not None:
+        emotion = analyze_emotion(text)
+        emoji = emoji_map.get(emotion, "🙂")  # 默认使用笑脸
+        await conn.websocket.send(
+            json.dumps(
+                {
+                    "type": "llm",
+                    "text": emoji,
+                    "emotion": emotion,
+                    "session_id": conn.session_id,
+                }
+            )
+        )
 
+    if text_index == conn.tts_first_text_index:
+        conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
+    await send_tts_message(conn, "sentence_start", text)
+
+    is_first_audio = text_index == conn.tts_first_text_index
+    await sendAudio(conn, audios, pre_buffer=is_first_audio)
+
+    await send_tts_message(conn, "sentence_end", text)
+
+    # 发送结束消息（如果是最后一个文本）
+    if conn.llm_finish_task and text_index == conn.tts_last_text_index:
+        await send_tts_message(conn, "stop", None)
+        if conn.close_after_chat:
+            await conn.close()
+
+
+# 播放音频
+async def sendAudio(conn, audios, pre_buffer=True):
     # 流控参数优化
-    original_frame_duration = 60  # 原始帧时长（毫秒）
-    adjusted_frame_duration = int(original_frame_duration * 1)  # 缩短20%
-    total_frames = len(ttsMessageDTO.content)  # 获取总帧数
-    compensation = (
-        total_frames * (original_frame_duration - adjusted_frame_duration) / 1000
-    )  # 补偿时间（秒）
-
+    frame_duration = 60  # 帧时长（毫秒），匹配 Opus 编码
     start_time = time.perf_counter()
-    play_position = 0  # 已播放时长（毫秒）
+    play_position = 0
+    last_reset_time = time.perf_counter()  # 记录最后的重置时间
 
-    for opus_packet in ttsMessageDTO.content:
+    # 仅当第一句话时执行预缓冲
+    if pre_buffer:
+        pre_buffer_frames = min(3, len(audios))
+        for i in range(pre_buffer_frames):
+            await conn.websocket.send(audios[i])
+        remaining_audios = audios[pre_buffer_frames:]
+    else:
+        remaining_audios = audios
+
+    # 播放剩余音频帧
+    for opus_packet in remaining_audios:
         if conn.client_abort:
             return
 
-        # 计算带加速因子的预期时间
+        # 每分钟重置一次计时器
+        if time.perf_counter() - last_reset_time > 60:
+            await conn.reset_timeout()
+            last_reset_time = time.perf_counter()
+
+        # 计算预期发送时间
         expected_time = start_time + (play_position / 1000)
         current_time = time.perf_counter()
-
-        # 流控等待（使用加速后的帧时长）
         delay = expected_time - current_time
         if delay > 0:
             await asyncio.sleep(delay)
 
         await conn.websocket.send(opus_packet)
-        play_position += adjusted_frame_duration  # 使用调整后的帧时长
 
-    # 补偿因加速损失的时长
-    # if compensation > 0:
-    #     await asyncio.sleep(compensation)
-    if SentenceType.SENTENCE_END == ttsMessageDTO.sentence_type:
-        logger.bind(tag=TAG).info(f"发送最后一段语音: {ttsMessageDTO.tts_finish_text}")
-        await send_tts_message(conn, "sentence_end", ttsMessageDTO.tts_finish_text)
-
-    # 发送结束消息（如果是最后一个文本）
-    if conn.llm_finish_task and MsgType.STOP_TTS_RESPONSE == ttsMessageDTO.msg_type:
-        await send_tts_message(conn, "stop", None)
-        if conn.close_after_chat:
-            await conn.close()
+        play_position += frame_duration
 
 
 async def send_tts_message(conn, state, text=None):
@@ -73,10 +107,22 @@ async def send_tts_message(conn, state, text=None):
     if text is not None:
         message["text"] = text
 
-    await conn.websocket.send(json.dumps(message))
+    # TTS播放结束
     if state == "stop":
+        # 播放提示音
+        tts_notify = conn.config.get("enable_stop_tts_notify", False)
+        if tts_notify:
+            stop_tts_notify_voice = conn.config.get(
+                "stop_tts_notify_voice", "config/assets/tts_notify.mp3"
+            )
+            audios, _ = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
+            await sendAudio(conn, audios)
+        # 清除服务端讲话状态
         conn.clearSpeakStatus()
 
+    # 发送消息到客户端
+    await conn.websocket.send(json.dumps(message))
+
 
 async def send_stt_message(conn, text):
     """发送 STT 状态消息"""
@@ -84,14 +130,4 @@ async def send_stt_message(conn, text):
     await conn.websocket.send(
         json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id})
     )
-    await conn.websocket.send(
-        json.dumps(
-            {
-                "type": "llm",
-                "text": "😊",
-                "emotion": "happy",
-                "session_id": conn.session_id,
-            }
-        )
-    )
     await send_tts_message(conn, "start")