update:优化

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -1,70 +1,104 @@
-import traceback
-
-from config.logger import setup_logging
 import json
 import asyncio
 import time
-
-from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType, MsgType
-from core.utils.util import (
-    remove_punctuation_and_length,
-    get_string_no_punctuation_or_emoji,
-)
+from core.utils.util import get_string_no_punctuation_or_emoji, analyze_emotion
 
 TAG = __name__
-logger = setup_logging()
+
+emoji_map = {
+    "neutral": "😶",
+    "happy": "🙂",
+    "laughing": "😆",
+    "funny": "😂",
+    "sad": "😔",
+    "angry": "😠",
+    "crying": "😭",
+    "loving": "😍",
+    "embarrassed": "😳",
+    "surprised": "😲",
+    "shocked": "😱",
+    "thinking": "🤔",
+    "winking": "😉",
+    "cool": "😎",
+    "relaxed": "😌",
+    "delicious": "🤤",
+    "kissy": "😘",
+    "confident": "😏",
+    "sleepy": "😴",
+    "silly": "😜",
+    "confused": "🙄",
+}
 
 
-async def sendAudioMessage(conn, ttsMessageDTO: TTSMessageDTO):
-    if ttsMessageDTO.u_id != conn.u_id:
-        logger.bind(tag=TAG).info(
-            f"msg id:{ttsMessageDTO.u_id},不是当前对话，当前对话id：{conn.u_id}"
-        )
-        return
+async def sendAudioMessage(conn, audios, text, text_index=0):
     # 发送句子开始消息
-    if SentenceType.SENTENCE_START == ttsMessageDTO.sentence_type:
-        logger.bind(tag=TAG).info(f"发送第一段语音: {ttsMessageDTO.tts_finish_text}")
-        await send_tts_message(conn, "sentence_start", ttsMessageDTO.tts_finish_text)
+    if text is not None:
+        emotion = analyze_emotion(text)
+        emoji = emoji_map.get(emotion, "🙂")  # 默认使用笑脸
+        await conn.websocket.send(
+            json.dumps(
+                {
+                    "type": "llm",
+                    "text": emoji,
+                    "emotion": emotion,
+                    "session_id": conn.session_id,
+                }
+            )
+        )
 
+    if text_index == conn.tts_first_text_index:
+        conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
+    await send_tts_message(conn, "sentence_start", text)
+
+    is_first_audio = text_index == conn.tts_first_text_index
+    await sendAudio(conn, audios, pre_buffer=is_first_audio)
+
+    await send_tts_message(conn, "sentence_end", text)
+
+    # 发送结束消息（如果是最后一个文本）
+    if conn.llm_finish_task and text_index == conn.tts_last_text_index:
+        await send_tts_message(conn, "stop", None)
+        if conn.close_after_chat:
+            await conn.close()
+
+
+# 播放音频
+async def sendAudio(conn, audios, pre_buffer=True):
     # 流控参数优化
-    original_frame_duration = 60  # 原始帧时长（毫秒）
-    adjusted_frame_duration = int(original_frame_duration * 1)  # 缩短20%
-    total_frames = len(ttsMessageDTO.content)  # 获取总帧数
-    compensation = (
-        total_frames * (original_frame_duration - adjusted_frame_duration) / 1000
-    )  # 补偿时间（秒）
-
+    frame_duration = 60  # 帧时长（毫秒），匹配 Opus 编码
     start_time = time.perf_counter()
-    play_position = 0  # 已播放时长（毫秒）
+    play_position = 0
+    last_reset_time = time.perf_counter()  # 记录最后的重置时间
 
-    for opus_packet in ttsMessageDTO.content:
+    # 仅当第一句话时执行预缓冲
+    if pre_buffer:
+        pre_buffer_frames = min(3, len(audios))
+        for i in range(pre_buffer_frames):
+            await conn.websocket.send(audios[i])
+        remaining_audios = audios[pre_buffer_frames:]
+    else:
+        remaining_audios = audios
+
+    # 播放剩余音频帧
+    for opus_packet in remaining_audios:
         if conn.client_abort:
             return
 
-        # 计算带加速因子的预期时间
+        # 每分钟重置一次计时器
+        if time.perf_counter() - last_reset_time > 60:
+            await conn.reset_timeout()
+            last_reset_time = time.perf_counter()
+
+        # 计算预期发送时间
         expected_time = start_time + (play_position / 1000)
         current_time = time.perf_counter()
-
-        # 流控等待（使用加速后的帧时长）
         delay = expected_time - current_time
         if delay > 0:
             await asyncio.sleep(delay)
 
         await conn.websocket.send(opus_packet)
-        play_position += adjusted_frame_duration  # 使用调整后的帧时长
 
-    # 补偿因加速损失的时长
-    # if compensation > 0:
-    #     await asyncio.sleep(compensation)
-    if SentenceType.SENTENCE_END == ttsMessageDTO.sentence_type:
-        logger.bind(tag=TAG).info(f"发送最后一段语音: {ttsMessageDTO.tts_finish_text}")
-        await send_tts_message(conn, "sentence_end", ttsMessageDTO.tts_finish_text)
-
-    # 发送结束消息（如果是最后一个文本）
-    if conn.llm_finish_task and MsgType.STOP_TTS_RESPONSE == ttsMessageDTO.msg_type:
-        await send_tts_message(conn, "stop", None)
-        if conn.close_after_chat:
-            await conn.close()
+        play_position += frame_duration
 
 
 async def send_tts_message(conn, state, text=None):
@@ -73,10 +107,22 @@ async def send_tts_message(conn, state, text=None):
     if text is not None:
         message["text"] = text
 
-    await conn.websocket.send(json.dumps(message))
+    # TTS播放结束
     if state == "stop":
+        # 播放提示音
+        tts_notify = conn.config.get("enable_stop_tts_notify", False)
+        if tts_notify:
+            stop_tts_notify_voice = conn.config.get(
+                "stop_tts_notify_voice", "config/assets/tts_notify.mp3"
+            )
+            audios, _ = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
+            await sendAudio(conn, audios)
+        # 清除服务端讲话状态
         conn.clearSpeakStatus()
 
+    # 发送消息到客户端
+    await conn.websocket.send(json.dumps(message))
+
 
 async def send_stt_message(conn, text):
     """发送 STT 状态消息"""
@@ -84,14 +130,4 @@ async def send_stt_message(conn, text):
     await conn.websocket.send(
         json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id})
     )
-    await conn.websocket.send(
-        json.dumps(
-            {
-                "type": "llm",
-                "text": "😊",
-                "emotion": "happy",
-                "session_id": conn.session_id,
-            }
-        )
-    )
     await send_tts_message(conn, "start")