update:优化音频播放

.gitignore

@@ -146,7 +146,7 @@ tmp
 .history
 .DS_Store
 main/xiaozhi-server/data
-main/xiaozhi-server/config/assets/wakeup_words.*
+
 main/manager-web/node_modules
 .config.yaml
 .secrets.yaml
@@ -163,6 +163,7 @@ main/xiaozhi-server/models/sherpa-onnx*
 /models/SenseVoiceSmall/model.pt
 my_wakeup_words.mp3
 !main/xiaozhi-server/config/assets/bind_code.wav
+!main/xiaozhi-server/config/assets/wakeup_words.wav
 !main/xiaozhi-server/config/assets/bind_not_found.wav
 !main/xiaozhi-server/config/assets/bind_code/*.wav
 !main/xiaozhi-server/config/assets/max_output_size.wav

main/xiaozhi-server/config.yaml

@@ -85,6 +85,7 @@ module_test:
 # 唤醒词，用于识别唤醒词还是讲话内容
 wakeup_words:
   - "你好小智"
+  - "嘿你好呀"
   - "你好小志"
   - "小爱同学"
   - "你好小鑫"

main/xiaozhi-server/core/connection.py

@@ -1,24 +1,15 @@
 import os
+import sys
 import copy
 import json
-import subprocess
-import sys
 import uuid
 import time
 import queue
 import asyncio
-import traceback
 import threading
+import traceback
+import subprocess
 import websockets
-from typing import Dict, Any
-from plugins_func.loadplugins import auto_import_modules
-from config.logger import setup_logging
-from config.config_loader import get_project_dir
-from core.utils import p3
-from core.utils.dialogue import Message, Dialogue
-from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType
-from core.providers.tts.default import DefaultTTS
-from core.handle.textHandle import handleTextMessage
 from core.utils.util import (
     extract_json_from_string,
     initialize_modules,
@@ -27,16 +18,23 @@ from core.utils.util import (
     filter_sensitive_info,
     initialize_tts,
 )
+from typing import Dict, Any
+from config.logger import setup_logging
+from core.mcp.manager import MCPManager
+from core.handle.reportHandle import report
+from core.providers.tts.default import DefaultTTS
 from concurrent.futures import ThreadPoolExecutor
-from core.handle.receiveAudioHandle import handleAudioMessage
+from core.utils.dialogue import Message, Dialogue
+from core.handle.textHandle import handleTextMessage
 from core.handle.functionHandler import FunctionHandler
+from plugins_func.loadplugins import auto_import_modules
 from plugins_func.register import Action, ActionResponse
 from core.auth import AuthMiddleware, AuthenticationError
-from core.mcp.manager import MCPManager
 from config.config_loader import get_private_config_from_api
+from core.handle.receiveAudioHandle import handleAudioMessage
+from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType
 from config.manage_api_client import DeviceNotFoundException, DeviceBindException
-from core.utils.output_counter import add_device_output
+
-from core.handle.reportHandle import report
 
 TAG = __name__
 
@@ -860,23 +858,24 @@ class ConnectionHandler:
 
     def clear_queues(self):
         """清空所有任务队列"""
-        self.logger.bind(tag=TAG).debug(
+        if self.tts:
-            f"开始清理: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
+            self.logger.bind(tag=TAG).debug(
-        )
+                f"开始清理: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
+            )
 
-        # 使用非阻塞方式清空队列
+            # 使用非阻塞方式清空队列
-        for q in [self.tts.tts_text_queue, self.tts.tts_audio_queue]:
+            for q in [self.tts.tts_text_queue, self.tts.tts_audio_queue]:
-            if not q:
+                if not q:
-                continue
+                    continue
-            while True:
+                while True:
-                try:
+                    try:
-                    q.get_nowait()
+                        q.get_nowait()
-                except queue.Empty:
+                    except queue.Empty:
-                    break
+                        break
 
-        self.logger.bind(tag=TAG).debug(
+            self.logger.bind(tag=TAG).debug(
-            f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
+                f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
-        )
+            )
 
     def reset_vad_states(self):
         self.client_audio_buffer = bytearray()

main/xiaozhi-server/core/handle/helloHandle.py

@@ -1,11 +1,14 @@
+import os
+import time
 import json
-from core.handle.sendAudioHandle import send_stt_message
+import random
-from core.utils.util import remove_punctuation_and_length
 import shutil
 import asyncio
-import os
+from core.providers.tts.dto.dto import ContentType
-import random
+from core.providers.tts.dto.dto import SentenceType
-import time
+from core.handle.sendAudioHandle import send_stt_message
+from core.utils.util import remove_punctuation_and_length
+
 
 TAG = __name__
 
@@ -49,11 +52,12 @@ async def checkWakeupWords(conn, text):
         if file is None:
             asyncio.create_task(wakeupWordsResponse(conn))
             return False
-        opus_packets, _ = conn.tts.audio_to_opus_data(file)
         text_hello = WAKEUP_CONFIG["text"]
         if not text_hello:
             text_hello = text
-        conn.tts.tts_audio_queue.put((opus_packets, text_hello, 0))
+        conn.tts.tts_one_sentence(
+            conn, ContentType.FILE, content_file=file, content_detail=text_hello
+        )
         if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
             asyncio.create_task(wakeupWordsResponse(conn))
         return True

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -47,9 +47,8 @@ async def handleAudioMessage(conn, audio):
             text_len, _ = remove_punctuation_and_length(raw_text)
             if text_len > 0:
                 # 使用自定义模块进行上报
-                enqueue_asr_report(conn, raw_text, copy.deepcopy(conn.asr_audio))
-
                 await startToChat(conn, raw_text)
+                enqueue_asr_report(conn, raw_text, copy.deepcopy(conn.asr_audio))
             else:
                 conn.asr_server_receive = True
         conn.asr_audio.clear()

main/xiaozhi-server/core/providers/tts/base.py

@@ -10,6 +10,8 @@ from abc import ABC, abstractmethod
 from config.logger import setup_logging
 from core.utils.util import audio_to_data
 from core.utils.tts import MarkdownCleaner
+from core.utils.output_counter import add_device_output
+from core.handle.reportHandle import enqueue_tts_report
 from core.handle.sendAudioHandle import sendAudioMessage
 from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType, ContentType
 
@@ -226,6 +228,9 @@ class TTSProviderBase(ABC):
                     self.conn.loop,
                 )
                 future.result()
+                if self.conn.max_output_size > 0 and text:
+                    add_device_output(self.conn.headers.get("device-id"), len(text))
+                enqueue_tts_report(self.conn, text, audio_datas)
             except Exception as e:
                 logger.bind(tag=TAG).error(
                     f"audio_play_priority priority_thread: {text} {e}"
@@ -298,6 +303,14 @@ class TTSProviderBase(ABC):
             audio_datas, _ = self.audio_to_pcm_data(tts_file)
         else:
             audio_datas, _ = self.audio_to_opus_data(tts_file)
+
+        if (
+            self.delete_audio_file
+            and tts_file is not None
+            and os.path.exists(tts_file)
+            and tts_file.startswith(self.output_file)
+        ):
+            os.remove(tts_file)
         return audio_datas
 
     def _process_remaining_text(self):

main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py

@@ -1,3 +1,4 @@
+import os
 import asyncio
 import threading
 import traceback
@@ -149,7 +150,7 @@ class TTSProvider(TTSProviderBase):
         self.start_connection_flag = False
         self.tts_text = ""
         # 合成文字语音后，播放的音频文件列表
-        self.tts_audio_files = []
+        self.before_stop_play_files = []
         self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
             sample_rate=16000, channels=1, frame_size_ms=60
         )
@@ -188,6 +189,7 @@ class TTSProvider(TTSProviderBase):
                     )
                     future.result()
                     self.tts_audio_first_sentence = True
+                    self.before_stop_play_files.clear()
                 elif ContentType.TEXT == message.content_type:
                     if message.content_detail:
                         future = asyncio.run_coroutine_threadsafe(
@@ -196,9 +198,18 @@ class TTSProvider(TTSProviderBase):
                         )
                         future.result()
                 elif ContentType.FILE == message.content_type:
-                    pass
+                    self.before_stop_play_files.append(
+                        (message.content_file, message.content_detail)
+                    )
 
                 if message.sentence_type == SentenceType.LAST:
+                    for tts_file, text in self.before_stop_play_files:
+                        if tts_file and os.path.exists(tts_file):
+                            audio_datas = self._process_audio_file(tts_file)
+                            self.tts_audio_queue.put(
+                                (message.sentence_type, audio_datas, text)
+                            )
+                    self.before_stop_play_files.clear()
                     future = asyncio.run_coroutine_threadsafe(
                         self.finish_session(self.conn.sentence_id), loop=self.conn.loop
                     )