diff --git a/.gitignore b/.gitignore index 3de2252f..9469baac 100644 --- a/.gitignore +++ b/.gitignore @@ -146,7 +146,7 @@ tmp .history .DS_Store main/xiaozhi-server/data -main/xiaozhi-server/config/assets/wakeup_words.* + main/manager-web/node_modules .config.yaml .secrets.yaml @@ -163,6 +163,7 @@ main/xiaozhi-server/models/sherpa-onnx* /models/SenseVoiceSmall/model.pt my_wakeup_words.mp3 !main/xiaozhi-server/config/assets/bind_code.wav +!main/xiaozhi-server/config/assets/wakeup_words.wav !main/xiaozhi-server/config/assets/bind_not_found.wav !main/xiaozhi-server/config/assets/bind_code/*.wav !main/xiaozhi-server/config/assets/max_output_size.wav diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 5a474fd1..5b2b6884 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -85,6 +85,7 @@ module_test: # 唤醒词，用于识别唤醒词还是讲话内容 wakeup_words: - "你好小智" + - "嘿你好呀" - "你好小志" - "小爱同学" - "你好小鑫" diff --git a/main/xiaozhi-server/config/assets/wakeup_words.wav b/main/xiaozhi-server/config/assets/wakeup_words.wav new file mode 100644 index 00000000..e5422555 Binary files /dev/null and b/main/xiaozhi-server/config/assets/wakeup_words.wav differ diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index c29e8015..ccf67fff 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -1,24 +1,15 @@ import os +import sys import copy import json -import subprocess -import sys import uuid import time import queue import asyncio -import traceback import threading +import traceback +import subprocess import websockets -from typing import Dict, Any -from plugins_func.loadplugins import auto_import_modules -from config.logger import setup_logging -from config.config_loader import get_project_dir -from core.utils import p3 -from core.utils.dialogue import Message, Dialogue -from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType -from core.providers.tts.default import DefaultTTS -from core.handle.textHandle import handleTextMessage from core.utils.util import ( extract_json_from_string, initialize_modules, @@ -27,16 +18,23 @@ from core.utils.util import ( filter_sensitive_info, initialize_tts, ) +from typing import Dict, Any +from config.logger import setup_logging +from core.mcp.manager import MCPManager +from core.handle.reportHandle import report +from core.providers.tts.default import DefaultTTS from concurrent.futures import ThreadPoolExecutor -from core.handle.receiveAudioHandle import handleAudioMessage +from core.utils.dialogue import Message, Dialogue +from core.handle.textHandle import handleTextMessage from core.handle.functionHandler import FunctionHandler +from plugins_func.loadplugins import auto_import_modules from plugins_func.register import Action, ActionResponse from core.auth import AuthMiddleware, AuthenticationError -from core.mcp.manager import MCPManager from config.config_loader import get_private_config_from_api +from core.handle.receiveAudioHandle import handleAudioMessage +from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType from config.manage_api_client import DeviceNotFoundException, DeviceBindException -from core.utils.output_counter import add_device_output -from core.handle.reportHandle import report + TAG = __name__ @@ -860,23 +858,24 @@ class ConnectionHandler: def clear_queues(self): """清空所有任务队列""" - self.logger.bind(tag=TAG).debug( - f"开始清理: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}" - ) + if self.tts: + self.logger.bind(tag=TAG).debug( + f"开始清理: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}" + ) - # 使用非阻塞方式清空队列 - for q in [self.tts.tts_text_queue, self.tts.tts_audio_queue]: - if not q: - continue - while True: - try: - q.get_nowait() - except queue.Empty: - break + # 使用非阻塞方式清空队列 + for q in [self.tts.tts_text_queue, self.tts.tts_audio_queue]: + if not q: + continue + while True: + try: + q.get_nowait() + except queue.Empty: + break - self.logger.bind(tag=TAG).debug( - f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}" - ) + self.logger.bind(tag=TAG).debug( + f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}" + ) def reset_vad_states(self): self.client_audio_buffer = bytearray() diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 8a0b2ab9..422afe70 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -1,11 +1,14 @@ +import os +import time import json -from core.handle.sendAudioHandle import send_stt_message -from core.utils.util import remove_punctuation_and_length +import random import shutil import asyncio -import os -import random -import time +from core.providers.tts.dto.dto import ContentType +from core.providers.tts.dto.dto import SentenceType +from core.handle.sendAudioHandle import send_stt_message +from core.utils.util import remove_punctuation_and_length + TAG = __name__ @@ -49,11 +52,12 @@ async def checkWakeupWords(conn, text): if file is None: asyncio.create_task(wakeupWordsResponse(conn)) return False - opus_packets, _ = conn.tts.audio_to_opus_data(file) text_hello = WAKEUP_CONFIG["text"] if not text_hello: text_hello = text - conn.tts.tts_audio_queue.put((opus_packets, text_hello, 0)) + conn.tts.tts_one_sentence( + conn, ContentType.FILE, content_file=file, content_detail=text_hello + ) if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]: asyncio.create_task(wakeupWordsResponse(conn)) return True diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 14e1bc5a..18482a04 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -47,9 +47,8 @@ async def handleAudioMessage(conn, audio): text_len, _ = remove_punctuation_and_length(raw_text) if text_len > 0: # 使用自定义模块进行上报 - enqueue_asr_report(conn, raw_text, copy.deepcopy(conn.asr_audio)) - await startToChat(conn, raw_text) + enqueue_asr_report(conn, raw_text, copy.deepcopy(conn.asr_audio)) else: conn.asr_server_receive = True conn.asr_audio.clear() diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index b68744fe..c44ba640 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -10,6 +10,8 @@ from abc import ABC, abstractmethod from config.logger import setup_logging from core.utils.util import audio_to_data from core.utils.tts import MarkdownCleaner +from core.utils.output_counter import add_device_output +from core.handle.reportHandle import enqueue_tts_report from core.handle.sendAudioHandle import sendAudioMessage from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType, ContentType @@ -226,6 +228,9 @@ class TTSProviderBase(ABC): self.conn.loop, ) future.result() + if self.conn.max_output_size > 0 and text: + add_device_output(self.conn.headers.get("device-id"), len(text)) + enqueue_tts_report(self.conn, text, audio_datas) except Exception as e: logger.bind(tag=TAG).error( f"audio_play_priority priority_thread: {text} {e}" @@ -298,6 +303,14 @@ class TTSProviderBase(ABC): audio_datas, _ = self.audio_to_pcm_data(tts_file) else: audio_datas, _ = self.audio_to_opus_data(tts_file) + + if ( + self.delete_audio_file + and tts_file is not None + and os.path.exists(tts_file) + and tts_file.startswith(self.output_file) + ): + os.remove(tts_file) return audio_datas def _process_remaining_text(self): diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index c26eec3a..1bd4f14d 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -1,3 +1,4 @@ +import os import asyncio import threading import traceback @@ -149,7 +150,7 @@ class TTSProvider(TTSProviderBase): self.start_connection_flag = False self.tts_text = "" # 合成文字语音后，播放的音频文件列表 - self.tts_audio_files = [] + self.before_stop_play_files = [] self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( sample_rate=16000, channels=1, frame_size_ms=60 ) @@ -188,6 +189,7 @@ class TTSProvider(TTSProviderBase): ) future.result() self.tts_audio_first_sentence = True + self.before_stop_play_files.clear() elif ContentType.TEXT == message.content_type: if message.content_detail: future = asyncio.run_coroutine_threadsafe( @@ -196,9 +198,18 @@ class TTSProvider(TTSProviderBase): ) future.result() elif ContentType.FILE == message.content_type: - pass + self.before_stop_play_files.append( + (message.content_file, message.content_detail) + ) if message.sentence_type == SentenceType.LAST: + for tts_file, text in self.before_stop_play_files: + if tts_file and os.path.exists(tts_file): + audio_datas = self._process_audio_file(tts_file) + self.tts_audio_queue.put( + (message.sentence_type, audio_datas, text) + ) + self.before_stop_play_files.clear() future = asyncio.run_coroutine_threadsafe( self.finish_session(self.conn.sentence_id), loop=self.conn.loop )