From 3130044909fbd61d512286536540da933d9ef9f4 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 29 May 2025 09:05:08 +0800 Subject: [PATCH 01/31] test --- main/xiaozhi-server/config.yaml | 7 ++ .../core/providers/tts/linkerai.py | 95 +++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 main/xiaozhi-server/core/providers/tts/linkerai.py diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 25c8e017..473c095b 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -713,4 +713,11 @@ TTS: headers: # 自定义请求头 # Authorization: Bearer xxxx format: mp3 # 接口返回的音频格式 + output_dir: tmp/ + LinkeraiTTS: + #各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get + type: linkerai + api_url: https://tts.linkerai.top/tts + access_token: "test" + voice: "OUeAo1mhq6IBExi" output_dir: tmp/ \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py new file mode 100644 index 00000000..8ad069bd --- /dev/null +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -0,0 +1,95 @@ +from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import ( + TTSMessageDTO, + SentenceType, + ContentType, + InterfaceType +) +from config.logger import setup_logging +from core.utils import opus_encoder_utils +import requests + +TAG = __name__ +logger = setup_logging() + + +class TTSProvider(TTSProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__(config, delete_audio_file) + self.interface_type = InterfaceType.SINGLE_STREAM + self.access_token = config.get("access_token") + self.voice = config.get("voice") + self.api_url = config.get("api_url") + + # 根据配置选择音频格式，优先使用opus + self.audio_format = config.get("audio_format", "opus") + + # 创建Opus编码器 + self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( + sample_rate=16000, + channels=1, + frame_size_ms=60 + ) + + async def text_to_speak(self, text, _): + """将文本转换为语音（流式）""" + await self.send_text(self.voice, text) + return + + async def send_text(self, speaker: str, text: str): + """向 TTS 服务发送文本并获取音频流""" + try: + # 构造请求参数 + params = { + "tts_text": text, + "spk_id": self.voice, + "frame_duration": 60, + "stream": "true", + "target_sr": 16000, + "audio_format": self.audio_format, + } + + # 构造请求头 + headers = { + "Authorization": f"Bearer {self.access_token}", + } + + # 发送流式请求 + response = requests.get( + self.api_url, + params=params, + headers=headers, + stream=True + ) + + # 检查响应状态 + if response.status_code != 200: + logger.bind(tag=TAG).error(f"TTS 请求失败: {response.status_code}, {response.text}") + return + + # 处理音频流 + audio_frames = [] + for chunk in response.iter_content(chunk_size=1024): + if chunk: + if self.audio_format == "pcm": + # 将PCM转换为Opus + opus_frames = self.opus_encoder.encode_pcm_to_opus(chunk, False) + audio_frames.extend(opus_frames) + else: + # 直接使用Opus帧 + audio_frames.append(chunk) + + # 将音频帧放入队列 + self.tts_audio_queue.put( + (SentenceType.MIDDLE, audio_frames, text) + ) + + except Exception as e: + logger.bind(tag=TAG).error(f"TTS 流式处理异常：{str(e)}") + raise + + async def close(self): + """资源清理方法""" + await super().close() + if hasattr(self, "opus_encoder"): + self.opus_encoder.close() From 92affd6e1345374285be93aaae69596d47daa77e Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 30 May 2025 15:22:41 +0800 Subject: [PATCH 02/31] =?UTF-8?q?=E5=BE=85=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 1 + .../xiaozhi-server/core/providers/tts/base.py | 11 +- .../core/providers/tts/linkerai.py | 198 ++++++++++++++---- 3 files changed, 162 insertions(+), 48 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 473c095b..4230715a 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -718,6 +718,7 @@ TTS: #各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get type: linkerai api_url: https://tts.linkerai.top/tts + audio_format: "pcm" access_token: "test" voice: "OUeAo1mhq6IBExi" output_dir: tmp/ \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 0b17eef5..6bc77dff 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -331,10 +331,11 @@ class TTSProviderBase(ABC): segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) if segment_text: tts_file = self.to_tts(segment_text) - audio_datas = self._process_audio_file(tts_file) - self.tts_audio_queue.put( - (SentenceType.MIDDLE, audio_datas, segment_text) - ) - self.processed_chars += len(full_text) + if tts_file: + audio_datas = self._process_audio_file(tts_file) + self.tts_audio_queue.put( + (SentenceType.MIDDLE, audio_datas, segment_text) + ) + self.processed_chars += len(full_text) return True return False diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index 8ad069bd..cdb7296e 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -1,3 +1,8 @@ +import asyncio +import traceback +import queue +import requests +from pathlib import Path from core.providers.tts.base import TTSProviderBase from core.providers.tts.dto.dto import ( TTSMessageDTO, @@ -6,8 +11,7 @@ from core.providers.tts.dto.dto import ( InterfaceType ) from config.logger import setup_logging -from core.utils import opus_encoder_utils -import requests +from core.utils import opus_encoder_utils, textUtils TAG = __name__ logger = setup_logging() @@ -20,8 +24,6 @@ class TTSProvider(TTSProviderBase): self.access_token = config.get("access_token") self.voice = config.get("voice") self.api_url = config.get("api_url") - - # 根据配置选择音频格式，优先使用opus self.audio_format = config.get("audio_format", "opus") # 创建Opus编码器 @@ -31,15 +33,135 @@ class TTSProvider(TTSProviderBase): frame_size_ms=60 ) - async def text_to_speak(self, text, _): - """将文本转换为语音（流式）""" - await self.send_text(self.voice, text) + # 添加文本缓冲区 + self.text_buffer = "" + # 句子结束标点集合 + self.sentence_endings = ("。", "？", "！", "；", ":", ".", "?", "!", ";","……") + # 逗号类标点（用于第一句话分割） + self.comma_endings = ("，", "~", "、", ",", "。", ".", "？", "?", "！", "!", "；", ";", "：",) + + # PCM缓冲区 + self.pcm_buffer = bytearray() + + ################################################################################### + # linkerai单流式TTS重写父类的方法--开始 + ################################################################################### + + def tts_text_priority_thread(self): + """流式文本处理线程""" + while not self.conn.stop_event.is_set(): + try: + message = self.tts_text_queue.get(timeout=1) + logger.bind(tag=TAG).debug( + f"TTS任务｜{message.sentence_type.name}｜{message.content_type.name}" + ) + + if message.sentence_type == SentenceType.FIRST: + # 初始化流式状态 + self.tts_audio_first_sentence = True + self.pcm_buffer = bytearray() + self.text_buffer = "" # 重置文本缓冲区 + + elif ContentType.TEXT == message.content_type: + # 将文本添加到缓冲区 + self.text_buffer += message.content_detail + # 尝试分割并发送完整句子 + self._process_text_buffer() + + elif ContentType.FILE == message.content_type: + # 先处理缓冲区中的剩余文本 + self._flush_text_buffer() + # 处理文件类型 + if message.content_file and Path(message.content_file).exists(): + audio_datas = self._process_audio_file(message.content_file) + self.tts_audio_queue.put( + (SentenceType.MIDDLE, audio_datas, message.content_detail) + ) + + if message.sentence_type == SentenceType.LAST: + # 处理缓冲区中的剩余文本 + self._flush_text_buffer() + # 发送结束帧 + if self.pcm_buffer: + opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, is_end=True) + self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, "")) + self.pcm_buffer = bytearray() + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + self.text_buffer = "" # 重置文本缓冲区 + + except queue.Empty: + continue + except Exception as e: + logger.bind(tag=TAG).error( + f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" + ) + + def _process_text_buffer(self): + """处理文本缓冲区，分割并发送完整句子""" + while True: + # 查找最近的句子结束位置 + end_pos = -1 + for punct in self.sentence_endings: + pos = self.text_buffer.find(punct) + if pos != -1 and (end_pos == -1 or pos < end_pos): + end_pos = pos + + # 如果是第一句话，也允许在逗号处分隔 + if self.tts_audio_first_sentence and end_pos == -1: + for punct in self.comma_endings: + pos = self.text_buffer.find(punct) + if pos != -1 and (end_pos == -1 or pos < end_pos): + end_pos = pos + + # 找到分割点 + if end_pos != -1: + # 提取完整句子 + sentence = self.text_buffer[:end_pos + 1] + sentence = textUtils.get_string_no_punctuation_or_emoji(sentence) + + if not sentence.strip(): # 检查是否为空文本 + self.text_buffer = self.text_buffer[end_pos + 1:] + continue + + self.text_buffer = self.text_buffer[end_pos + 1:] + + # 发送句子 + future = asyncio.run_coroutine_threadsafe( + self.text_to_speak(sentence), + loop=self.conn.loop + ) + future.result() + + # 更新第一句话标志 + if self.tts_audio_first_sentence: + self.tts_audio_first_sentence = False + else: + break + + def _flush_text_buffer(self): + """处理缓冲区中剩余的文本""" + if self.text_buffer: + clean_text = textUtils.get_string_no_punctuation_or_emoji(self.text_buffer) + if clean_text.strip(): # 检查是否为空文本 + future = asyncio.run_coroutine_threadsafe( + self.text_to_speak(clean_text), + loop=self.conn.loop + ) + future.result() + self.text_buffer = "" + + async def text_to_speak(self, text): + # 发送文本 + await self.send_text(text) return - async def send_text(self, speaker: str, text: str): - """向 TTS 服务发送文本并获取音频流""" + ################################################################################### + # linkerai单流式TTS重写父类的方法--结束 + ################################################################################### + + async def send_text(self, text: str): + """流式处理TTS音频""" try: - # 构造请求参数 params = { "tts_text": text, "spk_id": self.voice, @@ -48,48 +170,38 @@ class TTSProvider(TTSProviderBase): "target_sr": 16000, "audio_format": self.audio_format, } + headers = {"Authorization": f"Bearer {self.access_token}"} - # 构造请求头 - headers = { - "Authorization": f"Bearer {self.access_token}", - } + with requests.get(self.api_url, params=params, headers=headers, stream=True) as response: + if response.status_code != 200: + logger.error(f"TTS请求失败: {response.status_code}, {response.text}") + return - # 发送流式请求 - response = requests.get( - self.api_url, - params=params, - headers=headers, - stream=True - ) + logger.debug(f"处理TTS文本: {text}") - # 检查响应状态 - if response.status_code != 200: - logger.bind(tag=TAG).error(f"TTS 请求失败: {response.status_code}, {response.text}") - return + # 流式处理音频数据 + for chunk in response.iter_content(chunk_size=1024): + if chunk: + # 实时编码并发送音频帧 + self.pcm_buffer.extend(chunk) - # 处理音频流 - audio_frames = [] - for chunk in response.iter_content(chunk_size=1024): - if chunk: - if self.audio_format == "pcm": - # 将PCM转换为Opus - opus_frames = self.opus_encoder.encode_pcm_to_opus(chunk, False) - audio_frames.extend(opus_frames) - else: - # 直接使用Opus帧 - audio_frames.append(chunk) - - # 将音频帧放入队列 - self.tts_audio_queue.put( - (SentenceType.MIDDLE, audio_frames, text) - ) + # 处理剩余缓冲区数据 + if self.pcm_buffer: + opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, is_end=True) + self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, text)) + self.pcm_buffer = bytearray() except Exception as e: - logger.bind(tag=TAG).error(f"TTS 流式处理异常：{str(e)}") + logger.error(f"TTS流式处理异常：{str(e)}") raise + # 保持原有方法 + def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False): + opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end) + return opus_datas + async def close(self): - """资源清理方法""" + """资源清理""" await super().close() if hasattr(self, "opus_encoder"): self.opus_encoder.close() From 0bc609cac47c1b33b2d85dcb927d09330899bc6d Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Tue, 3 Jun 2025 11:20:54 +0800 Subject: [PATCH 03/31] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=AE=9E=E6=97=B6?= =?UTF-8?q?=E7=BC=96=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/linkerai.py | 61 +++++++++++-------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index cdb7296e..ab585ef3 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -35,10 +35,6 @@ class TTSProvider(TTSProviderBase): # 添加文本缓冲区 self.text_buffer = "" - # 句子结束标点集合 - self.sentence_endings = ("。", "？", "！", "；", ":", ".", "?", "!", ";","……") - # 逗号类标点（用于第一句话分割） - self.comma_endings = ("，", "~", "、", ",", "。", ".", "？", "?", "！", "!", "；", ";", "：",) # PCM缓冲区 self.pcm_buffer = bytearray() @@ -52,9 +48,9 @@ class TTSProvider(TTSProviderBase): while not self.conn.stop_event.is_set(): try: message = self.tts_text_queue.get(timeout=1) - logger.bind(tag=TAG).debug( - f"TTS任务｜{message.sentence_type.name}｜{message.content_type.name}" - ) + # logger.bind(tag=TAG).debug( + # f"TTS任务｜{message.sentence_type.name}｜{message.content_type.name}" + # ) if message.sentence_type == SentenceType.FIRST: # 初始化流式状态 @@ -83,7 +79,7 @@ class TTSProvider(TTSProviderBase): self._flush_text_buffer() # 发送结束帧 if self.pcm_buffer: - opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, is_end=True) + opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, end_of_stream=True) self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, "")) self.pcm_buffer = bytearray() self.tts_audio_queue.put((SentenceType.LAST, [], None)) @@ -98,17 +94,20 @@ class TTSProvider(TTSProviderBase): def _process_text_buffer(self): """处理文本缓冲区，分割并发送完整句子""" + # 使用父类的标点集合 + sentence_endings = self.punctuations + comma_endings = self.first_sentence_punctuations while True: # 查找最近的句子结束位置 end_pos = -1 - for punct in self.sentence_endings: + for punct in sentence_endings: pos = self.text_buffer.find(punct) if pos != -1 and (end_pos == -1 or pos < end_pos): end_pos = pos # 如果是第一句话，也允许在逗号处分隔 if self.tts_audio_first_sentence and end_pos == -1: - for punct in self.comma_endings: + for punct in comma_endings: pos = self.text_buffer.find(punct) if pos != -1 and (end_pos == -1 or pos < end_pos): end_pos = pos @@ -160,7 +159,7 @@ class TTSProvider(TTSProviderBase): ################################################################################### async def send_text(self, text: str): - """流式处理TTS音频""" + """流式处理TTS音频，每句只推送一次音频列表""" try: params = { "tts_text": text, @@ -172,32 +171,44 @@ class TTSProvider(TTSProviderBase): } headers = {"Authorization": f"Bearer {self.access_token}"} - with requests.get(self.api_url, params=params, headers=headers, stream=True) as response: + with requests.get(self.api_url, params=params, headers=headers, stream=True, timeout=5) as response: if response.status_code != 200: logger.error(f"TTS请求失败: {response.status_code}, {response.text}") + # 推送空LAST，防止播放端卡死 + self.tts_audio_queue.put((SentenceType.LAST, [], None)) return logger.debug(f"处理TTS文本: {text}") - # 流式处理音频数据 - for chunk in response.iter_content(chunk_size=1024): + pcm_buffer = bytearray() + frame_bytes = self.opus_encoder.frame_size * 4 # 每帧字节数（int16=2字节） + opus_datas = [] + for chunk in response.iter_content(chunk_size=960): if chunk: - # 实时编码并发送音频帧 - self.pcm_buffer.extend(chunk) - - # 处理剩余缓冲区数据 - if self.pcm_buffer: - opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, is_end=True) - self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, text)) - self.pcm_buffer = bytearray() + pcm_buffer.extend(chunk) + # 只要够帧就编码 + while len(pcm_buffer) >= frame_bytes: + frame = pcm_buffer[:frame_bytes] + opus_chunk = self.opus_encoder.encode_pcm_to_opus(frame, end_of_stream=False) + if opus_chunk: + opus_datas.extend(opus_chunk) + pcm_buffer = pcm_buffer[frame_bytes:] # 剩余部分继续累积 + # 处理最后剩余数据 + if pcm_buffer: + opus_chunk = self.opus_encoder.encode_pcm_to_opus(pcm_buffer, end_of_stream=True) + if opus_chunk: + opus_datas.extend(opus_chunk) + # 推送本句所有音频帧 + self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, text)) except Exception as e: logger.error(f"TTS流式处理异常：{str(e)}") - raise + # 推送空LAST，防止播放端卡死 + self.tts_audio_queue.put((SentenceType.LAST, [], None)) # 保持原有方法 - def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False): - opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end) + def wav_to_opus_data_audio_raw(self, raw_data_var): + opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, end_of_stream=True) return opus_datas async def close(self): From 4cc7247c3733c9c88f343c7432731e9b296e6a88 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Wed, 4 Jun 2025 14:35:21 +0800 Subject: [PATCH 04/31] =?UTF-8?q?update:funasr=E5=AE=9E=E4=BE=8B=E4=B9=8B?= =?UTF-8?q?=E5=89=8D=E6=B7=BB=E5=8A=A0=E5=86=85=E5=AD=98=E6=A3=80=E6=B5=8B?= =?UTF-8?q?=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/providers/asr/fun_local.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 8c7305c6..4b25a265 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -2,6 +2,7 @@ import time import os import sys import io +import psutil from config.logger import setup_logging from typing import Optional, Tuple, List from core.providers.asr.base import ASRProviderBase @@ -37,6 +38,13 @@ class CaptureOutput: class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): super().__init__() + + # 内存检测，要求大于2G + min_mem_bytes = 2 * 1024 * 1024 * 1024 + total_mem = psutil.virtual_memory().total + if total_mem < min_mem_bytes: + logger.bind(tag=TAG).error(f"可用内存不足2G，当前仅有 {total_mem / (1024*1024):.2f} MB") + self.interface_type = InterfaceType.LOCAL self.model_dir = config.get("model_dir") self.output_dir = config.get("output_dir") # 修正配置键名 From 0268f90e9c69ae0ac9556cdcbebf59946ce76938 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Wed, 4 Jun 2025 14:46:52 +0800 Subject: [PATCH 05/31] =?UTF-8?q?update=EF=BC=9A=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E6=A8=A1=E5=9D=97=EF=BC=8C=E4=BD=BF=E5=85=B6?= =?UTF-8?q?=E6=8C=89=E6=97=A5=E6=9C=9F=E5=92=8C=E5=A4=A7=E5=B0=8F=E5=88=86?= =?UTF-8?q?=E5=89=B2=E7=9A=84=E6=97=A5=E5=BF=97=E6=96=87=E4=BB=B6=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config/logger.py | 91 +++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 16 deletions(-) diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index 5f107f5d..e5bf8466 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -3,15 +3,16 @@ import sys from loguru import logger from config.config_loader import load_config from config.settings import check_config_file +from datetime import datetime SERVER_VERSION = "0.5.2" _logger_initialized = False - +_current_log_file = None +_current_log_size = 0 +_max_log_size = 10 * 1024 * 1024 # 10MB def get_module_abbreviation(module_name, module_dict): - """获取模块名称的缩写，如果为空则返回00 - 如果名称中包含下划线，则返回下划线后面的前两个字符 - """ + """获取模块名称的缩写，如果为空则返回00""" module_value = module_dict.get(module_name, "") if not module_value: return "00" @@ -20,7 +21,6 @@ def get_module_abbreviation(module_name, module_dict): return parts[-1][:2] if parts[-1] else "00" return module_value[:2] - def build_module_string(selected_module): """构建模块字符串""" return ( @@ -32,12 +32,59 @@ def build_module_string(selected_module): + get_module_abbreviation("Intent", selected_module) ) - def formatter(record): """为没有 tag 的日志添加默认值""" record["extra"].setdefault("tag", record["name"]) return record["message"] +def get_log_filename(log_dir, base_name): + """生成按日期和大小分割的日志文件名""" + global _current_log_file, _current_log_size + + now = datetime.now() + date_part = f"{now.month}.{now.day}" + + # 检查是否需要创建新文件 + if _current_log_file is None or not os.path.exists(_current_log_file): + file_index = 1 + while True: + new_filename = os.path.join(log_dir, f"{base_name}.{date_part}.{file_index}") + if not os.path.exists(new_filename): + _current_log_file = new_filename + _current_log_size = 0 + return new_filename + # 如果文件已存在，增加索引 + file_index += 1 + else: + # 检查文件大小 + if _current_log_size > _max_log_size: + # 文件过大，创建新文件 + base_path = os.path.splitext(_current_log_file)[0] + parts = base_path.split('.') + if len(parts) >= 3: + try: + file_index = int(parts[-1]) + 1 + except ValueError: + file_index = 1 + else: + file_index = 1 + + new_filename = f"{base_path.rsplit('.', 1)[0]}.{file_index}" + _current_log_file = new_filename + _current_log_size = 0 + return new_filename + else: + return _current_log_file + +def update_log_size(message): + """更新当前日志文件大小""" + global _current_log_size + # 如果当前日志文件存在，获取其大小 + if _current_log_file and os.path.exists(_current_log_file): + _current_log_size = os.path.getsize(_current_log_file) + else: + # 如果当前日志文件不存在，重置大小 + _current_log_size = len(message.encode('utf-8')) def setup_logging(): check_config_file() @@ -52,7 +99,7 @@ def setup_logging(): extra={ "selected_module": log_config.get("selected_module", "00000000000000") } - ) # 新增配置 + ) log_format = log_config.get( "log_format", "{time:YYMMDD HH:mm:ss}[{version}_{extra[selected_module]}][{extra[tag]}]-{level}-{message}", @@ -72,7 +119,7 @@ def setup_logging(): log_level = log_config.get("log_level", "INFO") log_dir = log_config.get("log_dir", "tmp") - log_file = log_config.get("log_file", "server.log") + log_file = log_config.get("log_file", "server") data_dir = log_config.get("data_dir", "data") os.makedirs(log_dir, exist_ok=True) @@ -84,18 +131,23 @@ def setup_logging(): # 输出到控制台 logger.add(sys.stdout, format=log_format, level=log_level, filter=formatter) - # 输出到文件 + # 输出到文件，使用自定义的日志文件名生成器 + def sink(message): + filename = get_log_filename(log_dir, log_file) + with open(filename, "a", encoding="utf-8") as f: + f.write(message + "\n") + update_log_size(message) + logger.add( - os.path.join(log_dir, log_file), + sink, format=log_format_file, level=log_level, filter=formatter, ) - _logger_initialized = True # 标记为已初始化 + _logger_initialized = True return logger - def update_module_string(selected_module_str): """更新模块字符串并重新配置日志处理器""" logger.debug(f"更新日志配置组件") @@ -126,6 +178,9 @@ def update_module_string(selected_module_str): "{selected_module}", selected_module_str ) + log_dir = log_config.get("log_dir", "tmp") + log_file = log_config.get("log_file", "server") + logger.remove() logger.add( sys.stdout, @@ -133,11 +188,15 @@ def update_module_string(selected_module_str): level=log_config.get("log_level", "INFO"), filter=formatter, ) + + def sink(message): + filename = get_log_filename(log_dir, log_file) + with open(filename, "a", encoding="utf-8") as f: + f.write(message + "\n") + update_log_size(message) + logger.add( - os.path.join( - log_config.get("log_dir", "tmp"), - log_config.get("log_file", "server.log"), - ), + sink, format=log_format_file, level=log_config.get("log_level", "INFO"), filter=formatter, From fba758ea36024629236e80359fc9d8f8dbc9f68f Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Wed, 4 Jun 2025 15:42:52 +0800 Subject: [PATCH 06/31] =?UTF-8?q?update:=E6=81=A2=E5=A4=8D=E6=9C=80?= =?UTF-8?q?=E5=88=9D=E7=9A=84=E6=97=A5=E5=BF=97=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config/logger.py | 93 +++++----------------------- 1 file changed, 17 insertions(+), 76 deletions(-) diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index e5bf8466..643952dd 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -3,16 +3,15 @@ import sys from loguru import logger from config.config_loader import load_config from config.settings import check_config_file -from datetime import datetime -SERVER_VERSION = "0.5.2" +SERVER_VERSION = "0.5.4" _logger_initialized = False -_current_log_file = None -_current_log_size = 0 -_max_log_size = 10 * 1024 * 1024 # 10MB + def get_module_abbreviation(module_name, module_dict): - """获取模块名称的缩写，如果为空则返回00""" + """获取模块名称的缩写，如果为空则返回00 + 如果名称中包含下划线，则返回下划线后面的前两个字符 + """ module_value = module_dict.get(module_name, "") if not module_value: return "00" @@ -21,6 +20,7 @@ def get_module_abbreviation(module_name, module_dict): return parts[-1][:2] if parts[-1] else "00" return module_value[:2] + def build_module_string(selected_module): """构建模块字符串""" return ( @@ -32,59 +32,12 @@ def build_module_string(selected_module): + get_module_abbreviation("Intent", selected_module) ) + def formatter(record): """为没有 tag 的日志添加默认值""" record["extra"].setdefault("tag", record["name"]) return record["message"] -def get_log_filename(log_dir, base_name): - """生成按日期和大小分割的日志文件名""" - global _current_log_file, _current_log_size - - now = datetime.now() - date_part = f"{now.month}.{now.day}" - - # 检查是否需要创建新文件 - if _current_log_file is None or not os.path.exists(_current_log_file): - file_index = 1 - while True: - new_filename = os.path.join(log_dir, f"{base_name}.{date_part}.{file_index}") - if not os.path.exists(new_filename): - _current_log_file = new_filename - _current_log_size = 0 - return new_filename - # 如果文件已存在，增加索引 - file_index += 1 - else: - # 检查文件大小 - if _current_log_size > _max_log_size: - # 文件过大，创建新文件 - base_path = os.path.splitext(_current_log_file)[0] - parts = base_path.split('.') - if len(parts) >= 3: - try: - file_index = int(parts[-1]) + 1 - except ValueError: - file_index = 1 - else: - file_index = 1 - - new_filename = f"{base_path.rsplit('.', 1)[0]}.{file_index}" - _current_log_file = new_filename - _current_log_size = 0 - return new_filename - else: - return _current_log_file - -def update_log_size(message): - """更新当前日志文件大小""" - global _current_log_size - # 如果当前日志文件存在，获取其大小 - if _current_log_file and os.path.exists(_current_log_file): - _current_log_size = os.path.getsize(_current_log_file) - else: - # 如果当前日志文件不存在，重置大小 - _current_log_size = len(message.encode('utf-8')) def setup_logging(): check_config_file() @@ -99,7 +52,7 @@ def setup_logging(): extra={ "selected_module": log_config.get("selected_module", "00000000000000") } - ) + ) # 新增配置 log_format = log_config.get( "log_format", "{time:YYMMDD HH:mm:ss}[{version}_{extra[selected_module]}][{extra[tag]}]-{level}-{message}", @@ -119,7 +72,7 @@ def setup_logging(): log_level = log_config.get("log_level", "INFO") log_dir = log_config.get("log_dir", "tmp") - log_file = log_config.get("log_file", "server") + log_file = log_config.get("log_file", "server.log") data_dir = log_config.get("data_dir", "data") os.makedirs(log_dir, exist_ok=True) @@ -131,23 +84,18 @@ def setup_logging(): # 输出到控制台 logger.add(sys.stdout, format=log_format, level=log_level, filter=formatter) - # 输出到文件，使用自定义的日志文件名生成器 - def sink(message): - filename = get_log_filename(log_dir, log_file) - with open(filename, "a", encoding="utf-8") as f: - f.write(message + "\n") - update_log_size(message) - + # 输出到文件 logger.add( - sink, + os.path.join(log_dir, log_file), format=log_format_file, level=log_level, filter=formatter, ) - _logger_initialized = True + _logger_initialized = True # 标记为已初始化 return logger + def update_module_string(selected_module_str): """更新模块字符串并重新配置日志处理器""" logger.debug(f"更新日志配置组件") @@ -178,9 +126,6 @@ def update_module_string(selected_module_str): "{selected_module}", selected_module_str ) - log_dir = log_config.get("log_dir", "tmp") - log_file = log_config.get("log_file", "server") - logger.remove() logger.add( sys.stdout, @@ -188,15 +133,11 @@ def update_module_string(selected_module_str): level=log_config.get("log_level", "INFO"), filter=formatter, ) - - def sink(message): - filename = get_log_filename(log_dir, log_file) - with open(filename, "a", encoding="utf-8") as f: - f.write(message + "\n") - update_log_size(message) - logger.add( - sink, + os.path.join( + log_config.get("log_dir", "tmp"), + log_config.get("log_file", "server.log"), + ), format=log_format_file, level=log_config.get("log_level", "INFO"), filter=formatter, From 23cb7616d920dc3b07a6ae813fe7b334ca1cd4ad Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 4 Jun 2025 16:46:49 +0800 Subject: [PATCH 07/31] =?UTF-8?q?update:=20=E6=9B=B4=E6=94=B9to=5Ftts?= =?UTF-8?q?=E4=BF=9D=E5=AD=98=E4=B8=B4=E6=97=B6=E6=96=87=E4=BB=B6=E5=88=A4?= =?UTF-8?q?=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 32 +++---- .../core/providers/tts/aliyun.py | 11 ++- .../xiaozhi-server/core/providers/tts/base.py | 95 +++++++++++++------ .../core/providers/tts/cozecn.py | 11 ++- .../core/providers/tts/custom.py | 9 +- .../core/providers/tts/doubao.py | 12 ++- .../xiaozhi-server/core/providers/tts/edge.py | 25 +++-- .../core/providers/tts/fishspeech.py | 9 +- .../core/providers/tts/gpt_sovits_v2.py | 8 +- .../core/providers/tts/gpt_sovits_v3.py | 8 +- .../core/providers/tts/minimax.py | 9 +- .../core/providers/tts/openai.py | 10 +- .../core/providers/tts/siliconflow.py | 10 +- .../core/providers/tts/tencent.py | 15 +-- .../core/providers/tts/ttson.py | 9 +- main/xiaozhi-server/core/utils/p3.py | 26 +++++ main/xiaozhi-server/core/utils/util.py | 46 +++++++++ 17 files changed, 253 insertions(+), 92 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 6b8cb4fd..859059b6 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -2,10 +2,9 @@ import os import time import json import random -import shutil import asyncio from core.handle.sendAudioHandle import send_stt_message -from core.utils.util import remove_punctuation_and_length +from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes from core.providers.tts.dto.dto import ContentType, InterfaceType from core.handle.mcpHandle import ( MCPClient, @@ -119,19 +118,18 @@ async def wakeupWordsResponse(conn): result = conn.llm.response_no_stream(conn.config["prompt"], question) if result is None or result == "": return - tts_file = await asyncio.to_thread(conn.tts.to_tts, result) - if tts_file is not None and os.path.exists(tts_file): - file_type = os.path.splitext(tts_file)[1] - if file_type: - file_type = file_type.lstrip(".") - old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"]) - if old_file is not None: - os.remove(old_file) - """将文件挪到"wakeup_words.mp3""" - shutil.move( - tts_file, - WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type, - ) - WAKEUP_CONFIG["create_time"] = time.time() - WAKEUP_CONFIG["text"] = result + opus_datas = await asyncio.to_thread(conn.tts.to_tts, result) + if not opus_datas: + return + + wav_bytes = opus_datas_to_wav_bytes(opus_datas, sample_rate=16000) + file_path = os.path.join( + WAKEUP_CONFIG["dir"], "my_" + WAKEUP_CONFIG["file_name"] + ".wav" + ) + # 写入wav数据 + with open(file_path, "wb") as f: + f.write(wav_bytes) + + WAKEUP_CONFIG["create_time"] = time.time() + WAKEUP_CONFIG["text"] = result diff --git a/main/xiaozhi-server/core/providers/tts/aliyun.py b/main/xiaozhi-server/core/providers/tts/aliyun.py index 1fda8d77..c9b3d130 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun.py @@ -91,7 +91,7 @@ class TTSProvider(TTSProviderBase): self.appkey = config.get("appkey") self.format = config.get("format", "wav") - + self.audio_file_type = config.get("format", "wav") sample_rate = config.get("sample_rate", "16000") self.sample_rate = int(sample_rate) if sample_rate else 16000 @@ -188,9 +188,12 @@ class TTSProvider(TTSProviderBase): ) # 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的 if resp.headers["Content-Type"].startswith("audio/"): - with open(output_file, "wb") as f: - f.write(resp.content) - return output_file + if output_file: + with open(output_file, "wb") as f: + f.write(resp.content) + return output_file + else: + return resp.content else: raise Exception( f"{__name__} status_code: {resp.status_code} response: {resp.content}" diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 9c094f24..b711659b 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -8,7 +8,7 @@ from datetime import datetime from core.utils import textUtils from abc import ABC, abstractmethod from config.logger import setup_logging -from core.utils.util import audio_to_data +from core.utils.util import audio_to_data, audio_bytes_to_data from core.utils.tts import MarkdownCleaner from core.utils.output_counter import add_device_output from core.handle.reportHandle import enqueue_tts_report @@ -20,7 +20,6 @@ from core.providers.tts.dto.dto import ( InterfaceType, ) - import traceback TAG = __name__ @@ -33,6 +32,7 @@ class TTSProviderBase(ABC): self.conn = None self.tts_timeout = 10 self.delete_audio_file = delete_audio_file + self.audio_file_type = "wav" self.output_file = config.get("output_dir", "tmp/") self.tts_text_queue = queue.Queue() self.tts_audio_queue = queue.Queue() @@ -76,35 +76,60 @@ class TTSProviderBase(ABC): ) def to_tts(self, text): - tmp_file = self.generate_filename() - try: - max_repeat_time = 5 - text = MarkdownCleaner.clean_markdown(text) - while not os.path.exists(tmp_file) and max_repeat_time > 0: + text = MarkdownCleaner.clean_markdown(text) + max_repeat_time = 5 + if self.delete_audio_file: + # 需要删除文件的直接转为音频数据 + while max_repeat_time > 0: try: - asyncio.run(self.text_to_speak(text, tmp_file)) + audio_bytes = asyncio.run(self.text_to_speak(text, None)) + if audio_bytes: + audio_datas, _ = audio_bytes_to_data(audio_bytes, file_type=self.audio_file_type, is_opus=True) + return audio_datas + else: + max_repeat_time -= 1 except Exception as e: logger.bind(tag=TAG).warning( f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}" ) - # 未执行成功，删除文件 - if os.path.exists(tmp_file): - os.remove(tmp_file) max_repeat_time -= 1 - if max_repeat_time > 0: logger.bind(tag=TAG).info( - f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次" + f"语音生成成功: {text}，重试{5 - max_repeat_time}次" ) else: logger.bind(tag=TAG).error( f"语音生成失败: {text}，请检查网络或服务是否正常" ) - - return tmp_file - except Exception as e: - logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}") return None + else: + tmp_file = self.generate_filename() + try: + while not os.path.exists(tmp_file) and max_repeat_time > 0: + try: + asyncio.run(self.text_to_speak(text, tmp_file)) + except Exception as e: + logger.bind(tag=TAG).warning( + f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}" + ) + # 未执行成功，删除文件 + if os.path.exists(tmp_file): + os.remove(tmp_file) + max_repeat_time -= 1 + + if max_repeat_time > 0: + logger.bind(tag=TAG).info( + f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次" + ) + else: + logger.bind(tag=TAG).error( + f"语音生成失败: {text}，请检查网络或服务是否正常" + ) + + return tmp_file + except Exception as e: + logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}") + return None @abstractmethod async def text_to_speak(self, text, output_file): @@ -192,12 +217,19 @@ class TTSProviderBase(ABC): self.tts_text_buff.append(message.content_detail) segment_text = self._get_segment_text() if segment_text: - tts_file = self.to_tts(segment_text) - if tts_file: - audio_datas = self._process_audio_file(tts_file) - self.tts_audio_queue.put( - (message.sentence_type, audio_datas, segment_text) - ) + if self.delete_audio_file: + audio_datas = self.to_tts(segment_text) + if audio_datas: + self.tts_audio_queue.put( + (message.sentence_type, audio_datas, segment_text) + ) + else: + tts_file = self.to_tts(segment_text) + if tts_file: + audio_datas = self._process_audio_file(tts_file) + self.tts_audio_queue.put( + (message.sentence_type, audio_datas, segment_text) + ) elif ContentType.FILE == message.content_type: self._process_remaining_text() tts_file = message.content_file @@ -334,11 +366,18 @@ class TTSProviderBase(ABC): if remaining_text: segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) if segment_text: - tts_file = self.to_tts(segment_text) - audio_datas = self._process_audio_file(tts_file) - self.tts_audio_queue.put( - (SentenceType.MIDDLE, audio_datas, segment_text) - ) + if self.delete_audio_file: + audio_datas = self.to_tts(segment_text) + if audio_datas: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, audio_datas, segment_text) + ) + else: + tts_file = self.to_tts(segment_text) + audio_datas = self._process_audio_file(tts_file) + self.tts_audio_queue.put( + (SentenceType.MIDDLE, audio_datas, segment_text) + ) self.processed_chars += len(full_text) return True return False diff --git a/main/xiaozhi-server/core/providers/tts/cozecn.py b/main/xiaozhi-server/core/providers/tts/cozecn.py index c3019b67..ede08928 100644 --- a/main/xiaozhi-server/core/providers/tts/cozecn.py +++ b/main/xiaozhi-server/core/providers/tts/cozecn.py @@ -11,8 +11,8 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("private_voice") else: self.voice = config.get("voice") - self.response_format = config.get("response_format") - + self.response_format = config.get("response_format", "mp3") + self.audio_file_type = config.get("response_format", "mp3") self.host = "api.coze.cn" self.api_url = f"https://{self.host}/v1/audio/speech" @@ -33,7 +33,10 @@ class TTSProvider(TTSProviderBase): "POST", self.api_url, json=request_json, headers=headers ) data = response.content - file_to_save = open(output_file, "wb") - file_to_save.write(data) + if output_file: + with open(output_file, "wb") as file_to_save: + file_to_save.write(data) + else: + return data except Exception as e: raise Exception(f"{__name__} error: {e}") diff --git a/main/xiaozhi-server/core/providers/tts/custom.py b/main/xiaozhi-server/core/providers/tts/custom.py index a5300a3e..b417825e 100644 --- a/main/xiaozhi-server/core/providers/tts/custom.py +++ b/main/xiaozhi-server/core/providers/tts/custom.py @@ -16,8 +16,8 @@ class TTSProvider(TTSProviderBase): self.method = config.get("method", "GET") self.headers = config.get("headers", {}) self.format = config.get("format", "wav") + self.audio_file_type = config.get("format", "wav") self.output_file = config.get("output_dir", "tmp/") - self.params = config.get("params") if isinstance(self.params, str): @@ -43,8 +43,11 @@ class TTSProvider(TTSProviderBase): else: resp = requests.get(self.url, params=request_params, headers=self.headers) if resp.status_code == 200: - with open(output_file, "wb") as file: - file.write(resp.content) + if output_file: + with open(output_file, "wb") as file: + file.write(resp.content) + else: + return resp.content else: error_msg = f"Custom TTS请求失败: {resp.status_code} - {resp.text}" logger.bind(tag=TAG).error(error_msg) diff --git a/main/xiaozhi-server/core/providers/tts/doubao.py b/main/xiaozhi-server/core/providers/tts/doubao.py index 78058c31..1a21ffa1 100644 --- a/main/xiaozhi-server/core/providers/tts/doubao.py +++ b/main/xiaozhi-server/core/providers/tts/doubao.py @@ -29,7 +29,7 @@ class TTSProvider(TTSProviderBase): speed_ratio = config.get("speed_ratio", "1.0") volume_ratio = config.get("volume_ratio", "1.0") pitch_ratio = config.get("pitch_ratio", "1.0") - + self.audio_file_type = config.get("format", "wav") self.speed_ratio = float(speed_ratio) if speed_ratio else 1.0 self.volume_ratio = float(volume_ratio) if volume_ratio else 1.0 self.pitch_ratio = float(pitch_ratio) if pitch_ratio else 1.0 @@ -49,7 +49,7 @@ class TTSProvider(TTSProviderBase): "user": {"uid": "1"}, "audio": { "voice_type": self.voice, - "encoding": "wav", + "encoding": self.audio_file_type, "speed_ratio": self.speed_ratio, "volume_ratio": self.volume_ratio, "pitch_ratio": self.pitch_ratio, @@ -70,8 +70,12 @@ class TTSProvider(TTSProviderBase): ) if "data" in resp.json(): data = resp.json()["data"] - file_to_save = open(output_file, "wb") - file_to_save.write(base64.b64decode(data)) + audio_bytes = base64.b64decode(data) + if output_file: + with open(output_file, "wb") as file_to_save: + file_to_save.write(audio_bytes) + else: + return audio_bytes else: raise Exception( f"{__name__} status_code: {resp.status_code} response: {resp.content}" diff --git a/main/xiaozhi-server/core/providers/tts/edge.py b/main/xiaozhi-server/core/providers/tts/edge.py index 3d9b2547..ffa1c3aa 100644 --- a/main/xiaozhi-server/core/providers/tts/edge.py +++ b/main/xiaozhi-server/core/providers/tts/edge.py @@ -12,6 +12,7 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("private_voice") else: self.voice = config.get("voice") + self.audio_file_type = config.get("format", "mp3") def generate_filename(self, extension=".mp3"): return os.path.join( @@ -22,16 +23,24 @@ class TTSProvider(TTSProviderBase): async def text_to_speak(self, text, output_file): try: communicate = edge_tts.Communicate(text, voice=self.voice) - # 确保目录存在并创建空文件 - os.makedirs(os.path.dirname(output_file), exist_ok=True) - with open(output_file, "wb") as f: - pass + if output_file: + # 确保目录存在并创建空文件 + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, "wb") as f: + pass - # 流式写入音频数据 - with open(output_file, "ab") as f: # 改为追加模式避免覆盖 + # 流式写入音频数据 + with open(output_file, "ab") as f: # 改为追加模式避免覆盖 + async for chunk in communicate.stream(): + if chunk["type"] == "audio": # 只处理音频数据块 + f.write(chunk["data"]) + else: + # 返回音频二进制数据 + audio_bytes = b"" async for chunk in communicate.stream(): - if chunk["type"] == "audio": # 只处理音频数据块 - f.write(chunk["data"]) + if chunk["type"] == "audio": + audio_bytes += chunk["data"] + return audio_bytes except Exception as e: error_msg = f"Edge TTS请求失败: {e}" raise Exception(error_msg) # 抛出异常，让调用方捕获 \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/fishspeech.py b/main/xiaozhi-server/core/providers/tts/fishspeech.py index 23b20e76..3bcb1229 100644 --- a/main/xiaozhi-server/core/providers/tts/fishspeech.py +++ b/main/xiaozhi-server/core/providers/tts/fishspeech.py @@ -88,7 +88,7 @@ class TTSProvider(TTSProviderBase): self.reference_audio = parse_string_to_list(config.get("reference_audio")) self.reference_text = parse_string_to_list(config.get("reference_text")) self.format = config.get("response_format", "wav") - + self.audio_file_type = config.get("response_format", "wav") self.api_key = config.get("api_key", "YOUR_API_KEY") have_key = check_model_key("FishSpeech TTS", self.api_key) if not have_key: @@ -170,8 +170,11 @@ class TTSProvider(TTSProviderBase): if response.status_code == 200: audio_content = response.content - with open(output_file, "wb") as audio_file: - audio_file.write(audio_content) + if output_file: + with open(output_file, "wb") as audio_file: + audio_file.write(audio_content) + else: + return audio_content else: error_msg = f"Request failed with status code {response.status_code}" diff --git a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py index e76b3f22..7f43ad7d 100644 --- a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py +++ b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py @@ -65,6 +65,7 @@ class TTSProvider(TTSProviderBase): self.aux_ref_audio_paths = parse_string_to_list( config.get("aux_ref_audio_paths") ) + self.audio_file_type = config.get("format", "wav") async def text_to_speak(self, text, output_file): request_json = { @@ -91,8 +92,11 @@ class TTSProvider(TTSProviderBase): resp = requests.post(self.url, json=request_json) if resp.status_code == 200: - with open(output_file, "wb") as file: - file.write(resp.content) + if output_file: + with open(output_file, "wb") as file: + file.write(resp.content) + else: + return resp.content else: error_msg = f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}" logger.bind(tag=TAG).error(error_msg) diff --git a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py index 6b66efc3..ae41fc4e 100644 --- a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py +++ b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py @@ -32,6 +32,7 @@ class TTSProvider(TTSProviderBase): self.cut_punc = config.get("cut_punc", "") self.inp_refs = parse_string_to_list(config.get("inp_refs")) self.if_sr = str(config.get("if_sr", False)).lower() in ("true", "1", "yes") + self.audio_file_type = config.get("format", "wav") async def text_to_speak(self, text, output_file): request_params = { @@ -52,8 +53,11 @@ class TTSProvider(TTSProviderBase): resp = requests.get(self.url, params=request_params) if resp.status_code == 200: - with open(output_file, "wb") as file: - file.write(resp.content) + if output_file: + with open(output_file, "wb") as file: + file.write(resp.content) + else: + return resp.content else: error_msg = f"GPT_SoVITS_V3 TTS请求失败: {resp.status_code} - {resp.text}" logger.bind(tag=TAG).error(error_msg) diff --git a/main/xiaozhi-server/core/providers/tts/minimax.py b/main/xiaozhi-server/core/providers/tts/minimax.py index dd406b64..3741d169 100644 --- a/main/xiaozhi-server/core/providers/tts/minimax.py +++ b/main/xiaozhi-server/core/providers/tts/minimax.py @@ -52,6 +52,7 @@ class TTSProvider(TTSProviderBase): "Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}", } + self.audio_file_type = defult_audio_setting.get("format", "mp3") def generate_filename(self, extension=".mp3"): return os.path.join( @@ -80,8 +81,12 @@ class TTSProvider(TTSProviderBase): # 检查返回请求数据的status_code是否为0 if resp.json()["base_resp"]["status_code"] == 0: data = resp.json()["data"]["audio"] - file_to_save = open(output_file, "wb") - file_to_save.write(bytes.fromhex(data)) + audio_bytes = bytes.fromhex(data) + if output_file: + with open(output_file, "wb") as file_to_save: + file_to_save.write(audio_bytes) + else: + return audio_bytes else: raise Exception( f"{__name__} status_code: {resp.status_code} response: {resp.content}" diff --git a/main/xiaozhi-server/core/providers/tts/openai.py b/main/xiaozhi-server/core/providers/tts/openai.py index bdbc9174..e5155513 100644 --- a/main/xiaozhi-server/core/providers/tts/openai.py +++ b/main/xiaozhi-server/core/providers/tts/openai.py @@ -17,7 +17,8 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("private_voice") else: self.voice = config.get("voice", "alloy") - self.response_format = "wav" + self.response_format = config.get("format", "wav") + self.audio_file_type = config.get("format", "wav") # 处理空字符串的情况 speed = config.get("speed", "1.0") @@ -40,8 +41,11 @@ class TTSProvider(TTSProviderBase): } response = requests.post(self.api_url, json=data, headers=headers) if response.status_code == 200: - with open(output_file, "wb") as audio_file: - audio_file.write(response.content) + if output_file: + with open(output_file, "wb") as audio_file: + audio_file.write(response.content) + else: + return response.content else: raise Exception( f"OpenAI TTS请求失败: {response.status_code} - {response.text}" diff --git a/main/xiaozhi-server/core/providers/tts/siliconflow.py b/main/xiaozhi-server/core/providers/tts/siliconflow.py index 8f7fd641..b85cebf4 100644 --- a/main/xiaozhi-server/core/providers/tts/siliconflow.py +++ b/main/xiaozhi-server/core/providers/tts/siliconflow.py @@ -11,7 +11,8 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("private_voice") else: self.voice = config.get("voice") - self.response_format = config.get("response_format") + self.response_format = config.get("response_format", "mp3") + self.audio_file_type = config.get("response_format", "mp3") self.sample_rate = config.get("sample_rate") self.speed = float(config.get("speed", 1.0)) self.gain = config.get("gain") @@ -35,7 +36,10 @@ class TTSProvider(TTSProviderBase): "POST", self.api_url, json=request_json, headers=headers ) data = response.content - file_to_save = open(output_file, "wb") - file_to_save.write(data) + if output_file: + with open(output_file, "wb") as file_to_save: + file_to_save.write(data) + else: + return data except Exception as e: raise Exception(f"{__name__} error: {e}") diff --git a/main/xiaozhi-server/core/providers/tts/tencent.py b/main/xiaozhi-server/core/providers/tts/tencent.py index 7a55328a..a2493632 100644 --- a/main/xiaozhi-server/core/providers/tts/tencent.py +++ b/main/xiaozhi-server/core/providers/tts/tencent.py @@ -22,6 +22,7 @@ class TTSProvider(TTSProviderBase): self.api_url = "https://tts.tencentcloudapi.com" # 正确的API端点 self.region = config.get("region") self.output_file = config.get("output_dir") + self.audio_file_type = config.get("format", "wav") def _get_auth_headers(self, request_body): """生成鉴权请求头""" @@ -148,12 +149,14 @@ class TTSProvider(TTSProviderBase): f"API返回错误: {error_info['Code']}: {error_info['Message']}" ) - # 提取音频数据 - audio_data = response_data["Response"].get("Audio") - if audio_data: - # 解码Base64音频数据并保存 - with open(output_file, "wb") as f: - f.write(base64.b64decode(audio_data)) + # 解码Base64音频数据 + audio_bytes = base64.b64decode(response_data["Response"].get("Audio")) + if audio_bytes: + if output_file: + with open(output_file, "wb") as f: + f.write(audio_bytes) + else: + return audio_bytes else: raise Exception(f"{__name__}: 没有返回音频数据: {response_data}") else: diff --git a/main/xiaozhi-server/core/providers/tts/ttson.py b/main/xiaozhi-server/core/providers/tts/ttson.py index b3c7797f..c11206a6 100644 --- a/main/xiaozhi-server/core/providers/tts/ttson.py +++ b/main/xiaozhi-server/core/providers/tts/ttson.py @@ -30,6 +30,7 @@ class TTSProvider(TTSProviderBase): self.output_file = config.get("output_dir") self.pitch_factor = int(config.get("pitch_factor", 0)) self.format = config.get("format", "mp3") + self.audio_file_type = config.get("format", "mp3") self.emotion = int(config.get("emotion", 1)) self.header = {"Content-Type": "application/json"} @@ -73,9 +74,11 @@ class TTSProvider(TTSProviderBase): ) audio_content = requests.get(result) - with open(output_file, "wb") as f: - f.write(audio_content.content) - return True + if output_file: + with open(output_file, "wb") as f: + f.write(audio_content.content) + else: + return audio_content.content voice_path = resp_json.get("voice_path") des_path = output_file shutil.move(voice_path, des_path) diff --git a/main/xiaozhi-server/core/utils/p3.py b/main/xiaozhi-server/core/utils/p3.py index a022a2d2..c75b968e 100644 --- a/main/xiaozhi-server/core/utils/p3.py +++ b/main/xiaozhi-server/core/utils/p3.py @@ -29,5 +29,31 @@ def decode_opus_from_file(input_file): total_frames += 1 # 计算总时长 + total_duration = (total_frames * frame_duration_ms) / 1000.0 + return opus_datas, total_duration + +def decode_opus_from_bytes(input_bytes): + """ + 从p3二进制数据中解码 Opus 数据，并返回一个 Opus 数据包的列表以及总时长。 + """ + import io + opus_datas = [] + total_frames = 0 + sample_rate = 16000 # 文件采样率 + frame_duration_ms = 60 # 帧时长 + frame_size = int(sample_rate * frame_duration_ms / 1000) + + f = io.BytesIO(input_bytes) + while True: + header = f.read(4) + if not header: + break + _, _, data_len = struct.unpack('>BBH', header) + opus_data = f.read(data_len) + if len(opus_data) != data_len: + raise ValueError(f"Data length({len(opus_data)}) mismatch({data_len}) in the bytes.") + opus_datas.append(opus_data) + total_frames += 1 + total_duration = (total_frames * frame_duration_ms) / 1000.0 return opus_datas, total_duration \ No newline at end of file diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 82b5d800..b492529c 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -3,6 +3,9 @@ import socket import subprocess import re import os +import wave +from io import BytesIO +from core.utils import p3 import numpy as np import requests import opuslib_next @@ -773,6 +776,22 @@ def audio_to_data(audio_file_path, is_opus=True): return pcm_to_data(raw_data, is_opus), duration +def audio_bytes_to_data(audio_bytes, file_type, is_opus=True): + """ + 直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3 + """ + if file_type == "p3": + # 直接用p3解码 + return p3.decode_opus_from_bytes(audio_bytes) + else: + # 其他格式用pydub + audio = AudioSegment.from_file(BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"]) + audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) + duration = len(audio) / 1000.0 + raw_data = audio.raw_data + return pcm_to_data(raw_data, is_opus), duration + + def pcm_to_data(raw_data, is_opus=True): # 初始化Opus编码器 encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO) @@ -804,6 +823,33 @@ def pcm_to_data(raw_data, is_opus=True): return datas +def opus_datas_to_wav_bytes(opus_datas, sample_rate=16000, channels=1): + """ + 将opus帧列表解码为wav字节流 + """ + decoder = opuslib_next.Decoder(sample_rate, channels) + pcm_datas = [] + + frame_duration = 60 # ms + frame_size = int(sample_rate * frame_duration / 1000) # 960 + + for opus_frame in opus_datas: + # 解码为PCM（返回bytes，2字节/采样点） + pcm = decoder.decode(opus_frame, frame_size) + pcm_datas.append(pcm) + + pcm_bytes = b''.join(pcm_datas) + + # 写入wav字节流 + wav_buffer = BytesIO() + with wave.open(wav_buffer, 'wb') as wf: + wf.setnchannels(channels) + wf.setsampwidth(2) # 16bit + wf.setframerate(sample_rate) + wf.writeframes(pcm_bytes) + return wav_buffer.getvalue() + + def check_vad_update(before_config, new_config): if ( new_config.get("selected_module") is None From adf1a4794553ac5514202e69399d612baaf2b583 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 5 Jun 2025 09:25:30 +0800 Subject: [PATCH 08/31] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 4230715a..eb3ec309 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -715,10 +715,12 @@ TTS: format: mp3 # 接口返回的音频格式 output_dir: tmp/ LinkeraiTTS: - #各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get type: linkerai api_url: https://tts.linkerai.top/tts audio_format: "pcm" - access_token: "test" + # 默认的access_token供大家测试时免费使用的，此access_token请勿用于商业用途 + # 如果效果不错，可自行申请token，申请地址：https://linkerai.top + # 各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get + access_token: "U4YdYXVfpwWnk2t5Gp822zWPCuORyeJL" voice: "OUeAo1mhq6IBExi" output_dir: tmp/ \ No newline at end of file From 8df9846ad7b89c4d91528599d3bbe5b9c4922e84 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Thu, 5 Jun 2025 09:46:54 +0800 Subject: [PATCH 09/31] =?UTF-8?q?update:=E6=94=B9=E4=B8=BA=E9=87=87?= =?UTF-8?q?=E7=94=A8=E6=97=A5=E5=BF=97=E8=BD=AE=E8=BD=AC=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=EF=BC=8C=E5=90=8C=E6=97=B6=E6=B7=BB=E5=8A=A0=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config/logger.py | 40 ++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index 643952dd..7bd22c43 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -3,6 +3,7 @@ import sys from loguru import logger from config.config_loader import load_config from config.settings import check_config_file +from datetime import datetime SERVER_VERSION = "0.5.4" _logger_initialized = False @@ -59,7 +60,7 @@ def setup_logging(): ) log_format_file = log_config.get( "log_format_file", - "{time:YYYY-MM-DD HH:mm:ss} - {version_{extra[selected_module]}} - {name} - {level} - {extra[tag]} - {message}", + "{time:YYYY-MM-DD HH:mm:ss} - {version}_{extra[selected_module]} - {name} - {level} - {extra[tag]} - {message}", ) selected_module_str = logger._core.extra["selected_module"] @@ -84,12 +85,23 @@ def setup_logging(): # 输出到控制台 logger.add(sys.stdout, format=log_format, level=log_level, filter=formatter) - # 输出到文件 + # 输出到文件 - 统一目录，按大小轮转 + # 日志文件完整路径 + log_file_path = os.path.join(log_dir, log_file) + + # 添加日志处理器 logger.add( - os.path.join(log_dir, log_file), + log_file_path, format=log_format_file, level=log_level, filter=formatter, + rotation="10 MB", # 每个文件最大10MB + retention="30 days", # 保留30天 + compression=None, + encoding="utf-8", + enqueue=True, # 异步安全 + backtrace=True, + diagnose=True, ) _logger_initialized = True # 标记为已初始化 @@ -116,7 +128,7 @@ def update_module_string(selected_module_str): ) log_format_file = log_config.get( "log_format_file", - "{time:YYYY-MM-DD HH:mm:ss} - {version_{extra[selected_module]}} - {name} - {level} - {extra[tag]} - {message}", + "{time:YYYY-MM-DD HH:mm:ss} - {version}_{extra[selected_module]} - {name} - {level} - {extra[tag]} - {message}", ) log_format = log_format.replace("{version}", SERVER_VERSION) @@ -133,14 +145,26 @@ def update_module_string(selected_module_str): level=log_config.get("log_level", "INFO"), filter=formatter, ) + + # 更新文件日志配置 - 统一目录，按大小轮转 + log_dir = log_config.get("log_dir", "tmp") + log_file = log_config.get("log_file", "server.log") + + # 日志文件完整路径 + log_file_path = os.path.join(log_dir, log_file) + logger.add( - os.path.join( - log_config.get("log_dir", "tmp"), - log_config.get("log_file", "server.log"), - ), + log_file_path, format=log_format_file, level=log_config.get("log_level", "INFO"), filter=formatter, + rotation="10 MB", # 每个文件最大10MB + retention="30 days", # 保留30天 + compression=None, + encoding="utf-8", + enqueue=True, # 异步安全 + backtrace=True, + diagnose=True, ) except Exception as e: From d7eecfdceaf2200a3d8432e7bf7b962fe683eafb Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 5 Jun 2025 10:49:25 +0800 Subject: [PATCH 10/31] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 4 +- .../core/providers/tts/linkerai.py | 201 +++++++----------- 2 files changed, 77 insertions(+), 128 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 5d2c47a6..a7ff54f3 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -759,10 +759,10 @@ TTS: LinkeraiTTS: type: linkerai api_url: https://tts.linkerai.top/tts - audio_format: "pcm" + audio_format: "opus" # 默认的access_token供大家测试时免费使用的，此access_token请勿用于商业用途 # 如果效果不错，可自行申请token，申请地址：https://linkerai.top # 各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get access_token: "U4YdYXVfpwWnk2t5Gp822zWPCuORyeJL" - voice: "OUeAo1mhq6IBExi" + voice: "7OEPouTL46bS2Qe" output_dir: tmp/ \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index ab585ef3..f570cfb0 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -1,17 +1,14 @@ -import asyncio -import traceback +import os +import time import queue +import asyncio import requests -from pathlib import Path -from core.providers.tts.base import TTSProviderBase -from core.providers.tts.dto.dto import ( - TTSMessageDTO, - SentenceType, - ContentType, - InterfaceType -) +import traceback from config.logger import setup_logging +from core.utils.tts import MarkdownCleaner +from core.providers.tts.base import TTSProviderBase from core.utils import opus_encoder_utils, textUtils +from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType TAG = __name__ logger = setup_logging() @@ -24,13 +21,12 @@ class TTSProvider(TTSProviderBase): self.access_token = config.get("access_token") self.voice = config.get("voice") self.api_url = config.get("api_url") - self.audio_format = config.get("audio_format", "opus") + self.audio_format = "pcm" + self.before_stop_play_files = [] # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=16000, - channels=1, - frame_size_ms=60 + sample_rate=16000, channels=1, frame_size_ms=60 ) # 添加文本缓冲区 @@ -48,42 +44,31 @@ class TTSProvider(TTSProviderBase): while not self.conn.stop_event.is_set(): try: message = self.tts_text_queue.get(timeout=1) - # logger.bind(tag=TAG).debug( - # f"TTS任务｜{message.sentence_type.name}｜{message.content_type.name}" - # ) - if message.sentence_type == SentenceType.FIRST: - # 初始化流式状态 + # 初始化参数 + self.tts_stop_request = False + self.processed_chars = 0 + self.tts_text_buff = [] + self.is_first_sentence = True self.tts_audio_first_sentence = True - self.pcm_buffer = bytearray() - self.text_buffer = "" # 重置文本缓冲区 - + self.before_stop_play_files.clear() elif ContentType.TEXT == message.content_type: - # 将文本添加到缓冲区 - self.text_buffer += message.content_detail - # 尝试分割并发送完整句子 - self._process_text_buffer() + self.tts_text_buff.append(message.content_detail) + segment_text = self._get_segment_text() + if segment_text: + self.to_tts(segment_text) elif ContentType.FILE == message.content_type: - # 先处理缓冲区中的剩余文本 - self._flush_text_buffer() - # 处理文件类型 - if message.content_file and Path(message.content_file).exists(): - audio_datas = self._process_audio_file(message.content_file) - self.tts_audio_queue.put( - (SentenceType.MIDDLE, audio_datas, message.content_detail) - ) + self._process_remaining_text() + logger.bind(tag=TAG).info( + f"添加音频文件到待播放列表: {message.content_file}" + ) + self.before_stop_play_files.append( + (message.content_file, message.content_detail) + ) if message.sentence_type == SentenceType.LAST: - # 处理缓冲区中的剩余文本 - self._flush_text_buffer() - # 发送结束帧 - if self.pcm_buffer: - opus_datas = self.wav_to_opus_data_audio_raw(self.pcm_buffer, end_of_stream=True) - self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, "")) - self.pcm_buffer = bytearray() - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - self.text_buffer = "" # 重置文本缓冲区 + self._process_remaining_text() except queue.Empty: continue @@ -92,115 +77,77 @@ class TTSProvider(TTSProviderBase): f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" ) - def _process_text_buffer(self): - """处理文本缓冲区，分割并发送完整句子""" - # 使用父类的标点集合 - sentence_endings = self.punctuations - comma_endings = self.first_sentence_punctuations - while True: - # 查找最近的句子结束位置 - end_pos = -1 - for punct in sentence_endings: - pos = self.text_buffer.find(punct) - if pos != -1 and (end_pos == -1 or pos < end_pos): - end_pos = pos + def _process_remaining_text(self): + """处理剩余的文本并生成语音 - # 如果是第一句话，也允许在逗号处分隔 - if self.tts_audio_first_sentence and end_pos == -1: - for punct in comma_endings: - pos = self.text_buffer.find(punct) - if pos != -1 and (end_pos == -1 or pos < end_pos): - end_pos = pos + Returns: + bool: 是否成功处理了文本 + """ + full_text = "".join(self.tts_text_buff) + remaining_text = full_text[self.processed_chars :] + if remaining_text: + segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) + if segment_text: + self.to_tts(segment_text) + self.processed_chars += len(full_text) - # 找到分割点 - if end_pos != -1: - # 提取完整句子 - sentence = self.text_buffer[:end_pos + 1] - sentence = textUtils.get_string_no_punctuation_or_emoji(sentence) - - if not sentence.strip(): # 检查是否为空文本 - self.text_buffer = self.text_buffer[end_pos + 1:] - continue - - self.text_buffer = self.text_buffer[end_pos + 1:] - - # 发送句子 - future = asyncio.run_coroutine_threadsafe( - self.text_to_speak(sentence), - loop=self.conn.loop + def to_tts(self, text): + try: + max_repeat_time = 5 + text = MarkdownCleaner.clean_markdown(text) + try: + asyncio.run(self.text_to_speak(text, None)) + except Exception as e: + logger.bind(tag=TAG).warning( + f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}" ) - future.result() + max_repeat_time -= 1 - # 更新第一句话标志 - if self.tts_audio_first_sentence: - self.tts_audio_first_sentence = False + if max_repeat_time > 0: + logger.bind(tag=TAG).info( + f"语音生成成功: {text}，重试{5 - max_repeat_time}次" + ) else: - break - - def _flush_text_buffer(self): - """处理缓冲区中剩余的文本""" - if self.text_buffer: - clean_text = textUtils.get_string_no_punctuation_or_emoji(self.text_buffer) - if clean_text.strip(): # 检查是否为空文本 - future = asyncio.run_coroutine_threadsafe( - self.text_to_speak(clean_text), - loop=self.conn.loop + logger.bind(tag=TAG).error( + f"语音生成失败: {text}，请检查网络或服务是否正常" ) - future.result() - self.text_buffer = "" - - async def text_to_speak(self, text): - # 发送文本 - await self.send_text(text) - return + except Exception as e: + logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}") + finally: + return None ################################################################################### # linkerai单流式TTS重写父类的方法--结束 ################################################################################### - async def send_text(self, text: str): + async def text_to_speak(self, text, _): """流式处理TTS音频，每句只推送一次音频列表""" + start_time = time.time() + logger.info(f"TTS请求: {text}") try: params = { "tts_text": text, "spk_id": self.voice, "frame_duration": 60, - "stream": "true", + "stream": True, "target_sr": 16000, "audio_format": self.audio_format, } headers = {"Authorization": f"Bearer {self.access_token}"} - with requests.get(self.api_url, params=params, headers=headers, stream=True, timeout=5) as response: + with requests.get( + self.api_url, params=params, headers=headers, timeout=5 + ) as response: if response.status_code != 200: - logger.error(f"TTS请求失败: {response.status_code}, {response.text}") + logger.error( + f"TTS请求失败: {response.status_code}, {response.text}" + ) # 推送空LAST，防止播放端卡死 self.tts_audio_queue.put((SentenceType.LAST, [], None)) return - - logger.debug(f"处理TTS文本: {text}") - - pcm_buffer = bytearray() - frame_bytes = self.opus_encoder.frame_size * 4 # 每帧字节数（int16=2字节） - opus_datas = [] - for chunk in response.iter_content(chunk_size=960): - if chunk: - pcm_buffer.extend(chunk) - # 只要够帧就编码 - while len(pcm_buffer) >= frame_bytes: - frame = pcm_buffer[:frame_bytes] - opus_chunk = self.opus_encoder.encode_pcm_to_opus(frame, end_of_stream=False) - if opus_chunk: - opus_datas.extend(opus_chunk) - pcm_buffer = pcm_buffer[frame_bytes:] # 剩余部分继续累积 - # 处理最后剩余数据 - if pcm_buffer: - opus_chunk = self.opus_encoder.encode_pcm_to_opus(pcm_buffer, end_of_stream=True) - if opus_chunk: - opus_datas.extend(opus_chunk) - # 推送本句所有音频帧 + logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") + opus_datas = self.wav_to_opus_data_audio_raw(response.content) self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, text)) - except Exception as e: logger.error(f"TTS流式处理异常：{str(e)}") # 推送空LAST，防止播放端卡死 @@ -208,7 +155,9 @@ class TTSProvider(TTSProviderBase): # 保持原有方法 def wav_to_opus_data_audio_raw(self, raw_data_var): - opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, end_of_stream=True) + opus_datas = self.opus_encoder.encode_pcm_to_opus( + raw_data_var, end_of_stream=True + ) return opus_datas async def close(self): From 2654802bb079ac98662c1636d6b29c695629ece9 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 5 Jun 2025 11:09:09 +0800 Subject: [PATCH 11/31] =?UTF-8?q?update:=E6=B5=81=E5=BC=8F=E6=92=AD?= =?UTF-8?q?=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/linkerai.py | 88 +++++++++++-------- 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index f570cfb0..767d2662 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -4,6 +4,7 @@ import queue import asyncio import requests import traceback +import aiohttp from config.logger import setup_logging from core.utils.tts import MarkdownCleaner from core.providers.tts.base import TTSProviderBase @@ -122,46 +123,59 @@ class TTSProvider(TTSProviderBase): async def text_to_speak(self, text, _): """流式处理TTS音频，每句只推送一次音频列表""" - start_time = time.time() - logger.info(f"TTS请求: {text}") - try: - params = { - "tts_text": text, - "spk_id": self.voice, - "frame_duration": 60, - "stream": True, - "target_sr": 16000, - "audio_format": self.audio_format, - } - headers = {"Authorization": f"Bearer {self.access_token}"} - - with requests.get( - self.api_url, params=params, headers=headers, timeout=5 - ) as response: - if response.status_code != 200: - logger.error( - f"TTS请求失败: {response.status_code}, {response.text}" - ) - # 推送空LAST，防止播放端卡死 - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - return - logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") - opus_datas = self.wav_to_opus_data_audio_raw(response.content) - self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, text)) - except Exception as e: - logger.error(f"TTS流式处理异常：{str(e)}") - # 推送空LAST，防止播放端卡死 - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - - # 保持原有方法 - def wav_to_opus_data_audio_raw(self, raw_data_var): - opus_datas = self.opus_encoder.encode_pcm_to_opus( - raw_data_var, end_of_stream=True - ) - return opus_datas + await self._tts_request(text) async def close(self): """资源清理""" await super().close() if hasattr(self, "opus_encoder"): self.opus_encoder.close() + + async def _tts_request(self, text: str) -> None: + """发送TTS请求""" + start_time = time.time() + params = { + "tts_text": text, + "spk_id": self.voice, + "frame_durition": 60, + "stream": "true", + "target_sr": 16000, + "audio_format": "pcm", + "instruct_text": "", + } + headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + } + + try: + async with aiohttp.ClientSession() as session: + async with session.get( + self.api_url, params=params, headers=headers, timeout=10 + ) as response: + if response.status != 200: + logger.error( + f"TTS请求失败: {response.status}, {await response.text()}" + ) + # 推送空LAST，防止播放端卡死 + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + return + + logger.info( + f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒" + ) + + # 流式处理音频数据 + async for chunk in response.content.iter_chunks(): + if chunk[0]: # 确保数据不为空 + opus_data = self.opus_encoder.encode_pcm_to_opus( + chunk[0], end_of_stream=True + ) + if opus_data: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_data, text) + ) + + except Exception as e: + logger.error(f"TTS请求异常: {str(e)}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) From dca02c1f4b8c0380673fa2ca8902f261f781d7f6 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 5 Jun 2025 14:11:50 +0800 Subject: [PATCH 12/31] =?UTF-8?q?update:=20=E5=94=A4=E9=86=92=E9=9F=B3?= =?UTF-8?q?=E9=A2=91=E5=85=BC=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 859059b6..c41baba0 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -1,4 +1,5 @@ import os +import shutil import time import json import random @@ -119,17 +120,31 @@ async def wakeupWordsResponse(conn): if result is None or result == "": return - opus_datas = await asyncio.to_thread(conn.tts.to_tts, result) - if not opus_datas: - return - - wav_bytes = opus_datas_to_wav_bytes(opus_datas, sample_rate=16000) - file_path = os.path.join( - WAKEUP_CONFIG["dir"], "my_" + WAKEUP_CONFIG["file_name"] + ".wav" - ) - # 写入wav数据 - with open(file_path, "wb") as f: - f.write(wav_bytes) + tts_result = await asyncio.to_thread(conn.tts.to_tts, result) + if conn.tts.delete_audio_file: + # 返回的是opus数据流 + if not tts_result: + return + wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000) + file_path = os.path.join( + WAKEUP_CONFIG["dir"], "my_" + WAKEUP_CONFIG["file_name"] + ".wav" + ) + with open(file_path, "wb") as f: + f.write(wav_bytes) + else: + # 返回为文件 + if tts_result is not None and os.path.exists(tts_result): + file_type = os.path.splitext(tts_result)[1] + if file_type: + file_type = file_type.lstrip(".") + old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"]) + if old_file is not None: + os.remove(old_file) + """将文件挪到wakeup_words目录下""" + shutil.move( + tts_result, + WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + '.' + file_type, + ) WAKEUP_CONFIG["create_time"] = time.time() WAKEUP_CONFIG["text"] = result From 01eb416a034db4db541b1bce8bef16a5d615d0db Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 5 Jun 2025 16:02:16 +0800 Subject: [PATCH 13/31] =?UTF-8?q?update:=E6=99=BA=E6=8E=A7=E5=8F=B0?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=85=8D=E8=B4=B9=E6=B5=81=E5=BC=8FTTS(linke?= =?UTF-8?q?rai)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202506051538.sql | 20 ++++ .../db/changelog/db.changelog-master.yaml | 9 +- main/xiaozhi-server/config.yaml | 13 +- .../xiaozhi-server/core/providers/tts/base.py | 9 ++ .../providers/tts/huoshan_double_stream.py | 10 +- .../core/providers/tts/linkerai.py | 112 ++++++++++++------ 6 files changed, 124 insertions(+), 49 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202506051538.sql diff --git a/main/manager-api/src/main/resources/db/changelog/202506051538.sql b/main/manager-api/src/main/resources/db/changelog/202506051538.sql new file mode 100644 index 00000000..cfe65da7 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202506051538.sql @@ -0,0 +1,20 @@ +-- 增加LinkeraiTTS供应器和模型配置 +delete from `ai_model_provider` where id = 'SYSTEM_TTS_LinkeraiTTS'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_TTS_LinkeraiTTS', 'TTS', 'linkerai', 'Linkerai语音合成', '[{"key":"api_url","label":"API地址","type":"string"},{"key":"audio_format","label":"音频格式","type":"string"},{"key":"access_token","label":"访问令牌","type":"string"},{"key":"voice","label":"默认音色","type":"string"}]', 14, 1, NOW(), 1, NOW()); + +delete from `ai_model_config` where id = 'TTS_LinkeraiTTS'; +INSERT INTO `ai_model_config` VALUES ('TTS_LinkeraiTTS', 'TTS', 'LinkeraiTTS', 'Linkerai语音合成', 0, 1, '{\"type\": \"linkerai\", \"api_url\": \"https://tts.linkerai.cn/tts\", \"audio_format\": \"pcm\", \"access_token\": \"U4YdYXVfpwWnk2t5Gp822zWPCuORyeJL\", \"voice\": \"OUeAo1mhq6IBExi\"}', NULL, NULL, 17, NULL, NULL, NULL, NULL); + +-- LinkeraiTTS模型配置说明文档 +UPDATE `ai_model_config` SET +`doc_link` = 'https://tts.linkerai.cn/docs', +`remark` = 'Linkerai语音合成服务配置说明： +1. 访问 https://linkerai.cn 注册并获取访问令牌 +2. 默认的access_token供测试使用，请勿用于商业用途 +3. 支持声音克隆功能，可自行上传音频，填入voice参数 +4. 如果voice参数为空，将使用默认声音' WHERE `id` = 'TTS_LinkeraiTTS'; + + +delete from `ai_tts_voice` where tts_model_id = 'TTS_LinkeraiTTS'; +INSERT INTO `ai_tts_voice` VALUES ('TTS_LinkeraiTTS_0001', 'TTS_LinkeraiTTS', '芷若', 'OUeAo1mhq6IBExi', '中文', NULL, NULL, 1, NULL, NULL, NULL, NULL); diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 0583fce9..cb3007aa 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -190,4 +190,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202506032232.sql \ No newline at end of file + path: classpath:db/changelog/202506032232.sql + - changeSet: + id: 202506051538 + author: hrz + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202506051538.sql \ No newline at end of file diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index a7ff54f3..7721b132 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -324,7 +324,7 @@ VAD: type: silero threshold: 0.5 model_dir: models/snakers4_silero-vad - min_silence_duration_ms: 700 # 如果说话停顿比较长，可以把这个值设置大一些 + min_silence_duration_ms: 200 # 如果说话停顿比较长，可以把这个值设置大一些 LLM: # 所有openai类型均可以修改超参，以AliLLM为例 @@ -758,11 +758,12 @@ TTS: output_dir: tmp/ LinkeraiTTS: type: linkerai - api_url: https://tts.linkerai.top/tts - audio_format: "opus" + api_url: https://tts.linkerai.cn/tts + audio_format: "pcm" # 默认的access_token供大家测试时免费使用的，此access_token请勿用于商业用途 - # 如果效果不错，可自行申请token，申请地址：https://linkerai.top - # 各参数意义见开发文档：https://tts.linkerai.top/docs#/default/text_to_speech_tts_get + # 如果效果不错，可自行申请token，申请地址：https://linkerai.cn + # 各参数意义见开发文档：https://tts.linkerai.cn/docs + # 支持声音克隆，可自行上传音频，填入voice参数，voice参数为空时，使用默认声音 access_token: "U4YdYXVfpwWnk2t5Gp822zWPCuORyeJL" - voice: "7OEPouTL46bS2Qe" + voice: "OUeAo1mhq6IBExi" output_dir: tmp/ \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index e82a8725..c467e99a 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -37,6 +37,7 @@ class TTSProviderBase(ABC): self.tts_text_queue = queue.Queue() self.tts_audio_queue = queue.Queue() self.tts_audio_first_sentence = True + self.before_stop_play_files = [] self.tts_text_buff = [] self.punctuations = ( @@ -324,6 +325,14 @@ class TTSProviderBase(ABC): os.remove(tts_file) return audio_datas + def _process_before_stop_play_files(self): + for tts_file, text in self.before_stop_play_files: + if tts_file and os.path.exists(tts_file): + audio_datas = self._process_audio_file(tts_file) + self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text)) + self.before_stop_play_files.clear() + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + def _process_remaining_text(self): """处理剩余的文本并生成语音 diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index b9dc399b..b248f5ce 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -154,7 +154,6 @@ class TTSProvider(TTSProviderBase): self.header = {"Authorization": f"{self.authorization}{self.access_token}"} self.enable_two_way = True self.tts_text = "" - self.before_stop_play_files = [] self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( sample_rate=16000, channels=1, frame_size_ms=60 ) @@ -432,14 +431,7 @@ class TTSProvider(TTSProviderBase): is_first_sentence = False elif res.optional.event == EVENT_SessionFinished: logger.bind(tag=TAG).debug(f"会话结束～～") - for tts_file, text in self.before_stop_play_files: - if tts_file and os.path.exists(tts_file): - audio_datas = self._process_audio_file(tts_file) - self.tts_audio_queue.put( - (SentenceType.MIDDLE, audio_datas, text) - ) - self.before_stop_play_files.clear() - self.tts_audio_queue.put((SentenceType.LAST, [], None)) + self._process_before_stop_play_files() break except websockets.ConnectionClosed: logger.bind(tag=TAG).warning("WebSocket连接已关闭") diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index 767d2662..dd8d0e6b 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -1,8 +1,5 @@ -import os -import time import queue import asyncio -import requests import traceback import aiohttp from config.logger import setup_logging @@ -24,6 +21,7 @@ class TTSProvider(TTSProviderBase): self.api_url = config.get("api_url") self.audio_format = "pcm" self.before_stop_play_files = [] + self.segment_count = 0 # 添加片段计数器 # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( @@ -50,7 +48,7 @@ class TTSProvider(TTSProviderBase): self.tts_stop_request = False self.processed_chars = 0 self.tts_text_buff = [] - self.is_first_sentence = True + self.segment_count = 0 self.tts_audio_first_sentence = True self.before_stop_play_files.clear() elif ContentType.TEXT == message.content_type: @@ -60,7 +58,6 @@ class TTSProvider(TTSProviderBase): self.to_tts(segment_text) elif ContentType.FILE == message.content_type: - self._process_remaining_text() logger.bind(tag=TAG).info( f"添加音频文件到待播放列表: {message.content_file}" ) @@ -69,7 +66,8 @@ class TTSProvider(TTSProviderBase): ) if message.sentence_type == SentenceType.LAST: - self._process_remaining_text() + # 处理剩余的文本 + self._process_remaining_text(True) except queue.Empty: continue @@ -78,7 +76,7 @@ class TTSProvider(TTSProviderBase): f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" ) - def _process_remaining_text(self): + def _process_remaining_text(self, is_last=False): """处理剩余的文本并生成语音 Returns: @@ -89,15 +87,17 @@ class TTSProvider(TTSProviderBase): if remaining_text: segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) if segment_text: - self.to_tts(segment_text) + self.to_tts(segment_text, is_last) self.processed_chars += len(full_text) + else: + self._process_before_stop_play_files() - def to_tts(self, text): + def to_tts(self, text, is_last=False): try: max_repeat_time = 5 text = MarkdownCleaner.clean_markdown(text) try: - asyncio.run(self.text_to_speak(text, None)) + asyncio.run(self.text_to_speak(text, is_last)) except Exception as e: logger.bind(tag=TAG).warning( f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}" @@ -121,9 +121,9 @@ class TTSProvider(TTSProviderBase): # linkerai单流式TTS重写父类的方法--结束 ################################################################################### - async def text_to_speak(self, text, _): + async def text_to_speak(self, text, is_last): """流式处理TTS音频，每句只推送一次音频列表""" - await self._tts_request(text) + await self._tts_request(text, is_last) async def close(self): """资源清理""" @@ -131,9 +131,7 @@ class TTSProvider(TTSProviderBase): if hasattr(self, "opus_encoder"): self.opus_encoder.close() - async def _tts_request(self, text: str) -> None: - """发送TTS请求""" - start_time = time.time() + async def _tts_request(self, text: str, is_last: bool) -> None: params = { "tts_text": text, "spk_id": self.voice, @@ -141,41 +139,89 @@ class TTSProvider(TTSProviderBase): "stream": "true", "target_sr": 16000, "audio_format": "pcm", - "instruct_text": "", + "instruct_text": "请生成一段自然流畅的语音", } headers = { "Authorization": f"Bearer {self.access_token}", "Content-Type": "application/json", } + # 一帧 PCM 所需字节数：60 ms × 16 kHz × 1 ch × 2 B = 1 920 + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels # 1 + * self.opus_encoder.frame_size_ms + / 1000 + * 2 + ) # 16-bit = 2 bytes + try: async with aiohttp.ClientSession() as session: async with session.get( self.api_url, params=params, headers=headers, timeout=10 - ) as response: - if response.status != 200: - logger.error( - f"TTS请求失败: {response.status}, {await response.text()}" - ) - # 推送空LAST，防止播放端卡死 + ) as resp: + + if resp.status != 200: + logger.error(f"TTS请求失败: {resp.status}, {await resp.text()}") self.tts_audio_queue.put((SentenceType.LAST, [], None)) return - logger.info( - f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒" - ) + self.pcm_buffer.clear() + opus_datas_cache = [] - # 流式处理音频数据 - async for chunk in response.content.iter_chunks(): - if chunk[0]: # 确保数据不为空 - opus_data = self.opus_encoder.encode_pcm_to_opus( - chunk[0], end_of_stream=True + # 兼容 iter_chunked / iter_chunks / iter_any + async for chunk in resp.content.iter_any(): + data = chunk[0] if isinstance(chunk, (list, tuple)) else chunk + if not data: + continue + + # 拼到 buffer + self.pcm_buffer.extend(data) + + # 够一帧就编码 + while len(self.pcm_buffer) >= frame_bytes: + frame = bytes(self.pcm_buffer[:frame_bytes]) + del self.pcm_buffer[:frame_bytes] + + opus = self.opus_encoder.encode_pcm_to_opus( + frame, end_of_stream=False ) - if opus_data: + if opus: + if self.segment_count < 10: # 前10个片段直接发送 + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus, text) + ) + self.segment_count += 1 + else: + opus_datas_cache.extend(opus) + + # flush 剩余不足一帧的数据 + if self.pcm_buffer: + opus = self.opus_encoder.encode_pcm_to_opus( + bytes(self.pcm_buffer), end_of_stream=True + ) + if opus: + if self.segment_count < 10: # 前10个片段直接发送 + # 直接发送 self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_data, text) + (SentenceType.MIDDLE, opus, text) ) + self.segment_count += 1 + else: + # 后续片段缓存 + opus_datas_cache.extend(opus) + self.pcm_buffer.clear() + + # 如果不是前10个片段，发送缓存的数据 + if self.segment_count >= 10 and opus_datas_cache: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas_cache, text) + ) + + # 如果是最后一段，输出音频获取完毕 + if is_last: + self._process_before_stop_play_files() except Exception as e: - logger.error(f"TTS请求异常: {str(e)}") + logger.error(f"TTS请求异常: {e}") self.tts_audio_queue.put((SentenceType.LAST, [], None)) From 8357d7abc37f402ec0f41036b27462542b804e8b Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 5 Jun 2025 16:25:40 +0800 Subject: [PATCH 14/31] =?UTF-8?q?update=EF=BC=9A=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=85=8D=E8=B4=B9=E6=B5=81=E5=BC=8Ftts=EF=BC=88=E7=81=B5?= =?UTF-8?q?=E7=8A=80=E6=B5=81=E5=BC=8F=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 25907967..0737c834 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ Websocket接口地址: wss://2662r3426b.vicp.fun/xiaozhi/v1/ | ASR(语音识别) | FunASR(本地) | 👍DoubaoStreamASR(火山流式语音识别) | | LLM(大模型) | ChatGLMLLM(智谱glm-4-flash) | 👍DoubaoLLM(火山doubao-1-5-pro-32k-250115) | | VLLM(视觉大模型) | ChatGLMVLLM(智谱glm-4v-flash) | 👍QwenVLVLLM(千问qwen2.5-vl-3b-instructh) | -| TTS(语音合成) | EdgeTTS(微软语音) | 👍HuoshanDoubleStreamTTS(火山双流式语音合成) | +| TTS(语音合成) | 👍LinkeraiTTS(灵犀流式) | 👍HuoshanDoubleStreamTTS(火山双流式语音合成) | | Intent(意图识别) | function_call(函数调用) | ✅function_call(函数调用) | | Memory(记忆功能) | mem_local_short(本地短期记忆） | ✅mem_local_short（本地短期记忆） | @@ -204,7 +204,7 @@ Websocket接口地址: wss://2662r3426b.vicp.fun/xiaozhi/v1/ | 工具名称 | 位置 | 使用方法 | 功能说明 | |:---:|:---|:---:|:---:| -| 音频交互测试工具 | main》xiaozhi-server》test/》est_page.html | 使用谷歌浏览器直接打开 | 测试音频播放和接收功能，验证Python端音频处理是否正常 | +| 音频交互测试工具 | main》xiaozhi-server》test》test_page.html | 使用谷歌浏览器直接打开 | 测试音频播放和接收功能，验证Python端音频处理是否正常 | | 模型响应测试工具1 | main》xiaozhi-server》performance_tester.py | 执行 `python performance_tester.py` | 测试ASR(语音识别)、LLM(大模型)、TTS(语音合成)三个核心模块的响应速度 | | 模型响应测试工具2 | main》xiaozhi-server》performance_tester_vllm.py | 执行 `python performance_tester_vllm.py` | 测试VLLM(视觉模型)的响应速度 | @@ -277,7 +277,7 @@ Websocket接口地址: wss://2662r3426b.vicp.fun/xiaozhi/v1/ | 使用方式 | 支持平台 | 免费平台 | |:---:|:---:|:---:| -| 接口调用 | EdgeTTS、火山引擎豆包TTS、腾讯云、阿里云TTS、CosyVoiceSiliconflow、TTS302AI、CozeCnTTS、GizwitsTTS、ACGNTTS、OpenAITTS | EdgeTTS、CosyVoiceSiliconflow(部分) | +| 接口调用 | EdgeTTS、火山引擎豆包TTS、腾讯云、阿里云TTS、CosyVoiceSiliconflow、TTS302AI、CozeCnTTS、GizwitsTTS、ACGNTTS、OpenAITTS、灵犀流式TTS | 灵犀流式TTS、EdgeTTS、CosyVoiceSiliconflow(部分) | | 本地服务 | FishSpeech、GPT_SOVITS_V2、GPT_SOVITS_V3、MinimaxTTS | FishSpeech、GPT_SOVITS_V2、GPT_SOVITS_V3、MinimaxTTS | --- From f8de052d541858568a08fd34f808c06d1c55975f Mon Sep 17 00:00:00 2001 From: chan Date: Thu, 5 Jun 2025 16:33:47 +0800 Subject: [PATCH 15/31] =?UTF-8?q?fix:=20=E4=BD=BF=E7=94=A8deepseek=20r1-05?= =?UTF-8?q?28=E6=A8=A1=E5=9E=8B=E6=97=B6=EF=BC=8Ctools=5Fcall=E4=B8=BA?= =?UTF-8?q?=E7=A9=BAlist=EF=BC=8Ctools=5Fcall[0]=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index aa1aaf42..bf9b9c1f 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -651,7 +651,7 @@ class ConnectionHandler: # print("content_arguments", content_arguments) tool_call_flag = True - if tools_call is not None: + if tools_call: tool_call_flag = True if tools_call[0].id is not None: function_id = tools_call[0].id From bc44ea87576dfa0c3f64c1df77595e9ddf1643b1 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 5 Jun 2025 17:07:29 +0800 Subject: [PATCH 16/31] Update connection.py --- main/xiaozhi-server/core/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index bf9b9c1f..9102b76d 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -651,7 +651,7 @@ class ConnectionHandler: # print("content_arguments", content_arguments) tool_call_flag = True - if tools_call: + if tools_call is not None and len(tools_call) > 0: tool_call_flag = True if tools_call[0].id is not None: function_id = tools_call[0].id From 2cd90d80664752533b23c03c5b8a65905dc1206c Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 5 Jun 2025 20:54:53 +0800 Subject: [PATCH 17/31] =?UTF-8?q?update:=E4=BF=AE=E5=A4=8D=E8=A2=AB?= =?UTF-8?q?=E5=94=A4=E9=86=92=E8=AF=8D=E6=89=93=E6=96=ADbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/handle/receiveAudioHandle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 4c8c2c2e..90600ce2 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -18,6 +18,7 @@ async def handleAudioMessage(conn, audio): if hasattr(conn, "just_woken_up") and conn.just_woken_up: have_voice = False # 设置一个短暂延迟后恢复VAD检测 + conn.asr_audio.clear() asyncio.create_task(resume_vad_detection(conn)) if have_voice: From 3eb3a4502f7b651b460576feb552c15670dd4b2a Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Fri, 6 Jun 2025 14:08:38 +0800 Subject: [PATCH 18/31] Update fun_local.py --- main/xiaozhi-server/core/providers/asr/fun_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 4b25a265..e907bdce 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -43,7 +43,7 @@ class ASRProvider(ASRProviderBase): min_mem_bytes = 2 * 1024 * 1024 * 1024 total_mem = psutil.virtual_memory().total if total_mem < min_mem_bytes: - logger.bind(tag=TAG).error(f"可用内存不足2G，当前仅有 {total_mem / (1024*1024):.2f} MB") + logger.bind(tag=TAG).error(f"可用内存不足2G，当前仅有 {total_mem / (1024*1024):.2f} MB，可能无法启动FunASR") self.interface_type = InterfaceType.LOCAL self.model_dir = config.get("model_dir") From 50e6e4817b74cf40fa7c9aa6ec294ab79cf24c1c Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sat, 7 Jun 2025 22:08:22 +0800 Subject: [PATCH 19/31] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E5=94=A4?= =?UTF-8?q?=E9=86=92=E8=AF=8D=E7=AD=94=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 150 ++++++++---------- .../providers/tts/huoshan_double_stream.py | 9 +- .../core/providers/tts/minimax.py | 8 +- .../core/providers/tts/ttson.py | 6 +- main/xiaozhi-server/core/utils/wakeup_word.py | 143 +++++++++++++++++ 5 files changed, 221 insertions(+), 95 deletions(-) create mode 100644 main/xiaozhi-server/core/utils/wakeup_word.py diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index c41baba0..154a2096 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -12,19 +12,21 @@ from core.handle.mcpHandle import ( send_mcp_initialize_message, send_mcp_tools_list_request, ) - +from core.utils.wakeup_word import WakeupWordsConfig TAG = __name__ WAKEUP_CONFIG = { - "dir": "config/assets/", - "file_name": "wakeup_words", - "create_time": time.time(), - "refresh_time": 10, + "refresh_time": 5, "words": ["你好小智", "你好啊小智", "小智你好", "小智"], - "text": "", } +# 创建全局的唤醒词配置管理器 +wakeup_words_config = WakeupWordsConfig() + +# 用于防止并发调用wakeupWordsResponse的锁 +_wakeup_response_lock = asyncio.Lock() + async def handleHelloMessage(conn, msg_json): """处理hello消息""" @@ -55,96 +57,78 @@ async def checkWakeupWords(conn, text): enable_wakeup_words_response_cache = conn.config[ "enable_wakeup_words_response_cache" ] - """是否用的是非流式tts""" - if conn.tts and conn.tts.interface_type != InterfaceType.NON_STREAM: + + if not enable_wakeup_words_response_cache or not conn.tts: return False - """是否开启唤醒词加速""" - if not enable_wakeup_words_response_cache: + if conn.tts.interface_type != InterfaceType.NON_STREAM: return False - """检查是否是唤醒词""" + _, filtered_text = remove_punctuation_and_length(text) - if filtered_text in conn.config.get("wakeup_words"): - await send_stt_message(conn, text) + if filtered_text not in conn.config.get("wakeup_words"): + return False - file = getWakeupWordFile(WAKEUP_CONFIG["file_name"]) - if file is None: + await send_stt_message(conn, text) + + # 获取当前音色 + voice = getattr(conn.tts, "voice", "default") + + # 获取唤醒词回复配置 + response = wakeup_words_config.get_wakeup_response(voice) + + # 播放唤醒词回复 + conn.client_abort = False + conn.tts.tts_one_sentence( + conn, + ContentType.FILE, + content_file=response["file_path"], + content_detail=response["text"], + ) + + # 检查是否需要更新唤醒词回复 + if time.time() - response["time"] > WAKEUP_CONFIG["refresh_time"]: + if not _wakeup_response_lock.locked(): asyncio.create_task(wakeupWordsResponse(conn)) - return False - text_hello = WAKEUP_CONFIG["text"] - if not text_hello: - text_hello = text - if conn.tts is None: - return False - conn.tts.tts_one_sentence( - conn, ContentType.FILE, content_file=file, content_detail=text_hello - ) - if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]: - asyncio.create_task(wakeupWordsResponse(conn)) - return True - return False - - -def getWakeupWordFile(file_name): - for file in os.listdir(WAKEUP_CONFIG["dir"]): - if file.startswith("my_" + file_name): - """避免缓存文件是一个空文件""" - if os.stat(f"config/assets/{file}").st_size > (15 * 1024): - return f"config/assets/{file}" - - """查找config/assets/目录下名称为wakeup_words的文件""" - for file in os.listdir(WAKEUP_CONFIG["dir"]): - if file.startswith(file_name): - return f"config/assets/{file}" - return None + return True async def wakeupWordsResponse(conn): - wait_max_time = 5 - while conn.llm is None or not conn.llm.response_no_stream: - await asyncio.sleep(1) - wait_max_time -= 1 - if wait_max_time <= 0: - conn.logger.bind(tag=TAG).error("连接对象没有llm") - return - - """唤醒词响应""" - wakeup_word = random.choice(WAKEUP_CONFIG["words"]) - question = ( - "此刻用户正在和你说```" - + wakeup_word - + "```。\n请你根据以上用户的内容，进行简短回复，文字内容控制在15个字以内。\n" - + "请勿对这条内容本身进行任何解释和回应，仅返回对用户的内容的回复。" - ) - result = conn.llm.response_no_stream(conn.config["prompt"], question) - if result is None or result == "": + if not conn.tts or not conn.llm or not conn.llm.response_no_stream: return - tts_result = await asyncio.to_thread(conn.tts.to_tts, result) - if conn.tts.delete_audio_file: - # 返回的是opus数据流 + try: + # 尝试获取锁，如果获取不到就返回 + if not await _wakeup_response_lock.acquire(): + return + + # 生成唤醒词回复 + wakeup_word = random.choice(WAKEUP_CONFIG["words"]) + question = ( + "此刻用户正在和你说```" + + wakeup_word + + "```。\n请你根据以上用户的内容进行简短回复。要像一个人正常人一样说话，不要像机器人一样说话。\n" + + "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。" + ) + + result = conn.llm.response_no_stream(conn.config["prompt"], question) + if not result or len(result) == 0: + return + + # 生成TTS音频 + tts_result = await asyncio.to_thread(conn.tts.to_tts, result) if not tts_result: return + + # 获取当前音色 + voice = getattr(conn.tts, "voice", "default") + wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000) - file_path = os.path.join( - WAKEUP_CONFIG["dir"], "my_" + WAKEUP_CONFIG["file_name"] + ".wav" - ) + file_path = wakeup_words_config.generate_file_path(voice) with open(file_path, "wb") as f: f.write(wav_bytes) - - else: - # 返回为文件 - if tts_result is not None and os.path.exists(tts_result): - file_type = os.path.splitext(tts_result)[1] - if file_type: - file_type = file_type.lstrip(".") - old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"]) - if old_file is not None: - os.remove(old_file) - """将文件挪到wakeup_words目录下""" - shutil.move( - tts_result, - WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + '.' + file_type, - ) - WAKEUP_CONFIG["create_time"] = time.time() - WAKEUP_CONFIG["text"] = result + # 更新配置 + wakeup_words_config.update_wakeup_response(voice, file_path, result) + finally: + # 确保在任何情况下都释放锁 + if _wakeup_response_lock.locked(): + _wakeup_response_lock.release() diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 4d15204e..5193e19e 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -145,10 +145,9 @@ class TTSProvider(TTSProviderBase): self.cluster = config.get("cluster") self.resource_id = config.get("resource_id") if config.get("private_voice"): - self.speaker = config.get("private_voice") + self.voice = config.get("private_voice") else: - self.speaker = config.get("speaker") - self.voice = config.get("voice") + self.voice = config.get("speaker") self.ws_url = config.get("ws_url") self.authorization = config.get("authorization") self.header = {"Authorization": f"{self.authorization}{self.access_token}"} @@ -272,7 +271,7 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).error(f"WebSocket连接不存在，终止发送文本") return # 发送文本 - await self.send_text(self.speaker, text, self.conn.sentence_id) + await self.send_text(self.voice, text, self.conn.sentence_id) return except Exception as e: logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") @@ -302,7 +301,7 @@ class TTSProvider(TTSProviderBase): event=EVENT_StartSession, sessionId=session_id ).as_bytes() payload = self.get_payload_bytes( - event=EVENT_StartSession, speaker=self.speaker + event=EVENT_StartSession, speaker=self.voice ) await self.send_event(header, optional, payload) logger.bind(tag=TAG).info("会话启动请求已发送") diff --git a/main/xiaozhi-server/core/providers/tts/minimax.py b/main/xiaozhi-server/core/providers/tts/minimax.py index 3741d169..75e25e2e 100644 --- a/main/xiaozhi-server/core/providers/tts/minimax.py +++ b/main/xiaozhi-server/core/providers/tts/minimax.py @@ -14,9 +14,9 @@ class TTSProvider(TTSProviderBase): self.api_key = config.get("api_key") self.model = config.get("model") if config.get("private_voice"): - self.voice_id = config.get("private_voice") + self.voice = config.get("private_voice") else: - self.voice_id = config.get("voice_id") + self.voice = config.get("voice_id") default_voice_setting = { "voice_id": "female-shaonv", @@ -43,8 +43,8 @@ class TTSProvider(TTSProviderBase): self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})} self.timber_weights = parse_string_to_list(config.get("timber_weights")) - if self.voice_id: - self.voice_setting["voice_id"] = self.voice_id + if self.voice: + self.voice_setting["voice_id"] = self.voice self.host = "api.minimax.chat" self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}" diff --git a/main/xiaozhi-server/core/providers/tts/ttson.py b/main/xiaozhi-server/core/providers/tts/ttson.py index c11206a6..91deeed9 100644 --- a/main/xiaozhi-server/core/providers/tts/ttson.py +++ b/main/xiaozhi-server/core/providers/tts/ttson.py @@ -19,9 +19,9 @@ class TTSProvider(TTSProviderBase): "https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=", ) if config.get("private_voice"): - self.voice_id = int(config.get("private_voice")) + self.voice = int(config.get("private_voice")) else: - self.voice_id = int(config.get("voice_id", 1695)) + self.voice = int(config.get("voice_id", 1695)) self.token = config.get("token") self.to_lang = config.get("to_lang") self.volume_change_dB = int(config.get("volume_change_dB", 0)) @@ -50,7 +50,7 @@ class TTSProvider(TTSProviderBase): "emotion": self.emotion, "format": self.format, "volume_change_dB": self.volume_change_dB, - "voice_id": self.voice_id, + "voice_id": self.voice, "pitch_factor": self.pitch_factor, "speed_factor": self.speed_factor, "token": self.token, diff --git a/main/xiaozhi-server/core/utils/wakeup_word.py b/main/xiaozhi-server/core/utils/wakeup_word.py new file mode 100644 index 00000000..258f879b --- /dev/null +++ b/main/xiaozhi-server/core/utils/wakeup_word.py @@ -0,0 +1,143 @@ +import os +import yaml +import time +import hashlib +import fcntl +from typing import Dict +from contextlib import contextmanager + + +class FileLock: + def __init__(self, file, timeout=5): + self.file = file + self.timeout = timeout + self.start_time = None + + def __enter__(self): + self.start_time = time.time() + while True: + try: + fcntl.flock(self.file, fcntl.LOCK_EX | fcntl.LOCK_NB) + return self.file + except IOError: + if time.time() - self.start_time > self.timeout: + raise TimeoutError("获取文件锁超时") + time.sleep(0.1) + + def __exit__(self, exc_type, exc_val, exc_tb): + fcntl.flock(self.file, fcntl.LOCK_UN) + + +class WakeupWordsConfig: + def __init__(self): + self.config_file = "data/.wakeup_words.yaml" + self.assets_dir = "config/assets/wakeup_words" + self._ensure_directories() + self._config_cache = None + self._last_load_time = 0 + self._cache_ttl = 1 # 缓存有效期（秒） + self._lock_timeout = 5 # 文件锁超时时间（秒） + + def _ensure_directories(self): + """确保必要的目录存在""" + os.makedirs(os.path.dirname(self.config_file), exist_ok=True) + os.makedirs(self.assets_dir, exist_ok=True) + + def _load_config(self) -> Dict: + """加载配置文件，使用缓存机制""" + current_time = time.time() + + # 如果缓存有效，直接返回缓存 + if ( + self._config_cache is not None + and current_time - self._last_load_time < self._cache_ttl + ): + return self._config_cache + + try: + with open(self.config_file, "a+") as f: + with FileLock(f, timeout=self._lock_timeout): + f.seek(0) + content = f.read() + config = yaml.safe_load(content) if content else {} + self._config_cache = config + self._last_load_time = current_time + return config + except (TimeoutError, IOError) as e: + print(f"加载配置文件失败: {e}") + return {} + except Exception as e: + print(f"加载配置文件时发生未知错误: {e}") + return {} + + def _save_config(self, config: Dict): + """保存配置到文件，使用文件锁保护""" + try: + with open(self.config_file, "w") as f: + with FileLock(f, timeout=self._lock_timeout): + yaml.dump(config, f, allow_unicode=True) + self._config_cache = config + self._last_load_time = time.time() + except (TimeoutError, IOError) as e: + print(f"保存配置文件失败: {e}") + raise + except Exception as e: + print(f"保存配置文件时发生未知错误: {e}") + raise + + def get_wakeup_response(self, voice: str) -> Dict: + voice = hashlib.md5(voice.encode()).hexdigest() + """获取唤醒词回复配置""" + config = self._load_config() + default_response = { + "voice": "default", + "file_path": "config/assets/wakeup_words.wav", + "time": 0, + "text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦", + } + + if not config or voice not in config: + return default_response + + # 检查文件大小 + file_path = config[voice]["file_path"] + if not os.path.exists(file_path) or os.stat(file_path).st_size < (15 * 1024): + return default_response + + return config[voice] + + def update_wakeup_response(self, voice: str, file_path: str, text: str): + """更新唤醒词回复配置""" + try: + config = self._load_config() + voice_hash = hashlib.md5(voice.encode()).hexdigest() + config[voice_hash] = { + "voice": voice, + "file_path": file_path, + "time": time.time(), + "text": text, + } + self._save_config(config) + except Exception as e: + print(f"更新唤醒词回复配置失败: {e}") + raise + + def generate_file_path(self, voice: str) -> str: + """生成音频文件路径，使用voice的哈希值作为文件名""" + try: + # 生成voice的哈希值 + voice_hash = hashlib.md5(voice.encode()).hexdigest() + file_path = os.path.join(self.assets_dir, f"{voice_hash}.wav") + + # 如果文件已存在，先删除 + if os.path.exists(file_path): + try: + os.remove(file_path) + except Exception as e: + print(f"删除已存在的音频文件失败: {e}") + raise + + return file_path + except Exception as e: + print(f"生成音频文件路径失败: {e}") + raise From 98174bcc162128a9e9258d077f47a3e122812a8a Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 8 Jun 2025 00:06:40 +0800 Subject: [PATCH 20/31] =?UTF-8?q?update:=E3=80=90=E6=B5=81=E5=BC=8Ftts?= =?UTF-8?q?=E3=80=91=E5=A2=9E=E5=8A=A0=E3=80=90=E9=9D=9E=E6=B5=81=E5=BC=8F?= =?UTF-8?q?=E3=80=91=E6=96=B9=E6=B3=95=EF=BC=8C=E7=94=A8=E4=BA=8E=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E5=8F=8A=E7=94=9F=E6=88=90=E6=96=87=E4=BB=B6=E7=9A=84?= =?UTF-8?q?=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 17 +-- .../providers/tts/huoshan_double_stream.py | 120 +++++++++++++++++- .../core/providers/tts/linkerai.py | 78 +++++++++++- 3 files changed, 194 insertions(+), 21 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index b19dc054..bb15382d 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -1,12 +1,11 @@ -import os -import shutil import time import json import random import asyncio +from core.utils.util import audio_to_data from core.handle.sendAudioHandle import send_stt_message from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes -from core.providers.tts.dto.dto import ContentType, InterfaceType +from core.providers.tts.dto.dto import ContentType, SentenceType from core.handle.mcpHandle import ( MCPClient, send_mcp_initialize_message, @@ -59,9 +58,6 @@ async def checkWakeupWords(conn, text): if not enable_wakeup_words_response_cache or not conn.tts: return False - if conn.tts.interface_type != InterfaceType.NON_STREAM: - return False - _, filtered_text = remove_punctuation_and_length(text) if filtered_text not in conn.config.get("wakeup_words"): return False @@ -77,12 +73,9 @@ async def checkWakeupWords(conn, text): # 播放唤醒词回复 conn.client_abort = False - conn.tts.tts_one_sentence( - conn, - ContentType.FILE, - content_file=response["file_path"], - content_detail=response["text"], - ) + opus_packets, _ = audio_to_data(response["file_path"]) + conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, response["text"])) + conn.tts.tts_audio_queue.put((SentenceType.LAST, [], None)) # 检查是否需要更新唤醒词回复 if time.time() - response["time"] > WAKEUP_CONFIG["refresh_time"]: diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 10a37b51..33a1c8ec 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -301,7 +301,7 @@ class TTSProvider(TTSProviderBase): payload = self.get_payload_bytes( event=EVENT_StartSession, speaker=self.voice ) - await self.send_event(header, optional, payload) + await self.send_event(self.ws, header, optional, payload) logger.bind(tag=TAG).info("会话启动请求已发送") except Exception as e: logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}") @@ -334,7 +334,7 @@ class TTSProvider(TTSProviderBase): event=EVENT_FinishSession, sessionId=session_id ).as_bytes() payload = str.encode("{}") - await self.send_event(header, optional, payload) + await self.send_event(self.ws, header, optional, payload) logger.bind(tag=TAG).info("会话结束请求已发送") # 等待监听任务完成 @@ -451,7 +451,11 @@ class TTSProvider(TTSProviderBase): self.ws = None async def send_event( - self, header: bytes, optional: bytes | None = None, payload: bytes = None + self, + ws: websockets.WebSocketClientProtocol, + header: bytes, + optional: bytes | None = None, + payload: bytes = None, ): try: full_client_request = bytearray(header) @@ -461,7 +465,7 @@ class TTSProvider(TTSProviderBase): payload_size = len(payload).to_bytes(4, "big", signed=True) full_client_request.extend(payload_size) full_client_request.extend(payload) - await self.ws.send(full_client_request) + await ws.send(full_client_request) except websockets.ConnectionClosed: logger.bind(tag=TAG).error(f"ConnectionClosed") raise @@ -476,7 +480,7 @@ class TTSProvider(TTSProviderBase): payload = self.get_payload_bytes( event=EVENT_TaskRequest, text=text, speaker=speaker ) - return await self.send_event(header, optional, payload) + return await self.send_event(self.ws, header, optional, payload) # 读取 res 数组某段 字符串内容 def read_res_content(self, res: bytes, offset: int): @@ -554,7 +558,7 @@ class TTSProvider(TTSProviderBase): ).as_bytes() optional = Optional(event=EVENT_Start_Connection).as_bytes() payload = str.encode("{}") - return await self.send_event(header, optional, payload) + return await self.send_event(self.ws, header, optional, payload) def print_response(self, res, tag_msg: str): logger.bind(tag=TAG).debug(f"===>{tag_msg} header:{res.header.__dict__}") @@ -590,3 +594,107 @@ class TTSProvider(TTSProviderBase): def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False): opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end) return opus_datas + + def to_tts(self, text: str) -> list: + """非流式生成音频数据，用于生成音频及测试场景 + + Args: + text: 要转换的文本 + + Returns: + list: 音频数据列表 + """ + try: + # 创建事件循环 + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # 生成会话ID + session_id = uuid.uuid4().__str__().replace("-", "") + + # 存储音频数据 + audio_data = [] + + async def _generate_audio(): + # 创建新的WebSocket连接 + ws_header = { + "X-Api-App-Key": self.appId, + "X-Api-Access-Key": self.access_token, + "X-Api-Resource-Id": self.resource_id, + "X-Api-Connect-Id": uuid.uuid4(), + } + ws = await websockets.connect( + self.ws_url, additional_headers=ws_header, max_size=1000000000 + ) + + try: + # 启动会话 + header = Header( + message_type=FULL_CLIENT_REQUEST, + message_type_specific_flags=MsgTypeFlagWithEvent, + serial_method=JSON, + ).as_bytes() + optional = Optional( + event=EVENT_StartSession, sessionId=session_id + ).as_bytes() + payload = self.get_payload_bytes( + event=EVENT_StartSession, speaker=self.voice + ) + await self.send_event(ws, header, optional, payload) + + # 发送文本 + header = Header( + message_type=FULL_CLIENT_REQUEST, + message_type_specific_flags=MsgTypeFlagWithEvent, + serial_method=JSON, + ).as_bytes() + optional = Optional( + event=EVENT_TaskRequest, sessionId=session_id + ).as_bytes() + payload = self.get_payload_bytes( + event=EVENT_TaskRequest, text=text, speaker=self.voice + ) + await self.send_event(ws, header, optional, payload) + + # 发送结束会话请求 + header = Header( + message_type=FULL_CLIENT_REQUEST, + message_type_specific_flags=MsgTypeFlagWithEvent, + serial_method=JSON, + ).as_bytes() + optional = Optional( + event=EVENT_FinishSession, sessionId=session_id + ).as_bytes() + payload = str.encode("{}") + await self.send_event(ws, header, optional, payload) + + # 接收音频数据 + while True: + msg = await ws.recv() + res = self.parser_response(msg) + + if ( + res.optional.event == EVENT_TTSResponse + and res.header.message_type == AUDIO_ONLY_RESPONSE + ): + opus_datas = self.wav_to_opus_data_audio_raw(res.payload) + audio_data.extend(opus_datas) + elif res.optional.event == EVENT_SessionFinished: + break + + finally: + # 清理资源 + try: + await ws.close() + except: + pass + + # 运行异步任务 + loop.run_until_complete(_generate_audio()) + loop.close() + + return audio_data + + except Exception as e: + logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}") + return [] diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index dd8d0e6b..649ed51d 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -2,6 +2,8 @@ import queue import asyncio import traceback import aiohttp +import requests +import time from config.logger import setup_logging from core.utils.tts import MarkdownCleaner from core.providers.tts.base import TTSProviderBase @@ -55,7 +57,7 @@ class TTSProvider(TTSProviderBase): self.tts_text_buff.append(message.content_detail) segment_text = self._get_segment_text() if segment_text: - self.to_tts(segment_text) + self.to_tts_single_stream(segment_text) elif ContentType.FILE == message.content_type: logger.bind(tag=TAG).info( @@ -87,12 +89,12 @@ class TTSProvider(TTSProviderBase): if remaining_text: segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) if segment_text: - self.to_tts(segment_text, is_last) + self.to_tts_single_stream(segment_text, is_last) self.processed_chars += len(full_text) else: self._process_before_stop_play_files() - def to_tts(self, text, is_last=False): + def to_tts_single_stream(self, text, is_last=False): try: max_repeat_time = 5 text = MarkdownCleaner.clean_markdown(text) @@ -225,3 +227,73 @@ class TTSProvider(TTSProviderBase): except Exception as e: logger.error(f"TTS请求异常: {e}") self.tts_audio_queue.put((SentenceType.LAST, [], None)) + + def to_tts(self, text: str) -> list: + """非流式TTS处理，用于测试及保存音频文件的场景 + + Args: + text: 要转换的文本 + + Returns: + list: 返回opus编码后的音频数据列表 + """ + start_time = time.time() + text = MarkdownCleaner.clean_markdown(text) + + params = { + "tts_text": text, + "spk_id": self.voice, + "frame_duration": 60, + "stream": False, + "target_sr": 16000, + "audio_format": self.audio_format, + "instruct_text": "请生成一段自然流畅的语音", + } + headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + } + + try: + with requests.get( + self.api_url, params=params, headers=headers, timeout=5 + ) as response: + if response.status_code != 200: + logger.error( + f"TTS请求失败: {response.status_code}, {response.text}" + ) + return [] + + logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") + + # 使用opus编码器处理PCM数据 + opus_datas = [] + pcm_data = response.content + + # 计算每帧的字节数 + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels + * self.opus_encoder.frame_size_ms + / 1000 + * 2 + ) + + # 分帧处理PCM数据 + for i in range(0, len(pcm_data), frame_bytes): + frame = pcm_data[i : i + frame_bytes] + if len(frame) < frame_bytes: + # 最后一帧可能不足，用0填充 + frame = frame + b"\x00" * (frame_bytes - len(frame)) + + opus = self.opus_encoder.encode_pcm_to_opus( + frame, end_of_stream=(i + frame_bytes >= len(pcm_data)) + ) + if opus: + opus_datas.extend(opus) + + return opus_datas + + except Exception as e: + logger.error(f"TTS请求异常: {e}") + return [] From 594b4f1d75feaa8cd12422447f87dbf3a1d6c863 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 8 Jun 2025 00:21:30 +0800 Subject: [PATCH 21/31] =?UTF-8?q?add=EF=BC=9A=E5=A2=9E=E5=8A=A0psutil?= =?UTF-8?q?=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main/xiaozhi-server/requirements.txt b/main/xiaozhi-server/requirements.txt index ed5f9973..dc5a2294 100755 --- a/main/xiaozhi-server/requirements.txt +++ b/main/xiaozhi-server/requirements.txt @@ -31,4 +31,5 @@ chardet==5.2.0 aioconsole==0.8.1 markitdown==0.1.1 mcp-proxy==0.6.0 -PyJWT==2.8.0 \ No newline at end of file +PyJWT==2.8.0 +psutil==7.0.0 \ No newline at end of file From 61232e7ddabafd199ba2dea9107a5297b149192f Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 8 Jun 2025 01:09:32 +0800 Subject: [PATCH 22/31] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E7=A9=BAasr?= =?UTF-8?q?=E8=AF=AD=E9=9F=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/handle/helloHandle.py | 2 +- main/xiaozhi-server/core/handle/receiveAudioHandle.py | 8 +++++--- main/xiaozhi-server/core/providers/asr/doubao.py | 4 ++++ main/xiaozhi-server/core/providers/asr/doubao_stream.py | 5 +++++ 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index bb15382d..0232c9ac 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -17,7 +17,7 @@ TAG = __name__ WAKEUP_CONFIG = { "refresh_time": 5, - "words": ["你好小智", "你好啊小智", "小智你好", "小智"], + "words": ["你好", "你好啊", "嘿，你好", "嗨"], } # 创建全局的唤醒词配置管理器 diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 90600ce2..3b98a638 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -15,11 +15,13 @@ async def handleAudioMessage(conn, audio): have_voice = conn.vad.is_vad(conn, audio) # 如果设备刚刚被唤醒，短暂忽略VAD检测 - if hasattr(conn, "just_woken_up") and conn.just_woken_up: + if have_voice and hasattr(conn, "just_woken_up") and conn.just_woken_up: have_voice = False # 设置一个短暂延迟后恢复VAD检测 conn.asr_audio.clear() - asyncio.create_task(resume_vad_detection(conn)) + if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done(): + print("clear asr_audio") + conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn)) if have_voice: if conn.client_is_speaking: @@ -32,7 +34,7 @@ async def handleAudioMessage(conn, audio): async def resume_vad_detection(conn): # 等待2秒后恢复VAD检测 - await asyncio.sleep(2) + await asyncio.sleep(1) conn.just_woken_up = False diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index ea590aad..f6cd79c6 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -168,6 +168,7 @@ class ASRProvider(ASRProviderBase): if ( "payload_msg" in result and result["payload_msg"]["code"] != self.success_code + and result["payload_msg"]["code"] != 1013 # 忽略无有效语音的错误 ): logger.bind(tag=TAG).error(f"ASR error: {result}") return None @@ -203,6 +204,9 @@ class ASRProvider(ASRProviderBase): if len(result["payload_msg"]["result"]) > 0: return result["payload_msg"]["result"][0]["text"] return None + elif "payload_msg" in result and result["payload_msg"]["code"] == 1013: + # 无有效语音，返回空字符串 + return "" else: logger.bind(tag=TAG).error(f"ASR error: {result}") return None diff --git a/main/xiaozhi-server/core/providers/asr/doubao_stream.py b/main/xiaozhi-server/core/providers/asr/doubao_stream.py index 491fd785..f532ff27 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao_stream.py +++ b/main/xiaozhi-server/core/providers/asr/doubao_stream.py @@ -157,6 +157,11 @@ class ASRProvider(ASRProviderBase): if "payload_msg" in result: payload = result["payload_msg"] + # 检查是否是错误码1013（无有效语音） + if "code" in payload and payload["code"] == 1013: + # 静默处理，不记录错误日志 + continue + if "result" in payload: utterances = payload["result"].get("utterances", []) # 检查duration和空文本的情况 From d5fc4d48b44902f7d26025ab5e695d3f1717d42d Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 8 Jun 2025 02:33:07 +0800 Subject: [PATCH 23/31] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E6=B5=81?= =?UTF-8?q?=E5=A4=B1=E5=89=8D=E5=B8=A7=E5=8F=91=E9=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/handle/receiveAudioHandle.py | 3 +-- .../providers/tts/huoshan_double_stream.py | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 3b98a638..fd58f49d 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -13,15 +13,14 @@ TAG = __name__ async def handleAudioMessage(conn, audio): # 当前片段是否有人说话 have_voice = conn.vad.is_vad(conn, audio) - # 如果设备刚刚被唤醒，短暂忽略VAD检测 if have_voice and hasattr(conn, "just_woken_up") and conn.just_woken_up: have_voice = False # 设置一个短暂延迟后恢复VAD检测 conn.asr_audio.clear() if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done(): - print("clear asr_audio") conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn)) + return if have_voice: if conn.client_is_speaking: diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 33a1c8ec..8245b754 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -381,6 +381,7 @@ class TTSProvider(TTSProviderBase): """监听TTS响应""" opus_datas_cache = [] is_first_sentence = True + first_sentence_segment_count = 0 # 添加计数器 try: while not self.conn.stop_event.is_set(): try: @@ -402,6 +403,7 @@ class TTSProvider(TTSProviderBase): (SentenceType.FIRST, [], self.tts_text) ) opus_datas_cache = [] + first_sentence_segment_count = 0 # 重置计数器 elif ( res.optional.event == EVENT_TTSResponse and res.header.message_type == AUDIO_ONLY_RESPONSE @@ -412,19 +414,22 @@ class TTSProvider(TTSProviderBase): f"推送数据到队列里面帧数～～{len(opus_datas)}" ) if is_first_sentence: - # 第一句话直接发送 - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_datas, self.tts_text) - ) + first_sentence_segment_count += 1 + if first_sentence_segment_count <= 6: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas, None) + ) + else: + opus_datas_cache = opus_datas_cache + opus_datas else: # 后续句子缓存 opus_datas_cache = opus_datas_cache + opus_datas elif res.optional.event == EVENT_TTSSentenceEnd: logger.bind(tag=TAG).info(f"句子语音生成成功：{self.tts_text}") - if not is_first_sentence: - # 只有非第一句话才发送缓存的数据 + if not is_first_sentence or first_sentence_segment_count > 10: + # 发送缓存的数据 self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_datas_cache, self.tts_text) + (SentenceType.MIDDLE, opus_datas_cache, None) ) # 第一句话结束后，将标志设置为False is_first_sentence = False From ac1e254eedc523af92487a395c931dc8b5118de8 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 8 Jun 2025 11:50:09 +0800 Subject: [PATCH 24/31] =?UTF-8?q?update:=E6=99=BA=E6=8E=A7=E5=8F=B0?= =?UTF-8?q?=E5=BC=80=E5=90=AF=E5=94=A4=E9=86=92=E8=AF=8D=E5=8A=A0=E9=80=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/xiaozhi/common/constant/Constant.java | 2 +- .../src/main/resources/db/changelog/202506080955.sql | 3 +++ .../main/resources/db/changelog/db.changelog-master.yaml | 9 ++++++++- main/xiaozhi-server/config/logger.py | 2 +- main/xiaozhi-server/core/handle/helloHandle.py | 8 +++++--- main/xiaozhi-server/core/handle/sendAudioHandle.py | 1 + .../core/providers/tts/huoshan_double_stream.py | 2 -- main/xiaozhi-server/core/providers/tts/linkerai.py | 2 ++ 8 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202506080955.sql diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index 2590a566..44294756 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -227,7 +227,7 @@ public interface Constant { /** * 版本号 */ - public static final String VERSION = "0.5.4"; + public static final String VERSION = "0.5.5"; /** * 无效固件URL diff --git a/main/manager-api/src/main/resources/db/changelog/202506080955.sql b/main/manager-api/src/main/resources/db/changelog/202506080955.sql new file mode 100644 index 00000000..5f5e799e --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202506080955.sql @@ -0,0 +1,3 @@ +-- 智控台开启唤醒词加速 +update `sys_params` set param_value = '你好小智;你好小志;小爱同学;你好小鑫;你好小新;小美同学;小龙小龙;喵喵同学;小滨小滨;小冰小冰;嘿你好呀' where param_code = 'wakeup_words'; +update `sys_params` set param_value = 'true' where param_code = 'enable_wakeup_words_response_cache'; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index cb3007aa..f33d28a2 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -197,4 +197,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202506051538.sql \ No newline at end of file + path: classpath:db/changelog/202506051538.sql + - changeSet: + id: 202506080955 + author: hrz + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202506080955.sql \ No newline at end of file diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index 7bd22c43..a99013aa 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -5,7 +5,7 @@ from config.config_loader import load_config from config.settings import check_config_file from datetime import datetime -SERVER_VERSION = "0.5.4" +SERVER_VERSION = "0.5.5" _logger_initialized = False diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 0232c9ac..1edeefcd 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -3,7 +3,7 @@ import json import random import asyncio from core.utils.util import audio_to_data -from core.handle.sendAudioHandle import send_stt_message +from core.handle.sendAudioHandle import sendAudioMessage, send_stt_message from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes from core.providers.tts.dto.dto import ContentType, SentenceType from core.handle.mcpHandle import ( @@ -74,8 +74,10 @@ async def checkWakeupWords(conn, text): # 播放唤醒词回复 conn.client_abort = False opus_packets, _ = audio_to_data(response["file_path"]) - conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, response["text"])) - conn.tts.tts_audio_queue.put((SentenceType.LAST, [], None)) + + conn.logger.bind(tag=TAG).info(f"播放唤醒词回复: {response['text']}") + await sendAudioMessage(conn, SentenceType.FIRST, opus_packets, response["text"]) + await sendAudioMessage(conn, SentenceType.LAST, [], None) # 检查是否需要更新唤醒词回复 if time.time() - response["time"] > WAKEUP_CONFIG["refresh_time"]: diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index ea81b7ed..2bf55e65 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -34,6 +34,7 @@ emoji_map = { async def sendAudioMessage(conn, sentenceType, audios, text): # 发送句子开始消息 + conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}") if text is not None: emotion = analyze_emotion(text) emoji = emoji_map.get(emotion, "🙂") # 默认使用笑脸 diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 8245b754..83a2d83a 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -188,10 +188,8 @@ class TTSProvider(TTSProviderBase): def tts_text_priority_thread(self): """火山引擎双流式TTS的文本处理线程""" - logger.bind(tag=TAG).info("TTS文本处理线程启动") while not self.conn.stop_event.is_set(): try: - logger.bind(tag=TAG).debug("等待TTS文本队列消息...") message = self.tts_text_queue.get(timeout=1) logger.bind(tag=TAG).debug( f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}" diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index 649ed51d..69127e8a 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -93,6 +93,8 @@ class TTSProvider(TTSProviderBase): self.processed_chars += len(full_text) else: self._process_before_stop_play_files() + else: + self._process_before_stop_play_files() def to_tts_single_stream(self, text, is_last=False): try: From 1d57e75d28bc1da8632e0dd5e11d998ba6c1936e Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 8 Jun 2025 11:59:06 +0800 Subject: [PATCH 25/31] =?UTF-8?q?update:=E5=8D=95=E6=B5=81=E5=BC=8Ftts?= =?UTF-8?q?=E8=81=8A=E5=A4=A9=E8=AE=B0=E5=BD=95=E4=B8=8A=E6=8A=A5=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/providers/tts/linkerai.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index 69127e8a..7493764f 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -173,6 +173,8 @@ class TTSProvider(TTSProviderBase): self.pcm_buffer.clear() opus_datas_cache = [] + self.tts_audio_queue.put((SentenceType.FIRST, [], text)) + # 兼容 iter_chunked / iter_chunks / iter_any async for chunk in resp.content.iter_any(): data = chunk[0] if isinstance(chunk, (list, tuple)) else chunk @@ -193,7 +195,7 @@ class TTSProvider(TTSProviderBase): if opus: if self.segment_count < 10: # 前10个片段直接发送 self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus, text) + (SentenceType.MIDDLE, opus, None) ) self.segment_count += 1 else: @@ -208,7 +210,7 @@ class TTSProvider(TTSProviderBase): if self.segment_count < 10: # 前10个片段直接发送 # 直接发送 self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus, text) + (SentenceType.MIDDLE, opus, None) ) self.segment_count += 1 else: @@ -219,7 +221,7 @@ class TTSProvider(TTSProviderBase): # 如果不是前10个片段，发送缓存的数据 if self.segment_count >= 10 and opus_datas_cache: self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_datas_cache, text) + (SentenceType.MIDDLE, opus_datas_cache, None) ) # 如果是最后一段，输出音频获取完毕 From 4d0ddd7ff3f30ffa7d83bfc97671247da68bc145 Mon Sep 17 00:00:00 2001 From: whats2000 <60466660+whats2000@users.noreply.github.com> Date: Sat, 7 Jun 2025 13:46:35 +0800 Subject: [PATCH 26/31] fix: sanitize MCP tool names for OpenAI --- main/xiaozhi-server/core/handle/mcpHandle.py | 14 ++++++++---- main/xiaozhi-server/core/mcp/MCPClient.py | 24 +++++++++++++------- main/xiaozhi-server/core/utils/util.py | 5 ++++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/main/xiaozhi-server/core/handle/mcpHandle.py b/main/xiaozhi-server/core/handle/mcpHandle.py index cab8a96b..b5fd9372 100644 --- a/main/xiaozhi-server/core/handle/mcpHandle.py +++ b/main/xiaozhi-server/core/handle/mcpHandle.py @@ -1,7 +1,7 @@ import json import asyncio from concurrent.futures import Future -from core.utils.util import get_vision_url +from core.utils.util import get_vision_url, sanitize_tool_name from core.utils.auth import AuthToken TAG = __name__ @@ -11,7 +11,8 @@ class MCPClient: """MCPClient，用于管理MCP状态和工具""" def __init__(self): - self.tools = {} # Dictionary for O(1) lookup + self.tools = {} # sanitized_name -> tool_data + self.name_mapping = {} self.ready = False self.call_results = {} # To store Futures for tool call responses self.next_id = 1 @@ -30,7 +31,7 @@ class MCPClient: result = [] for tool_name, tool_data in self.tools.items(): function_def = { - "name": tool_data["name"], + "name": tool_name, "description": tool_data["description"], "parameters": { "type": tool_data["inputSchema"].get("type", "object"), @@ -53,7 +54,9 @@ class MCPClient: async def add_tool(self, tool_data: dict): async with self.lock: - self.tools[tool_data["name"]] = tool_data + sanitized_name = sanitize_tool_name(tool_data["name"]) + self.tools[sanitized_name] = tool_data + self.name_mapping[sanitized_name] = tool_data["name"] self._cached_available_tools = ( None # Invalidate the cache when a tool is added ) @@ -333,11 +336,12 @@ async def call_mcp_tool( raise ValueError(f"参数处理失败: {str(e)}") raise e + actual_name = mcp_client.name_mapping.get(tool_name, tool_name) payload = { "jsonrpc": "2.0", "id": tool_call_id, "method": "tools/call", - "params": {"name": tool_name, "arguments": arguments}, + "params": {"name": actual_name, "arguments": arguments}, } conn.logger.bind(tag=TAG).info( diff --git a/main/xiaozhi-server/core/mcp/MCPClient.py b/main/xiaozhi-server/core/mcp/MCPClient.py index b6b6335f..6af5f058 100644 --- a/main/xiaozhi-server/core/mcp/MCPClient.py +++ b/main/xiaozhi-server/core/mcp/MCPClient.py @@ -9,6 +9,7 @@ from mcp import ClientSession, StdioServerParameters from mcp.client.stdio import stdio_client from mcp.client.sse import sse_client from config.logger import setup_logging +from core.utils.util import sanitize_tool_name TAG = __name__ @@ -23,7 +24,9 @@ class MCPClient: self._shutdown_evt = asyncio.Event() self.session: Optional[ClientSession] = None - self.tools: List = [] + self.tools: List = [] # original tool objects + self.tools_dict: Dict[str, Any] = {} + self.name_mapping: Dict[str, str] = {} async def initialize(self): if self._worker_task: @@ -32,7 +35,7 @@ class MCPClient: await self._ready_evt.wait() self.logger.bind(tag=TAG).info( - f"Connected, tools = {[t.name for t in self.tools]}" + f"Connected, tools = {[name for name in self.name_mapping.values()]}" ) async def cleanup(self): @@ -48,27 +51,28 @@ class MCPClient: self._worker_task = None def has_tool(self, name: str) -> bool: - return any(t.name == name for t in self.tools) + return name in self.tools_dict def get_available_tools(self): return [ { "type": "function", "function": { - "name": t.name, - "description": t.description, - "parameters": t.inputSchema, + "name": name, + "description": tool.description, + "parameters": tool.inputSchema, }, } - for t in self.tools + for name, tool in self.tools_dict.items() ] async def call_tool(self, name: str, args: dict): if not self.session: raise RuntimeError("MCPClient not initialized") + real_name = self.name_mapping.get(name, name) loop = self._worker_task.get_loop() - coro = self.session.call_tool(name, args) + coro = self.session.call_tool(real_name, args) if loop is asyncio.get_running_loop(): return await coro @@ -123,6 +127,10 @@ class MCPClient: # 获取工具 self.tools = (await self.session.list_tools()).tools + for t in self.tools: + sanitized = sanitize_tool_name(t.name) + self.tools_dict[sanitized] = t + self.name_mapping[sanitized] = t.name self._ready_evt.set() diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index b492529c..cf52bfc2 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -976,3 +976,8 @@ def is_valid_image_file(file_data: bytes) -> bool: return True return False + + +def sanitize_tool_name(name: str) -> str: + """Sanitize tool names for OpenAI compatibility.""" + return re.sub(r"[^a-zA-Z0-9_-]", "_", name) From 182acc0787423e34011fd01e76cf6cbdc6d0c7b5 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 8 Jun 2025 15:51:57 +0800 Subject: [PATCH 27/31] =?UTF-8?q?update:=E6=9B=BF=E6=8D=A2=E6=8F=8F?= =?UTF-8?q?=E8=BF=B0=E9=87=8C=E6=B6=89=E5=8F=8A=E5=88=B0=E7=9A=84=E5=8E=9F?= =?UTF-8?q?=E5=A7=8B=E6=96=B9=E6=B3=95=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/handle/mcpHandle.py | 23 ++++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/main/xiaozhi-server/core/handle/mcpHandle.py b/main/xiaozhi-server/core/handle/mcpHandle.py index b5fd9372..3f3216ac 100644 --- a/main/xiaozhi-server/core/handle/mcpHandle.py +++ b/main/xiaozhi-server/core/handle/mcpHandle.py @@ -136,9 +136,6 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict): conn.logger.bind(tag=TAG).info( f"客户端MCP服务器信息: name={name}, version={version}" ) - await send_mcp_tools_list_request( - conn - ) # After initialization, request tool list return elif msg_id == 2: # mcpToolsListID @@ -177,6 +174,20 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict): await mcp_client.add_tool(new_tool) conn.logger.bind(tag=TAG).debug(f"客户端工具 #{i+1}: {name}") + # 替换所有工具描述中的工具名称 + for tool_data in mcp_client.tools.values(): + if "description" in tool_data: + description = tool_data["description"] + # 遍历所有工具名称进行替换 + for ( + sanitized_name, + original_name, + ) in mcp_client.name_mapping.items(): + description = description.replace( + original_name, sanitized_name + ) + tool_data["description"] = description + next_cursor = result.get("nextCursor", "") if next_cursor: conn.logger.bind(tag=TAG).info( @@ -222,8 +233,6 @@ async def send_mcp_initialize_message(conn): "token": token, } - conn.logger.bind(tag=TAG).info(f"视觉服务信息: {vision}") - payload = { "jsonrpc": "2.0", "id": 1, # mcpInitializeID @@ -345,7 +354,7 @@ async def call_mcp_tool( } conn.logger.bind(tag=TAG).info( - f"发送客户端mcp工具调用请求: {tool_name}，参数: {args}" + f"发送客户端mcp工具调用请求: {actual_name}，参数: {args}" ) await send_mcp_message(conn, payload) @@ -353,7 +362,7 @@ async def call_mcp_tool( # Wait for response or timeout raw_result = await asyncio.wait_for(result_future, timeout=timeout) conn.logger.bind(tag=TAG).info( - f"客户端mcp工具调用 {tool_name} 成功，原始结果: {raw_result}" + f"客户端mcp工具调用 {actual_name} 成功，原始结果: {raw_result}" ) if isinstance(raw_result, dict): From 7163abbbe32896d7acfb468cb4afbc87e60a6e82 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Mon, 9 Jun 2025 00:20:49 +0800 Subject: [PATCH 28/31] =?UTF-8?q?update:=E4=BF=AE=E5=A4=8Dwindows=E5=B9=B3?= =?UTF-8?q?=E5=8F=B0=E6=97=A0=E6=B3=95=E4=BD=BF=E7=94=A8fcntl=E6=A8=A1?= =?UTF-8?q?=E5=9D=97=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/utils/wakeup_word.py | 9 ++++----- main/xiaozhi-server/requirements.txt | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/main/xiaozhi-server/core/utils/wakeup_word.py b/main/xiaozhi-server/core/utils/wakeup_word.py index 258f879b..29929081 100644 --- a/main/xiaozhi-server/core/utils/wakeup_word.py +++ b/main/xiaozhi-server/core/utils/wakeup_word.py @@ -2,9 +2,8 @@ import os import yaml import time import hashlib -import fcntl +import portalocker from typing import Dict -from contextlib import contextmanager class FileLock: @@ -17,15 +16,15 @@ class FileLock: self.start_time = time.time() while True: try: - fcntl.flock(self.file, fcntl.LOCK_EX | fcntl.LOCK_NB) + portalocker.lock(self.file, portalocker.LOCK_EX | portalocker.LOCK_NB) return self.file - except IOError: + except portalocker.LockException: if time.time() - self.start_time > self.timeout: raise TimeoutError("获取文件锁超时") time.sleep(0.1) def __exit__(self, exc_type, exc_val, exc_tb): - fcntl.flock(self.file, fcntl.LOCK_UN) + portalocker.unlock(self.file) class WakeupWordsConfig: diff --git a/main/xiaozhi-server/requirements.txt b/main/xiaozhi-server/requirements.txt index dc5a2294..3ffbcfa4 100755 --- a/main/xiaozhi-server/requirements.txt +++ b/main/xiaozhi-server/requirements.txt @@ -32,4 +32,5 @@ aioconsole==0.8.1 markitdown==0.1.1 mcp-proxy==0.6.0 PyJWT==2.8.0 -psutil==7.0.0 \ No newline at end of file +psutil==7.0.0 +portalocker==2.10.1 \ No newline at end of file From 3d1768100f46b1419a1b8e13ed2120ea24857939 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Sun, 8 Jun 2025 15:40:18 +0800 Subject: [PATCH 29/31] =?UTF-8?q?fix:=E4=BD=BF=E7=94=A8=E2=80=9Cclear=5Fma?= =?UTF-8?q?rkdown=E2=80=9D=E6=96=B9=E6=B3=95=E4=BF=AE=E5=A4=8D=E2=80=9C?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E7=81=AB=E5=B1=B1=E6=B5=81=E5=BC=8FTTS?= =?UTF-8?q?=E4=B8=ADmarkdown=20=E8=AF=AD=E6=B3=95=E8=A2=AB=20TTS=20?= =?UTF-8?q?=E8=AF=BB=E5=87=BA=E2=80=9D=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/huoshan_double_stream.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 83a2d83a..6e4b9eac 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -5,6 +5,7 @@ import queue import asyncio import traceback import websockets +from core.utils.tts import MarkdownCleaner from config.logger import setup_logging from core.utils import opus_encoder_utils from core.utils.util import check_model_key @@ -266,8 +267,12 @@ class TTSProvider(TTSProviderBase): await handleAbortMessage(self.conn) logger.bind(tag=TAG).error(f"WebSocket连接不存在，终止发送文本") return + + # 过滤Markdown + filtered_text = MarkdownCleaner.clean_markdown(text) + # 发送文本 - await self.send_text(self.voice, text, self.conn.sentence_id) + await self.send_text(self.voice, filtered_text, self.conn.sentence_id) return except Exception as e: logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") From 7a23ae4b8431855ad77eeee9c1f9129551857446 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 8 Jun 2025 19:31:47 +0800 Subject: [PATCH 30/31] =?UTF-8?q?update:=E6=9B=B4=E6=96=B0=E6=8A=80?= =?UTF-8?q?=E6=9C=AF=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/README.md | 464 +++++++++++++++++++++++++++++++++++++++++++++- main/README_en.md | 369 ++++++++++++++++++++++++++++++++++++ 2 files changed, 826 insertions(+), 7 deletions(-) create mode 100644 main/README_en.md diff --git a/main/README.md b/main/README.md index 4255ab14..c1f23518 100644 --- a/main/README.md +++ b/main/README.md @@ -1,7 +1,70 @@ -本文档是开发类文档，如需部署小智服务端，[点击这里查看部署教程](../README.md#%E9%83%A8%E7%BD%B2%E6%96%87%E6%A1%A3) +# 技术文档：`xiaozhi-esp32-server` -# 项目目录介绍 -如果你是一名软件开发者，这里有一份[《致开发者的公开信》](../docs/contributor_open_letter.md)，欢迎归队！ +**目录：** + +1. [引言](#1-引言) +2. [整体架构](#2-整体架构) +3. [核心组件深度剖析](#3-核心组件深度剖析) + * [3.1. `xiaozhi-server` (核心AI引擎 - Python实现)](#31-xiaozhi-server-核心ai引擎---python实现) + * [3.2. `manager-api` (管理后端 - Java Spring Boot实现)](#32-manager-api-管理后端---java-spring-boot实现) + * [3.3. `manager-web` (Web管理前端 - Vue.js实现)](#33-manager-web-web管理前端---vuejs实现) +4. [数据流与交互机制](#4-数据流与交互机制) +5. [核心功能概要](#5-核心功能概要) +6. [部署与配置概述](#6-部署与配置概述) +--- + +## 1. 引言 + +`xiaozhi-esp32-server` 项目是一个专为基于ESP32的智能硬件提供支持的**综合性后端系统**。其核心目标是使开发人员能够快速构建一个强大的服务器基础设施，该设施不仅能够理解自然语言指令，还能与多种AI服务（用于语音识别、自然语言理解及语音合成）进行高效交互、管理物联网（IoT）设备，并提供一个基于Web的用户界面以进行系统配置和管理。通过将多种尖端技术整合到一个高内聚且可扩展的平台中，本项目旨在简化和加速可定制化语音助手及智能控制系统的开发进程。它不仅仅是一个简单的服务器，更是一个连接硬件、AI能力与用户管理的桥梁。 + +--- + +## 2. 整体架构 + +`xiaozhi-esp32-server` 系统采用了一种**分布式、多组件协作**的架构设计，确保了系统的模块化、可维护性和可扩展性。各个核心组件各司其职，协同工作。主要组件包括： + +1. **ESP32 硬件 (客户端设备):** + 这是终端用户直接与之交互的物理智能硬件设备。其主要职责包括： + * 捕捉用户的语音指令。 + * 将捕捉到的原始音频数据安全地发送至 `xiaozhi-server` 进行处理。 + * 接收来自 `xiaozhi-server` 合成的语音回复，并通过扬声器播放给用户。 + * 根据从 `xiaozhi-server` 收到的指令，控制与之连接的其他外围设备或IoT设备（例如智能灯泡、传感器等）。 + +2. **`xiaozhi-server` (核心AI引擎 - Python实现):** + 这个基于Python的服务器是整个系统的“大脑”，负责处理所有语音相关的逻辑和AI交互。其关键职责细化如下： + * 通过WebSocket协议与ESP32设备建立**稳定、低延迟的实时双向通信链路**。 + * 接收来自ESP32的音频流，并利用语音活动检测（VAD）技术精确切分有效的语音片段。 + * 集成并调用自动语音识别（ASR）服务（可配置本地或云端），将语音片段转换为文本。 + * 通过与大型语言模型（LLM）的交互来解析用户意图、生成智能回复，并支持复杂的自然语言理解任务。 + * 管理多轮对话中的上下文信息和用户记忆，以提供连贯的交互体验。 + * 调用文本转语音（TTS）服务，将LLM生成的文本回复合成为自然流畅的语音。 + * 通过一个灵活的**插件系统**执行自定义命令，包括对IoT设备的控制逻辑。 + * 从 `manager-api` 服务获取其详细的运行时操作配置。 + +3. **`manager-api` (管理后端 - Java实现):** + 这是一个基于Java Spring Boot框架构建的应用程序，它为整个系统的管理和配置提供了一套安全的RESTful API。它不仅是 `manager-web` 控制台的后端支撑，也是 `xiaozhi-server` 的配置数据来源。其核心功能包括： + * 为Web控制台提供用户认证（登录、权限验证）和用户账户管理功能。 + * ESP32设备的注册、信息管理以及设备特定配置的维护。 + * 在**MySQL数据库**中持久化存储系统配置，例如用户选择的AI服务提供商、API密钥、设备参数、插件设置等。 + * 提供特定的API端点，供 `xiaozhi-server` 拉取其所需的最新配置。 + * 管理TTS音色选项、处理OTA（Over-The-Air）固件更新流程及相关元数据。 + * 利用 **Redis** 作为高速缓存，存储热点数据（如会话信息、频繁访问的配置），以提升API响应速度和系统整体性能。 + +4. **`manager-web` (Web控制面板 - Vue.js实现):** + 这是一个基于Vue.js构建的单页应用（SPA），为系统管理员提供了一个图形化、用户友好的操作界面。其主要能力包括： + * 便捷地配置 `xiaozhi-server` 所使用的各项AI服务（如ASR、LLM、TTS的提供商切换、参数调整）。 + * 管理平台用户账户、角色分配及权限控制。 + * 管理已注册的ESP32设备及其相关设置。 + * （潜在功能）监控系统运行状态、查看日志、进行故障排查等。 + * 与 `manager-api` 提供的所有后端管理功能进行全面的交互。 + +**高层交互流程概述:** + +* **语音交互主线:** **ESP32设备**捕捉到用户语音后，通过**WebSocket**将音频数据实时传输给**`xiaozhi-server`**。`xiaozhi-server`完成一系列AI处理（VAD、ASR、LLM交互、TTS）后，再通过WebSocket将合成的语音回复发送回ESP32设备进行播放。所有与语音直接相关的实时交互均在此链路完成。 +* **管理配置主线:** 管理员通过浏览器访问**`manager-web`**控制台。`manager-web`通过调用**`manager-api`**提供的**RESTful HTTP接口**来执行各种管理操作（如修改配置、管理用户或设备）。数据以JSON格式在两者间传递。 +* **配置同步:** **`xiaozhi-server`**在启动或特定更新机制触发时，会主动通过HTTP请求从**`manager-api`**拉取其最新的操作配置。这确保了管理员在Web界面上所做的配置更改能够及时有效地应用到核心AI引擎的运行中。 + +这种**前后端分离、核心服务与管理服务分离**的架构设计，使得 `xiaozhi-server`能够专注于高效的实时AI处理任务，而 `manager-api` 和 `manager-web` 则共同提供了一个功能强大且易于使用的管理和配置平台。各组件职责清晰，有利于独立开发、测试、部署和扩展。 ``` xiaozhi-esp32-server @@ -10,10 +73,397 @@ xiaozhi-esp32-server ├─ manager-api 8002 端口 Java语言开发 负责提供控制台的api ``` -# xiaozhi-server 和ESP32通讯协议 +--- -https://ccnphfhqs21z.feishu.cn/wiki/M0XiwldO9iJwHikpXD5cEx71nKh +## 3. 核心组件深度剖析 -# manager-web 、manager-api接口协议 +### 3.1. `xiaozhi-server` (核心AI引擎 - Python实现) -https://2662r3426b.vicp.fun/xiaozhi/doc.html +`xiaozhi-server` 作为系统的智能核心，全权负责处理语音交互、对接各类AI服务以及管理与ESP32设备间的通信。其设计目标是实现高效、灵活且可扩展的语音AI处理能力。 + +* **核心目标:** + * 为ESP32设备提供实时的语音指令处理服务。 + * 深度集成各类AI服务，包括：自动语音识别 (ASR)、大型语言模型 (LLM) 进行自然语言理解 (NLU)、文本转语音 (TTS)、语音活动检测 (VAD)、意图识别 (Intent Recognition) 及对话记忆 (Memory)。 + * 精细管理用户与设备间的对话流程及上下文状态。 + * 基于用户指令，通过插件化机制执行自定义函数及控制物联网 (IoT) 设备。 + * 支持通过 `manager-api`进行动态配置加载与更新。 + +* **核心技术栈:** + * **Python 3:** 作为主要编程语言，Python以其丰富的AI/ML生态库和快速开发特性被选用。 + * **Asyncio:** Python的异步编程框架，是`xiaozhi-server`高性能的关键。它被广泛用于高效处理来自大量ESP32设备的并发WebSocket连接，以及执行与外部AI服务API通信时的非阻塞I/O操作，确保服务器在高并发下的响应能力。 + * **`websockets` 库:** 提供WebSocket服务器的具体实现，支持与ESP32客户端进行全双工实时通信。 + * **HTTP客户端 (如 `aiohttp`, `httpx`):** 用于异步执行HTTP请求，主要目的是从`manager-api`获取配置信息，以及与云端AI服务的API进行交互。 + * **YAML (通常通过 PyYAML 库):** 用于解析本地的 `config.yaml` 配置文件。 + * **FFmpeg (外部依赖):** 在 `app.py` 启动时会进行检查 (`check_ffmpeg_installed()`)。FFmpeg通常用于音频处理和格式转换，例如，确保音频数据符合特定AI服务的要求或进行内部处理。 + +* **关键实现细节:** + + 1. **AI服务提供者模式 (Provider Pattern - `core/providers/`):** + * **设计思想:** 这是`xiaozhi-server`集成不同AI服务的核心设计模式，极大地增强了系统的灵活性和可扩展性。针对每一种AI服务类型（ASR, TTS, LLM, VAD, Intent, Memory, VLLM），都在其对应子目录下定义了一个抽象基类 (ABC, Abstract Base Class)，例如 `core/providers/asr/base.py`。这个基类规定了该类型服务必须实现的通用接口方法（如ASR的 `async def transcribe(self, audio_chunk: bytes) -> str: pass`）。 + * **具体实现:** 各种具体的AI服务提供商或本地模型的实现，则以独立的Python类形式存在（例如 `core/providers/asr/fun_local.py` 实现了本地FunASR的逻辑，`core/providers/llm/openai.py` 实现了与OpenAI GPT模型的对接）。这些具体类继承自相应的抽象基类，并实现其定义的接口。部分提供者还使用DTOs (Data Transfer Objects, 存在于各自的 `dto/` 目录) 来结构化与外部服务交换的数据。 + * **优势:** 使得核心业务逻辑能够以统一的方式调用不同的AI服务，而无需关心其底层具体实现。用户可以通过配置文件轻松切换AI服务后端。添加对新AI服务的支持也变得相对简单，只需实现对应的Provider接口。 + * **动态加载与初始化:** `core/utils/modules_initialize.py` 脚本扮演了工厂的角色。它在服务器启动时，或在接收到配置更新指令时，会根据配置文件中 `selected_module` 及各项服务的具体provider设置，动态地导入并实例化相应的Provider类。 + + 2. **WebSocket通信与连接处理 (`app.py`, `core/websocket_server.py`, `core/connection.py`):** + * **服务器启动与入口 (`app.py`):** + * `app.py` 作为主入口，负责初始化应用环境（如检查FFmpeg、加载配置、设置日志）。 + * 它会生成或加载一个 `auth_key` (JWT密钥)，用于保护特定的HTTP接口（如视觉分析接口 `/mcp/vision/explain`）。若配置中 `manager-api.secret` 为空，则会生成一个UUID作为 `auth_key`。 + * 使用 `asyncio.create_task()` 并发启动 `WebSocketServer` (监听如 `ws://0.0.0.0:8000/xiaozhi/v1/`) 和 `SimpleHttpServer` (监听如 `http://0.0.0.0:8003/xiaozhi/ota/`)。 + * 包含一个 `monitor_stdin()` 协程，用于在某些环境下保持应用存活或处理终端输入。 + * **WebSocket服务器核心 (`core/websocket_server.py`):** + * `WebSocketServer` 类使用 `websockets` 库监听来自ESP32设备的连接请求。 + * 对于每一个成功的WebSocket连接，它都会创建一个**独立的 `ConnectionHandler` 实例** (推测定义于 `core/connection.py`)。这种每个连接一个处理程序实例的设计模式，是实现多设备状态隔离和并发处理的关键，确保每个设备的对话流程和上下文信息互不干扰。 + * 该服务器还提供一个 `_http_response` 方法，允许在同一端口上对非WebSocket升级的HTTP GET请求做出简单响应（例如返回 "Server is running"），便于进行健康检查。 + * **动态配置更新:** `WebSocketServer` 包含一个 `update_config()` 异步方法。此方法使用 `config_lock` (一个 `asyncio.Lock`) 保证配置更新的原子性。它调用 `get_config_from_api()` (可能在 `config_loader.py` 中实现，通过 `manage_api_client.py` 与 `manager-api` 通信) 来获取新的配置。通过 `check_vad_update()` 和 `check_asr_update()` 等辅助函数判断是否需要重新初始化特定的AI模块，避免不必要的开销。更新后的配置会用于重新调用 `initialize_modules()`，从而实现AI服务提供者的热切换。 + + 3. **消息处理与对话流程控制 (`core/handle/` 和 `ConnectionHandler`):** + * `ConnectionHandler` (推测) 作为每个连接的控制中心，负责接收来自ESP32的消息，并根据消息类型或当前对话状态，将其分发给 `core/handle/` 目录下的相应处理模块。这种模块化的处理器设计使得 `ConnectionHandler` 逻辑更清晰，易于扩展。 + * **主要处理模块及其职责:** + * `helloHandle.py`: 处理与ESP32初次连接时的握手协议、设备认证或初始化信息交换。 + * `receiveAudioHandle.py`: 接收音频流数据，调用VAD Provider进行语音活动检测，并将有效的音频片段传递给ASR Provider进行识别。 + * `textHandle.py` / `intentHandler.py`: 获取ASR识别出的文本后，与Intent Provider (可能利用LLM进行意图识别) 和LLM Provider交互，以理解用户意图并生成初步回复或决策。 + * `functionHandler.py`: 当LLM的响应包含执行特定“函数调用”的指令时，此模块负责从插件注册表中查找并执行对应的插件函数。 + * `sendAudioHandle.py`: 将LLM最终生成的文本回复交给TTS Provider合成语音，并将音频流通过WebSocket发送回ESP32。 + * `abortHandle.py`: 处理来自ESP32的中断请求，例如停止当前的TTS播报。 + * `iotHandle.py`, `mcpHandle.py`: 处理与IoT设备控制相关的特定指令或更复杂的模块通信协议 (MCP)。 + + 4. **插件化功能扩展系统 (`plugins_func/`):** + * **设计目的:** 提供一种标准化的方式来扩展语音助手的功能和“技能”，而无需修改核心代码。 + * **实现机制:** + * 各个具体功能以独立的Python脚本形式存在于 `plugins_func/functions/` 目录中（例如 `get_weather.py`, `hass_set_state.py` 用于Home Assistant集成）。 + * `loadplugins.py` 在服务器启动时负责扫描并加载这些插件模块。 + * `register.py` (或插件模块内部的特定装饰器/函数) 可能用于定义每个插件函数的元数据，包括： + * **函数名称 (Function Name):** LLM调用时使用的标识符。 + * **功能描述 (Description):** 供LLM理解此函数的作用。 + * **参数模式 (Parameters Schema):** 通常是一个JSON Schema，详细定义了函数所需的参数、类型、是否必需以及描述。这是LLM能够正确生成函数调用参数的关键。 + * **执行流程:** 当LLM在其思考过程中决定需要调用某个外部工具或函数来获取信息或执行操作时，它会依据预先提供的函数模式生成一个结构化的“函数调用”请求。`xiaozhi-server`中的`functionHandler.py`捕获此请求，从插件注册表中找到对应的Python函数并执行，然后将执行结果返回给LLM，LLM再基于此结果生成最终给用户的自然语言回复。 + + 5. **配置管理 (`config/`):** + * **加载机制:** `config_loader.py` (通过 `settings.py` 被调用) 负责从根目录的 `config.yaml` 文件加载基础配置。 + * **远程配置与合并:** 通过 `manage_api_client.py` (使用如`aiohttp`的库与`manager-api`通信) 可以从`manager-api`服务拉取配置。远程配置通常会覆盖本地 `config.yaml` 中的同名设置，从而实现通过Web界面动态调整服务器行为。 + * **日志系统:** `logger.py` 初始化应用日志系统（可能使用 `loguru` 或对标准 `logging` 模块进行封装，支持通过 `logger.bind(tag=TAG)` 添加标签，便于追踪和过滤）。 + * **静态资源:** `config/assets/` 目录下存放了用于系统提示音的静态音频文件（如设备绑定提示音 `bind_code.wav`、错误提示音等）。 + + 6. **辅助HTTP服务 (`core/http_server.py`):** + * 与WebSocket服务并行运行一个简单的HTTP服务器，用于处理特定的HTTP请求。最主要的功能是为ESP32设备提供OTA (Over-The-Air) 固件更新的下载服务 (通过 `/xiaozhi/ota/` 端点)。此外，也可能承载其他如 `/mcp/vision/explain` (视觉分析) 等工具性HTTP接口。 + +综上所述，`xiaozhi-server` 是一个采用现代Python异步编程模型构建的、高度模块化、配置驱动的AI应用服务器。其精心设计的Provider模式和插件架构赋予了它强大的适应性和扩展性，能够灵活接入不同的AI能力并支持日益增长的功能需求。 + +--- + +### 3.2. `manager-api` (管理后端 - Java Spring Boot实现) + +`manager-api` 组件是使用Java和Spring Boot框架构建的强大后端服务，作为整个`xiaozhi-esp32-server`生态系统的中央行政管理和配置中枢。 + +* **核心目标:** + * 为`manager-web`（Vue.js前端）提供一套安全、稳定、符合RESTful规范的API接口，使得管理员能够便捷地管理用户、设备、系统配置及其他相关资源。 + * 充当`xiaozhi-server`（Python核心AI引擎）的集中化配置数据提供者，允许`xiaozhi-server`实例在启动或运行时获取其最新的操作参数。 + * 持久化存储关键数据，例如：用户账户信息、设备注册详情、AI服务提供商配置（包括API密钥、选定的服务模型等）、TTS音色参数，以及OTA固件版本信息等。 + +* **核心技术栈:** + * **Java 21:** 项目采用的JDK版本，确保了对现代Java特性的支持。 + * **Spring Boot 3:** 作为核心开发框架，极大地简化了独立、生产级别的Spring应用的创建和部署。它提供了自动配置、内嵌Web服务器（默认为Tomcat）、依赖管理等关键功能。 + * **Spring MVC:** Spring框架中用于构建Web应用和RESTful API的模块。 + * **MyBatis-Plus:** 一个对MyBatis进行功能增强的ORM（对象关系映射）框架。它简化了数据库操作，提供了强大的CRUD（增删改查）功能、条件构造器、代码生成器等，并能很好地与Spring Boot集成。 + * **MySQL:** 作为主要的后端关系型数据库，用于存储所有需要持久化的管理数据和配置信息。 + * **Druid (Alibaba Druid):** 一个功能强大的JDBC连接池实现，提供了丰富的监控功能和优秀的性能，用于高效管理数据库连接。 + * **Redis (通过 Spring Data Redis):** 一个高性能的内存数据结构存储，常用于实现数据缓存（例如缓存热点配置数据、用户会话信息），以显著提升API的响应速度。 + * **Apache Shiro:** 一个成熟且易用的Java安全框架，负责处理应用的认证（用户身份验证）和授权（API访问权限控制）需求。 + * **Liquibase:** 一个用于跟踪、管理和应用数据库 schéma（模式）变更的开源工具。它允许开发者以数据库无关的方式定义和版本化数据库结构变更。 + * **Knife4j:** 一个集成了Swagger并增强了UI的API文档生成工具，专为Java MVC框架（尤其是Spring Boot）设计。它能生成美观且易于交互的API文档界面（通常通过 `/xiaozhi/doc.html` 访问）。 + * **Maven:** 用于项目的构建自动化和依赖项管理。 + * **Lombok:** 一个Java库，通过注解自动生成构造函数、getter/setter、equals/hashCode、toString等样板代码，减少冗余。 + * **HuTool / Google Guava:** 提供大量实用工具类，简化常见编程任务。 + * **Aliyun Dysmsapi:** 阿里云短信服务SDK，用于集成发送短信功能（如验证码、通知）。 + +* **关键实现细节:** + + 1. **模块化项目结构 (`modules/` 包):** + * `manager-api` 的核心业务逻辑被清晰地划分到 `src/main/java/xiaozhi/modules/` 目录下的不同模块中。这种按功能领域划分模块的方式（例如 `sys` 负责系统管理，`agent` 负责智能体配置，`device` 负责设备管理，`config` 负责为`xiaozhi-server`提供配置，`security` 负责安全，`timbre` 负责音色管理，`ota` 负责固件升级）极大地提高了代码的可维护性和可扩展性。 + * **各模块内部结构:** 每个业务模块通常遵循经典的三层架构或其变体： + * **Controller (控制层):** 位于 `xiaozhi.modules.[模块名].controller`。 + * **Service (服务层):** 位于 `xiaozhi.modules.[模块名].service`。 + * **DAO/Mapper (数据访问层):** 位于 `xiaozhi.modules.[模块名].dao`。 + * **Entity (实体类):** 位于 `xiaozhi.modules.[模块名].entity`。 + * **DTO (数据传输对象):** 位于 `xiaozhi.modules.[模块名].dto`。 + + 2. **分层架构实现:** + * **Controller层 (`@RestController`):** 这些类使用Spring MVC注解（如 `@GetMapping`, `@PostMapping` 等）来定义API的端点(endpoints)。它们负责接收HTTP请求，将请求体中的JSON数据反序列化为DTO对象，调用相应的Service层方法处理业务逻辑，最后将Service层的返回结果序列化为JSON并作为HTTP响应返回给客户端。 + * **Service层 (`@Service`):** 这些类（通常是接口及其实现类的组合）封装了核心的业务规则和操作流程。它们可能会调用一个或多个DAO/Mapper对象来与数据库交互，并常常使用 `@Transactional` 注解来管理数据库事务的原子性。 + * **Data Access (DAO/Mapper) 层 (MyBatis-Plus Mappers):** 这些是Java接口，继承自MyBatis-Plus提供的 `BaseMapper` 接口。MyBatis-Plus会为这些接口自动提供标准的CRUD方法。对于更复杂的数据库查询，开发者可以通过在Mapper接口中定义方法并使用注解（如 `@Select`, `@Update`）或编写对应的XML映射文件来实现。例如，`UserMapper.selectById(userId)` 会被MyBatis-Plus自动实现。 + * **Entity层 (`@TableName`, `@TableId` 等MyBatis-Plus注解):** 这些POJO（Plain Old Java Objects）类直接映射到数据库中的表结构。Lombok的 `@Data` 注解常用于自动生成getter/setter等。 + * **DTO层:** 用于在各层之间，特别是Controller层与Service层之间，以及API的请求/响应体中传递数据。使用DTO有助于解耦API接口的数据结构与数据库实体的数据结构，使API更稳定。 + + 3. **通用功能与配置 (`common/` 包):** + * `src/main/java/xiaozhi/common/` 包提供了一系列跨模块共享的通用组件和配置： + * **基类:** 如 `BaseDao`, `BaseEntity`, `BaseService`, `CrudService`，为各模块的相应组件提供通用的属性或方法。 + * **全局配置:** 包括 `MybatisPlusConfig` (MyBatis-Plus的配置，如分页插件、数据权限插件等)、`RedisConfig` (Redis连接及序列化配置)、`SwaggerConfig` (Knife4j的配置)、`AsyncConfig` (异步任务执行器配置)。 + * **自定义注解:** 例如 `@LogOperation` 用于通过AOP记录操作日志，`@DataFilter` 可能用于实现数据范围过滤。 + * **AOP切面:** 如 `RedisAspect` 可能用于实现方法级别的缓存逻辑。 + * **全局异常处理:** `RenExceptionHandler` (使用 `@ControllerAdvice` 注解) 捕获应用中抛出的特定或所有异常 (如自定义的 `RenException`)，并返回统一格式的JSON错误响应给客户端。`ErrorCode` 定义了标准化的错误码。 + * **工具类:** 提供了日期转换、JSON处理(Jackson)、IP地址获取、HTTP上下文操作、统一结果封装 (`Result` 类)等多种实用工具。 + * **校验工具:** `ValidatorUtils` 和 `AssertUtils` 用于简化参数校验逻辑。 + * **XSS防护:** `XssFilter` 等组件用于防止跨站脚本攻击。 + * **MyBatis-Plus自动填充:** `FieldMetaObjectHandler` 用于在执行插入或更新数据库操作时，自动填充如 `createTime`, `updateTime` 等公共字段。 + + 4. **安全机制 (Apache Shiro):** + * Shiro的配置（通常在 `modules/security/config/` 或 `common/config/` 下）定义了如何进行用户认证和授权。 + * **Realms (域):** 自定义的Shiro Realm类负责从数据库中查询用户信息（用户名、密码、盐值）进行身份验证，以及获取用户的角色和权限信息用于授权决策。 + * **Filters (过滤器):** Shiro过滤器链被应用于保护API端点，确保只有经过认证且拥有足够权限的用户才能访问特定资源。 + * **Session/Token Management:** Shiro管理用户会话。对于RESTful API，可能结合OAuth2或JWT等令牌机制实现无状态认证。 + + 5. **数据库版本控制 (Liquibase):** + * 数据库的表结构、索引、初始数据等变更，都通过Liquibase的 `changelog` 文件（通常是XML格式）进行定义和版本化管理。当应用启动时，Liquibase会自动检查并应用必要的数据库结构更新，确保开发、测试和生产环境数据库结构的一致性。 + + 6. **API文档:** + * 完整的API接口文档可通过以下地址访问: https://2662r3426b.vicp.fun/xiaozhi/doc.html + * 该文档使用Knife4j生成,提供了所有RESTful API端点的详细说明、请求/响应示例以及在线测试功能。 + +`manager-api` 通过这些精心选择的技术和设计模式，构建了一个功能全面、结构清晰、安全可靠且易于维护和扩展的Java后端服务。其模块化的设计特别适合处理具有多种管理功能需求的复杂系统。 + +--- + +### 3.3. `manager-web` (Web管理前端 - Vue.js实现) + +`manager-web` 组件是一个采用 Vue.js 2 框架构建的单页应用 (SPA - Single Page Application)。它为系统管理员提供了一个功能丰富、交互友好的图形用户界面，用于全面管理和配置 `xiaozhi-esp32-server` 生态系统。 + +* **核心目标:** + * 提供一个基于Web的集中式控制面板，供管理员进行系统操作与监控。 + * 实现对 `xiaozhi-server` 中AI服务提供商（ASR、LLM、TTS等）及其相关API密钥或许可配置的便捷管理。 + * 支持用户账户、角色及权限的精细化管理。 + * 提供ESP32设备的注册、配置及状态查看功能。 + * 允许管理员自定义TTS音色、管理OTA固件更新流程、调整系统级参数及字典数据等。 + * 作为 `manager-api` 所暴露各项功能的图形化交互前端。 + +* **核心技术栈:** + * **Vue.js 2:** 一个渐进式的JavaScript框架，用于构建用户界面。其核心特性包括声明式渲染、组件化系统、数据绑定等，非常适合构建复杂的SPA。 + * **Vue CLI (`@vue/cli-service`):** Vue.js的官方命令行工具，用于项目的快速搭建、开发服务器的运行（支持热模块替换HMR）、以及生产环境构建打包（内部集成并配置了Webpack）。 + * **Vue Router (`vue-router`):** Vue.js官方的路由管理器。它负责在SPA内部实现不同“页面”或视图组件之间的导航切换，而无需重新加载整个HTML页面，提供了流畅的用户体验。 + * **Vuex (`vuex`):** Vue.js官方的状态管理模式和库。它充当了应用中所有组件的“中央数据存储”，用于管理全局共享状态（例如当前登录用户信息、设备列表、应用配置等），特别适用于大型复杂应用。 + * **Element UI (`element-ui`):** 一个广受欢迎的基于Vue 2.0的桌面端UI组件库。它提供了大量预先设计和实现的组件（如表单、表格、对话框、导航菜单、按钮、提示等），帮助开发者快速构建出专业且一致的用户界面。 + * **JavaScript (ES6+):** 前端逻辑实现的主要编程语言，利用其现代特性进行开发。 + * **SCSS (Sassy CSS):** 一种CSS预处理器，它为CSS增加了变量、嵌套规则、混合(Mixin)、继承等高级特性，使得CSS代码更易于组织、维护和复用。 + * **HTTP客户端 (Flyio 或 Axios 通过 `vue-axios`):** 用于在浏览器端向 `manager-api` 后端发起异步HTTP（AJAX）请求，以获取数据或提交操作。 + * **Webpack:** 一个强大的模块打包工具（由Vue CLI在底层管理和配置）。它将项目中的各种资源（JavaScript文件、CSS、图片、字体等）视为模块，并将它们打包成浏览器可识别的静态文件。 + * **Workbox (通过 `workbox-webpack-plugin`):** Google开发的一个库，用于简化Service Worker的编写和PWA（Progressive Web App - 渐进式Web应用）的实现。它可以帮助生成Service Worker脚本，实现资源缓存、离线访问等功能。 + * **Opus库 (`opus-decoder`, `opus-recorder`):** 这些音频处理库表明前端可能具备一些直接在浏览器中处理Opus格式音频的能力，例如：用于测试麦克风输入、允许管理员录制自定义音频片段（可能用于TTS音色样本或语音指令测试），或播放在管理界面中预览的Opus编码音频。 + +* **关键实现细节:** + + 1. **单页应用 (SPA) 结构:** + * 整个前端应用加载一个主HTML文件 (`public/index.html`)。后续的所有页面切换和内容更新都在客户端由Vue Router动态完成，无需每次都从服务器请求新的HTML页面。这种模式能提供更快的页面加载速度和更流畅的交互体验。 + + 2. **组件化架构 (Component-Based Architecture):** + * 用户界面由一系列可复用的Vue组件 (`.vue` 单文件组件) 构成，形成一个组件树。这种方式提高了代码的模块化程度、可维护性和复用性。 + * **`src/main.js`:** 应用的入口JS文件。它负责创建和初始化根Vue实例，注册全局插件（如Vue Router, Vuex, Element UI），并把根Vue实例挂载到 `public/index.html` 中的某个DOM元素上（通常是 `#app`）。 + * **`src/App.vue`:** 应用的根组件。它通常定义了应用的基础布局结构（如包含导航栏、侧边栏、主内容区），并通过 `` 标签来显示当前路由匹配到的视图组件。 + * **视图组件 (`src/views/`):** 这些组件代表了应用中的各个“页面”或主要功能区（例如 `Login.vue` 登录页, `DeviceManagement.vue` 设备管理页, `UserManagement.vue` 用户管理页, `ModelConfig.vue` 模型配置页）。它们通常由Vue Router直接映射。 + * **可复用UI组件 (`src/components/`):** 包含了在不同视图之间共享的、更小粒度的UI组件（例如 `HeaderBar.vue` 顶部导航栏, `AddDeviceDialog.vue` 添加设备对话框, `AudioPlayer.vue` 音频播放器组件）。 + + 3. **客户端路由 (`src/router/index.js`):** + * Vue Router在此文件中进行配置，定义了应用的路由表。每个路由规则将一个特定的URL路径映射到一个视图组件。 + * 常常包含**导航守卫 (Navigation Guards)**，例如 `beforeEach` 守卫，用于在路由跳转前执行逻辑，如检查用户是否已登录，如果未登录则重定向到登录页面，从而保护需要认证才能访问的页面。 + + 4. **状态管理 (`src/store/index.js`):** + * Vuex被用来构建一个集中的状态管理中心（Store）。这个Store包含了： + * **State:** 存储应用级别的共享数据（例如，当前登录用户的详细信息、从API获取的设备列表、系统配置等）。 + * **Getters:** 类似于Vue组件中的计算属性，用于从State派生出一些状态值，方便组件使用。 + * **Mutations:** **唯一**可以同步修改State中数据的方法。它们必须是同步函数。 + * **Actions:** 用于处理异步操作（如API调用）或封装多个Mutation提交。Actions会调用API，获取数据后，通过 `commit` 一个或多个Mutation来更新State。 + * 例如，用户登录时，一个名为 `login` 的Action可能会被调用，它会向后端API发送登录请求，成功后获取到用户信息和token，然后 `commit` 一个名为 `SET_USER_INFO` 的Mutation来更新State中的用户信息和token。 + + 5. **API通信 (`src/apis/`):** + * 与 `manager-api` 后端的所有HTTP通信逻辑被封装在 `src/apis/` 目录下，通常会按照后端API的模块进行组织（例如 `src/apis/module/agent.js`, `src/apis/module/device.js`）。 + * 每个模块导出一系列函数，每个函数对应一个具体的API请求。这些函数内部使用配置好的HTTP客户端实例 (例如，在 `src/apis/api.js` 或 `src/apis/httpRequest.js` 中统一配置Axios或Flyio实例，可能包含设置请求基地址、请求/响应拦截器等)。 + * **拦截器 (Interceptors):** HTTP客户端的请求拦截器常用于在每个请求发送前自动添加认证令牌（如JWT）；响应拦截器则可用于全局处理API错误（如权限不足、服务器错误）或对响应数据进行预处理。 + + 6. **样式与资源 (`src/styles/`, `src/assets/`):** + * `Element UI` 提供了基础的组件样式。 + * `src/styles/global.scss` 文件用于定义全局共享的SCSS样式、变量、混合(Mixin)等。 + * Vue单文件组件内部的 `