From 4cc7247c3733c9c88f343c7432731e9b296e6a88 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Wed, 4 Jun 2025 14:35:21 +0800 Subject: [PATCH 1/9] =?UTF-8?q?update:funasr=E5=AE=9E=E4=BE=8B=E4=B9=8B?= =?UTF-8?q?=E5=89=8D=E6=B7=BB=E5=8A=A0=E5=86=85=E5=AD=98=E6=A3=80=E6=B5=8B?= =?UTF-8?q?=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/providers/asr/fun_local.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 8c7305c6..4b25a265 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -2,6 +2,7 @@ import time import os import sys import io +import psutil from config.logger import setup_logging from typing import Optional, Tuple, List from core.providers.asr.base import ASRProviderBase @@ -37,6 +38,13 @@ class CaptureOutput: class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): super().__init__() + + # 内存检测，要求大于2G + min_mem_bytes = 2 * 1024 * 1024 * 1024 + total_mem = psutil.virtual_memory().total + if total_mem < min_mem_bytes: + logger.bind(tag=TAG).error(f"可用内存不足2G，当前仅有 {total_mem / (1024*1024):.2f} MB") + self.interface_type = InterfaceType.LOCAL self.model_dir = config.get("model_dir") self.output_dir = config.get("output_dir") # 修正配置键名 From 0268f90e9c69ae0ac9556cdcbebf59946ce76938 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Wed, 4 Jun 2025 14:46:52 +0800 Subject: [PATCH 2/9] =?UTF-8?q?update=EF=BC=9A=E4=BF=AE=E6=94=B9=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E6=A8=A1=E5=9D=97=EF=BC=8C=E4=BD=BF=E5=85=B6=E6=8C=89?= =?UTF-8?q?=E6=97=A5=E6=9C=9F=E5=92=8C=E5=A4=A7=E5=B0=8F=E5=88=86=E5=89=B2?= =?UTF-8?q?=E7=9A=84=E6=97=A5=E5=BF=97=E6=96=87=E4=BB=B6=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config/logger.py | 91 +++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 16 deletions(-) diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index 5f107f5d..e5bf8466 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -3,15 +3,16 @@ import sys from loguru import logger from config.config_loader import load_config from config.settings import check_config_file +from datetime import datetime SERVER_VERSION = "0.5.2" _logger_initialized = False - +_current_log_file = None +_current_log_size = 0 +_max_log_size = 10 * 1024 * 1024 # 10MB def get_module_abbreviation(module_name, module_dict): - """获取模块名称的缩写，如果为空则返回00 - 如果名称中包含下划线，则返回下划线后面的前两个字符 - """ + """获取模块名称的缩写，如果为空则返回00""" module_value = module_dict.get(module_name, "") if not module_value: return "00" @@ -20,7 +21,6 @@ def get_module_abbreviation(module_name, module_dict): return parts[-1][:2] if parts[-1] else "00" return module_value[:2] - def build_module_string(selected_module): """构建模块字符串""" return ( @@ -32,12 +32,59 @@ def build_module_string(selected_module): + get_module_abbreviation("Intent", selected_module) ) - def formatter(record): """为没有 tag 的日志添加默认值""" record["extra"].setdefault("tag", record["name"]) return record["message"] +def get_log_filename(log_dir, base_name): + """生成按日期和大小分割的日志文件名""" + global _current_log_file, _current_log_size + + now = datetime.now() + date_part = f"{now.month}.{now.day}" + + # 检查是否需要创建新文件 + if _current_log_file is None or not os.path.exists(_current_log_file): + file_index = 1 + while True: + new_filename = os.path.join(log_dir, f"{base_name}.{date_part}.{file_index}") + if not os.path.exists(new_filename): + _current_log_file = new_filename + _current_log_size = 0 + return new_filename + # 如果文件已存在，增加索引 + file_index += 1 + else: + # 检查文件大小 + if _current_log_size > _max_log_size: + # 文件过大，创建新文件 + base_path = os.path.splitext(_current_log_file)[0] + parts = base_path.split('.') + if len(parts) >= 3: + try: + file_index = int(parts[-1]) + 1 + except ValueError: + file_index = 1 + else: + file_index = 1 + + new_filename = f"{base_path.rsplit('.', 1)[0]}.{file_index}" + _current_log_file = new_filename + _current_log_size = 0 + return new_filename + else: + return _current_log_file + +def update_log_size(message): + """更新当前日志文件大小""" + global _current_log_size + # 如果当前日志文件存在，获取其大小 + if _current_log_file and os.path.exists(_current_log_file): + _current_log_size = os.path.getsize(_current_log_file) + else: + # 如果当前日志文件不存在，重置大小 + _current_log_size = len(message.encode('utf-8')) def setup_logging(): check_config_file() @@ -52,7 +99,7 @@ def setup_logging(): extra={ "selected_module": log_config.get("selected_module", "00000000000000") } - ) # 新增配置 + ) log_format = log_config.get( "log_format", "{time:YYMMDD HH:mm:ss}[{version}_{extra[selected_module]}][{extra[tag]}]-{level}-{message}", @@ -72,7 +119,7 @@ def setup_logging(): log_level = log_config.get("log_level", "INFO") log_dir = log_config.get("log_dir", "tmp") - log_file = log_config.get("log_file", "server.log") + log_file = log_config.get("log_file", "server") data_dir = log_config.get("data_dir", "data") os.makedirs(log_dir, exist_ok=True) @@ -84,18 +131,23 @@ def setup_logging(): # 输出到控制台 logger.add(sys.stdout, format=log_format, level=log_level, filter=formatter) - # 输出到文件 + # 输出到文件，使用自定义的日志文件名生成器 + def sink(message): + filename = get_log_filename(log_dir, log_file) + with open(filename, "a", encoding="utf-8") as f: + f.write(message + "\n") + update_log_size(message) + logger.add( - os.path.join(log_dir, log_file), + sink, format=log_format_file, level=log_level, filter=formatter, ) - _logger_initialized = True # 标记为已初始化 + _logger_initialized = True return logger - def update_module_string(selected_module_str): """更新模块字符串并重新配置日志处理器""" logger.debug(f"更新日志配置组件") @@ -126,6 +178,9 @@ def update_module_string(selected_module_str): "{selected_module}", selected_module_str ) + log_dir = log_config.get("log_dir", "tmp") + log_file = log_config.get("log_file", "server") + logger.remove() logger.add( sys.stdout, @@ -133,11 +188,15 @@ def update_module_string(selected_module_str): level=log_config.get("log_level", "INFO"), filter=formatter, ) + + def sink(message): + filename = get_log_filename(log_dir, log_file) + with open(filename, "a", encoding="utf-8") as f: + f.write(message + "\n") + update_log_size(message) + logger.add( - os.path.join( - log_config.get("log_dir", "tmp"), - log_config.get("log_file", "server.log"), - ), + sink, format=log_format_file, level=log_config.get("log_level", "INFO"), filter=formatter, From fba758ea36024629236e80359fc9d8f8dbc9f68f Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Wed, 4 Jun 2025 15:42:52 +0800 Subject: [PATCH 3/9] =?UTF-8?q?update:=E6=81=A2=E5=A4=8D=E6=9C=80=E5=88=9D?= =?UTF-8?q?=E7=9A=84=E6=97=A5=E5=BF=97=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config/logger.py | 93 +++++----------------------- 1 file changed, 17 insertions(+), 76 deletions(-) diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index e5bf8466..643952dd 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -3,16 +3,15 @@ import sys from loguru import logger from config.config_loader import load_config from config.settings import check_config_file -from datetime import datetime -SERVER_VERSION = "0.5.2" +SERVER_VERSION = "0.5.4" _logger_initialized = False -_current_log_file = None -_current_log_size = 0 -_max_log_size = 10 * 1024 * 1024 # 10MB + def get_module_abbreviation(module_name, module_dict): - """获取模块名称的缩写，如果为空则返回00""" + """获取模块名称的缩写，如果为空则返回00 + 如果名称中包含下划线，则返回下划线后面的前两个字符 + """ module_value = module_dict.get(module_name, "") if not module_value: return "00" @@ -21,6 +20,7 @@ def get_module_abbreviation(module_name, module_dict): return parts[-1][:2] if parts[-1] else "00" return module_value[:2] + def build_module_string(selected_module): """构建模块字符串""" return ( @@ -32,59 +32,12 @@ def build_module_string(selected_module): + get_module_abbreviation("Intent", selected_module) ) + def formatter(record): """为没有 tag 的日志添加默认值""" record["extra"].setdefault("tag", record["name"]) return record["message"] -def get_log_filename(log_dir, base_name): - """生成按日期和大小分割的日志文件名""" - global _current_log_file, _current_log_size - - now = datetime.now() - date_part = f"{now.month}.{now.day}" - - # 检查是否需要创建新文件 - if _current_log_file is None or not os.path.exists(_current_log_file): - file_index = 1 - while True: - new_filename = os.path.join(log_dir, f"{base_name}.{date_part}.{file_index}") - if not os.path.exists(new_filename): - _current_log_file = new_filename - _current_log_size = 0 - return new_filename - # 如果文件已存在，增加索引 - file_index += 1 - else: - # 检查文件大小 - if _current_log_size > _max_log_size: - # 文件过大，创建新文件 - base_path = os.path.splitext(_current_log_file)[0] - parts = base_path.split('.') - if len(parts) >= 3: - try: - file_index = int(parts[-1]) + 1 - except ValueError: - file_index = 1 - else: - file_index = 1 - - new_filename = f"{base_path.rsplit('.', 1)[0]}.{file_index}" - _current_log_file = new_filename - _current_log_size = 0 - return new_filename - else: - return _current_log_file - -def update_log_size(message): - """更新当前日志文件大小""" - global _current_log_size - # 如果当前日志文件存在，获取其大小 - if _current_log_file and os.path.exists(_current_log_file): - _current_log_size = os.path.getsize(_current_log_file) - else: - # 如果当前日志文件不存在，重置大小 - _current_log_size = len(message.encode('utf-8')) def setup_logging(): check_config_file() @@ -99,7 +52,7 @@ def setup_logging(): extra={ "selected_module": log_config.get("selected_module", "00000000000000") } - ) + ) # 新增配置 log_format = log_config.get( "log_format", "{time:YYMMDD HH:mm:ss}[{version}_{extra[selected_module]}][{extra[tag]}]-{level}-{message}", @@ -119,7 +72,7 @@ def setup_logging(): log_level = log_config.get("log_level", "INFO") log_dir = log_config.get("log_dir", "tmp") - log_file = log_config.get("log_file", "server") + log_file = log_config.get("log_file", "server.log") data_dir = log_config.get("data_dir", "data") os.makedirs(log_dir, exist_ok=True) @@ -131,23 +84,18 @@ def setup_logging(): # 输出到控制台 logger.add(sys.stdout, format=log_format, level=log_level, filter=formatter) - # 输出到文件，使用自定义的日志文件名生成器 - def sink(message): - filename = get_log_filename(log_dir, log_file) - with open(filename, "a", encoding="utf-8") as f: - f.write(message + "\n") - update_log_size(message) - + # 输出到文件 logger.add( - sink, + os.path.join(log_dir, log_file), format=log_format_file, level=log_level, filter=formatter, ) - _logger_initialized = True + _logger_initialized = True # 标记为已初始化 return logger + def update_module_string(selected_module_str): """更新模块字符串并重新配置日志处理器""" logger.debug(f"更新日志配置组件") @@ -178,9 +126,6 @@ def update_module_string(selected_module_str): "{selected_module}", selected_module_str ) - log_dir = log_config.get("log_dir", "tmp") - log_file = log_config.get("log_file", "server") - logger.remove() logger.add( sys.stdout, @@ -188,15 +133,11 @@ def update_module_string(selected_module_str): level=log_config.get("log_level", "INFO"), filter=formatter, ) - - def sink(message): - filename = get_log_filename(log_dir, log_file) - with open(filename, "a", encoding="utf-8") as f: - f.write(message + "\n") - update_log_size(message) - logger.add( - sink, + os.path.join( + log_config.get("log_dir", "tmp"), + log_config.get("log_file", "server.log"), + ), format=log_format_file, level=log_config.get("log_level", "INFO"), filter=formatter, From 23cb7616d920dc3b07a6ae813fe7b334ca1cd4ad Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 4 Jun 2025 16:46:49 +0800 Subject: [PATCH 4/9] =?UTF-8?q?update:=20=E6=9B=B4=E6=94=B9to=5Ftts?= =?UTF-8?q?=E4=BF=9D=E5=AD=98=E4=B8=B4=E6=97=B6=E6=96=87=E4=BB=B6=E5=88=A4?= =?UTF-8?q?=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 32 +++---- .../core/providers/tts/aliyun.py | 11 ++- .../xiaozhi-server/core/providers/tts/base.py | 95 +++++++++++++------ .../core/providers/tts/cozecn.py | 11 ++- .../core/providers/tts/custom.py | 9 +- .../core/providers/tts/doubao.py | 12 ++- .../xiaozhi-server/core/providers/tts/edge.py | 25 +++-- .../core/providers/tts/fishspeech.py | 9 +- .../core/providers/tts/gpt_sovits_v2.py | 8 +- .../core/providers/tts/gpt_sovits_v3.py | 8 +- .../core/providers/tts/minimax.py | 9 +- .../core/providers/tts/openai.py | 10 +- .../core/providers/tts/siliconflow.py | 10 +- .../core/providers/tts/tencent.py | 15 +-- .../core/providers/tts/ttson.py | 9 +- main/xiaozhi-server/core/utils/p3.py | 26 +++++ main/xiaozhi-server/core/utils/util.py | 46 +++++++++ 17 files changed, 253 insertions(+), 92 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 6b8cb4fd..859059b6 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -2,10 +2,9 @@ import os import time import json import random -import shutil import asyncio from core.handle.sendAudioHandle import send_stt_message -from core.utils.util import remove_punctuation_and_length +from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes from core.providers.tts.dto.dto import ContentType, InterfaceType from core.handle.mcpHandle import ( MCPClient, @@ -119,19 +118,18 @@ async def wakeupWordsResponse(conn): result = conn.llm.response_no_stream(conn.config["prompt"], question) if result is None or result == "": return - tts_file = await asyncio.to_thread(conn.tts.to_tts, result) - if tts_file is not None and os.path.exists(tts_file): - file_type = os.path.splitext(tts_file)[1] - if file_type: - file_type = file_type.lstrip(".") - old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"]) - if old_file is not None: - os.remove(old_file) - """将文件挪到"wakeup_words.mp3""" - shutil.move( - tts_file, - WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type, - ) - WAKEUP_CONFIG["create_time"] = time.time() - WAKEUP_CONFIG["text"] = result + opus_datas = await asyncio.to_thread(conn.tts.to_tts, result) + if not opus_datas: + return + + wav_bytes = opus_datas_to_wav_bytes(opus_datas, sample_rate=16000) + file_path = os.path.join( + WAKEUP_CONFIG["dir"], "my_" + WAKEUP_CONFIG["file_name"] + ".wav" + ) + # 写入wav数据 + with open(file_path, "wb") as f: + f.write(wav_bytes) + + WAKEUP_CONFIG["create_time"] = time.time() + WAKEUP_CONFIG["text"] = result diff --git a/main/xiaozhi-server/core/providers/tts/aliyun.py b/main/xiaozhi-server/core/providers/tts/aliyun.py index 1fda8d77..c9b3d130 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun.py @@ -91,7 +91,7 @@ class TTSProvider(TTSProviderBase): self.appkey = config.get("appkey") self.format = config.get("format", "wav") - + self.audio_file_type = config.get("format", "wav") sample_rate = config.get("sample_rate", "16000") self.sample_rate = int(sample_rate) if sample_rate else 16000 @@ -188,9 +188,12 @@ class TTSProvider(TTSProviderBase): ) # 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的 if resp.headers["Content-Type"].startswith("audio/"): - with open(output_file, "wb") as f: - f.write(resp.content) - return output_file + if output_file: + with open(output_file, "wb") as f: + f.write(resp.content) + return output_file + else: + return resp.content else: raise Exception( f"{__name__} status_code: {resp.status_code} response: {resp.content}" diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 9c094f24..b711659b 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -8,7 +8,7 @@ from datetime import datetime from core.utils import textUtils from abc import ABC, abstractmethod from config.logger import setup_logging -from core.utils.util import audio_to_data +from core.utils.util import audio_to_data, audio_bytes_to_data from core.utils.tts import MarkdownCleaner from core.utils.output_counter import add_device_output from core.handle.reportHandle import enqueue_tts_report @@ -20,7 +20,6 @@ from core.providers.tts.dto.dto import ( InterfaceType, ) - import traceback TAG = __name__ @@ -33,6 +32,7 @@ class TTSProviderBase(ABC): self.conn = None self.tts_timeout = 10 self.delete_audio_file = delete_audio_file + self.audio_file_type = "wav" self.output_file = config.get("output_dir", "tmp/") self.tts_text_queue = queue.Queue() self.tts_audio_queue = queue.Queue() @@ -76,35 +76,60 @@ class TTSProviderBase(ABC): ) def to_tts(self, text): - tmp_file = self.generate_filename() - try: - max_repeat_time = 5 - text = MarkdownCleaner.clean_markdown(text) - while not os.path.exists(tmp_file) and max_repeat_time > 0: + text = MarkdownCleaner.clean_markdown(text) + max_repeat_time = 5 + if self.delete_audio_file: + # 需要删除文件的直接转为音频数据 + while max_repeat_time > 0: try: - asyncio.run(self.text_to_speak(text, tmp_file)) + audio_bytes = asyncio.run(self.text_to_speak(text, None)) + if audio_bytes: + audio_datas, _ = audio_bytes_to_data(audio_bytes, file_type=self.audio_file_type, is_opus=True) + return audio_datas + else: + max_repeat_time -= 1 except Exception as e: logger.bind(tag=TAG).warning( f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}" ) - # 未执行成功，删除文件 - if os.path.exists(tmp_file): - os.remove(tmp_file) max_repeat_time -= 1 - if max_repeat_time > 0: logger.bind(tag=TAG).info( - f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次" + f"语音生成成功: {text}，重试{5 - max_repeat_time}次" ) else: logger.bind(tag=TAG).error( f"语音生成失败: {text}，请检查网络或服务是否正常" ) - - return tmp_file - except Exception as e: - logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}") return None + else: + tmp_file = self.generate_filename() + try: + while not os.path.exists(tmp_file) and max_repeat_time > 0: + try: + asyncio.run(self.text_to_speak(text, tmp_file)) + except Exception as e: + logger.bind(tag=TAG).warning( + f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}" + ) + # 未执行成功，删除文件 + if os.path.exists(tmp_file): + os.remove(tmp_file) + max_repeat_time -= 1 + + if max_repeat_time > 0: + logger.bind(tag=TAG).info( + f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次" + ) + else: + logger.bind(tag=TAG).error( + f"语音生成失败: {text}，请检查网络或服务是否正常" + ) + + return tmp_file + except Exception as e: + logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}") + return None @abstractmethod async def text_to_speak(self, text, output_file): @@ -192,12 +217,19 @@ class TTSProviderBase(ABC): self.tts_text_buff.append(message.content_detail) segment_text = self._get_segment_text() if segment_text: - tts_file = self.to_tts(segment_text) - if tts_file: - audio_datas = self._process_audio_file(tts_file) - self.tts_audio_queue.put( - (message.sentence_type, audio_datas, segment_text) - ) + if self.delete_audio_file: + audio_datas = self.to_tts(segment_text) + if audio_datas: + self.tts_audio_queue.put( + (message.sentence_type, audio_datas, segment_text) + ) + else: + tts_file = self.to_tts(segment_text) + if tts_file: + audio_datas = self._process_audio_file(tts_file) + self.tts_audio_queue.put( + (message.sentence_type, audio_datas, segment_text) + ) elif ContentType.FILE == message.content_type: self._process_remaining_text() tts_file = message.content_file @@ -334,11 +366,18 @@ class TTSProviderBase(ABC): if remaining_text: segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) if segment_text: - tts_file = self.to_tts(segment_text) - audio_datas = self._process_audio_file(tts_file) - self.tts_audio_queue.put( - (SentenceType.MIDDLE, audio_datas, segment_text) - ) + if self.delete_audio_file: + audio_datas = self.to_tts(segment_text) + if audio_datas: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, audio_datas, segment_text) + ) + else: + tts_file = self.to_tts(segment_text) + audio_datas = self._process_audio_file(tts_file) + self.tts_audio_queue.put( + (SentenceType.MIDDLE, audio_datas, segment_text) + ) self.processed_chars += len(full_text) return True return False diff --git a/main/xiaozhi-server/core/providers/tts/cozecn.py b/main/xiaozhi-server/core/providers/tts/cozecn.py index c3019b67..ede08928 100644 --- a/main/xiaozhi-server/core/providers/tts/cozecn.py +++ b/main/xiaozhi-server/core/providers/tts/cozecn.py @@ -11,8 +11,8 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("private_voice") else: self.voice = config.get("voice") - self.response_format = config.get("response_format") - + self.response_format = config.get("response_format", "mp3") + self.audio_file_type = config.get("response_format", "mp3") self.host = "api.coze.cn" self.api_url = f"https://{self.host}/v1/audio/speech" @@ -33,7 +33,10 @@ class TTSProvider(TTSProviderBase): "POST", self.api_url, json=request_json, headers=headers ) data = response.content - file_to_save = open(output_file, "wb") - file_to_save.write(data) + if output_file: + with open(output_file, "wb") as file_to_save: + file_to_save.write(data) + else: + return data except Exception as e: raise Exception(f"{__name__} error: {e}") diff --git a/main/xiaozhi-server/core/providers/tts/custom.py b/main/xiaozhi-server/core/providers/tts/custom.py index a5300a3e..b417825e 100644 --- a/main/xiaozhi-server/core/providers/tts/custom.py +++ b/main/xiaozhi-server/core/providers/tts/custom.py @@ -16,8 +16,8 @@ class TTSProvider(TTSProviderBase): self.method = config.get("method", "GET") self.headers = config.get("headers", {}) self.format = config.get("format", "wav") + self.audio_file_type = config.get("format", "wav") self.output_file = config.get("output_dir", "tmp/") - self.params = config.get("params") if isinstance(self.params, str): @@ -43,8 +43,11 @@ class TTSProvider(TTSProviderBase): else: resp = requests.get(self.url, params=request_params, headers=self.headers) if resp.status_code == 200: - with open(output_file, "wb") as file: - file.write(resp.content) + if output_file: + with open(output_file, "wb") as file: + file.write(resp.content) + else: + return resp.content else: error_msg = f"Custom TTS请求失败: {resp.status_code} - {resp.text}" logger.bind(tag=TAG).error(error_msg) diff --git a/main/xiaozhi-server/core/providers/tts/doubao.py b/main/xiaozhi-server/core/providers/tts/doubao.py index 78058c31..1a21ffa1 100644 --- a/main/xiaozhi-server/core/providers/tts/doubao.py +++ b/main/xiaozhi-server/core/providers/tts/doubao.py @@ -29,7 +29,7 @@ class TTSProvider(TTSProviderBase): speed_ratio = config.get("speed_ratio", "1.0") volume_ratio = config.get("volume_ratio", "1.0") pitch_ratio = config.get("pitch_ratio", "1.0") - + self.audio_file_type = config.get("format", "wav") self.speed_ratio = float(speed_ratio) if speed_ratio else 1.0 self.volume_ratio = float(volume_ratio) if volume_ratio else 1.0 self.pitch_ratio = float(pitch_ratio) if pitch_ratio else 1.0 @@ -49,7 +49,7 @@ class TTSProvider(TTSProviderBase): "user": {"uid": "1"}, "audio": { "voice_type": self.voice, - "encoding": "wav", + "encoding": self.audio_file_type, "speed_ratio": self.speed_ratio, "volume_ratio": self.volume_ratio, "pitch_ratio": self.pitch_ratio, @@ -70,8 +70,12 @@ class TTSProvider(TTSProviderBase): ) if "data" in resp.json(): data = resp.json()["data"] - file_to_save = open(output_file, "wb") - file_to_save.write(base64.b64decode(data)) + audio_bytes = base64.b64decode(data) + if output_file: + with open(output_file, "wb") as file_to_save: + file_to_save.write(audio_bytes) + else: + return audio_bytes else: raise Exception( f"{__name__} status_code: {resp.status_code} response: {resp.content}" diff --git a/main/xiaozhi-server/core/providers/tts/edge.py b/main/xiaozhi-server/core/providers/tts/edge.py index 3d9b2547..ffa1c3aa 100644 --- a/main/xiaozhi-server/core/providers/tts/edge.py +++ b/main/xiaozhi-server/core/providers/tts/edge.py @@ -12,6 +12,7 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("private_voice") else: self.voice = config.get("voice") + self.audio_file_type = config.get("format", "mp3") def generate_filename(self, extension=".mp3"): return os.path.join( @@ -22,16 +23,24 @@ class TTSProvider(TTSProviderBase): async def text_to_speak(self, text, output_file): try: communicate = edge_tts.Communicate(text, voice=self.voice) - # 确保目录存在并创建空文件 - os.makedirs(os.path.dirname(output_file), exist_ok=True) - with open(output_file, "wb") as f: - pass + if output_file: + # 确保目录存在并创建空文件 + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, "wb") as f: + pass - # 流式写入音频数据 - with open(output_file, "ab") as f: # 改为追加模式避免覆盖 + # 流式写入音频数据 + with open(output_file, "ab") as f: # 改为追加模式避免覆盖 + async for chunk in communicate.stream(): + if chunk["type"] == "audio": # 只处理音频数据块 + f.write(chunk["data"]) + else: + # 返回音频二进制数据 + audio_bytes = b"" async for chunk in communicate.stream(): - if chunk["type"] == "audio": # 只处理音频数据块 - f.write(chunk["data"]) + if chunk["type"] == "audio": + audio_bytes += chunk["data"] + return audio_bytes except Exception as e: error_msg = f"Edge TTS请求失败: {e}" raise Exception(error_msg) # 抛出异常，让调用方捕获 \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/tts/fishspeech.py b/main/xiaozhi-server/core/providers/tts/fishspeech.py index 23b20e76..3bcb1229 100644 --- a/main/xiaozhi-server/core/providers/tts/fishspeech.py +++ b/main/xiaozhi-server/core/providers/tts/fishspeech.py @@ -88,7 +88,7 @@ class TTSProvider(TTSProviderBase): self.reference_audio = parse_string_to_list(config.get("reference_audio")) self.reference_text = parse_string_to_list(config.get("reference_text")) self.format = config.get("response_format", "wav") - + self.audio_file_type = config.get("response_format", "wav") self.api_key = config.get("api_key", "YOUR_API_KEY") have_key = check_model_key("FishSpeech TTS", self.api_key) if not have_key: @@ -170,8 +170,11 @@ class TTSProvider(TTSProviderBase): if response.status_code == 200: audio_content = response.content - with open(output_file, "wb") as audio_file: - audio_file.write(audio_content) + if output_file: + with open(output_file, "wb") as audio_file: + audio_file.write(audio_content) + else: + return audio_content else: error_msg = f"Request failed with status code {response.status_code}" diff --git a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py index e76b3f22..7f43ad7d 100644 --- a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py +++ b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py @@ -65,6 +65,7 @@ class TTSProvider(TTSProviderBase): self.aux_ref_audio_paths = parse_string_to_list( config.get("aux_ref_audio_paths") ) + self.audio_file_type = config.get("format", "wav") async def text_to_speak(self, text, output_file): request_json = { @@ -91,8 +92,11 @@ class TTSProvider(TTSProviderBase): resp = requests.post(self.url, json=request_json) if resp.status_code == 200: - with open(output_file, "wb") as file: - file.write(resp.content) + if output_file: + with open(output_file, "wb") as file: + file.write(resp.content) + else: + return resp.content else: error_msg = f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}" logger.bind(tag=TAG).error(error_msg) diff --git a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py index 6b66efc3..ae41fc4e 100644 --- a/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py +++ b/main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py @@ -32,6 +32,7 @@ class TTSProvider(TTSProviderBase): self.cut_punc = config.get("cut_punc", "") self.inp_refs = parse_string_to_list(config.get("inp_refs")) self.if_sr = str(config.get("if_sr", False)).lower() in ("true", "1", "yes") + self.audio_file_type = config.get("format", "wav") async def text_to_speak(self, text, output_file): request_params = { @@ -52,8 +53,11 @@ class TTSProvider(TTSProviderBase): resp = requests.get(self.url, params=request_params) if resp.status_code == 200: - with open(output_file, "wb") as file: - file.write(resp.content) + if output_file: + with open(output_file, "wb") as file: + file.write(resp.content) + else: + return resp.content else: error_msg = f"GPT_SoVITS_V3 TTS请求失败: {resp.status_code} - {resp.text}" logger.bind(tag=TAG).error(error_msg) diff --git a/main/xiaozhi-server/core/providers/tts/minimax.py b/main/xiaozhi-server/core/providers/tts/minimax.py index dd406b64..3741d169 100644 --- a/main/xiaozhi-server/core/providers/tts/minimax.py +++ b/main/xiaozhi-server/core/providers/tts/minimax.py @@ -52,6 +52,7 @@ class TTSProvider(TTSProviderBase): "Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}", } + self.audio_file_type = defult_audio_setting.get("format", "mp3") def generate_filename(self, extension=".mp3"): return os.path.join( @@ -80,8 +81,12 @@ class TTSProvider(TTSProviderBase): # 检查返回请求数据的status_code是否为0 if resp.json()["base_resp"]["status_code"] == 0: data = resp.json()["data"]["audio"] - file_to_save = open(output_file, "wb") - file_to_save.write(bytes.fromhex(data)) + audio_bytes = bytes.fromhex(data) + if output_file: + with open(output_file, "wb") as file_to_save: + file_to_save.write(audio_bytes) + else: + return audio_bytes else: raise Exception( f"{__name__} status_code: {resp.status_code} response: {resp.content}" diff --git a/main/xiaozhi-server/core/providers/tts/openai.py b/main/xiaozhi-server/core/providers/tts/openai.py index bdbc9174..e5155513 100644 --- a/main/xiaozhi-server/core/providers/tts/openai.py +++ b/main/xiaozhi-server/core/providers/tts/openai.py @@ -17,7 +17,8 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("private_voice") else: self.voice = config.get("voice", "alloy") - self.response_format = "wav" + self.response_format = config.get("format", "wav") + self.audio_file_type = config.get("format", "wav") # 处理空字符串的情况 speed = config.get("speed", "1.0") @@ -40,8 +41,11 @@ class TTSProvider(TTSProviderBase): } response = requests.post(self.api_url, json=data, headers=headers) if response.status_code == 200: - with open(output_file, "wb") as audio_file: - audio_file.write(response.content) + if output_file: + with open(output_file, "wb") as audio_file: + audio_file.write(response.content) + else: + return response.content else: raise Exception( f"OpenAI TTS请求失败: {response.status_code} - {response.text}" diff --git a/main/xiaozhi-server/core/providers/tts/siliconflow.py b/main/xiaozhi-server/core/providers/tts/siliconflow.py index 8f7fd641..b85cebf4 100644 --- a/main/xiaozhi-server/core/providers/tts/siliconflow.py +++ b/main/xiaozhi-server/core/providers/tts/siliconflow.py @@ -11,7 +11,8 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("private_voice") else: self.voice = config.get("voice") - self.response_format = config.get("response_format") + self.response_format = config.get("response_format", "mp3") + self.audio_file_type = config.get("response_format", "mp3") self.sample_rate = config.get("sample_rate") self.speed = float(config.get("speed", 1.0)) self.gain = config.get("gain") @@ -35,7 +36,10 @@ class TTSProvider(TTSProviderBase): "POST", self.api_url, json=request_json, headers=headers ) data = response.content - file_to_save = open(output_file, "wb") - file_to_save.write(data) + if output_file: + with open(output_file, "wb") as file_to_save: + file_to_save.write(data) + else: + return data except Exception as e: raise Exception(f"{__name__} error: {e}") diff --git a/main/xiaozhi-server/core/providers/tts/tencent.py b/main/xiaozhi-server/core/providers/tts/tencent.py index 7a55328a..a2493632 100644 --- a/main/xiaozhi-server/core/providers/tts/tencent.py +++ b/main/xiaozhi-server/core/providers/tts/tencent.py @@ -22,6 +22,7 @@ class TTSProvider(TTSProviderBase): self.api_url = "https://tts.tencentcloudapi.com" # 正确的API端点 self.region = config.get("region") self.output_file = config.get("output_dir") + self.audio_file_type = config.get("format", "wav") def _get_auth_headers(self, request_body): """生成鉴权请求头""" @@ -148,12 +149,14 @@ class TTSProvider(TTSProviderBase): f"API返回错误: {error_info['Code']}: {error_info['Message']}" ) - # 提取音频数据 - audio_data = response_data["Response"].get("Audio") - if audio_data: - # 解码Base64音频数据并保存 - with open(output_file, "wb") as f: - f.write(base64.b64decode(audio_data)) + # 解码Base64音频数据 + audio_bytes = base64.b64decode(response_data["Response"].get("Audio")) + if audio_bytes: + if output_file: + with open(output_file, "wb") as f: + f.write(audio_bytes) + else: + return audio_bytes else: raise Exception(f"{__name__}: 没有返回音频数据: {response_data}") else: diff --git a/main/xiaozhi-server/core/providers/tts/ttson.py b/main/xiaozhi-server/core/providers/tts/ttson.py index b3c7797f..c11206a6 100644 --- a/main/xiaozhi-server/core/providers/tts/ttson.py +++ b/main/xiaozhi-server/core/providers/tts/ttson.py @@ -30,6 +30,7 @@ class TTSProvider(TTSProviderBase): self.output_file = config.get("output_dir") self.pitch_factor = int(config.get("pitch_factor", 0)) self.format = config.get("format", "mp3") + self.audio_file_type = config.get("format", "mp3") self.emotion = int(config.get("emotion", 1)) self.header = {"Content-Type": "application/json"} @@ -73,9 +74,11 @@ class TTSProvider(TTSProviderBase): ) audio_content = requests.get(result) - with open(output_file, "wb") as f: - f.write(audio_content.content) - return True + if output_file: + with open(output_file, "wb") as f: + f.write(audio_content.content) + else: + return audio_content.content voice_path = resp_json.get("voice_path") des_path = output_file shutil.move(voice_path, des_path) diff --git a/main/xiaozhi-server/core/utils/p3.py b/main/xiaozhi-server/core/utils/p3.py index a022a2d2..c75b968e 100644 --- a/main/xiaozhi-server/core/utils/p3.py +++ b/main/xiaozhi-server/core/utils/p3.py @@ -29,5 +29,31 @@ def decode_opus_from_file(input_file): total_frames += 1 # 计算总时长 + total_duration = (total_frames * frame_duration_ms) / 1000.0 + return opus_datas, total_duration + +def decode_opus_from_bytes(input_bytes): + """ + 从p3二进制数据中解码 Opus 数据，并返回一个 Opus 数据包的列表以及总时长。 + """ + import io + opus_datas = [] + total_frames = 0 + sample_rate = 16000 # 文件采样率 + frame_duration_ms = 60 # 帧时长 + frame_size = int(sample_rate * frame_duration_ms / 1000) + + f = io.BytesIO(input_bytes) + while True: + header = f.read(4) + if not header: + break + _, _, data_len = struct.unpack('>BBH', header) + opus_data = f.read(data_len) + if len(opus_data) != data_len: + raise ValueError(f"Data length({len(opus_data)}) mismatch({data_len}) in the bytes.") + opus_datas.append(opus_data) + total_frames += 1 + total_duration = (total_frames * frame_duration_ms) / 1000.0 return opus_datas, total_duration \ No newline at end of file diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 82b5d800..b492529c 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -3,6 +3,9 @@ import socket import subprocess import re import os +import wave +from io import BytesIO +from core.utils import p3 import numpy as np import requests import opuslib_next @@ -773,6 +776,22 @@ def audio_to_data(audio_file_path, is_opus=True): return pcm_to_data(raw_data, is_opus), duration +def audio_bytes_to_data(audio_bytes, file_type, is_opus=True): + """ + 直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3 + """ + if file_type == "p3": + # 直接用p3解码 + return p3.decode_opus_from_bytes(audio_bytes) + else: + # 其他格式用pydub + audio = AudioSegment.from_file(BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"]) + audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) + duration = len(audio) / 1000.0 + raw_data = audio.raw_data + return pcm_to_data(raw_data, is_opus), duration + + def pcm_to_data(raw_data, is_opus=True): # 初始化Opus编码器 encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO) @@ -804,6 +823,33 @@ def pcm_to_data(raw_data, is_opus=True): return datas +def opus_datas_to_wav_bytes(opus_datas, sample_rate=16000, channels=1): + """ + 将opus帧列表解码为wav字节流 + """ + decoder = opuslib_next.Decoder(sample_rate, channels) + pcm_datas = [] + + frame_duration = 60 # ms + frame_size = int(sample_rate * frame_duration / 1000) # 960 + + for opus_frame in opus_datas: + # 解码为PCM（返回bytes，2字节/采样点） + pcm = decoder.decode(opus_frame, frame_size) + pcm_datas.append(pcm) + + pcm_bytes = b''.join(pcm_datas) + + # 写入wav字节流 + wav_buffer = BytesIO() + with wave.open(wav_buffer, 'wb') as wf: + wf.setnchannels(channels) + wf.setsampwidth(2) # 16bit + wf.setframerate(sample_rate) + wf.writeframes(pcm_bytes) + return wav_buffer.getvalue() + + def check_vad_update(before_config, new_config): if ( new_config.get("selected_module") is None From 8df9846ad7b89c4d91528599d3bbe5b9c4922e84 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Thu, 5 Jun 2025 09:46:54 +0800 Subject: [PATCH 5/9] =?UTF-8?q?update:=E6=94=B9=E4=B8=BA=E9=87=87=E7=94=A8?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=BD=AE=E8=BD=AC=E6=9C=BA=E5=88=B6=EF=BC=8C?= =?UTF-8?q?=E5=90=8C=E6=97=B6=E6=B7=BB=E5=8A=A0=E8=87=AA=E5=8A=A8=E6=B8=85?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config/logger.py | 40 ++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index 643952dd..7bd22c43 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -3,6 +3,7 @@ import sys from loguru import logger from config.config_loader import load_config from config.settings import check_config_file +from datetime import datetime SERVER_VERSION = "0.5.4" _logger_initialized = False @@ -59,7 +60,7 @@ def setup_logging(): ) log_format_file = log_config.get( "log_format_file", - "{time:YYYY-MM-DD HH:mm:ss} - {version_{extra[selected_module]}} - {name} - {level} - {extra[tag]} - {message}", + "{time:YYYY-MM-DD HH:mm:ss} - {version}_{extra[selected_module]} - {name} - {level} - {extra[tag]} - {message}", ) selected_module_str = logger._core.extra["selected_module"] @@ -84,12 +85,23 @@ def setup_logging(): # 输出到控制台 logger.add(sys.stdout, format=log_format, level=log_level, filter=formatter) - # 输出到文件 + # 输出到文件 - 统一目录，按大小轮转 + # 日志文件完整路径 + log_file_path = os.path.join(log_dir, log_file) + + # 添加日志处理器 logger.add( - os.path.join(log_dir, log_file), + log_file_path, format=log_format_file, level=log_level, filter=formatter, + rotation="10 MB", # 每个文件最大10MB + retention="30 days", # 保留30天 + compression=None, + encoding="utf-8", + enqueue=True, # 异步安全 + backtrace=True, + diagnose=True, ) _logger_initialized = True # 标记为已初始化 @@ -116,7 +128,7 @@ def update_module_string(selected_module_str): ) log_format_file = log_config.get( "log_format_file", - "{time:YYYY-MM-DD HH:mm:ss} - {version_{extra[selected_module]}} - {name} - {level} - {extra[tag]} - {message}", + "{time:YYYY-MM-DD HH:mm:ss} - {version}_{extra[selected_module]} - {name} - {level} - {extra[tag]} - {message}", ) log_format = log_format.replace("{version}", SERVER_VERSION) @@ -133,14 +145,26 @@ def update_module_string(selected_module_str): level=log_config.get("log_level", "INFO"), filter=formatter, ) + + # 更新文件日志配置 - 统一目录，按大小轮转 + log_dir = log_config.get("log_dir", "tmp") + log_file = log_config.get("log_file", "server.log") + + # 日志文件完整路径 + log_file_path = os.path.join(log_dir, log_file) + logger.add( - os.path.join( - log_config.get("log_dir", "tmp"), - log_config.get("log_file", "server.log"), - ), + log_file_path, format=log_format_file, level=log_config.get("log_level", "INFO"), filter=formatter, + rotation="10 MB", # 每个文件最大10MB + retention="30 days", # 保留30天 + compression=None, + encoding="utf-8", + enqueue=True, # 异步安全 + backtrace=True, + diagnose=True, ) except Exception as e: From dca02c1f4b8c0380673fa2ca8902f261f781d7f6 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 5 Jun 2025 14:11:50 +0800 Subject: [PATCH 6/9] =?UTF-8?q?update:=20=E5=94=A4=E9=86=92=E9=9F=B3?= =?UTF-8?q?=E9=A2=91=E5=85=BC=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 859059b6..c41baba0 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -1,4 +1,5 @@ import os +import shutil import time import json import random @@ -119,17 +120,31 @@ async def wakeupWordsResponse(conn): if result is None or result == "": return - opus_datas = await asyncio.to_thread(conn.tts.to_tts, result) - if not opus_datas: - return - - wav_bytes = opus_datas_to_wav_bytes(opus_datas, sample_rate=16000) - file_path = os.path.join( - WAKEUP_CONFIG["dir"], "my_" + WAKEUP_CONFIG["file_name"] + ".wav" - ) - # 写入wav数据 - with open(file_path, "wb") as f: - f.write(wav_bytes) + tts_result = await asyncio.to_thread(conn.tts.to_tts, result) + if conn.tts.delete_audio_file: + # 返回的是opus数据流 + if not tts_result: + return + wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000) + file_path = os.path.join( + WAKEUP_CONFIG["dir"], "my_" + WAKEUP_CONFIG["file_name"] + ".wav" + ) + with open(file_path, "wb") as f: + f.write(wav_bytes) + else: + # 返回为文件 + if tts_result is not None and os.path.exists(tts_result): + file_type = os.path.splitext(tts_result)[1] + if file_type: + file_type = file_type.lstrip(".") + old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"]) + if old_file is not None: + os.remove(old_file) + """将文件挪到wakeup_words目录下""" + shutil.move( + tts_result, + WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + '.' + file_type, + ) WAKEUP_CONFIG["create_time"] = time.time() WAKEUP_CONFIG["text"] = result From 3eb3a4502f7b651b460576feb552c15670dd4b2a Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Fri, 6 Jun 2025 14:08:38 +0800 Subject: [PATCH 7/9] Update fun_local.py --- main/xiaozhi-server/core/providers/asr/fun_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 4b25a265..e907bdce 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -43,7 +43,7 @@ class ASRProvider(ASRProviderBase): min_mem_bytes = 2 * 1024 * 1024 * 1024 total_mem = psutil.virtual_memory().total if total_mem < min_mem_bytes: - logger.bind(tag=TAG).error(f"可用内存不足2G，当前仅有 {total_mem / (1024*1024):.2f} MB") + logger.bind(tag=TAG).error(f"可用内存不足2G，当前仅有 {total_mem / (1024*1024):.2f} MB，可能无法启动FunASR") self.interface_type = InterfaceType.LOCAL self.model_dir = config.get("model_dir") From 50e6e4817b74cf40fa7c9aa6ec294ab79cf24c1c Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sat, 7 Jun 2025 22:08:22 +0800 Subject: [PATCH 8/9] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E5=94=A4=E9=86=92?= =?UTF-8?q?=E8=AF=8D=E7=AD=94=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 150 ++++++++---------- .../providers/tts/huoshan_double_stream.py | 9 +- .../core/providers/tts/minimax.py | 8 +- .../core/providers/tts/ttson.py | 6 +- main/xiaozhi-server/core/utils/wakeup_word.py | 143 +++++++++++++++++ 5 files changed, 221 insertions(+), 95 deletions(-) create mode 100644 main/xiaozhi-server/core/utils/wakeup_word.py diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index c41baba0..154a2096 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -12,19 +12,21 @@ from core.handle.mcpHandle import ( send_mcp_initialize_message, send_mcp_tools_list_request, ) - +from core.utils.wakeup_word import WakeupWordsConfig TAG = __name__ WAKEUP_CONFIG = { - "dir": "config/assets/", - "file_name": "wakeup_words", - "create_time": time.time(), - "refresh_time": 10, + "refresh_time": 5, "words": ["你好小智", "你好啊小智", "小智你好", "小智"], - "text": "", } +# 创建全局的唤醒词配置管理器 +wakeup_words_config = WakeupWordsConfig() + +# 用于防止并发调用wakeupWordsResponse的锁 +_wakeup_response_lock = asyncio.Lock() + async def handleHelloMessage(conn, msg_json): """处理hello消息""" @@ -55,96 +57,78 @@ async def checkWakeupWords(conn, text): enable_wakeup_words_response_cache = conn.config[ "enable_wakeup_words_response_cache" ] - """是否用的是非流式tts""" - if conn.tts and conn.tts.interface_type != InterfaceType.NON_STREAM: + + if not enable_wakeup_words_response_cache or not conn.tts: return False - """是否开启唤醒词加速""" - if not enable_wakeup_words_response_cache: + if conn.tts.interface_type != InterfaceType.NON_STREAM: return False - """检查是否是唤醒词""" + _, filtered_text = remove_punctuation_and_length(text) - if filtered_text in conn.config.get("wakeup_words"): - await send_stt_message(conn, text) + if filtered_text not in conn.config.get("wakeup_words"): + return False - file = getWakeupWordFile(WAKEUP_CONFIG["file_name"]) - if file is None: + await send_stt_message(conn, text) + + # 获取当前音色 + voice = getattr(conn.tts, "voice", "default") + + # 获取唤醒词回复配置 + response = wakeup_words_config.get_wakeup_response(voice) + + # 播放唤醒词回复 + conn.client_abort = False + conn.tts.tts_one_sentence( + conn, + ContentType.FILE, + content_file=response["file_path"], + content_detail=response["text"], + ) + + # 检查是否需要更新唤醒词回复 + if time.time() - response["time"] > WAKEUP_CONFIG["refresh_time"]: + if not _wakeup_response_lock.locked(): asyncio.create_task(wakeupWordsResponse(conn)) - return False - text_hello = WAKEUP_CONFIG["text"] - if not text_hello: - text_hello = text - if conn.tts is None: - return False - conn.tts.tts_one_sentence( - conn, ContentType.FILE, content_file=file, content_detail=text_hello - ) - if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]: - asyncio.create_task(wakeupWordsResponse(conn)) - return True - return False - - -def getWakeupWordFile(file_name): - for file in os.listdir(WAKEUP_CONFIG["dir"]): - if file.startswith("my_" + file_name): - """避免缓存文件是一个空文件""" - if os.stat(f"config/assets/{file}").st_size > (15 * 1024): - return f"config/assets/{file}" - - """查找config/assets/目录下名称为wakeup_words的文件""" - for file in os.listdir(WAKEUP_CONFIG["dir"]): - if file.startswith(file_name): - return f"config/assets/{file}" - return None + return True async def wakeupWordsResponse(conn): - wait_max_time = 5 - while conn.llm is None or not conn.llm.response_no_stream: - await asyncio.sleep(1) - wait_max_time -= 1 - if wait_max_time <= 0: - conn.logger.bind(tag=TAG).error("连接对象没有llm") - return - - """唤醒词响应""" - wakeup_word = random.choice(WAKEUP_CONFIG["words"]) - question = ( - "此刻用户正在和你说```" - + wakeup_word - + "```。\n请你根据以上用户的内容，进行简短回复，文字内容控制在15个字以内。\n" - + "请勿对这条内容本身进行任何解释和回应，仅返回对用户的内容的回复。" - ) - result = conn.llm.response_no_stream(conn.config["prompt"], question) - if result is None or result == "": + if not conn.tts or not conn.llm or not conn.llm.response_no_stream: return - tts_result = await asyncio.to_thread(conn.tts.to_tts, result) - if conn.tts.delete_audio_file: - # 返回的是opus数据流 + try: + # 尝试获取锁，如果获取不到就返回 + if not await _wakeup_response_lock.acquire(): + return + + # 生成唤醒词回复 + wakeup_word = random.choice(WAKEUP_CONFIG["words"]) + question = ( + "此刻用户正在和你说```" + + wakeup_word + + "```。\n请你根据以上用户的内容进行简短回复。要像一个人正常人一样说话，不要像机器人一样说话。\n" + + "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。" + ) + + result = conn.llm.response_no_stream(conn.config["prompt"], question) + if not result or len(result) == 0: + return + + # 生成TTS音频 + tts_result = await asyncio.to_thread(conn.tts.to_tts, result) if not tts_result: return + + # 获取当前音色 + voice = getattr(conn.tts, "voice", "default") + wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000) - file_path = os.path.join( - WAKEUP_CONFIG["dir"], "my_" + WAKEUP_CONFIG["file_name"] + ".wav" - ) + file_path = wakeup_words_config.generate_file_path(voice) with open(file_path, "wb") as f: f.write(wav_bytes) - - else: - # 返回为文件 - if tts_result is not None and os.path.exists(tts_result): - file_type = os.path.splitext(tts_result)[1] - if file_type: - file_type = file_type.lstrip(".") - old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"]) - if old_file is not None: - os.remove(old_file) - """将文件挪到wakeup_words目录下""" - shutil.move( - tts_result, - WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + '.' + file_type, - ) - WAKEUP_CONFIG["create_time"] = time.time() - WAKEUP_CONFIG["text"] = result + # 更新配置 + wakeup_words_config.update_wakeup_response(voice, file_path, result) + finally: + # 确保在任何情况下都释放锁 + if _wakeup_response_lock.locked(): + _wakeup_response_lock.release() diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 4d15204e..5193e19e 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -145,10 +145,9 @@ class TTSProvider(TTSProviderBase): self.cluster = config.get("cluster") self.resource_id = config.get("resource_id") if config.get("private_voice"): - self.speaker = config.get("private_voice") + self.voice = config.get("private_voice") else: - self.speaker = config.get("speaker") - self.voice = config.get("voice") + self.voice = config.get("speaker") self.ws_url = config.get("ws_url") self.authorization = config.get("authorization") self.header = {"Authorization": f"{self.authorization}{self.access_token}"} @@ -272,7 +271,7 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).error(f"WebSocket连接不存在，终止发送文本") return # 发送文本 - await self.send_text(self.speaker, text, self.conn.sentence_id) + await self.send_text(self.voice, text, self.conn.sentence_id) return except Exception as e: logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") @@ -302,7 +301,7 @@ class TTSProvider(TTSProviderBase): event=EVENT_StartSession, sessionId=session_id ).as_bytes() payload = self.get_payload_bytes( - event=EVENT_StartSession, speaker=self.speaker + event=EVENT_StartSession, speaker=self.voice ) await self.send_event(header, optional, payload) logger.bind(tag=TAG).info("会话启动请求已发送") diff --git a/main/xiaozhi-server/core/providers/tts/minimax.py b/main/xiaozhi-server/core/providers/tts/minimax.py index 3741d169..75e25e2e 100644 --- a/main/xiaozhi-server/core/providers/tts/minimax.py +++ b/main/xiaozhi-server/core/providers/tts/minimax.py @@ -14,9 +14,9 @@ class TTSProvider(TTSProviderBase): self.api_key = config.get("api_key") self.model = config.get("model") if config.get("private_voice"): - self.voice_id = config.get("private_voice") + self.voice = config.get("private_voice") else: - self.voice_id = config.get("voice_id") + self.voice = config.get("voice_id") default_voice_setting = { "voice_id": "female-shaonv", @@ -43,8 +43,8 @@ class TTSProvider(TTSProviderBase): self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})} self.timber_weights = parse_string_to_list(config.get("timber_weights")) - if self.voice_id: - self.voice_setting["voice_id"] = self.voice_id + if self.voice: + self.voice_setting["voice_id"] = self.voice self.host = "api.minimax.chat" self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}" diff --git a/main/xiaozhi-server/core/providers/tts/ttson.py b/main/xiaozhi-server/core/providers/tts/ttson.py index c11206a6..91deeed9 100644 --- a/main/xiaozhi-server/core/providers/tts/ttson.py +++ b/main/xiaozhi-server/core/providers/tts/ttson.py @@ -19,9 +19,9 @@ class TTSProvider(TTSProviderBase): "https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=", ) if config.get("private_voice"): - self.voice_id = int(config.get("private_voice")) + self.voice = int(config.get("private_voice")) else: - self.voice_id = int(config.get("voice_id", 1695)) + self.voice = int(config.get("voice_id", 1695)) self.token = config.get("token") self.to_lang = config.get("to_lang") self.volume_change_dB = int(config.get("volume_change_dB", 0)) @@ -50,7 +50,7 @@ class TTSProvider(TTSProviderBase): "emotion": self.emotion, "format": self.format, "volume_change_dB": self.volume_change_dB, - "voice_id": self.voice_id, + "voice_id": self.voice, "pitch_factor": self.pitch_factor, "speed_factor": self.speed_factor, "token": self.token, diff --git a/main/xiaozhi-server/core/utils/wakeup_word.py b/main/xiaozhi-server/core/utils/wakeup_word.py new file mode 100644 index 00000000..258f879b --- /dev/null +++ b/main/xiaozhi-server/core/utils/wakeup_word.py @@ -0,0 +1,143 @@ +import os +import yaml +import time +import hashlib +import fcntl +from typing import Dict +from contextlib import contextmanager + + +class FileLock: + def __init__(self, file, timeout=5): + self.file = file + self.timeout = timeout + self.start_time = None + + def __enter__(self): + self.start_time = time.time() + while True: + try: + fcntl.flock(self.file, fcntl.LOCK_EX | fcntl.LOCK_NB) + return self.file + except IOError: + if time.time() - self.start_time > self.timeout: + raise TimeoutError("获取文件锁超时") + time.sleep(0.1) + + def __exit__(self, exc_type, exc_val, exc_tb): + fcntl.flock(self.file, fcntl.LOCK_UN) + + +class WakeupWordsConfig: + def __init__(self): + self.config_file = "data/.wakeup_words.yaml" + self.assets_dir = "config/assets/wakeup_words" + self._ensure_directories() + self._config_cache = None + self._last_load_time = 0 + self._cache_ttl = 1 # 缓存有效期（秒） + self._lock_timeout = 5 # 文件锁超时时间（秒） + + def _ensure_directories(self): + """确保必要的目录存在""" + os.makedirs(os.path.dirname(self.config_file), exist_ok=True) + os.makedirs(self.assets_dir, exist_ok=True) + + def _load_config(self) -> Dict: + """加载配置文件，使用缓存机制""" + current_time = time.time() + + # 如果缓存有效，直接返回缓存 + if ( + self._config_cache is not None + and current_time - self._last_load_time < self._cache_ttl + ): + return self._config_cache + + try: + with open(self.config_file, "a+") as f: + with FileLock(f, timeout=self._lock_timeout): + f.seek(0) + content = f.read() + config = yaml.safe_load(content) if content else {} + self._config_cache = config + self._last_load_time = current_time + return config + except (TimeoutError, IOError) as e: + print(f"加载配置文件失败: {e}") + return {} + except Exception as e: + print(f"加载配置文件时发生未知错误: {e}") + return {} + + def _save_config(self, config: Dict): + """保存配置到文件，使用文件锁保护""" + try: + with open(self.config_file, "w") as f: + with FileLock(f, timeout=self._lock_timeout): + yaml.dump(config, f, allow_unicode=True) + self._config_cache = config + self._last_load_time = time.time() + except (TimeoutError, IOError) as e: + print(f"保存配置文件失败: {e}") + raise + except Exception as e: + print(f"保存配置文件时发生未知错误: {e}") + raise + + def get_wakeup_response(self, voice: str) -> Dict: + voice = hashlib.md5(voice.encode()).hexdigest() + """获取唤醒词回复配置""" + config = self._load_config() + default_response = { + "voice": "default", + "file_path": "config/assets/wakeup_words.wav", + "time": 0, + "text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦", + } + + if not config or voice not in config: + return default_response + + # 检查文件大小 + file_path = config[voice]["file_path"] + if not os.path.exists(file_path) or os.stat(file_path).st_size < (15 * 1024): + return default_response + + return config[voice] + + def update_wakeup_response(self, voice: str, file_path: str, text: str): + """更新唤醒词回复配置""" + try: + config = self._load_config() + voice_hash = hashlib.md5(voice.encode()).hexdigest() + config[voice_hash] = { + "voice": voice, + "file_path": file_path, + "time": time.time(), + "text": text, + } + self._save_config(config) + except Exception as e: + print(f"更新唤醒词回复配置失败: {e}") + raise + + def generate_file_path(self, voice: str) -> str: + """生成音频文件路径，使用voice的哈希值作为文件名""" + try: + # 生成voice的哈希值 + voice_hash = hashlib.md5(voice.encode()).hexdigest() + file_path = os.path.join(self.assets_dir, f"{voice_hash}.wav") + + # 如果文件已存在，先删除 + if os.path.exists(file_path): + try: + os.remove(file_path) + except Exception as e: + print(f"删除已存在的音频文件失败: {e}") + raise + + return file_path + except Exception as e: + print(f"生成音频文件路径失败: {e}") + raise From 98174bcc162128a9e9258d077f47a3e122812a8a Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 8 Jun 2025 00:06:40 +0800 Subject: [PATCH 9/9] =?UTF-8?q?update:=E3=80=90=E6=B5=81=E5=BC=8Ftts?= =?UTF-8?q?=E3=80=91=E5=A2=9E=E5=8A=A0=E3=80=90=E9=9D=9E=E6=B5=81=E5=BC=8F?= =?UTF-8?q?=E3=80=91=E6=96=B9=E6=B3=95=EF=BC=8C=E7=94=A8=E4=BA=8E=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E5=8F=8A=E7=94=9F=E6=88=90=E6=96=87=E4=BB=B6=E7=9A=84?= =?UTF-8?q?=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 17 +-- .../providers/tts/huoshan_double_stream.py | 120 +++++++++++++++++- .../core/providers/tts/linkerai.py | 78 +++++++++++- 3 files changed, 194 insertions(+), 21 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index b19dc054..bb15382d 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -1,12 +1,11 @@ -import os -import shutil import time import json import random import asyncio +from core.utils.util import audio_to_data from core.handle.sendAudioHandle import send_stt_message from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes -from core.providers.tts.dto.dto import ContentType, InterfaceType +from core.providers.tts.dto.dto import ContentType, SentenceType from core.handle.mcpHandle import ( MCPClient, send_mcp_initialize_message, @@ -59,9 +58,6 @@ async def checkWakeupWords(conn, text): if not enable_wakeup_words_response_cache or not conn.tts: return False - if conn.tts.interface_type != InterfaceType.NON_STREAM: - return False - _, filtered_text = remove_punctuation_and_length(text) if filtered_text not in conn.config.get("wakeup_words"): return False @@ -77,12 +73,9 @@ async def checkWakeupWords(conn, text): # 播放唤醒词回复 conn.client_abort = False - conn.tts.tts_one_sentence( - conn, - ContentType.FILE, - content_file=response["file_path"], - content_detail=response["text"], - ) + opus_packets, _ = audio_to_data(response["file_path"]) + conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, response["text"])) + conn.tts.tts_audio_queue.put((SentenceType.LAST, [], None)) # 检查是否需要更新唤醒词回复 if time.time() - response["time"] > WAKEUP_CONFIG["refresh_time"]: diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 10a37b51..33a1c8ec 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -301,7 +301,7 @@ class TTSProvider(TTSProviderBase): payload = self.get_payload_bytes( event=EVENT_StartSession, speaker=self.voice ) - await self.send_event(header, optional, payload) + await self.send_event(self.ws, header, optional, payload) logger.bind(tag=TAG).info("会话启动请求已发送") except Exception as e: logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}") @@ -334,7 +334,7 @@ class TTSProvider(TTSProviderBase): event=EVENT_FinishSession, sessionId=session_id ).as_bytes() payload = str.encode("{}") - await self.send_event(header, optional, payload) + await self.send_event(self.ws, header, optional, payload) logger.bind(tag=TAG).info("会话结束请求已发送") # 等待监听任务完成 @@ -451,7 +451,11 @@ class TTSProvider(TTSProviderBase): self.ws = None async def send_event( - self, header: bytes, optional: bytes | None = None, payload: bytes = None + self, + ws: websockets.WebSocketClientProtocol, + header: bytes, + optional: bytes | None = None, + payload: bytes = None, ): try: full_client_request = bytearray(header) @@ -461,7 +465,7 @@ class TTSProvider(TTSProviderBase): payload_size = len(payload).to_bytes(4, "big", signed=True) full_client_request.extend(payload_size) full_client_request.extend(payload) - await self.ws.send(full_client_request) + await ws.send(full_client_request) except websockets.ConnectionClosed: logger.bind(tag=TAG).error(f"ConnectionClosed") raise @@ -476,7 +480,7 @@ class TTSProvider(TTSProviderBase): payload = self.get_payload_bytes( event=EVENT_TaskRequest, text=text, speaker=speaker ) - return await self.send_event(header, optional, payload) + return await self.send_event(self.ws, header, optional, payload) # 读取 res 数组某段 字符串内容 def read_res_content(self, res: bytes, offset: int): @@ -554,7 +558,7 @@ class TTSProvider(TTSProviderBase): ).as_bytes() optional = Optional(event=EVENT_Start_Connection).as_bytes() payload = str.encode("{}") - return await self.send_event(header, optional, payload) + return await self.send_event(self.ws, header, optional, payload) def print_response(self, res, tag_msg: str): logger.bind(tag=TAG).debug(f"===>{tag_msg} header:{res.header.__dict__}") @@ -590,3 +594,107 @@ class TTSProvider(TTSProviderBase): def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False): opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end) return opus_datas + + def to_tts(self, text: str) -> list: + """非流式生成音频数据，用于生成音频及测试场景 + + Args: + text: 要转换的文本 + + Returns: + list: 音频数据列表 + """ + try: + # 创建事件循环 + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # 生成会话ID + session_id = uuid.uuid4().__str__().replace("-", "") + + # 存储音频数据 + audio_data = [] + + async def _generate_audio(): + # 创建新的WebSocket连接 + ws_header = { + "X-Api-App-Key": self.appId, + "X-Api-Access-Key": self.access_token, + "X-Api-Resource-Id": self.resource_id, + "X-Api-Connect-Id": uuid.uuid4(), + } + ws = await websockets.connect( + self.ws_url, additional_headers=ws_header, max_size=1000000000 + ) + + try: + # 启动会话 + header = Header( + message_type=FULL_CLIENT_REQUEST, + message_type_specific_flags=MsgTypeFlagWithEvent, + serial_method=JSON, + ).as_bytes() + optional = Optional( + event=EVENT_StartSession, sessionId=session_id + ).as_bytes() + payload = self.get_payload_bytes( + event=EVENT_StartSession, speaker=self.voice + ) + await self.send_event(ws, header, optional, payload) + + # 发送文本 + header = Header( + message_type=FULL_CLIENT_REQUEST, + message_type_specific_flags=MsgTypeFlagWithEvent, + serial_method=JSON, + ).as_bytes() + optional = Optional( + event=EVENT_TaskRequest, sessionId=session_id + ).as_bytes() + payload = self.get_payload_bytes( + event=EVENT_TaskRequest, text=text, speaker=self.voice + ) + await self.send_event(ws, header, optional, payload) + + # 发送结束会话请求 + header = Header( + message_type=FULL_CLIENT_REQUEST, + message_type_specific_flags=MsgTypeFlagWithEvent, + serial_method=JSON, + ).as_bytes() + optional = Optional( + event=EVENT_FinishSession, sessionId=session_id + ).as_bytes() + payload = str.encode("{}") + await self.send_event(ws, header, optional, payload) + + # 接收音频数据 + while True: + msg = await ws.recv() + res = self.parser_response(msg) + + if ( + res.optional.event == EVENT_TTSResponse + and res.header.message_type == AUDIO_ONLY_RESPONSE + ): + opus_datas = self.wav_to_opus_data_audio_raw(res.payload) + audio_data.extend(opus_datas) + elif res.optional.event == EVENT_SessionFinished: + break + + finally: + # 清理资源 + try: + await ws.close() + except: + pass + + # 运行异步任务 + loop.run_until_complete(_generate_audio()) + loop.close() + + return audio_data + + except Exception as e: + logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}") + return [] diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index dd8d0e6b..649ed51d 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -2,6 +2,8 @@ import queue import asyncio import traceback import aiohttp +import requests +import time from config.logger import setup_logging from core.utils.tts import MarkdownCleaner from core.providers.tts.base import TTSProviderBase @@ -55,7 +57,7 @@ class TTSProvider(TTSProviderBase): self.tts_text_buff.append(message.content_detail) segment_text = self._get_segment_text() if segment_text: - self.to_tts(segment_text) + self.to_tts_single_stream(segment_text) elif ContentType.FILE == message.content_type: logger.bind(tag=TAG).info( @@ -87,12 +89,12 @@ class TTSProvider(TTSProviderBase): if remaining_text: segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) if segment_text: - self.to_tts(segment_text, is_last) + self.to_tts_single_stream(segment_text, is_last) self.processed_chars += len(full_text) else: self._process_before_stop_play_files() - def to_tts(self, text, is_last=False): + def to_tts_single_stream(self, text, is_last=False): try: max_repeat_time = 5 text = MarkdownCleaner.clean_markdown(text) @@ -225,3 +227,73 @@ class TTSProvider(TTSProviderBase): except Exception as e: logger.error(f"TTS请求异常: {e}") self.tts_audio_queue.put((SentenceType.LAST, [], None)) + + def to_tts(self, text: str) -> list: + """非流式TTS处理，用于测试及保存音频文件的场景 + + Args: + text: 要转换的文本 + + Returns: + list: 返回opus编码后的音频数据列表 + """ + start_time = time.time() + text = MarkdownCleaner.clean_markdown(text) + + params = { + "tts_text": text, + "spk_id": self.voice, + "frame_duration": 60, + "stream": False, + "target_sr": 16000, + "audio_format": self.audio_format, + "instruct_text": "请生成一段自然流畅的语音", + } + headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + } + + try: + with requests.get( + self.api_url, params=params, headers=headers, timeout=5 + ) as response: + if response.status_code != 200: + logger.error( + f"TTS请求失败: {response.status_code}, {response.text}" + ) + return [] + + logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") + + # 使用opus编码器处理PCM数据 + opus_datas = [] + pcm_data = response.content + + # 计算每帧的字节数 + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels + * self.opus_encoder.frame_size_ms + / 1000 + * 2 + ) + + # 分帧处理PCM数据 + for i in range(0, len(pcm_data), frame_bytes): + frame = pcm_data[i : i + frame_bytes] + if len(frame) < frame_bytes: + # 最后一帧可能不足，用0填充 + frame = frame + b"\x00" * (frame_bytes - len(frame)) + + opus = self.opus_encoder.encode_pcm_to_opus( + frame, end_of_stream=(i + frame_bytes >= len(pcm_data)) + ) + if opus: + opus_datas.extend(opus) + + return opus_datas + + except Exception as e: + logger.error(f"TTS请求异常: {e}") + return []