From a81f4332284763ea55a0eee2669689f41ea88520 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 5 Sep 2025 17:52:30 +0800 Subject: [PATCH] =?UTF-8?q?update:=20=E5=A2=9E=E5=8A=A0=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202508200911.sql | 33 ------ .../resources/db/changelog/202509051745.sql | 45 ++++++++ .../db/changelog/db.changelog-master.yaml | 4 +- .../core/providers/tts/minimax_httpstream.py | 100 ++++++++++++++++-- 4 files changed, 137 insertions(+), 45 deletions(-) delete mode 100644 main/manager-api/src/main/resources/db/changelog/202508200911.sql create mode 100644 main/manager-api/src/main/resources/db/changelog/202509051745.sql diff --git a/main/manager-api/src/main/resources/db/changelog/202508200911.sql b/main/manager-api/src/main/resources/db/changelog/202508200911.sql deleted file mode 100644 index 9ecc64df..00000000 --- a/main/manager-api/src/main/resources/db/changelog/202508200911.sql +++ /dev/null @@ -1,33 +0,0 @@ --- 添加 MinimaxHTTPStream 流式 TTS 供应器 -delete from `ai_model_provider` where id = 'SYSTEM_TTS_MinimaxStreamTTS'; -INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES -('SYSTEM_TTS_MinimaxStreamTTS', 'TTS', 'minimax_stream', 'Minimax流式语音合成', '[{"key":"group_id","label":"组ID","type":"string"},{"key":"api_key","label":"API密钥","type":"string"},{"key":"model","label":"模型","type":"string"},{"key":"voice_id","label":"音色ID","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"},{"key":"voice_setting","label":"音色设置","type":"dict","dict_name":"voice_setting"},{"key":"pronunciation_dict","label":"发音字典","type":"dict","dict_name":"pronunciation_dict"},{"key":"audio_setting","label":"音频设置","type":"dict","dict_name":"audio_setting"},{"key":"timber_weights","label":"音色权重","type":"string"}]', 18, 1, NOW(), 1, NOW()); - --- 添加Minimax流式TTS模型配置 -delete from `ai_model_config` where id = 'TTS_MinimaxStreamTTS'; -INSERT INTO `ai_model_config` VALUES ('TTS_MinimaxStreamTTS', 'TTS', 'MinimaxStreamTTS', 'Minimax流式语音合成', 0, 1, '{"type": "minimax_stream", "group_id": "", "api_key": "", "model": "speech-01", "voice_id": "female-shaonv", "output_dir": "tmp/", "voice_setting": {"speed": 1, "vol": 1, "pitch": 0, "emotion": "happy"}, "pronunciation_dict": {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}, "audio_setting": {"sample_rate": 24000, "bitrate": 128000, "format": "pcm", "channel": 1}}', NULL, NULL, 21, NULL, NULL, NULL, NULL); - --- 更新Minimax流式TTS配置说明 -UPDATE `ai_model_config` SET -`doc_link` = 'https://platform.minimaxi.com/', -`remark` = 'Minimax流式TTS配置说明： -1. 需要先申请Minimax API Key -2. 需要填写Group ID -3. 支持多种音色设置和音频参数调整 -4. 支持实时流式合成，具有较低的延迟 -5. 支持自定义发音字典和音色权重 -6. 隐藏参数配置：声音设定(voice_setting)、发音字典(pronunciation_dict)、音色权重(timber_weights) - - 语速(speed): 范围[0.5,2]，默认1.0，取值越大语速越快 - - 音量(vol): 范围(0,10]，默认1.0，取值越大音量越高 - - 音调(pitch): 范围[-12,12]，默认0，取值需为整数 - - timbre_weights与voice_id二选一必填 - - voice_id(请求的音色id，须和weight参数同步填写) - - weight(权重，最多支持4种音色混合。范围[1,100]) -' WHERE `id` = 'TTS_MinimaxStreamTTS'; - --- 添加Minimax流式TTS音色 -delete from `ai_tts_voice` where tts_model_id = 'TTS_MinimaxStreamTTS'; --- 默认音色 -INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0001', 'TTS_MinimaxStreamTTS', '少女音', 'female-shaonv', '中文', NULL, NULL, NULL, NULL, 1, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0002', 'TTS_MinimaxStreamTTS', '成熟女声', 'female-chengshu', '中文', NULL, NULL, NULL, NULL, 2, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0003', 'TTS_MinimaxStreamTTS', '少年音', 'male-shaonian', '中文', NULL, NULL, NULL, NULL, 3, NULL, NULL, NULL, NULL); \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/202509051745.sql b/main/manager-api/src/main/resources/db/changelog/202509051745.sql new file mode 100644 index 00000000..6dc92222 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202509051745.sql @@ -0,0 +1,45 @@ +-- 添加 MinimaxHTTPStream 流式 TTS 供应器 +delete from `ai_model_provider` where id = 'SYSTEM_TTS_MinimaxStreamTTS'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_TTS_MinimaxStreamTTS', 'TTS', 'minimax_httpstream', 'Minimax流式语音合成', '[{"key":"group_id","label":"组ID","type":"string"},{"key":"api_key","label":"API密钥","type":"string"},{"key":"model","label":"模型","type":"string"},{"key":"voice_id","label":"音色ID","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"},{"key":"voice_setting","label":"音色设置","type":"dict","dict_name":"voice_setting"},{"key":"pronunciation_dict","label":"发音字典","type":"dict","dict_name":"pronunciation_dict"},{"key":"audio_setting","label":"音频设置","type":"dict","dict_name":"audio_setting"},{"key":"timber_weights","label":"音色权重","type":"string"}]', 18, 1, NOW(), 1, NOW()); + +-- 添加Minimax流式TTS模型配置 +delete from `ai_model_config` where id = 'TTS_MinimaxStreamTTS'; +INSERT INTO `ai_model_config` VALUES ('TTS_MinimaxStreamTTS', 'TTS', 'MinimaxStreamTTS', 'Minimax流式语音合成', 0, 1, '{"type": "minimax_httpstream", "group_id": "", "api_key": "", "model": "speech-01-turbo", "voice_id": "female-shaonv", "output_dir": "tmp/", "voice_setting": {"speed": 1, "vol": 1, "pitch": 0, "emotion": "happy"}, "pronunciation_dict": {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}, "audio_setting": {"sample_rate": 24000, "bitrate": 128000, "format": "pcm", "channel": 1}}', NULL, NULL, 21, NULL, NULL, NULL, NULL); + +-- 更新Minimax流式TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://platform.minimaxi.com/', +`remark` = 'Minimax流式TTS配置说明： +1. 需要先申请Minimax API Key +2. 需要填写Group ID +3. 支持多种音色设置和音频参数调整 +4. 支持实时流式合成，具有较低的延迟 +5. 支持自定义发音字典和音色权重 +6. 隐藏参数配置：声音设定(voice_setting)、发音字典(pronunciation_dict)、音色权重(timber_weights) + - 语速(speed): 范围[0.5,2]，默认1.0，取值越大语速越快 + - 音量(vol): 范围(0,10]，默认1.0，取值越大音量越高 + - 音调(pitch): 范围[-12,12]，默认0，取值需为整数 + - 情绪(emotion): 控制合成语音的情绪，支持7种值：["happy", "sad", "angry", "fearful", "disgusted", "surprised", "calm"]，该参数仅对 speech-2.5-hd-preview、speech-2.5-turbo-preview、speech-02-hd、speech-02-turbo、speech-01-turbo、speech-01-hd 生效 + - timbre_weights与voice_id二选一必填 + - voice_id(请求的音色id，须和weight参数同步填写) + - weight(权重，最多支持4种音色混合。范围[1,100]) +' WHERE `id` = 'TTS_MinimaxStreamTTS'; + +-- 添加Minimax流式TTS音色 +delete from `ai_tts_voice` where tts_model_id = 'TTS_MinimaxStreamTTS'; + +-- 默认音色 +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0001', 'TTS_MinimaxStreamTTS', '少女音', 'female-shaonv', '中文', NULL, NULL, NULL, NULL, 1, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0002', 'TTS_MinimaxStreamTTS', '成熟女声', 'female-chengshu', '中文', NULL, NULL, NULL, NULL, 2, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0003', 'TTS_MinimaxStreamTTS', '霸道少爷', 'badao_shaoye', '中文', NULL, NULL, NULL, NULL, 3, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0004', 'TTS_MinimaxStreamTTS', '病娇弟弟', 'bingjiao_didi', '中文', NULL, NULL, NULL, NULL, 4, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0005', 'TTS_MinimaxStreamTTS', '纯真学弟', 'chunzhen_xuedi', '中文', NULL, NULL, NULL, NULL, 5, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0006', 'TTS_MinimaxStreamTTS', '冷淡学长', 'lengdan_xiongzhang', '中文', NULL, NULL, NULL, NULL, 6, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0007', 'TTS_MinimaxStreamTTS', '甜美小玲', 'tianxin_xiaoling', '中文', NULL, NULL, NULL, NULL, 7, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0008', 'TTS_MinimaxStreamTTS', '俏皮萌妹', 'qiaopi_mengmei', '中文', NULL, NULL, NULL, NULL, 8, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0009', 'TTS_MinimaxStreamTTS', '妩媚御姐', 'wumei_yujie', '中文', NULL, NULL, NULL, NULL, 9, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0010', 'TTS_MinimaxStreamTTS', '嗲嗲学妹', 'diadia_xuemei', '中文', NULL, NULL, NULL, NULL, 7, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0011', 'TTS_MinimaxStreamTTS', '淡雅学姐', 'danya_xuejie', '中文', NULL, NULL, NULL, NULL, 8, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0012', 'TTS_MinimaxStreamTTS', 'Santa Claus', 'Santa_Claus', '中文', NULL, NULL, NULL, NULL, 9, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_MinimaxStreamTTS_0013', 'TTS_MinimaxStreamTTS', 'Grinch', 'Grinch', '中文', NULL, NULL, NULL, NULL, 10, NULL, NULL, NULL, NULL); diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 86102570..243f42ad 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -304,9 +304,9 @@ databaseChangeLog: encoding: utf8 path: classpath:db/changelog/202508131557.sql - changeSet: - id: 202508200911 + id: 202509051745 author: RanChen changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202508200911.sql + path: classpath:db/changelog/202509051745.sql diff --git a/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py b/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py index be944521..514d8912 100644 --- a/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py +++ b/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py @@ -1,12 +1,14 @@ import os import json +import time import queue import asyncio -import traceback import aiohttp -from core.utils.util import parse_string_to_list +import requests +import traceback from config.logger import setup_logging from core.utils.tts import MarkdownCleaner +from core.utils.util import parse_string_to_list from core.providers.tts.base import TTSProviderBase from core.utils import opus_encoder_utils, textUtils from core.providers.tts.dto.dto import SentenceType, ContentType @@ -80,7 +82,6 @@ class TTSProvider(TTSProviderBase): self.processed_chars = 0 self.tts_text_buff = [] self.before_stop_play_files.clear() - self.reset_flow_controller() elif ContentType.TEXT == message.content_type: self.tts_text_buff.append(message.content_detail) segment_text = self._get_segment_text() @@ -93,13 +94,7 @@ class TTSProvider(TTSProviderBase): ) if message.content_file and os.path.exists(message.content_file): # 先处理文件音频数据 - self._process_audio_file_stream( - message.content_file, - callback=lambda audio_data: self.handle_audio_file( - audio_data, message.content_detail - ), - ) - + self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail)) if message.sentence_type == SentenceType.LAST: # 处理剩余的文本 self._process_remaining_text_stream(True) @@ -259,3 +254,88 @@ class TTSProvider(TTSProviderBase): await super().close() if hasattr(self, "opus_encoder"): self.opus_encoder.close() + + def to_tts(self, text: str) -> list: + """非流式TTS处理，用于测试及保存音频文件的场景 + Args: + text: 要转换的文本 + Returns: + list: 返回opus编码后的音频数据列表 + """ + start_time = time.time() + text = MarkdownCleaner.clean_markdown(text) + + payload = { + "model": self.model, + "text": text, + "stream": True, + "voice_setting": self.voice_setting, + "pronunciation_dict": self.pronunciation_dict, + "audio_setting": self.audio_setting, + } + + if type(self.timber_weights) is list and len(self.timber_weights) > 0: + payload["timber_weights"] = self.timber_weights + payload["voice_setting"]["voice_id"] = "" + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}", + } + + try: + with requests.post( + self.api_url, data=json.dumps(payload), headers=headers, timeout=5 + ) as response: + if response.status_code != 200: + logger.bind(tag=TAG).error( + f"TTS请求失败: {response.status_code}, {response.text}" + ) + return [] + + logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") + + # 使用opus编码器处理PCM数据 + opus_datas = [] + full_content = response.content.decode('utf-8') + pcm_data = bytearray() + for data_block in full_content.split('\n\n'): + if not data_block.startswith('data: '): + continue + + try: + json_str = data_block[6:] # 去除'data: '前缀 + data = json.loads(json_str) + if data.get('data', {}).get('status') == 1: + audio_hex = data['data']['audio'] + pcm_data.extend(bytes.fromhex(audio_hex)) + except (json.JSONDecodeError, KeyError) as e: + logger.bind(tag=TAG).warning(f"无效数据块: {e}") + continue + + # 计算每帧的字节数 + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels + * self.opus_encoder.frame_size_ms + / 1000 + * 2 + ) + + # 分帧处理合并后的PCM数据 + for i in range(0, len(pcm_data), frame_bytes): + frame = bytes(pcm_data[i:i+frame_bytes]) + if len(frame) < frame_bytes: + frame += b"\x00" * (frame_bytes - len(frame)) + + self.opus_encoder.encode_pcm_to_opus_stream( + frame, + end_of_stream=(i + frame_bytes >= len(pcm_data)), + callback=lambda opus: opus_datas.append(opus) + ) + + return opus_datas + + except Exception as e: + logger.bind(tag=TAG).error(f"TTS请求异常: {e}") + return []