diff --git a/main/manager-api/src/main/resources/db/changelog/202601141645.sql b/main/manager-api/src/main/resources/db/changelog/202601141645.sql new file mode 100644 index 00000000..8b0fca88 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202601141645.sql @@ -0,0 +1,31 @@ +-- 批量清理 ai_model_provider 中的 sample_rate 字段定义 +UPDATE `ai_model_provider` ap +JOIN ( + SELECT + id, + JSON_ARRAYAGG( + JSON_OBJECT('key', jt.k, 'label', jt.l, 'type', jt.t) + ) AS new_fields + FROM `ai_model_provider`, + JSON_TABLE(`fields`, '$[*]' COLUMNS ( + k VARCHAR(50) PATH '$.key', + l VARCHAR(100) PATH '$.label', + t VARCHAR(20) PATH '$.type' + )) AS jt + WHERE `model_type` = 'TTS' + AND jt.k != 'sample_rate' + GROUP BY id +) filtered ON ap.id = filtered.id +SET ap.fields = filtered.new_fields; + +-- 清理 config_json 顶层的 sample_rate +UPDATE `ai_model_config` +SET `config_json` = JSON_REMOVE(`config_json`, '$.sample_rate') +WHERE `model_type` = 'TTS' + AND JSON_EXTRACT(`config_json`, '$.sample_rate') IS NOT NULL; + +-- 清理Minimax流式TTS的sample_rate参数（位于audio_setting内部） +UPDATE `ai_model_config` SET +`config_json` = JSON_SET(`config_json`, '$.audio_setting', JSON_REMOVE(JSON_EXTRACT(`config_json`, '$.audio_setting'), '$.sample_rate')) +WHERE `id` = 'TTS_MinimaxStreamTTS' +AND JSON_EXTRACT(`config_json`, '$.audio_setting.sample_rate') IS NOT NULL; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/202601231530.sql b/main/manager-api/src/main/resources/db/changelog/202601231530.sql new file mode 100644 index 00000000..79be4310 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202601231530.sql @@ -0,0 +1,87 @@ +-- 更新HuoshanDoubleStreamTTS供应器配置，将分散的参数改为JSON字典配置 +-- 将 speech_rate, loudness_rate, pitch, emotion, emotion_scale 等参数整合为 audio_params, additions, mix_speaker 三个JSON字典 + +UPDATE `ai_model_provider` +SET `fields` = '[ + {"key": "ws_url", "type": "string", "label": "WebSocket地址"}, + {"key": "appid", "type": "string", "label": "应用ID"}, + {"key": "access_token", "type": "string", "label": "访问令牌"}, + {"key": "resource_id", "type": "string", "label": "资源ID"}, + {"key": "speaker", "type": "string", "label": "默认音色"}, + {"key": "enable_ws_reuse", "type": "boolean", "label": "是否开启链接复用", "default": true}, + {"key": "audio_params", "type": "dict", "label": "音频输出配置"}, + {"key": "additions", "type": "dict", "label": "高级文本处理配置"}, + {"key": "mix_speaker", "type": "dict", "label": "混音控制配置"} +]' +WHERE `id` = 'SYSTEM_TTS_HSDSTTS'; + +-- 更新现有配置，将旧的分散参数迁移到新的JSON字典结构 +UPDATE `ai_model_config` +SET `config_json` = JSON_SET( + `config_json`, + '$.audio_params', JSON_OBJECT( + 'speech_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.speech_rate')), ''), '0') AS SIGNED), + 'loudness_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.loudness_rate')), ''), '0') AS SIGNED) + ), + '$.additions', JSON_OBJECT( + 'aigc_metadata', JSON_OBJECT(), + 'cache_config', JSON_OBJECT(), + 'post_process', JSON_OBJECT( + 'pitch', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.pitch')), ''), '0') AS SIGNED) + ) + ), + '$.mix_speaker', JSON_OBJECT() +) +WHERE `id` = 'TTS_HuoshanDoubleStreamTTS'; + +-- 删除旧的分散参数字段 +UPDATE `ai_model_config` +SET `config_json` = JSON_REMOVE( + `config_json`, + '$.speech_rate', + '$.loudness_rate', + '$.pitch', + '$.emotion', + '$.emotion_scale' +) +WHERE `id` = 'TTS_HuoshanDoubleStreamTTS'; + +-- 更新文档链接和备注说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://www.volcengine.com/docs/6561/1329505', +`remark` = '火山引擎双向流式TTS配置说明： +1. 访问 https://www.volcengine.com/ 注册并开通火山引擎账号 +2. 访问 https://console.volcengine.com/speech/service/10007 开通语音合成大模型，购买音色 +3. 在页面底部获取appid和access_token +4. 资源ID固定为：volc.service_type.10029（大模型语音合成及混音） +5. 链接复用：开启WebSocket连接复用，默认true减少链接损耗（注意：复用后设备处于聆听状态时空闲链接会占并发数） + +详细参数文档：https://www.volcengine.com/docs/6561/1329505 +【audio_params】音频输出配置 - 用户可自定义添加火山引擎支持的任何音频参数 + - speech_rate: 语速(-50~100)，默认0 + - loudness_rate: 音量(-50~100)，默认0 + - emotion: 情感类型（仅部分音色支持），可选值：neutral、happy、sad、angry、fearful、disgusted、surprised + - emotion_scale: 情感强度(1~5)，默认4 + 示例：{"speech_rate": 10, "loudness_rate": 5, "emotion": "happy", "emotion_scale": 4} + +【additions】高级文本处理配置 - 用户可自定义添加火山引擎支持的任何高级参数 + - post_process.pitch: 音高(-12~12)，默认0 + - aigc_metadata: AIGC元数据配置 + - cache_config: 缓存配置 + 示例：{"post_process": {"pitch": 2}, "aigc_metadata": {}, "cache_config": {}} + +【mix_speaker】混音控制配置 - 多音色混合（仅 TTS 1.0） + 示例： + {"speakers": [ + {"source_speaker": "zh_male_bvlazysheep","mix_factor": 0.3}, + {"source_speaker": "BV120_streaming","mix_factor": 0.3}, + {"source_speaker": "zh_male_ahu_conversation_wvae_bigtts","mix_factor": 0.4} + ]} + +注意： +- 多情感音色参数（emotion、emotion_scale）仅部分音色支持 +- 相关音色列表：https://www.volcengine.com/docs/6561/1257544 +- 用户可根据火山引擎API文档自行添加更多参数 +- 混音功能主要适用于豆包语音合成模型1.0的音色，使用时需要将req_params.speaker设置为custom_mix_bigtts +' +WHERE `id` = 'TTS_HuoshanDoubleStreamTTS'; diff --git a/main/manager-api/src/main/resources/db/changelog/202601261730.sql b/main/manager-api/src/main/resources/db/changelog/202601261730.sql new file mode 100644 index 00000000..5c2238ad --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202601261730.sql @@ -0,0 +1,14 @@ +-- 更新小智参数中的默认采样率从 16000 改为 24000 +UPDATE `sys_params` +SET `param_value` = '{ + "type": "hello", + "version": 1, + "transport": "websocket", + "audio_params": { + "format": "opus", + "sample_rate": 24000, + "channels": 1, + "frame_duration": 60 + } +}' +WHERE `id` = 309 AND `param_code` = 'xiaozhi'; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 9a8b23cc..8efe61b6 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -487,3 +487,24 @@ databaseChangeLog: - sqlFile: encoding: utf8 path: classpath:db/changelog/202601051433.sql + - changeSet: + id: 202601141645 + author: RanChen + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202601141645.sql + - changeSet: + id: 202601231530 + author: RanChen + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202601231530.sql + - changeSet: + id: 202601261730 + author: RanChen + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202601261730.sql diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index a3c54c17..ae7f0972 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -89,7 +89,8 @@ xiaozhi: transport: websocket audio_params: format: opus - sample_rate: 16000 + # Opus支持的采样率范围为[8000, 12000, 16000, 24000, 48000] + sample_rate: 24000 channels: 1 frame_duration: 60 @@ -714,13 +715,31 @@ TTS: speaker: zh_female_wanwanxiaohe_moon_bigtts # 开启WebSocket连接复用，默认复用（注意：复用后设备处于聆听状态时空闲链接会占并发数） enable_ws_reuse: True - speech_rate: 0 - loudness_rate: 0 - pitch: 0 - # 多情感音色参数，注意：当前仅部分音色支持设置情感。 - # 相关音色列表：https://www.volcengine.com/docs/6561/1257544 - emotion: "neutral" # 情感类型，可选值为：neutral、happy、sad、angry、fearful、disgusted、surprised - emotion_scale: 4 # 情感强度，可选值为：1~5，默认值为4 + # 相关参数文档：https://www.volcengine.com/docs/6561/1329505 + # 音频输出配置（audio_params）- 用户可自定义添加火山引擎支持的任何音频参数 + audio_params: + speech_rate: 0 # 语速(-50~100) + loudness_rate: 0 # 音量(-50~100) + # 多情感音色参数，注意：当前仅部分音色支持设置情感。 + # 相关音色列表：https://www.volcengine.com/docs/6561/1257544 + # emotion: "neutral" # 情感类型（仅部分音色支持）：neutral、happy、sad、angry、fearful、disgusted、surprised + # emotion_scale: 4 # 情感强度(1~5) + # 高级文本处理配置（additions）- 用户可自定义添加火山引擎支持的任何高级参数 + additions: + post_process: + pitch: 0 # 音高(-12~12) + # aigc_metadata: {} # AIGC元数据配置 + # cache_config: {} # 缓存配置 + # 混音控制配置（mix_speaker）- 多音色混合（仅 TTS 1.0） + # 混音功能主要适用于豆包语音合成模型1.0的音色，使用时需要将req_params.speaker设置为custom_mix_bigtts + # mix_speaker: + # speakers: + # - source_speaker: zh_male_bvlazysheep + # mix_factor: 0.3 + # - source_speaker: BV120_streaming + # mix_factor: 0.3 + # - source_speaker: zh_male_ahu_conversation_wvae_bigtts + # mix_factor: 0.4 CosyVoiceSiliconflow: type: siliconflow # 硅基流动TTS @@ -837,7 +856,6 @@ TTS: # - "处理/(chu3)(li3)" # - "危险/dangerous" # audio_setting: - # sample_rate: 24000 # bitrate: 128000 # format: "mp3" # channel: 1 @@ -865,7 +883,6 @@ TTS: # 以下可不用设置，使用默认设置 # format: wav - # sample_rate: 16000 # volume: 50 # speech_rate: 0 # pitch_rate: 0 @@ -889,7 +906,6 @@ TTS: host: nls-gateway-cn-beijing.aliyuncs.com # 以下可不用设置，使用默认设置 # format: pcm # 音频格式：pcm、wav、mp3 - # sample_rate: 16000 # 采样率：8000、16000、24000 # volume: 50 # 音量：0-100 # speech_rate: 0 # 语速：-500到500 # pitch_rate: 0 # 语调：-500到500 @@ -1004,7 +1020,6 @@ TTS: protocol: websocket # protocol choices = ['websocket', 'http'] url: ws://127.0.0.1:8092/paddlespeech/tts/streaming # TTS 服务的 URL 地址，指向本地服务器 [websocket默认ws://127.0.0.1:8092/paddlespeech/tts/streaming，http默认http://127.0.0.1:8090/paddlespeech/tts] spk_id: 0 # 发音人 ID，0 通常表示默认的发音人 - sample_rate: 24000 # 采样率 [websocket默认24000，http默认0 自动选择] speed: 1.0 # 语速，1.0 表示正常语速，>1 表示加快，<1 表示减慢 volume: 1.0 # 音量，1.0 表示正常音量，>1 表示增大，<1 表示减小 save_path: # 保存路径 @@ -1028,7 +1043,6 @@ TTS: output_dir: tmp/ # 以下可不用设置，使用默认设置 # format: pcm # 音频格式：pcm、wav、mp3、opus - # sample_rate: 24000 # 采样率：16000, 24000, 48000 # volume: 50 # 音量：0-100 # rate: 1 # 语速：0.5~2 # pitch: 1 # 语调：0.5~2 @@ -1050,7 +1064,6 @@ TTS: # stop_split: 0 # 关闭服务端拆句 不关闭：0，关闭：1 # remain: 0 # 是否保留原书面语的样子 保留:1, 不保留:0 # format: raw # 音频格式：raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb - # sample_rate: 24000 # 采样率：16000, 8000, 24000 # volume: 50 # 音量：0-100 # speed: 50 # 语速：0-100 # pitch: 50 # 语调：0-100 diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 449fbd2d..68900bd4 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -85,6 +85,7 @@ class ConnectionHandler: self.max_output_size = 0 self.chat_history_conf = 0 self.audio_format = "opus" + self.sample_rate = 24000 # 默认采样率，从客户端 hello 消息中动态更新 # 客户端状态相关 self.client_abort = False @@ -206,6 +207,10 @@ class ConnectionHandler: self.welcome_msg = self.config["xiaozhi"] self.welcome_msg["session_id"] = self.session_id + # 从配置中读取采样率 + self.sample_rate = self.welcome_msg["audio_params"]["sample_rate"] + self.logger.bind(tag=TAG).info(f"配置输出音频采样率为: {self.sample_rate}") + # 在后台初始化配置和组件（完全不阻塞主循环） asyncio.create_task(self._background_initialize()) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index cda7d17f..f28eb0dd 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -142,7 +142,8 @@ async def wakeupWordsResponse(conn): # 获取当前音色 voice = getattr(conn.tts, "voice", "default") - wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000) + # 使用链接的sample_rate + wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=conn.sample_rate) file_path = wakeup_words_config.generate_file_path(voice) with open(file_path, "wb") as f: f.write(wav_bytes) diff --git a/main/xiaozhi-server/core/providers/tts/alibl_stream.py b/main/xiaozhi-server/core/providers/tts/alibl_stream.py index 42658008..c636d0cb 100644 --- a/main/xiaozhi-server/core/providers/tts/alibl_stream.py +++ b/main/xiaozhi-server/core/providers/tts/alibl_stream.py @@ -41,8 +41,6 @@ class TTSProvider(TTSProviderBase): # 音频参数配置 self.format = config.get("format", "pcm") - sample_rate = config.get("sample_rate", "24000") - self.sample_rate = int(sample_rate) if sample_rate else 24000 volume = config.get("volume", "50") self.volume = int(volume) if volume else 50 @@ -60,11 +58,6 @@ class TTSProvider(TTSProviderBase): "X-DashScope-DataInspection": "enable", } - # 创建Opus编码器 - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=self.sample_rate, channels=1, frame_size_ms=60 - ) - async def _ensure_connection(self): """确保WebSocket连接可用，支持60秒内连接复用""" try: @@ -245,7 +238,7 @@ class TTSProvider(TTSProviderBase): "text_type": "PlainText", "voice": self.voice, "format": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "volume": self.volume, "rate": self.rate, "pitch": self.pitch, @@ -429,7 +422,7 @@ class TTSProvider(TTSProviderBase): "text_type": "PlainText", "voice": self.voice, "format": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "volume": self.volume, "rate": self.rate, "pitch": self.pitch, diff --git a/main/xiaozhi-server/core/providers/tts/aliyun.py b/main/xiaozhi-server/core/providers/tts/aliyun.py index 1a4cab88..e97c13e4 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun.py @@ -95,8 +95,6 @@ class TTSProvider(TTSProviderBase): self.appkey = config.get("appkey") self.format = config.get("format", "wav") self.audio_file_type = config.get("format", "wav") - sample_rate = config.get("sample_rate", "16000") - self.sample_rate = int(sample_rate) if sample_rate else 16000 if config.get("private_voice"): self.voice = config.get("private_voice") @@ -172,7 +170,7 @@ class TTSProvider(TTSProviderBase): "token": self.token, "text": text, "format": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "voice": self.voice, "volume": self.volume, "speech_rate": self.speech_rate, diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 52284466..6026eafa 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -99,10 +99,6 @@ class TTSProvider(TTSProviderBase): self.format = config.get("format", "pcm") self.audio_file_type = config.get("format", "pcm") - # 采样率配置 - sample_rate = config.get("sample_rate", "16000") - self.sample_rate = int(sample_rate) if sample_rate else 16000 - # 音色配置 - CosyVoice大模型音色 if config.get("private_voice"): self.voice = config.get("private_voice") @@ -134,11 +130,6 @@ class TTSProvider(TTSProviderBase): # 专属tts设置 self.task_id = uuid.uuid4().hex - # 创建Opus编码器 - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=16000, channels=1, frame_size_ms=60 - ) - # Token管理 if self.access_key_id and self.access_key_secret: self._refresh_token() @@ -344,7 +335,7 @@ class TTSProvider(TTSProviderBase): "payload": { "voice": self.voice, "format": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "volume": self.volume, "speech_rate": self.speech_rate, "pitch_rate": self.pitch_rate, @@ -508,7 +499,7 @@ class TTSProvider(TTSProviderBase): "payload": { "voice": self.voice, "format": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "volume": self.volume, "speech_rate": self.speech_rate, "pitch_rate": self.pitch_rate, diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 018215d5..2ad36e05 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -1,17 +1,18 @@ import os import re -import time import uuid import queue import asyncio import threading import traceback + from core.utils import p3 from datetime import datetime from core.utils import textUtils from typing import Callable, Any from abc import ABC, abstractmethod from config.logger import setup_logging +from core.utils import opus_encoder_utils from core.utils.tts import MarkdownCleaner from core.utils.output_counter import add_device_output from core.handle.reportHandle import enqueue_tts_report @@ -97,6 +98,8 @@ class TTSProviderBase(ABC): file_type=self.audio_file_type, is_opus=True, callback=opus_handler, + sample_rate=self.conn.sample_rate, + opus_encoder=self.opus_encoder, ) break else: @@ -158,7 +161,8 @@ class TTSProviderBase(ABC): audio_bytes, file_type=self.audio_file_type, is_opus=True, - callback=lambda data: audio_datas.append(data) + callback=lambda data: audio_datas.append(data), + sample_rate=self.conn.sample_rate, ) return audio_datas else: @@ -214,13 +218,13 @@ class TTSProviderBase(ABC): self, audio_file_path, callback: Callable[[Any], Any] = None ): """音频文件转换为PCM编码""" - return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback) + return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=None) def audio_to_opus_data_stream( self, audio_file_path, callback: Callable[[Any], Any] = None ): """音频文件转换为Opus编码""" - return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback) + return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=self.opus_encoder) def tts_one_sentence( self, @@ -252,6 +256,13 @@ class TTSProviderBase(ABC): async def open_audio_channels(self, conn): self.conn = conn + + # 根据conn的sample_rate创建编码器，如果子类已经创建则不覆盖（IndexTTS接口返回为24kHZ-待重采样处理） + if not hasattr(self, 'opus_encoder') or self.opus_encoder is None: + self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( + sample_rate=conn.sample_rate, channels=1, frame_size_ms=60 + ) + # tts 消化线程 self.tts_priority_thread = threading.Thread( target=self.tts_text_priority_thread, daemon=True diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index d7829e35..b1ba1bed 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -154,16 +154,29 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("private_voice") else: self.voice = config.get("speaker") - speech_rate = config.get("speech_rate", "0") - loudness_rate = config.get("loudness_rate", "0") - pitch = config.get("pitch", "0") - self.speech_rate = int(speech_rate) if speech_rate else 0 - self.loudness_rate = int(loudness_rate) if loudness_rate else 0 - self.pitch = int(pitch) if pitch else 0 - # 多情感音色参数 - self.emotion = config.get("emotion", "neutral") - emotion_scale = config.get("emotion_scale", "4") - self.emotion_scale = int(emotion_scale) if emotion_scale else 4 + + # 默认 audio_params 配置 + default_audio_params = { + "speech_rate": 0, + "loudness_rate": 0 + } + + # 默认 additions 配置 + default_additions = { + "aigc_metadata": {}, + "cache_config": {}, + "post_process": { + "pitch": 0 + } + } + + # 默认 mix_speaker 配置 + default_mix_speaker = {} + + # 合并用户配置 + self.audio_params = {**default_audio_params, **config.get("audio_params", {})} + self.additions = {**default_additions, **config.get("additions", {})} + self.mix_speaker = {**default_mix_speaker, **config.get("mix_speaker", {})} self.ws_url = config.get("ws_url") self.authorization = config.get("authorization") @@ -171,9 +184,7 @@ class TTSProvider(TTSProviderBase): enable_ws_reuse_value = config.get("enable_ws_reuse", True) self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True self.tts_text = "" - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=16000, channels=1, frame_size_ms=60 - ) + model_key_msg = check_model_key("TTS", self.access_token) if model_key_msg: logger.bind(tag=TAG).error(model_key_msg) @@ -181,6 +192,8 @@ class TTSProvider(TTSProviderBase): async def open_audio_channels(self, conn): try: await super().open_audio_channels(conn) + # 更新 audio_params 中的采样率为实际的 conn.sample_rate + self.audio_params["sample_rate"] = conn.sample_rate except Exception as e: logger.bind(tag=TAG).error(f"Failed to open audio channels: {str(e)}") self.ws = None @@ -646,20 +659,18 @@ class TTSProvider(TTSProviderBase): text="", speaker="", audio_format="pcm", - audio_sample_rate=16000, ): - audio_params = { - "format": audio_format, - "sample_rate": audio_sample_rate, - "speech_rate": self.speech_rate, - "loudness_rate": self.loudness_rate + # 构建 req_params + req_params = { + "text": text, + "speaker": speaker, + "audio_params": {**self.audio_params, "format": audio_format}, + "additions": json.dumps(self.additions) } - - # 如果是多情感音色,添加情感参数 - if '_emo_' in self.voice: - if self.emotion: - audio_params["emotion"] = self.emotion - audio_params["emotion_scale"] = self.emotion_scale + + # 如果有 mix_speaker 配置，添加到 req_params + if self.mix_speaker: + req_params["mix_speaker"] = self.mix_speaker return str.encode( json.dumps( @@ -667,17 +678,7 @@ class TTSProvider(TTSProviderBase): "user": {"uid": uid}, "event": event, "namespace": "BidirectionalTTS", - "req_params": { - "text": text, - "speaker": speaker, - "audio_params": audio_params, - "additions": json.dumps({ - "post_process": { - "pitch": self.pitch - } - }) - }, - + "req_params": req_params } ) ) diff --git a/main/xiaozhi-server/core/providers/tts/index_stream.py b/main/xiaozhi-server/core/providers/tts/index_stream.py index 186e1fc8..72d2d68d 100644 --- a/main/xiaozhi-server/core/providers/tts/index_stream.py +++ b/main/xiaozhi-server/core/providers/tts/index_stream.py @@ -174,6 +174,20 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).error(f"TTS请求异常: {e}") self.tts_audio_queue.put((SentenceType.LAST, [], None)) + def audio_to_pcm_data_stream( + self, audio_file_path, callback=None + ): + """音频文件转换为PCM编码，使用24kHz采样率""" + from core.utils.util import audio_to_data_stream + return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=24000, opus_encoder=None) + + def audio_to_opus_data_stream( + self, audio_file_path, callback=None + ): + """音频文件转换为Opus编码，使用24kHz采样率和自己的编码器""" + from core.utils.util import audio_to_data_stream + return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=24000, opus_encoder=self.opus_encoder) + async def close(self): """资源清理""" await super().close() diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index 9b5e946f..7de5f3cd 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -25,11 +25,6 @@ class TTSProvider(TTSProviderBase): self.audio_format = "pcm" self.before_stop_play_files = [] - # 创建Opus编码器 - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=16000, channels=1, frame_size_ms=60 - ) - # PCM缓冲区 self.pcm_buffer = bytearray() @@ -127,7 +122,7 @@ class TTSProvider(TTSProviderBase): "spk_id": self.voice, "frame_durition": 60, "stream": "true", - "target_sr": 16000, + "target_sr": self.conn.sample_rate, "audio_format": "pcm", "instruct_text": "请生成一段自然流畅的语音", } @@ -136,7 +131,7 @@ class TTSProvider(TTSProviderBase): "Content-Type": "application/json", } - # 一帧 PCM 所需字节数：60 ms × 16 kHz × 1 ch × 2 B = 1 920 + # 一帧 PCM 所需字节数：60 ms × sample_rate × 1 ch × 2 B frame_bytes = int( self.opus_encoder.sample_rate * self.opus_encoder.channels # 1 @@ -213,7 +208,7 @@ class TTSProvider(TTSProviderBase): "spk_id": self.voice, "frame_duration": 60, "stream": False, - "target_sr": 16000, + "target_sr": self.conn.sample_rate, "audio_format": self.audio_format, "instruct_text": "请生成一段自然流畅的语音", } diff --git a/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py b/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py index 514d8912..73ce1c1c 100644 --- a/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py +++ b/main/xiaozhi-server/core/providers/tts/minimax_httpstream.py @@ -64,13 +64,17 @@ class TTSProvider(TTSProviderBase): } self.audio_file_type = defult_audio_setting.get("format", "pcm") - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=24000, channels=1, frame_size_ms=60 - ) - # PCM缓冲区 self.pcm_buffer = bytearray() + async def open_audio_channels(self, conn): + """初始化音频通道,并根据conn.sample_rate更新配置""" + # 调用父类方法 + await super().open_audio_channels(conn) + + # 更新audio_setting中的采样率为实际的conn.sample_rate + self.audio_setting["sample_rate"] = conn.sample_rate + def tts_text_priority_thread(self): """流式文本处理线程""" while not self.conn.stop_event.is_set(): diff --git a/main/xiaozhi-server/core/providers/tts/paddle_speech.py b/main/xiaozhi-server/core/providers/tts/paddle_speech.py index 864cf505..48b0bb0c 100644 --- a/main/xiaozhi-server/core/providers/tts/paddle_speech.py +++ b/main/xiaozhi-server/core/providers/tts/paddle_speech.py @@ -25,10 +25,7 @@ class TTSProvider(TTSProviderBase): self.spk_id = int(config.get("private_voice")) else: self.spk_id = int(config.get("spk_id", "0")) - - sample_rate = config.get("sample_rate", 24000) - self.sample_rate = float(sample_rate) if sample_rate else 24000 - + speed = config.get("speed", 1.0) self.speed = float(speed) if speed else 1.0 diff --git a/main/xiaozhi-server/core/providers/tts/siliconflow.py b/main/xiaozhi-server/core/providers/tts/siliconflow.py index b85cebf4..ebc3dbb9 100644 --- a/main/xiaozhi-server/core/providers/tts/siliconflow.py +++ b/main/xiaozhi-server/core/providers/tts/siliconflow.py @@ -13,7 +13,6 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("voice") self.response_format = config.get("response_format", "mp3") self.audio_file_type = config.get("response_format", "mp3") - self.sample_rate = config.get("sample_rate") self.speed = float(config.get("speed", 1.0)) self.gain = config.get("gain") diff --git a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py index dc8d833f..e7d4d4e9 100644 --- a/main/xiaozhi-server/core/providers/tts/xunfei_stream.py +++ b/main/xiaozhi-server/core/providers/tts/xunfei_stream.py @@ -91,9 +91,6 @@ class TTSProvider(TTSProviderBase): # 音频编码配置 self.format = config.get("format", "raw") - sample_rate = config.get("sample_rate", "24000") - self.sample_rate = int(sample_rate) if sample_rate else 24000 - # 口语化配置 self.oral_level = config.get("oral_level", "mid") @@ -113,11 +110,6 @@ class TTSProvider(TTSProviderBase): # 序列号管理 self.text_seq = 0 - # 创建Opus编码器 - self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( - sample_rate=self.sample_rate, channels=1, frame_size_ms=60 - ) - # 验证必需参数 if not all([self.app_id, self.api_key, self.api_secret]): raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret") @@ -507,7 +499,7 @@ class TTSProvider(TTSProviderBase): "rhy": 0, "audio": { "encoding": self.format, - "sample_rate": self.sample_rate, + "sample_rate": self.conn.sample_rate, "channels": 1, "bit_depth": 16, "frame_size": 0 diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 9339da7e..97158b13 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -227,7 +227,7 @@ def extract_json_from_string(input_string): def audio_to_data_stream( - audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None + audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None ) -> None: # 获取文件后缀名 file_type = os.path.splitext(audio_file_path)[1] @@ -238,12 +238,12 @@ def audio_to_data_stream( audio_file_path, format=file_type, parameters=["-nostdin"] ) - # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配） - audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) + # 转换为单声道/指定采样率/16位小端编码（确保与编码器匹配） + audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2) # 获取原始PCM数据（16位小端） raw_data = audio.raw_data - pcm_to_data_stream(raw_data, is_opus, callback) + pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder) async def audio_to_data( @@ -325,7 +325,7 @@ async def audio_to_data( def audio_bytes_to_data_stream( - audio_bytes, file_type, is_opus, callback: Callable[[Any], Any] + audio_bytes, file_type, is_opus, callback: Callable[[Any], Any], sample_rate=16000, opus_encoder=None ) -> None: """ 直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3 @@ -338,18 +338,30 @@ def audio_bytes_to_data_stream( audio = AudioSegment.from_file( BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"] ) - audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) + audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2) raw_data = audio.raw_data - pcm_to_data_stream(raw_data, is_opus, callback) + pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder) -def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None): - # 初始化Opus编码器 - encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO) +def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None): + """ + 将PCM数据流式编码为Opus或直接输出PCM + + Args: + raw_data: PCM原始数据 + is_opus: 是否编码为Opus + callback: 回调函数 + sample_rate: 采样率 + opus_encoder: OpusEncoderUtils对象(推荐提供以保持编码器状态连续) + """ + using_temp_encoder = False + if is_opus and opus_encoder is None: + encoder = opuslib_next.Encoder(sample_rate, 1, opuslib_next.APPLICATION_AUDIO) + using_temp_encoder = True # 编码参数 frame_duration = 60 # 60ms per frame - frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame + frame_size = int(sample_rate * frame_duration / 1000) # samples/frame # 按帧处理所有音频数据（包括最后一帧可能补零） for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample @@ -361,12 +373,17 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = chunk += b"\x00" * (frame_size * 2 - len(chunk)) if is_opus: - # 转换为numpy数组处理 - np_frame = np.frombuffer(chunk, dtype=np.int16) - # 编码Opus数据 - frame_data = encoder.encode(np_frame.tobytes(), frame_size) - callback(frame_data) + if using_temp_encoder: + # 使用临时编码器（仅用于独立音频场景） + np_frame = np.frombuffer(chunk, dtype=np.int16) + frame_data = encoder.encode(np_frame.tobytes(), frame_size) + callback(frame_data) + else: + # 使用外部编码器（TTS流式场景,保持状态连续） + is_last = (i + frame_size * 2 >= len(raw_data)) + opus_encoder.encode_pcm_to_opus_stream(chunk, end_of_stream=is_last, callback=callback) else: + # PCM模式,直接输出 frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk) callback(frame_data)