Merge pull request #2836 from xinnan-tech/py_audio_rate

update: 自定义配置输出音频采样率

main/manager-api/src/main/resources/db/changelog/202601141645.sql

@@ -0,0 +1,31 @@
+-- 批量清理 ai_model_provider 中的 sample_rate 字段定义
+UPDATE `ai_model_provider` ap
+JOIN (
+    SELECT 
+        id,
+        JSON_ARRAYAGG(
+            JSON_OBJECT('key', jt.k, 'label', jt.l, 'type', jt.t)
+        ) AS new_fields
+    FROM `ai_model_provider`,
+         JSON_TABLE(`fields`, '$[*]' COLUMNS (
+             k VARCHAR(50) PATH '$.key',
+             l VARCHAR(100) PATH '$.label',
+             t VARCHAR(20) PATH '$.type'
+         )) AS jt
+    WHERE `model_type` = 'TTS' 
+      AND jt.k != 'sample_rate'
+    GROUP BY id
+) filtered ON ap.id = filtered.id
+SET ap.fields = filtered.new_fields;
+
+-- 清理 config_json 顶层的 sample_rate
+UPDATE `ai_model_config`
+SET `config_json` = JSON_REMOVE(`config_json`, '$.sample_rate')
+WHERE `model_type` = 'TTS'
+  AND JSON_EXTRACT(`config_json`, '$.sample_rate') IS NOT NULL;
+
+-- 清理Minimax流式TTS的sample_rate参数（位于audio_setting内部）
+UPDATE `ai_model_config` SET 
+`config_json` = JSON_SET(`config_json`, '$.audio_setting', JSON_REMOVE(JSON_EXTRACT(`config_json`, '$.audio_setting'), '$.sample_rate'))
+WHERE `id` = 'TTS_MinimaxStreamTTS'
+AND JSON_EXTRACT(`config_json`, '$.audio_setting.sample_rate') IS NOT NULL;

main/manager-api/src/main/resources/db/changelog/202601231530.sql

@@ -0,0 +1,87 @@
+-- 更新HuoshanDoubleStreamTTS供应器配置，将分散的参数改为JSON字典配置
+-- 将 speech_rate, loudness_rate, pitch, emotion, emotion_scale 等参数整合为 audio_params, additions, mix_speaker 三个JSON字典
+
+UPDATE `ai_model_provider`
+SET `fields` = '[
+  {"key": "ws_url", "type": "string", "label": "WebSocket地址"},
+  {"key": "appid", "type": "string", "label": "应用ID"},
+  {"key": "access_token", "type": "string", "label": "访问令牌"},
+  {"key": "resource_id", "type": "string", "label": "资源ID"},
+  {"key": "speaker", "type": "string", "label": "默认音色"},
+  {"key": "enable_ws_reuse", "type": "boolean", "label": "是否开启链接复用", "default": true},
+  {"key": "audio_params", "type": "dict", "label": "音频输出配置"},
+  {"key": "additions", "type": "dict", "label": "高级文本处理配置"},
+  {"key": "mix_speaker", "type": "dict", "label": "混音控制配置"}
+]'
+WHERE `id` = 'SYSTEM_TTS_HSDSTTS';
+
+-- 更新现有配置，将旧的分散参数迁移到新的JSON字典结构
+UPDATE `ai_model_config`
+SET `config_json` = JSON_SET(
+    `config_json`,
+    '$.audio_params', JSON_OBJECT(
+        'speech_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.speech_rate')), ''), '0') AS SIGNED),
+        'loudness_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.loudness_rate')), ''), '0') AS SIGNED)
+    ),
+    '$.additions', JSON_OBJECT(
+        'aigc_metadata', JSON_OBJECT(),
+        'cache_config', JSON_OBJECT(),
+        'post_process', JSON_OBJECT(
+            'pitch', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.pitch')), ''), '0') AS SIGNED)
+        )
+    ),
+    '$.mix_speaker', JSON_OBJECT()
+)
+WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
+
+-- 删除旧的分散参数字段
+UPDATE `ai_model_config`
+SET `config_json` = JSON_REMOVE(
+    `config_json`,
+    '$.speech_rate',
+    '$.loudness_rate',
+    '$.pitch',
+    '$.emotion',
+    '$.emotion_scale'
+)
+WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
+
+-- 更新文档链接和备注说明
+UPDATE `ai_model_config` SET
+`doc_link` = 'https://www.volcengine.com/docs/6561/1329505',
+`remark` = '火山引擎双向流式TTS配置说明：
+1. 访问 https://www.volcengine.com/ 注册并开通火山引擎账号
+2. 访问 https://console.volcengine.com/speech/service/10007 开通语音合成大模型，购买音色
+3. 在页面底部获取appid和access_token
+4. 资源ID固定为：volc.service_type.10029（大模型语音合成及混音）
+5. 链接复用：开启WebSocket连接复用，默认true减少链接损耗（注意：复用后设备处于聆听状态时空闲链接会占并发数）
+
+详细参数文档：https://www.volcengine.com/docs/6561/1329505
+【audio_params】音频输出配置 - 用户可自定义添加火山引擎支持的任何音频参数
+  - speech_rate: 语速(-50~100)，默认0
+  - loudness_rate: 音量(-50~100)，默认0
+  - emotion: 情感类型（仅部分音色支持），可选值：neutral、happy、sad、angry、fearful、disgusted、surprised
+  - emotion_scale: 情感强度(1~5)，默认4
+  示例：{"speech_rate": 10, "loudness_rate": 5, "emotion": "happy", "emotion_scale": 4}
+
+【additions】高级文本处理配置 - 用户可自定义添加火山引擎支持的任何高级参数
+  - post_process.pitch: 音高(-12~12)，默认0
+  - aigc_metadata: AIGC元数据配置
+  - cache_config: 缓存配置
+  示例：{"post_process": {"pitch": 2}, "aigc_metadata": {}, "cache_config": {}}
+
+【mix_speaker】混音控制配置 - 多音色混合（仅 TTS 1.0）
+  示例：
+    {"speakers": [
+      {"source_speaker": "zh_male_bvlazysheep","mix_factor": 0.3}, 
+      {"source_speaker": "BV120_streaming","mix_factor": 0.3}, 
+      {"source_speaker": "zh_male_ahu_conversation_wvae_bigtts","mix_factor": 0.4}
+    ]}
+
+注意：
+- 多情感音色参数（emotion、emotion_scale）仅部分音色支持
+- 相关音色列表：https://www.volcengine.com/docs/6561/1257544
+- 用户可根据火山引擎API文档自行添加更多参数
+- 混音功能主要适用于豆包语音合成模型1.0的音色，使用时需要将req_params.speaker设置为custom_mix_bigtts
+'
+WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';

main/manager-api/src/main/resources/db/changelog/202601261730.sql

@@ -0,0 +1,14 @@
+-- 更新小智参数中的默认采样率从 16000 改为 24000
+UPDATE `sys_params`
+SET `param_value` = '{
+  "type": "hello",
+  "version": 1,
+  "transport": "websocket",
+  "audio_params": {
+    "format": "opus",
+    "sample_rate": 24000,
+    "channels": 1,
+    "frame_duration": 60
+  }
+}'
+WHERE `id` = 309 AND `param_code` = 'xiaozhi';

main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -487,3 +487,24 @@ databaseChangeLog:
         - sqlFile:
             encoding: utf8
             path: classpath:db/changelog/202601051433.sql
+  - changeSet:
+      id: 202601141645
+      author: RanChen
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202601141645.sql
+  - changeSet:
+      id: 202601231530
+      author: RanChen
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202601231530.sql
+  - changeSet:
+      id: 202601261730
+      author: RanChen
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202601261730.sql

main/xiaozhi-server/config.yaml

@@ -89,7 +89,8 @@ xiaozhi:
   transport: websocket
   audio_params:
     format: opus
-    sample_rate: 16000
+    # Opus支持的采样率范围为[8000, 12000, 16000, 24000, 48000]
+    sample_rate: 24000
     channels: 1
     frame_duration: 60
 
@@ -714,13 +715,31 @@ TTS:
     speaker: zh_female_wanwanxiaohe_moon_bigtts
     # 开启WebSocket连接复用，默认复用（注意：复用后设备处于聆听状态时空闲链接会占并发数）
     enable_ws_reuse: True
-    speech_rate: 0
-    loudness_rate: 0
-    pitch: 0
-    # 多情感音色参数，注意：当前仅部分音色支持设置情感。
-    # 相关音色列表：https://www.volcengine.com/docs/6561/1257544
-    emotion: "neutral"  # 情感类型，可选值为：neutral、happy、sad、angry、fearful、disgusted、surprised
-    emotion_scale: 4  # 情感强度，可选值为：1~5，默认值为4
+    # 相关参数文档：https://www.volcengine.com/docs/6561/1329505
+    # 音频输出配置（audio_params）- 用户可自定义添加火山引擎支持的任何音频参数
+    audio_params:
+      speech_rate: 0  # 语速(-50~100)
+      loudness_rate: 0  # 音量(-50~100)
+      # 多情感音色参数，注意：当前仅部分音色支持设置情感。
+      # 相关音色列表：https://www.volcengine.com/docs/6561/1257544
+      # emotion: "neutral"  # 情感类型（仅部分音色支持）：neutral、happy、sad、angry、fearful、disgusted、surprised
+      # emotion_scale: 4  # 情感强度(1~5)
+    # 高级文本处理配置（additions）- 用户可自定义添加火山引擎支持的任何高级参数
+    additions:
+      post_process:
+        pitch: 0  # 音高(-12~12)
+      # aigc_metadata: {}  # AIGC元数据配置
+      # cache_config: {}  # 缓存配置
+    # 混音控制配置（mix_speaker）- 多音色混合（仅 TTS 1.0）
+    # 混音功能主要适用于豆包语音合成模型1.0的音色，使用时需要将req_params.speaker设置为custom_mix_bigtts
+    # mix_speaker:
+    #   speakers:
+    #     - source_speaker: zh_male_bvlazysheep
+    #       mix_factor: 0.3
+    #     - source_speaker: BV120_streaming
+    #       mix_factor: 0.3
+    #     - source_speaker: zh_male_ahu_conversation_wvae_bigtts
+    #       mix_factor: 0.4
   CosyVoiceSiliconflow:
     type: siliconflow
     # 硅基流动TTS
@@ -837,7 +856,6 @@ TTS:
     #       - "处理/(chu3)(li3)"
     #       - "危险/dangerous"
     # audio_setting:
-    #     sample_rate: 24000
     #     bitrate: 128000
     #     format: "mp3"
     #     channel: 1
@@ -865,7 +883,6 @@ TTS:
 
     # 以下可不用设置，使用默认设置
     # format: wav
-    # sample_rate: 16000
     # volume: 50
     # speech_rate: 0
     # pitch_rate: 0
@@ -889,7 +906,6 @@ TTS:
     host: nls-gateway-cn-beijing.aliyuncs.com
     # 以下可不用设置，使用默认设置
     # format: pcm  # 音频格式：pcm、wav、mp3
-    # sample_rate: 16000  # 采样率：8000、16000、24000
     # volume: 50  # 音量：0-100
     # speech_rate: 0  # 语速：-500到500
     # pitch_rate: 0  # 语调：-500到500
@@ -1004,7 +1020,6 @@ TTS:
     protocol: websocket # protocol choices = ['websocket', 'http']
     url: ws://127.0.0.1:8092/paddlespeech/tts/streaming  # TTS 服务的 URL 地址，指向本地服务器 [websocket默认ws://127.0.0.1:8092/paddlespeech/tts/streaming，http默认http://127.0.0.1:8090/paddlespeech/tts]
     spk_id: 0  # 发音人 ID，0 通常表示默认的发音人
-    sample_rate: 24000  # 采样率 [websocket默认24000，http默认0 自动选择]
     speed: 1.0  # 语速，1.0 表示正常语速，>1 表示加快，<1 表示减慢
     volume: 1.0  # 音量，1.0 表示正常音量，>1 表示增大，<1 表示减小
     save_path:   # 保存路径
@@ -1028,7 +1043,6 @@ TTS:
     output_dir: tmp/
     # 以下可不用设置，使用默认设置
     # format: pcm  # 音频格式：pcm、wav、mp3、opus
-    # sample_rate: 24000  # 采样率：16000, 24000, 48000
     # volume: 50  # 音量：0-100
     # rate: 1  # 语速：0.5~2
     # pitch: 1  # 语调：0.5~2
@@ -1050,7 +1064,6 @@ TTS:
     # stop_split: 0  # 关闭服务端拆句 不关闭：0，关闭：1
     # remain: 0  # 是否保留原书面语的样子	保留:1, 不保留:0
     # format: raw  # 音频格式：raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb
-    # sample_rate: 24000  # 采样率：16000, 8000, 24000
     # volume: 50  # 音量：0-100
     # speed: 50  # 语速：0-100
     # pitch: 50  # 语调：0-100

main/xiaozhi-server/core/connection.py

@@ -85,6 +85,7 @@ class ConnectionHandler:
         self.max_output_size = 0
         self.chat_history_conf = 0
         self.audio_format = "opus"
+        self.sample_rate = 24000  # 默认采样率，从客户端 hello 消息中动态更新
 
         # 客户端状态相关
         self.client_abort = False
@@ -206,6 +207,10 @@ class ConnectionHandler:
             self.welcome_msg = self.config["xiaozhi"]
             self.welcome_msg["session_id"] = self.session_id
 
+            # 从配置中读取采样率
+            self.sample_rate = self.welcome_msg["audio_params"]["sample_rate"]
+            self.logger.bind(tag=TAG).info(f"配置输出音频采样率为: {self.sample_rate}")
+
             # 在后台初始化配置和组件（完全不阻塞主循环）
             asyncio.create_task(self._background_initialize())
 

main/xiaozhi-server/core/handle/helloHandle.py

@@ -142,7 +142,8 @@ async def wakeupWordsResponse(conn):
         # 获取当前音色
         voice = getattr(conn.tts, "voice", "default")
 
-        wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
+        # 使用链接的sample_rate
+        wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=conn.sample_rate)
         file_path = wakeup_words_config.generate_file_path(voice)
         with open(file_path, "wb") as f:
             f.write(wav_bytes)

main/xiaozhi-server/core/providers/tts/alibl_stream.py

@@ -41,8 +41,6 @@ class TTSProvider(TTSProviderBase):
 
         # 音频参数配置
         self.format = config.get("format", "pcm")
-        sample_rate = config.get("sample_rate", "24000")
-        self.sample_rate = int(sample_rate) if sample_rate else 24000
 
         volume = config.get("volume", "50")
         self.volume = int(volume) if volume else 50
@@ -60,11 +58,6 @@ class TTSProvider(TTSProviderBase):
             "X-DashScope-DataInspection": "enable",
         }
 
-        # 创建Opus编码器
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=self.sample_rate, channels=1, frame_size_ms=60
-        )
-
     async def _ensure_connection(self):
         """确保WebSocket连接可用，支持60秒内连接复用"""
         try:
@@ -245,7 +238,7 @@ class TTSProvider(TTSProviderBase):
                         "text_type": "PlainText",
                         "voice": self.voice,
                         "format": self.format,
-                        "sample_rate": self.sample_rate,
+                        "sample_rate": self.conn.sample_rate,
                         "volume": self.volume,
                         "rate": self.rate,
                         "pitch": self.pitch,
@@ -429,7 +422,7 @@ class TTSProvider(TTSProviderBase):
                                 "text_type": "PlainText",
                                 "voice": self.voice,
                                 "format": self.format,
-                                "sample_rate": self.sample_rate,
+                                "sample_rate": self.conn.sample_rate,
                                 "volume": self.volume,
                                 "rate": self.rate,
                                 "pitch": self.pitch,

main/xiaozhi-server/core/providers/tts/aliyun.py

@@ -95,8 +95,6 @@ class TTSProvider(TTSProviderBase):
         self.appkey = config.get("appkey")
         self.format = config.get("format", "wav")
         self.audio_file_type = config.get("format", "wav")
-        sample_rate = config.get("sample_rate", "16000")
-        self.sample_rate = int(sample_rate) if sample_rate else 16000
 
         if config.get("private_voice"):
             self.voice = config.get("private_voice")
@@ -172,7 +170,7 @@ class TTSProvider(TTSProviderBase):
             "token": self.token,
             "text": text,
             "format": self.format,
-            "sample_rate": self.sample_rate,
+            "sample_rate": self.conn.sample_rate,
             "voice": self.voice,
             "volume": self.volume,
             "speech_rate": self.speech_rate,

main/xiaozhi-server/core/providers/tts/aliyun_stream.py

@@ -99,10 +99,6 @@ class TTSProvider(TTSProviderBase):
         self.format = config.get("format", "pcm")
         self.audio_file_type = config.get("format", "pcm")
 
-        # 采样率配置
-        sample_rate = config.get("sample_rate", "16000")
-        self.sample_rate = int(sample_rate) if sample_rate else 16000
-
         # 音色配置 - CosyVoice大模型音色
         if config.get("private_voice"):
             self.voice = config.get("private_voice")
@@ -134,11 +130,6 @@ class TTSProvider(TTSProviderBase):
         # 专属tts设置
         self.task_id = uuid.uuid4().hex
 
-        # 创建Opus编码器
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=16000, channels=1, frame_size_ms=60
-        )
-
         # Token管理
         if self.access_key_id and self.access_key_secret:
             self._refresh_token()
@@ -344,7 +335,7 @@ class TTSProvider(TTSProviderBase):
                 "payload": {
                     "voice": self.voice,
                     "format": self.format,
-                    "sample_rate": self.sample_rate,
+                    "sample_rate": self.conn.sample_rate,
                     "volume": self.volume,
                     "speech_rate": self.speech_rate,
                     "pitch_rate": self.pitch_rate,
@@ -508,7 +499,7 @@ class TTSProvider(TTSProviderBase):
                         "payload": {
                             "voice": self.voice,
                             "format": self.format,
-                            "sample_rate": self.sample_rate,
+                            "sample_rate": self.conn.sample_rate,
                             "volume": self.volume,
                             "speech_rate": self.speech_rate,
                             "pitch_rate": self.pitch_rate,

main/xiaozhi-server/core/providers/tts/base.py

@@ -1,17 +1,18 @@
 import os
 import re
-import time
 import uuid
 import queue
 import asyncio
 import threading
 import traceback
+
 from core.utils import p3
 from datetime import datetime
 from core.utils import textUtils
 from typing import Callable, Any
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
+from core.utils import opus_encoder_utils
 from core.utils.tts import MarkdownCleaner
 from core.utils.output_counter import add_device_output
 from core.handle.reportHandle import enqueue_tts_report
@@ -97,6 +98,8 @@ class TTSProviderBase(ABC):
                             file_type=self.audio_file_type,
                             is_opus=True,
                             callback=opus_handler,
+                            sample_rate=self.conn.sample_rate,
+                            opus_encoder=self.opus_encoder,
                         )
                         break
                     else:
@@ -158,7 +161,8 @@ class TTSProviderBase(ABC):
                             audio_bytes,
                             file_type=self.audio_file_type,
                             is_opus=True,
-                            callback=lambda data: audio_datas.append(data)
+                            callback=lambda data: audio_datas.append(data),
+                            sample_rate=self.conn.sample_rate,
                         )
                         return audio_datas
                     else:
@@ -214,13 +218,13 @@ class TTSProviderBase(ABC):
         self, audio_file_path, callback: Callable[[Any], Any] = None
     ):
         """音频文件转换为PCM编码"""
-        return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback)
+        return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=None)
 
     def audio_to_opus_data_stream(
         self, audio_file_path, callback: Callable[[Any], Any] = None
     ):
         """音频文件转换为Opus编码"""
-        return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback)
+        return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=self.opus_encoder)
 
     def tts_one_sentence(
         self,
@@ -252,6 +256,13 @@ class TTSProviderBase(ABC):
 
     async def open_audio_channels(self, conn):
         self.conn = conn
+
+        # 根据conn的sample_rate创建编码器，如果子类已经创建则不覆盖（IndexTTS接口返回为24kHZ-待重采样处理）
+        if not hasattr(self, 'opus_encoder') or self.opus_encoder is None:
+            self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
+                sample_rate=conn.sample_rate, channels=1, frame_size_ms=60
+            )
+
         # tts 消化线程
         self.tts_priority_thread = threading.Thread(
             target=self.tts_text_priority_thread, daemon=True

main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py

@@ -154,16 +154,29 @@ class TTSProvider(TTSProviderBase):
             self.voice = config.get("private_voice")
         else:
             self.voice = config.get("speaker")
-        speech_rate = config.get("speech_rate", "0")
-        loudness_rate = config.get("loudness_rate", "0")
-        pitch = config.get("pitch", "0")
-        self.speech_rate = int(speech_rate) if speech_rate else 0
-        self.loudness_rate = int(loudness_rate) if loudness_rate else 0
-        self.pitch = int(pitch) if pitch else 0
-        # 多情感音色参数
-        self.emotion = config.get("emotion", "neutral")  
-        emotion_scale = config.get("emotion_scale", "4")
-        self.emotion_scale = int(emotion_scale) if emotion_scale else 4
+
+        # 默认 audio_params 配置
+        default_audio_params = {
+            "speech_rate": 0,
+            "loudness_rate": 0
+        }
+
+        # 默认 additions 配置
+        default_additions = {
+            "aigc_metadata": {},
+            "cache_config": {},
+            "post_process": {
+                "pitch": 0
+            }
+        }
+
+        # 默认 mix_speaker 配置
+        default_mix_speaker = {}
+
+        # 合并用户配置
+        self.audio_params = {**default_audio_params, **config.get("audio_params", {})}
+        self.additions = {**default_additions, **config.get("additions", {})}
+        self.mix_speaker = {**default_mix_speaker, **config.get("mix_speaker", {})}
 
         self.ws_url = config.get("ws_url")
         self.authorization = config.get("authorization")
@@ -171,9 +184,7 @@ class TTSProvider(TTSProviderBase):
         enable_ws_reuse_value = config.get("enable_ws_reuse", True)
         self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True
         self.tts_text = ""
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=16000, channels=1, frame_size_ms=60
-        )
+
         model_key_msg = check_model_key("TTS", self.access_token)
         if model_key_msg:
             logger.bind(tag=TAG).error(model_key_msg)
@@ -181,6 +192,8 @@ class TTSProvider(TTSProviderBase):
     async def open_audio_channels(self, conn):
         try:
             await super().open_audio_channels(conn)
+            # 更新 audio_params 中的采样率为实际的 conn.sample_rate
+            self.audio_params["sample_rate"] = conn.sample_rate
         except Exception as e:
             logger.bind(tag=TAG).error(f"Failed to open audio channels: {str(e)}")
             self.ws = None
@@ -646,20 +659,18 @@ class TTSProvider(TTSProviderBase):
         text="",
         speaker="",
         audio_format="pcm",
-        audio_sample_rate=16000,
     ):
-        audio_params = {
-            "format": audio_format,
-            "sample_rate": audio_sample_rate,
-            "speech_rate": self.speech_rate,
-            "loudness_rate": self.loudness_rate
+        # 构建 req_params
+        req_params = {
+            "text": text,
+            "speaker": speaker,
+            "audio_params": {**self.audio_params, "format": audio_format},
+            "additions": json.dumps(self.additions)
         }
-
-        # 如果是多情感音色,添加情感参数
-        if '_emo_' in self.voice:
-            if self.emotion:
-                audio_params["emotion"] = self.emotion
-                audio_params["emotion_scale"] = self.emotion_scale
+        
+        # 如果有 mix_speaker 配置，添加到 req_params
+        if self.mix_speaker:
+            req_params["mix_speaker"] = self.mix_speaker
 
         return str.encode(
             json.dumps(
@@ -667,17 +678,7 @@ class TTSProvider(TTSProviderBase):
                     "user": {"uid": uid},
                     "event": event,
                     "namespace": "BidirectionalTTS",
-                    "req_params": {
-                        "text": text,
-                        "speaker": speaker,
-                        "audio_params": audio_params,
-                        "additions": json.dumps({
-                            "post_process": {
-                                "pitch": self.pitch
-                            }
-                        })
-                    },
-
+                    "req_params": req_params
                 }
             )
         )

main/xiaozhi-server/core/providers/tts/index_stream.py

@@ -174,6 +174,20 @@ class TTSProvider(TTSProviderBase):
             logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
             self.tts_audio_queue.put((SentenceType.LAST, [], None))
 
+    def audio_to_pcm_data_stream(
+        self, audio_file_path, callback=None
+    ):
+        """音频文件转换为PCM编码，使用24kHz采样率"""
+        from core.utils.util import audio_to_data_stream
+        return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=24000, opus_encoder=None)
+
+    def audio_to_opus_data_stream(
+        self, audio_file_path, callback=None
+    ):
+        """音频文件转换为Opus编码，使用24kHz采样率和自己的编码器"""
+        from core.utils.util import audio_to_data_stream
+        return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=24000, opus_encoder=self.opus_encoder)
+
     async def close(self):
         """资源清理"""
         await super().close()

main/xiaozhi-server/core/providers/tts/linkerai.py

@@ -25,11 +25,6 @@ class TTSProvider(TTSProviderBase):
         self.audio_format = "pcm"
         self.before_stop_play_files = []
 
-        # 创建Opus编码器
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=16000, channels=1, frame_size_ms=60
-        )
-
         # PCM缓冲区
         self.pcm_buffer = bytearray()
 
@@ -127,7 +122,7 @@ class TTSProvider(TTSProviderBase):
             "spk_id": self.voice,
             "frame_durition": 60,
             "stream": "true",
-            "target_sr": 16000,
+            "target_sr": self.conn.sample_rate,
             "audio_format": "pcm",
             "instruct_text": "请生成一段自然流畅的语音",
         }
@@ -136,7 +131,7 @@ class TTSProvider(TTSProviderBase):
             "Content-Type": "application/json",
         }
 
-        # 一帧 PCM 所需字节数：60 ms × 16 kHz × 1 ch × 2 B = 1 920
+        # 一帧 PCM 所需字节数：60 ms × sample_rate × 1 ch × 2 B
         frame_bytes = int(
             self.opus_encoder.sample_rate
             * self.opus_encoder.channels  # 1
@@ -213,7 +208,7 @@ class TTSProvider(TTSProviderBase):
             "spk_id": self.voice,
             "frame_duration": 60,
             "stream": False,
-            "target_sr": 16000,
+            "target_sr": self.conn.sample_rate,
             "audio_format": self.audio_format,
             "instruct_text": "请生成一段自然流畅的语音",
         }

main/xiaozhi-server/core/providers/tts/minimax_httpstream.py

@@ -64,13 +64,17 @@ class TTSProvider(TTSProviderBase):
         }
         self.audio_file_type = defult_audio_setting.get("format", "pcm")
 
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=24000, channels=1, frame_size_ms=60
-        )
-
         # PCM缓冲区
         self.pcm_buffer = bytearray()
 
+    async def open_audio_channels(self, conn):
+        """初始化音频通道,并根据conn.sample_rate更新配置"""
+        # 调用父类方法
+        await super().open_audio_channels(conn)
+
+        # 更新audio_setting中的采样率为实际的conn.sample_rate
+        self.audio_setting["sample_rate"] = conn.sample_rate
+
     def tts_text_priority_thread(self):
         """流式文本处理线程"""
         while not self.conn.stop_event.is_set():

main/xiaozhi-server/core/providers/tts/paddle_speech.py

@@ -25,10 +25,7 @@ class TTSProvider(TTSProviderBase):
             self.spk_id = int(config.get("private_voice"))
         else:
             self.spk_id = int(config.get("spk_id", "0"))
-            
-        sample_rate = config.get("sample_rate", 24000)
-        self.sample_rate = float(sample_rate) if sample_rate else 24000
-        
+
         speed = config.get("speed", 1.0)
         self.speed = float(speed) if speed else 1.0
         

main/xiaozhi-server/core/providers/tts/siliconflow.py

@@ -13,7 +13,6 @@ class TTSProvider(TTSProviderBase):
             self.voice = config.get("voice")
         self.response_format = config.get("response_format", "mp3")
         self.audio_file_type = config.get("response_format", "mp3")
-        self.sample_rate = config.get("sample_rate")
         self.speed = float(config.get("speed", 1.0))
         self.gain = config.get("gain")
 

main/xiaozhi-server/core/providers/tts/xunfei_stream.py

@@ -91,9 +91,6 @@ class TTSProvider(TTSProviderBase):
         # 音频编码配置
         self.format = config.get("format", "raw")
 
-        sample_rate = config.get("sample_rate", "24000")
-        self.sample_rate = int(sample_rate) if sample_rate else 24000
-
         # 口语化配置
         self.oral_level = config.get("oral_level", "mid")
 
@@ -113,11 +110,6 @@ class TTSProvider(TTSProviderBase):
         # 序列号管理
         self.text_seq = 0
 
-        # 创建Opus编码器
-        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
-            sample_rate=self.sample_rate, channels=1, frame_size_ms=60
-        )
-
         # 验证必需参数
         if not all([self.app_id, self.api_key, self.api_secret]):
             raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
@@ -507,7 +499,7 @@ class TTSProvider(TTSProviderBase):
                     "rhy": 0,
                     "audio": {
                         "encoding": self.format,
-                        "sample_rate": self.sample_rate,
+                        "sample_rate": self.conn.sample_rate,
                         "channels": 1,
                         "bit_depth": 16,
                         "frame_size": 0

main/xiaozhi-server/core/utils/util.py

@@ -227,7 +227,7 @@ def extract_json_from_string(input_string):
 
 
 def audio_to_data_stream(
-    audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None
+    audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None
 ) -> None:
     # 获取文件后缀名
     file_type = os.path.splitext(audio_file_path)[1]
@@ -238,12 +238,12 @@ def audio_to_data_stream(
         audio_file_path, format=file_type, parameters=["-nostdin"]
     )
 
-    # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
-    audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
+    # 转换为单声道/指定采样率/16位小端编码（确保与编码器匹配）
+    audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
 
     # 获取原始PCM数据（16位小端）
     raw_data = audio.raw_data
-    pcm_to_data_stream(raw_data, is_opus, callback)
+    pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
 
 
 async def audio_to_data(
@@ -325,7 +325,7 @@ async def audio_to_data(
 
 
 def audio_bytes_to_data_stream(
-    audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]
+    audio_bytes, file_type, is_opus, callback: Callable[[Any], Any], sample_rate=16000, opus_encoder=None
 ) -> None:
     """
     直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3
@@ -338,18 +338,30 @@ def audio_bytes_to_data_stream(
         audio = AudioSegment.from_file(
             BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"]
         )
-        audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
+        audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
         raw_data = audio.raw_data
-        pcm_to_data_stream(raw_data, is_opus, callback)
+        pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
 
 
-def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None):
-    # 初始化Opus编码器
-    encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None):
+    """
+    将PCM数据流式编码为Opus或直接输出PCM
+
+    Args:
+        raw_data: PCM原始数据
+        is_opus: 是否编码为Opus
+        callback: 回调函数
+        sample_rate: 采样率
+        opus_encoder: OpusEncoderUtils对象(推荐提供以保持编码器状态连续)
+    """
+    using_temp_encoder = False
+    if is_opus and opus_encoder is None:
+        encoder = opuslib_next.Encoder(sample_rate, 1, opuslib_next.APPLICATION_AUDIO)
+        using_temp_encoder = True
 
     # 编码参数
     frame_duration = 60  # 60ms per frame
-    frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+    frame_size = int(sample_rate * frame_duration / 1000)  # samples/frame
 
     # 按帧处理所有音频数据（包括最后一帧可能补零）
     for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
@@ -361,12 +373,17 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] =
             chunk += b"\x00" * (frame_size * 2 - len(chunk))
 
         if is_opus:
-            # 转换为numpy数组处理
-            np_frame = np.frombuffer(chunk, dtype=np.int16)
-            # 编码Opus数据
-            frame_data = encoder.encode(np_frame.tobytes(), frame_size)
-            callback(frame_data)
+            if using_temp_encoder:
+                # 使用临时编码器（仅用于独立音频场景）
+                np_frame = np.frombuffer(chunk, dtype=np.int16)
+                frame_data = encoder.encode(np_frame.tobytes(), frame_size)
+                callback(frame_data)
+            else:
+                # 使用外部编码器（TTS流式场景,保持状态连续）
+                is_last = (i + frame_size * 2 >= len(raw_data))
+                opus_encoder.encode_pcm_to_opus_stream(chunk, end_of_stream=is_last, callback=callback)
         else:
+            # PCM模式,直接输出
             frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
             callback(frame_data)