Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

update: 自定义配置输出音频采样率

Browse Source
This commit is contained in:
Sakura-RanChen
2026-01-14 17:54:50 +08:00
parent 0a8121e849
commit fd1dd39463
17 changed files with 129 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/manager-api/src/main/resources/db/changelog/202601141645.sql
+31
View File
@@ -0,0 +1,31 @@
-- 批量清理 ai_model_provider 中的 sample_rate 字段定义
UPDATE `ai_model_provider` ap
JOIN (
SELECT
id,
JSON_ARRAYAGG(
JSON_OBJECT('key', jt.k, 'label', jt.l, 'type', jt.t)
) AS new_fields
FROM `ai_model_provider`,
JSON_TABLE(`fields`, '$[*]' COLUMNS (
k VARCHAR(50) PATH '$.key',
l VARCHAR(100) PATH '$.label',
t VARCHAR(20) PATH '$.type'
)) AS jt
WHERE `model_type` = 'TTS'
AND jt.k != 'sample_rate'
GROUP BY id
) filtered ON ap.id = filtered.id
SET ap.fields = filtered.new_fields;
-- 清理 config_json 顶层的 sample_rate
UPDATE `ai_model_config`
SET `config_json` = JSON_REMOVE(`config_json`, '$.sample_rate')
WHERE `model_type` = 'TTS'
AND JSON_EXTRACT(`config_json`, '$.sample_rate') IS NOT NULL;
-- 清理Minimax流式TTS的sample_rate参数（位于audio_setting内部）
UPDATE `ai_model_config` SET
`config_json` = JSON_SET(`config_json`, '$.audio_setting', JSON_REMOVE(JSON_EXTRACT(`config_json`, '$.audio_setting'), '$.sample_rate'))
WHERE `id` = 'TTS_MinimaxStreamTTS'
AND JSON_EXTRACT(`config_json`, '$.audio_setting.sample_rate') IS NOT NULL;
main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml
+7
View File
@@ -487,3 +487,10 @@ databaseChangeLog:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202601051433.sql
- changeSet:
id: 202601141645
author: RanChen
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202601141645.sql
main/xiaozhi-server/config.yaml
+2 -7
View File
@@ -89,7 +89,8 @@ xiaozhi:
transport: websocket
audio_params:
format: opus
sample_rate: 16000
# Opus支持的采样率范围为[8000, 12000, 16000, 24000, 48000]
sample_rate: 24000
channels: 1
frame_duration: 60
@@ -837,7 +838,6 @@ TTS:
# - "处理/(chu3)(li3)"
# - "危险/dangerous"
# audio_setting:
# sample_rate: 24000
# bitrate: 128000
# format: "mp3"
# channel: 1
@@ -865,7 +865,6 @@ TTS:
# 以下可不用设置，使用默认设置
# format: wav
# sample_rate: 16000
# volume: 50
# speech_rate: 0
# pitch_rate: 0
@@ -889,7 +888,6 @@ TTS:
host: nls-gateway-cn-beijing.aliyuncs.com
# 以下可不用设置，使用默认设置
# format: pcm # 音频格式：pcm、wav、mp3
# sample_rate: 16000 # 采样率：8000、16000、24000
# volume: 50 # 音量：0-100
# speech_rate: 0 # 语速：-500到500
# pitch_rate: 0 # 语调：-500到500
@@ -1004,7 +1002,6 @@ TTS:
protocol: websocket # protocol choices = ['websocket', 'http']
url: ws://127.0.0.1:8092/paddlespeech/tts/streaming # TTS 服务的 URL 地址，指向本地服务器 [websocket默认ws://127.0.0.1:8092/paddlespeech/tts/streaming，http默认http://127.0.0.1:8090/paddlespeech/tts]
spk_id: 0 # 发音人 ID，0 通常表示默认的发音人
sample_rate: 24000 # 采样率 [websocket默认24000，http默认0 自动选择]
speed: 1.0 # 语速，1.0 表示正常语速，>1 表示加快，<1 表示减慢
volume: 1.0 # 音量，1.0 表示正常音量，>1 表示增大，<1 表示减小
save_path: # 保存路径
@@ -1028,7 +1025,6 @@ TTS:
output_dir: tmp/
# 以下可不用设置，使用默认设置
# format: pcm # 音频格式：pcm、wav、mp3、opus
# sample_rate: 24000 # 采样率：16000, 24000, 48000
# volume: 50 # 音量：0-100
# rate: 1 # 语速：0.5~2
# pitch: 1 # 语调：0.5~2
@@ -1050,7 +1046,6 @@ TTS:
# stop_split: 0 # 关闭服务端拆句 不关闭：0，关闭：1
# remain: 0 # 是否保留原书面语的样子 保留:1, 不保留:0
# format: raw # 音频格式：raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb
# sample_rate: 24000 # 采样率：16000, 8000, 24000
# volume: 50 # 音量：0-100
# speed: 50 # 语速：0-100
# pitch: 50 # 语调：0-100
main/xiaozhi-server/core/connection.py
+5
View File
@@ -85,6 +85,7 @@ class ConnectionHandler:
self.max_output_size = 0
self.chat_history_conf = 0
self.audio_format = "opus"
self.sample_rate = 16000 # 默认采样率，从客户端 hello 消息中动态更新
# 客户端状态相关
self.client_abort = False
@@ -206,6 +207,10 @@ class ConnectionHandler:
self.welcome_msg = self.config["xiaozhi"]
self.welcome_msg["session_id"] = self.session_id
# 从配置中读取采样率
self.sample_rate = self.welcome_msg["audio_params"]["sample_rate"]
self.logger.bind(tag=TAG).info(f"配置输出音频采样率为: {self.sample_rate}")
# 在后台初始化配置和组件（完全不阻塞主循环）
asyncio.create_task(self._background_initialize())
main/xiaozhi-server/core/handle/helloHandle.py
+2 -1
View File
@@ -145,7 +145,8 @@ async def wakeupWordsResponse(conn):
# 获取当前音色
voice = getattr(conn.tts, "voice", "default")
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
# 使用链接的sample_rate
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=conn.sample_rate)
file_path = wakeup_words_config.generate_file_path(voice)
with open(file_path, "wb") as f:
f.write(wav_bytes)
main/xiaozhi-server/core/providers/tts/alibl_stream.py
+2 -9
View File
@@ -41,8 +41,6 @@ class TTSProvider(TTSProviderBase):
# 音频参数配置
self.format = config.get("format", "pcm")
sample_rate = config.get("sample_rate", "24000")
self.sample_rate = int(sample_rate) if sample_rate else 24000
volume = config.get("volume", "50")
self.volume = int(volume) if volume else 50
@@ -60,11 +58,6 @@ class TTSProvider(TTSProviderBase):
"X-DashScope-DataInspection": "enable",
}
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
)
async def _ensure_connection(self):
"""确保WebSocket连接可用，支持60秒内连接复用"""
try:
@@ -245,7 +238,7 @@ class TTSProvider(TTSProviderBase):
"text_type": "PlainText",
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"rate": self.rate,
"pitch": self.pitch,
@@ -429,7 +422,7 @@ class TTSProvider(TTSProviderBase):
"text_type": "PlainText",
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"rate": self.rate,
"pitch": self.pitch,
main/xiaozhi-server/core/providers/tts/aliyun.py
+1 -3
View File
@@ -95,8 +95,6 @@ class TTSProvider(TTSProviderBase):
self.appkey = config.get("appkey")
self.format = config.get("format", "wav")
self.audio_file_type = config.get("format", "wav")
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
if config.get("private_voice"):
self.voice = config.get("private_voice")
@@ -172,7 +170,7 @@ class TTSProvider(TTSProviderBase):
"token": self.token,
"text": text,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"voice": self.voice,
"volume": self.volume,
"speech_rate": self.speech_rate,
main/xiaozhi-server/core/providers/tts/aliyun_stream.py
+2 -11
View File
@@ -99,10 +99,6 @@ class TTSProvider(TTSProviderBase):
self.format = config.get("format", "pcm")
self.audio_file_type = config.get("format", "pcm")
# 采样率配置
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
# 音色配置 - CosyVoice大模型音色
if config.get("private_voice"):
self.voice = config.get("private_voice")
@@ -134,11 +130,6 @@ class TTSProvider(TTSProviderBase):
# 专属tts设置
self.task_id = uuid.uuid4().hex
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# Token管理
if self.access_key_id and self.access_key_secret:
self._refresh_token()
@@ -344,7 +335,7 @@ class TTSProvider(TTSProviderBase):
"payload": {
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"speech_rate": self.speech_rate,
"pitch_rate": self.pitch_rate,
@@ -508,7 +499,7 @@ class TTSProvider(TTSProviderBase):
"payload": {
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"speech_rate": self.speech_rate,
"pitch_rate": self.pitch_rate,
main/xiaozhi-server/core/providers/tts/base.py
+15 -4
View File
@@ -1,17 +1,18 @@
import os
import re
import time
import uuid
import queue
import asyncio
import threading
import traceback
from core.utils import p3
from datetime import datetime
from core.utils import textUtils
from typing import Callable, Any
from abc import ABC, abstractmethod
from config.logger import setup_logging
from core.utils import opus_encoder_utils
from core.utils.tts import MarkdownCleaner
from core.utils.output_counter import add_device_output
from core.handle.reportHandle import enqueue_tts_report
@@ -97,6 +98,8 @@ class TTSProviderBase(ABC):
file_type=self.audio_file_type,
is_opus=True,
callback=opus_handler,
sample_rate=self.conn.sample_rate,
opus_encoder=self.opus_encoder,
)
break
else:
@@ -158,7 +161,8 @@ class TTSProviderBase(ABC):
audio_bytes,
file_type=self.audio_file_type,
is_opus=True,
callback=lambda data: audio_datas.append(data)
callback=lambda data: audio_datas.append(data),
sample_rate=self.conn.sample_rate,
)
return audio_datas
else:
@@ -214,13 +218,13 @@ class TTSProviderBase(ABC):
self, audio_file_path, callback: Callable[[Any], Any] = None
):
"""音频文件转换为PCM编码"""
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback)
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=None)
def audio_to_opus_data_stream(
self, audio_file_path, callback: Callable[[Any], Any] = None
):
"""音频文件转换为Opus编码"""
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback)
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=self.opus_encoder)
def tts_one_sentence(
self,
@@ -252,6 +256,13 @@ class TTSProviderBase(ABC):
async def open_audio_channels(self, conn):
self.conn = conn
# 根据conn的sample_rate创建编码器，如果子类已经创建则不覆盖（IndexTTS接口返回为24kHZ-待重采样处理）
if not hasattr(self, 'opus_encoder') or self.opus_encoder is None:
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=conn.sample_rate, channels=1, frame_size_ms=60
)
# tts 消化线程
self.tts_priority_thread = threading.Thread(
target=self.tts_text_priority_thread, daemon=True
main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py
+2 -5
View File
@@ -171,9 +171,7 @@ class TTSProvider(TTSProviderBase):
enable_ws_reuse_value = config.get("enable_ws_reuse", True)
self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True
self.tts_text = ""
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
model_key_msg = check_model_key("TTS", self.access_token)
if model_key_msg:
logger.bind(tag=TAG).error(model_key_msg)
@@ -646,11 +644,10 @@ class TTSProvider(TTSProviderBase):
text="",
speaker="",
audio_format="pcm",
audio_sample_rate=16000,
):
audio_params = {
"format": audio_format,
"sample_rate": audio_sample_rate,
"sample_rate": self.conn.sample_rate,
"speech_rate": self.speech_rate,
"loudness_rate": self.loudness_rate
}
main/xiaozhi-server/core/providers/tts/index_stream.py
+14
View File
@@ -174,6 +174,20 @@ class TTSProvider(TTSProviderBase):
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
self.tts_audio_queue.put((SentenceType.LAST, [], None))
def audio_to_pcm_data_stream(
self, audio_file_path, callback=None
):
"""音频文件转换为PCM编码，使用24kHz采样率"""
from core.utils.util import audio_to_data_stream
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=24000, opus_encoder=None)
def audio_to_opus_data_stream(
self, audio_file_path, callback=None
):
"""音频文件转换为Opus编码，使用24kHz采样率和自己的编码器"""
from core.utils.util import audio_to_data_stream
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=24000, opus_encoder=self.opus_encoder)
async def close(self):
"""资源清理"""
await super().close()
main/xiaozhi-server/core/providers/tts/linkerai.py
+3 -8
View File
@@ -25,11 +25,6 @@ class TTSProvider(TTSProviderBase):
self.audio_format = "pcm"
self.before_stop_play_files = []
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# PCM缓冲区
self.pcm_buffer = bytearray()
@@ -127,7 +122,7 @@ class TTSProvider(TTSProviderBase):
"spk_id": self.voice,
"frame_durition": 60,
"stream": "true",
"target_sr": 16000,
"target_sr": self.conn.sample_rate,
"audio_format": "pcm",
"instruct_text": "请生成一段自然流畅的语音",
}
@@ -136,7 +131,7 @@ class TTSProvider(TTSProviderBase):
"Content-Type": "application/json",
}
# 一帧 PCM 所需字节数：60 ms &times; 16 kHz &times; 1 ch &times; 2 B = 1 920
# 一帧 PCM 所需字节数：60 ms × sample_rate × 1 ch × 2 B
frame_bytes = int(
self.opus_encoder.sample_rate
* self.opus_encoder.channels # 1
@@ -213,7 +208,7 @@ class TTSProvider(TTSProviderBase):
"spk_id": self.voice,
"frame_duration": 60,
"stream": False,
"target_sr": 16000,
"target_sr": self.conn.sample_rate,
"audio_format": self.audio_format,
"instruct_text": "请生成一段自然流畅的语音",
}
main/xiaozhi-server/core/providers/tts/minimax_httpstream.py
+8 -4
View File
@@ -64,13 +64,17 @@ class TTSProvider(TTSProviderBase):
}
self.audio_file_type = defult_audio_setting.get("format", "pcm")
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=24000, channels=1, frame_size_ms=60
)
# PCM缓冲区
self.pcm_buffer = bytearray()
async def open_audio_channels(self, conn):
"""初始化音频通道,并根据conn.sample_rate更新配置"""
# 调用父类方法
await super().open_audio_channels(conn)
# 更新audio_setting中的采样率为实际的conn.sample_rate
self.audio_setting["sample_rate"] = conn.sample_rate
def tts_text_priority_thread(self):
"""流式文本处理线程"""
while not self.conn.stop_event.is_set():
main/xiaozhi-server/core/providers/tts/paddle_speech.py
+1 -4
View File
@@ -25,10 +25,7 @@ class TTSProvider(TTSProviderBase):
self.spk_id = int(config.get("private_voice"))
else:
self.spk_id = int(config.get("spk_id", "0"))
sample_rate = config.get("sample_rate", 24000)
self.sample_rate = float(sample_rate) if sample_rate else 24000
speed = config.get("speed", 1.0)
self.speed = float(speed) if speed else 1.0
main/xiaozhi-server/core/providers/tts/siliconflow.py
-1
View File
@@ -13,7 +13,6 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("voice")
self.response_format = config.get("response_format", "mp3")
self.audio_file_type = config.get("response_format", "mp3")
self.sample_rate = config.get("sample_rate")
self.speed = float(config.get("speed", 1.0))
self.gain = config.get("gain")
main/xiaozhi-server/core/providers/tts/xunfei_stream.py
+1 -9
View File
@@ -91,9 +91,6 @@ class TTSProvider(TTSProviderBase):
# 音频编码配置
self.format = config.get("format", "raw")
sample_rate = config.get("sample_rate", "24000")
self.sample_rate = int(sample_rate) if sample_rate else 24000
# 口语化配置
self.oral_level = config.get("oral_level", "mid")
@@ -113,11 +110,6 @@ class TTSProvider(TTSProviderBase):
# 序列号管理
self.text_seq = 0
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
)
# 验证必需参数
if not all([self.app_id, self.api_key, self.api_secret]):
raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
@@ -507,7 +499,7 @@ class TTSProvider(TTSProviderBase):
"rhy": 0,
"audio": {
"encoding": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"channels": 1,
"bit_depth": 16,
"frame_size": 0
main/xiaozhi-server/core/utils/util.py
+33 -16
View File
@@ -227,7 +227,7 @@ def extract_json_from_string(input_string):
def audio_to_data_stream(
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None
) -> None:
# 获取文件后缀名
file_type = os.path.splitext(audio_file_path)[1]
@@ -238,12 +238,12 @@ def audio_to_data_stream(
audio_file_path, format=file_type, parameters=["-nostdin"]
)
# 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
# 转换为单声道/指定采样率/16位小端编码（确保与编码器匹配）
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
# 获取原始PCM数据（16位小端）
raw_data = audio.raw_data
pcm_to_data_stream(raw_data, is_opus, callback)
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
async def audio_to_data(
@@ -325,7 +325,7 @@ async def audio_to_data(
def audio_bytes_to_data_stream(
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any], sample_rate=16000, opus_encoder=None
) -> None:
"""
直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3
@@ -338,18 +338,30 @@ def audio_bytes_to_data_stream(
audio = AudioSegment.from_file(
BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"]
)
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
raw_data = audio.raw_data
pcm_to_data_stream(raw_data, is_opus, callback)
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None):
# 初始化Opus编码器
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None):
"""
将PCM数据流式编码为Opus或直接输出PCM
Args:
raw_data: PCM原始数据
is_opus: 是否编码为Opus
callback: 回调函数
sample_rate: 采样率
opus_encoder: OpusEncoderUtils对象(推荐提供以保持编码器状态连续)
"""
using_temp_encoder = False
if is_opus and opus_encoder is None:
encoder = opuslib_next.Encoder(sample_rate, 1, opuslib_next.APPLICATION_AUDIO)
using_temp_encoder = True
# 编码参数
frame_duration = 60 # 60ms per frame
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
frame_size = int(sample_rate * frame_duration / 1000) # samples/frame
# 按帧处理所有音频数据（包括最后一帧可能补零）
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
@@ -361,12 +373,17 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] =
chunk += b"\x00" * (frame_size * 2 - len(chunk))
if is_opus:
# 转换为numpy数组处理
np_frame = np.frombuffer(chunk, dtype=np.int16)
# 编码Opus数据
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
callback(frame_data)
if using_temp_encoder:
# 使用临时编码器（仅用于独立音频场景）
np_frame = np.frombuffer(chunk, dtype=np.int16)
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
callback(frame_data)
else:
# 使用外部编码器（TTS流式场景,保持状态连续）
is_last = (i + frame_size * 2 >= len(raw_data))
opus_encoder.encode_pcm_to_opus_stream(chunk, end_of_stream=is_last, callback=callback)
else:
# PCM模式,直接输出
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
callback(frame_data)