Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-28 01:53:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'py_audio_rate' of https://github.com/xinnan-tech/xiaozhi-esp32-server into new

Browse Source
This commit is contained in:
rainv123
2026-01-26 17:39:29 +08:00
parent 5f39e5c308 3051dc8327
commit 0f16a8d31f
19 changed files with 304 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/providers/tts/alibl_stream.py
+2 -9
View File
@@ -41,8 +41,6 @@ class TTSProvider(TTSProviderBase):
# 音频参数配置
self.format = config.get("format", "pcm")
sample_rate = config.get("sample_rate", "24000")
self.sample_rate = int(sample_rate) if sample_rate else 24000
volume = config.get("volume", "50")
self.volume = int(volume) if volume else 50
@@ -60,11 +58,6 @@ class TTSProvider(TTSProviderBase):
"X-DashScope-DataInspection": "enable",
}
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
)
async def _ensure_connection(self):
"""确保WebSocket连接可用，支持60秒内连接复用"""
try:
@@ -245,7 +238,7 @@ class TTSProvider(TTSProviderBase):
"text_type": "PlainText",
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"rate": self.rate,
"pitch": self.pitch,
@@ -429,7 +422,7 @@ class TTSProvider(TTSProviderBase):
"text_type": "PlainText",
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"rate": self.rate,
"pitch": self.pitch,
main/xiaozhi-server/core/providers/tts/aliyun.py
+1 -3
View File
@@ -95,8 +95,6 @@ class TTSProvider(TTSProviderBase):
self.appkey = config.get("appkey")
self.format = config.get("format", "wav")
self.audio_file_type = config.get("format", "wav")
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
if config.get("private_voice"):
self.voice = config.get("private_voice")
@@ -172,7 +170,7 @@ class TTSProvider(TTSProviderBase):
"token": self.token,
"text": text,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"voice": self.voice,
"volume": self.volume,
"speech_rate": self.speech_rate,
main/xiaozhi-server/core/providers/tts/aliyun_stream.py
+2 -11
View File
@@ -99,10 +99,6 @@ class TTSProvider(TTSProviderBase):
self.format = config.get("format", "pcm")
self.audio_file_type = config.get("format", "pcm")
# 采样率配置
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
# 音色配置 - CosyVoice大模型音色
if config.get("private_voice"):
self.voice = config.get("private_voice")
@@ -134,11 +130,6 @@ class TTSProvider(TTSProviderBase):
# 专属tts设置
self.task_id = uuid.uuid4().hex
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# Token管理
if self.access_key_id and self.access_key_secret:
self._refresh_token()
@@ -344,7 +335,7 @@ class TTSProvider(TTSProviderBase):
"payload": {
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"speech_rate": self.speech_rate,
"pitch_rate": self.pitch_rate,
@@ -508,7 +499,7 @@ class TTSProvider(TTSProviderBase):
"payload": {
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"speech_rate": self.speech_rate,
"pitch_rate": self.pitch_rate,
main/xiaozhi-server/core/providers/tts/base.py
+15 -4
View File
@@ -1,17 +1,18 @@
import os
import re
import time
import uuid
import queue
import asyncio
import threading
import traceback
from core.utils import p3
from datetime import datetime
from core.utils import textUtils
from typing import Callable, Any
from abc import ABC, abstractmethod
from config.logger import setup_logging
from core.utils import opus_encoder_utils
from core.utils.tts import MarkdownCleaner
from core.utils.output_counter import add_device_output
from core.handle.reportHandle import enqueue_tts_report
@@ -97,6 +98,8 @@ class TTSProviderBase(ABC):
file_type=self.audio_file_type,
is_opus=True,
callback=opus_handler,
sample_rate=self.conn.sample_rate,
opus_encoder=self.opus_encoder,
)
break
else:
@@ -158,7 +161,8 @@ class TTSProviderBase(ABC):
audio_bytes,
file_type=self.audio_file_type,
is_opus=True,
callback=lambda data: audio_datas.append(data)
callback=lambda data: audio_datas.append(data),
sample_rate=self.conn.sample_rate,
)
return audio_datas
else:
@@ -214,13 +218,13 @@ class TTSProviderBase(ABC):
self, audio_file_path, callback: Callable[[Any], Any] = None
):
"""音频文件转换为PCM编码"""
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback)
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=None)
def audio_to_opus_data_stream(
self, audio_file_path, callback: Callable[[Any], Any] = None
):
"""音频文件转换为Opus编码"""
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback)
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=self.opus_encoder)
def tts_one_sentence(
self,
@@ -252,6 +256,13 @@ class TTSProviderBase(ABC):
async def open_audio_channels(self, conn):
self.conn = conn
# 根据conn的sample_rate创建编码器，如果子类已经创建则不覆盖（IndexTTS接口返回为24kHZ-待重采样处理）
if not hasattr(self, 'opus_encoder') or self.opus_encoder is None:
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=conn.sample_rate, channels=1, frame_size_ms=60
)
# tts 消化线程
self.tts_priority_thread = threading.Thread(
target=self.tts_text_priority_thread, daemon=True
main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py
+37 -36
View File
@@ -154,16 +154,29 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("private_voice")
else:
self.voice = config.get("speaker")
speech_rate = config.get("speech_rate", "0")
loudness_rate = config.get("loudness_rate", "0")
pitch = config.get("pitch", "0")
self.speech_rate = int(speech_rate) if speech_rate else 0
self.loudness_rate = int(loudness_rate) if loudness_rate else 0
self.pitch = int(pitch) if pitch else 0
# 多情感音色参数
self.emotion = config.get("emotion", "neutral")
emotion_scale = config.get("emotion_scale", "4")
self.emotion_scale = int(emotion_scale) if emotion_scale else 4
# 默认 audio_params 配置
default_audio_params = {
"speech_rate": 0,
"loudness_rate": 0
}
# 默认 additions 配置
default_additions = {
"aigc_metadata": {},
"cache_config": {},
"post_process": {
"pitch": 0
}
}
# 默认 mix_speaker 配置
default_mix_speaker = {}
# 合并用户配置
self.audio_params = {**default_audio_params, **config.get("audio_params", {})}
self.additions = {**default_additions, **config.get("additions", {})}
self.mix_speaker = {**default_mix_speaker, **config.get("mix_speaker", {})}
self.ws_url = config.get("ws_url")
self.authorization = config.get("authorization")
@@ -171,9 +184,7 @@ class TTSProvider(TTSProviderBase):
enable_ws_reuse_value = config.get("enable_ws_reuse", True)
self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True
self.tts_text = ""
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
model_key_msg = check_model_key("TTS", self.access_token)
if model_key_msg:
logger.bind(tag=TAG).error(model_key_msg)
@@ -181,6 +192,8 @@ class TTSProvider(TTSProviderBase):
async def open_audio_channels(self, conn):
try:
await super().open_audio_channels(conn)
# 更新 audio_params 中的采样率为实际的 conn.sample_rate
self.audio_params["sample_rate"] = conn.sample_rate
except Exception as e:
logger.bind(tag=TAG).error(f"Failed to open audio channels: {str(e)}")
self.ws = None
@@ -646,20 +659,18 @@ class TTSProvider(TTSProviderBase):
text="",
speaker="",
audio_format="pcm",
audio_sample_rate=16000,
):
audio_params = {
"format": audio_format,
"sample_rate": audio_sample_rate,
"speech_rate": self.speech_rate,
"loudness_rate": self.loudness_rate
# 构建 req_params
req_params = {
"text": text,
"speaker": speaker,
"audio_params": {**self.audio_params, "format": audio_format},
"additions": json.dumps(self.additions)
}
# 如果是多情感音色,添加情感参数
if '_emo_' in self.voice:
if self.emotion:
audio_params["emotion"] = self.emotion
audio_params["emotion_scale"] = self.emotion_scale
# 如果有 mix_speaker 配置，添加到 req_params
if self.mix_speaker:
req_params["mix_speaker"] = self.mix_speaker
return str.encode(
json.dumps(
@@ -667,17 +678,7 @@ class TTSProvider(TTSProviderBase):
"user": {"uid": uid},
"event": event,
"namespace": "BidirectionalTTS",
"req_params": {
"text": text,
"speaker": speaker,
"audio_params": audio_params,
"additions": json.dumps({
"post_process": {
"pitch": self.pitch
}
})
},
"req_params": req_params
}
)
)
main/xiaozhi-server/core/providers/tts/index_stream.py
+14
View File
@@ -174,6 +174,20 @@ class TTSProvider(TTSProviderBase):
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
self.tts_audio_queue.put((SentenceType.LAST, [], None))
def audio_to_pcm_data_stream(
self, audio_file_path, callback=None
):
"""音频文件转换为PCM编码，使用24kHz采样率"""
from core.utils.util import audio_to_data_stream
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=24000, opus_encoder=None)
def audio_to_opus_data_stream(
self, audio_file_path, callback=None
):
"""音频文件转换为Opus编码，使用24kHz采样率和自己的编码器"""
from core.utils.util import audio_to_data_stream
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=24000, opus_encoder=self.opus_encoder)
async def close(self):
"""资源清理"""
await super().close()
main/xiaozhi-server/core/providers/tts/linkerai.py
+3 -8
View File
@@ -25,11 +25,6 @@ class TTSProvider(TTSProviderBase):
self.audio_format = "pcm"
self.before_stop_play_files = []
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# PCM缓冲区
self.pcm_buffer = bytearray()
@@ -127,7 +122,7 @@ class TTSProvider(TTSProviderBase):
"spk_id": self.voice,
"frame_durition": 60,
"stream": "true",
"target_sr": 16000,
"target_sr": self.conn.sample_rate,
"audio_format": "pcm",
"instruct_text": "请生成一段自然流畅的语音",
}
@@ -136,7 +131,7 @@ class TTSProvider(TTSProviderBase):
"Content-Type": "application/json",
}
# 一帧 PCM 所需字节数：60 ms × 16 kHz × 1 ch × 2 B = 1 920
# 一帧 PCM 所需字节数：60 ms × sample_rate × 1 ch × 2 B
frame_bytes = int(
self.opus_encoder.sample_rate
* self.opus_encoder.channels # 1
@@ -213,7 +208,7 @@ class TTSProvider(TTSProviderBase):
"spk_id": self.voice,
"frame_duration": 60,
"stream": False,
"target_sr": 16000,
"target_sr": self.conn.sample_rate,
"audio_format": self.audio_format,
"instruct_text": "请生成一段自然流畅的语音",
}
main/xiaozhi-server/core/providers/tts/minimax_httpstream.py
+8 -4
View File
@@ -64,13 +64,17 @@ class TTSProvider(TTSProviderBase):
}
self.audio_file_type = defult_audio_setting.get("format", "pcm")
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=24000, channels=1, frame_size_ms=60
)
# PCM缓冲区
self.pcm_buffer = bytearray()
async def open_audio_channels(self, conn):
"""初始化音频通道,并根据conn.sample_rate更新配置"""
# 调用父类方法
await super().open_audio_channels(conn)
# 更新audio_setting中的采样率为实际的conn.sample_rate
self.audio_setting["sample_rate"] = conn.sample_rate
def tts_text_priority_thread(self):
"""流式文本处理线程"""
while not self.conn.stop_event.is_set():
main/xiaozhi-server/core/providers/tts/paddle_speech.py
+1 -4
View File
@@ -25,10 +25,7 @@ class TTSProvider(TTSProviderBase):
self.spk_id = int(config.get("private_voice"))
else:
self.spk_id = int(config.get("spk_id", "0"))
sample_rate = config.get("sample_rate", 24000)
self.sample_rate = float(sample_rate) if sample_rate else 24000
speed = config.get("speed", 1.0)
self.speed = float(speed) if speed else 1.0
main/xiaozhi-server/core/providers/tts/siliconflow.py
-1
View File
@@ -13,7 +13,6 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("voice")
self.response_format = config.get("response_format", "mp3")
self.audio_file_type = config.get("response_format", "mp3")
self.sample_rate = config.get("sample_rate")
self.speed = float(config.get("speed", 1.0))
self.gain = config.get("gain")
main/xiaozhi-server/core/providers/tts/xunfei_stream.py
+1 -9
View File
@@ -91,9 +91,6 @@ class TTSProvider(TTSProviderBase):
# 音频编码配置
self.format = config.get("format", "raw")
sample_rate = config.get("sample_rate", "24000")
self.sample_rate = int(sample_rate) if sample_rate else 24000
# 口语化配置
self.oral_level = config.get("oral_level", "mid")
@@ -113,11 +110,6 @@ class TTSProvider(TTSProviderBase):
# 序列号管理
self.text_seq = 0
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
)
# 验证必需参数
if not all([self.app_id, self.api_key, self.api_secret]):
raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
@@ -507,7 +499,7 @@ class TTSProvider(TTSProviderBase):
"rhy": 0,
"audio": {
"encoding": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"channels": 1,
"bit_depth": 16,
"frame_size": 0