Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-26 00:53:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'py_audio_rate' of https://github.com/xinnan-tech/xiaozhi-esp32-server into new

Browse Source
This commit is contained in:
rainv123
2026-01-26 17:39:29 +08:00
parent 5f39e5c308 3051dc8327
commit 0f16a8d31f
19 changed files with 304 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/connection.py
+5
View File
@@ -85,6 +85,7 @@ class ConnectionHandler:
self.max_output_size = 0
self.chat_history_conf = 0
self.audio_format = "opus"
self.sample_rate = 24000 # 默认采样率，从客户端 hello 消息中动态更新
# 客户端状态相关
self.client_abort = False
@@ -206,6 +207,10 @@ class ConnectionHandler:
self.welcome_msg = self.config["xiaozhi"]
self.welcome_msg["session_id"] = self.session_id
# 从配置中读取采样率
self.sample_rate = self.welcome_msg["audio_params"]["sample_rate"]
self.logger.bind(tag=TAG).info(f"配置输出音频采样率为: {self.sample_rate}")
# 在后台初始化配置和组件（完全不阻塞主循环）
asyncio.create_task(self._background_initialize())
main/xiaozhi-server/core/handle/helloHandle.py
+2 -1
View File
@@ -142,7 +142,8 @@ async def wakeupWordsResponse(conn):
# 获取当前音色
voice = getattr(conn.tts, "voice", "default")
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
# 使用链接的sample_rate
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=conn.sample_rate)
file_path = wakeup_words_config.generate_file_path(voice)
with open(file_path, "wb") as f:
f.write(wav_bytes)
main/xiaozhi-server/core/providers/tts/alibl_stream.py
+2 -9
View File
@@ -41,8 +41,6 @@ class TTSProvider(TTSProviderBase):
# 音频参数配置
self.format = config.get("format", "pcm")
sample_rate = config.get("sample_rate", "24000")
self.sample_rate = int(sample_rate) if sample_rate else 24000
volume = config.get("volume", "50")
self.volume = int(volume) if volume else 50
@@ -60,11 +58,6 @@ class TTSProvider(TTSProviderBase):
"X-DashScope-DataInspection": "enable",
}
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
)
async def _ensure_connection(self):
"""确保WebSocket连接可用，支持60秒内连接复用"""
try:
@@ -245,7 +238,7 @@ class TTSProvider(TTSProviderBase):
"text_type": "PlainText",
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"rate": self.rate,
"pitch": self.pitch,
@@ -429,7 +422,7 @@ class TTSProvider(TTSProviderBase):
"text_type": "PlainText",
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"rate": self.rate,
"pitch": self.pitch,
main/xiaozhi-server/core/providers/tts/aliyun.py
+1 -3
View File
@@ -95,8 +95,6 @@ class TTSProvider(TTSProviderBase):
self.appkey = config.get("appkey")
self.format = config.get("format", "wav")
self.audio_file_type = config.get("format", "wav")
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
if config.get("private_voice"):
self.voice = config.get("private_voice")
@@ -172,7 +170,7 @@ class TTSProvider(TTSProviderBase):
"token": self.token,
"text": text,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"voice": self.voice,
"volume": self.volume,
"speech_rate": self.speech_rate,
main/xiaozhi-server/core/providers/tts/aliyun_stream.py
+2 -11
View File
@@ -99,10 +99,6 @@ class TTSProvider(TTSProviderBase):
self.format = config.get("format", "pcm")
self.audio_file_type = config.get("format", "pcm")
# 采样率配置
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
# 音色配置 - CosyVoice大模型音色
if config.get("private_voice"):
self.voice = config.get("private_voice")
@@ -134,11 +130,6 @@ class TTSProvider(TTSProviderBase):
# 专属tts设置
self.task_id = uuid.uuid4().hex
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# Token管理
if self.access_key_id and self.access_key_secret:
self._refresh_token()
@@ -344,7 +335,7 @@ class TTSProvider(TTSProviderBase):
"payload": {
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"speech_rate": self.speech_rate,
"pitch_rate": self.pitch_rate,
@@ -508,7 +499,7 @@ class TTSProvider(TTSProviderBase):
"payload": {
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"speech_rate": self.speech_rate,
"pitch_rate": self.pitch_rate,
main/xiaozhi-server/core/providers/tts/base.py
+15 -4
View File
@@ -1,17 +1,18 @@
import os
import re
import time
import uuid
import queue
import asyncio
import threading
import traceback
from core.utils import p3
from datetime import datetime
from core.utils import textUtils
from typing import Callable, Any
from abc import ABC, abstractmethod
from config.logger import setup_logging
from core.utils import opus_encoder_utils
from core.utils.tts import MarkdownCleaner
from core.utils.output_counter import add_device_output
from core.handle.reportHandle import enqueue_tts_report
@@ -97,6 +98,8 @@ class TTSProviderBase(ABC):
file_type=self.audio_file_type,
is_opus=True,
callback=opus_handler,
sample_rate=self.conn.sample_rate,
opus_encoder=self.opus_encoder,
)
break
else:
@@ -158,7 +161,8 @@ class TTSProviderBase(ABC):
audio_bytes,
file_type=self.audio_file_type,
is_opus=True,
callback=lambda data: audio_datas.append(data)
callback=lambda data: audio_datas.append(data),
sample_rate=self.conn.sample_rate,
)
return audio_datas
else:
@@ -214,13 +218,13 @@ class TTSProviderBase(ABC):
self, audio_file_path, callback: Callable[[Any], Any] = None
):
"""音频文件转换为PCM编码"""
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback)
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=None)
def audio_to_opus_data_stream(
self, audio_file_path, callback: Callable[[Any], Any] = None
):
"""音频文件转换为Opus编码"""
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback)
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=self.opus_encoder)
def tts_one_sentence(
self,
@@ -252,6 +256,13 @@ class TTSProviderBase(ABC):
async def open_audio_channels(self, conn):
self.conn = conn
# 根据conn的sample_rate创建编码器，如果子类已经创建则不覆盖（IndexTTS接口返回为24kHZ-待重采样处理）
if not hasattr(self, 'opus_encoder') or self.opus_encoder is None:
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=conn.sample_rate, channels=1, frame_size_ms=60
)
# tts 消化线程
self.tts_priority_thread = threading.Thread(
target=self.tts_text_priority_thread, daemon=True
main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py
+37 -36
View File
@@ -154,16 +154,29 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("private_voice")
else:
self.voice = config.get("speaker")
speech_rate = config.get("speech_rate", "0")
loudness_rate = config.get("loudness_rate", "0")
pitch = config.get("pitch", "0")
self.speech_rate = int(speech_rate) if speech_rate else 0
self.loudness_rate = int(loudness_rate) if loudness_rate else 0
self.pitch = int(pitch) if pitch else 0
# 多情感音色参数
self.emotion = config.get("emotion", "neutral")
emotion_scale = config.get("emotion_scale", "4")
self.emotion_scale = int(emotion_scale) if emotion_scale else 4
# 默认 audio_params 配置
default_audio_params = {
"speech_rate": 0,
"loudness_rate": 0
}
# 默认 additions 配置
default_additions = {
"aigc_metadata": {},
"cache_config": {},
"post_process": {
"pitch": 0
}
}
# 默认 mix_speaker 配置
default_mix_speaker = {}
# 合并用户配置
self.audio_params = {**default_audio_params, **config.get("audio_params", {})}
self.additions = {**default_additions, **config.get("additions", {})}
self.mix_speaker = {**default_mix_speaker, **config.get("mix_speaker", {})}
self.ws_url = config.get("ws_url")
self.authorization = config.get("authorization")
@@ -171,9 +184,7 @@ class TTSProvider(TTSProviderBase):
enable_ws_reuse_value = config.get("enable_ws_reuse", True)
self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True
self.tts_text = ""
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
model_key_msg = check_model_key("TTS", self.access_token)
if model_key_msg:
logger.bind(tag=TAG).error(model_key_msg)
@@ -181,6 +192,8 @@ class TTSProvider(TTSProviderBase):
async def open_audio_channels(self, conn):
try:
await super().open_audio_channels(conn)
# 更新 audio_params 中的采样率为实际的 conn.sample_rate
self.audio_params["sample_rate"] = conn.sample_rate
except Exception as e:
logger.bind(tag=TAG).error(f"Failed to open audio channels: {str(e)}")
self.ws = None
@@ -646,20 +659,18 @@ class TTSProvider(TTSProviderBase):
text="",
speaker="",
audio_format="pcm",
audio_sample_rate=16000,
):
audio_params = {
"format": audio_format,
"sample_rate": audio_sample_rate,
"speech_rate": self.speech_rate,
"loudness_rate": self.loudness_rate
# 构建 req_params
req_params = {
"text": text,
"speaker": speaker,
"audio_params": {**self.audio_params, "format": audio_format},
"additions": json.dumps(self.additions)
}
# 如果是多情感音色,添加情感参数
if '_emo_' in self.voice:
if self.emotion:
audio_params["emotion"] = self.emotion
audio_params["emotion_scale"] = self.emotion_scale
# 如果有 mix_speaker 配置，添加到 req_params
if self.mix_speaker:
req_params["mix_speaker"] = self.mix_speaker
return str.encode(
json.dumps(
@@ -667,17 +678,7 @@ class TTSProvider(TTSProviderBase):
"user": {"uid": uid},
"event": event,
"namespace": "BidirectionalTTS",
"req_params": {
"text": text,
"speaker": speaker,
"audio_params": audio_params,
"additions": json.dumps({
"post_process": {
"pitch": self.pitch
}
})
},
"req_params": req_params
}
)
)
main/xiaozhi-server/core/providers/tts/index_stream.py
+14
View File
@@ -174,6 +174,20 @@ class TTSProvider(TTSProviderBase):
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
self.tts_audio_queue.put((SentenceType.LAST, [], None))
def audio_to_pcm_data_stream(
self, audio_file_path, callback=None
):
"""音频文件转换为PCM编码，使用24kHz采样率"""
from core.utils.util import audio_to_data_stream
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=24000, opus_encoder=None)
def audio_to_opus_data_stream(
self, audio_file_path, callback=None
):
"""音频文件转换为Opus编码，使用24kHz采样率和自己的编码器"""
from core.utils.util import audio_to_data_stream
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=24000, opus_encoder=self.opus_encoder)
async def close(self):
"""资源清理"""
await super().close()
main/xiaozhi-server/core/providers/tts/linkerai.py
+3 -8
View File
@@ -25,11 +25,6 @@ class TTSProvider(TTSProviderBase):
self.audio_format = "pcm"
self.before_stop_play_files = []
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# PCM缓冲区
self.pcm_buffer = bytearray()
@@ -127,7 +122,7 @@ class TTSProvider(TTSProviderBase):
"spk_id": self.voice,
"frame_durition": 60,
"stream": "true",
"target_sr": 16000,
"target_sr": self.conn.sample_rate,
"audio_format": "pcm",
"instruct_text": "请生成一段自然流畅的语音",
}
@@ -136,7 +131,7 @@ class TTSProvider(TTSProviderBase):
"Content-Type": "application/json",
}
# 一帧 PCM 所需字节数：60 ms × 16 kHz × 1 ch × 2 B = 1 920
# 一帧 PCM 所需字节数：60 ms × sample_rate × 1 ch × 2 B
frame_bytes = int(
self.opus_encoder.sample_rate
* self.opus_encoder.channels # 1
@@ -213,7 +208,7 @@ class TTSProvider(TTSProviderBase):
"spk_id": self.voice,
"frame_duration": 60,
"stream": False,
"target_sr": 16000,
"target_sr": self.conn.sample_rate,
"audio_format": self.audio_format,
"instruct_text": "请生成一段自然流畅的语音",
}
main/xiaozhi-server/core/providers/tts/minimax_httpstream.py
+8 -4
View File
@@ -64,13 +64,17 @@ class TTSProvider(TTSProviderBase):
}
self.audio_file_type = defult_audio_setting.get("format", "pcm")
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=24000, channels=1, frame_size_ms=60
)
# PCM缓冲区
self.pcm_buffer = bytearray()
async def open_audio_channels(self, conn):
"""初始化音频通道,并根据conn.sample_rate更新配置"""
# 调用父类方法
await super().open_audio_channels(conn)
# 更新audio_setting中的采样率为实际的conn.sample_rate
self.audio_setting["sample_rate"] = conn.sample_rate
def tts_text_priority_thread(self):
"""流式文本处理线程"""
while not self.conn.stop_event.is_set():
main/xiaozhi-server/core/providers/tts/paddle_speech.py
+1 -4
View File
@@ -25,10 +25,7 @@ class TTSProvider(TTSProviderBase):
self.spk_id = int(config.get("private_voice"))
else:
self.spk_id = int(config.get("spk_id", "0"))
sample_rate = config.get("sample_rate", 24000)
self.sample_rate = float(sample_rate) if sample_rate else 24000
speed = config.get("speed", 1.0)
self.speed = float(speed) if speed else 1.0
main/xiaozhi-server/core/providers/tts/siliconflow.py
-1
View File
@@ -13,7 +13,6 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("voice")
self.response_format = config.get("response_format", "mp3")
self.audio_file_type = config.get("response_format", "mp3")
self.sample_rate = config.get("sample_rate")
self.speed = float(config.get("speed", 1.0))
self.gain = config.get("gain")
main/xiaozhi-server/core/providers/tts/xunfei_stream.py
+1 -9
View File
@@ -91,9 +91,6 @@ class TTSProvider(TTSProviderBase):
# 音频编码配置
self.format = config.get("format", "raw")
sample_rate = config.get("sample_rate", "24000")
self.sample_rate = int(sample_rate) if sample_rate else 24000
# 口语化配置
self.oral_level = config.get("oral_level", "mid")
@@ -113,11 +110,6 @@ class TTSProvider(TTSProviderBase):
# 序列号管理
self.text_seq = 0
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
)
# 验证必需参数
if not all([self.app_id, self.api_key, self.api_secret]):
raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
@@ -507,7 +499,7 @@ class TTSProvider(TTSProviderBase):
"rhy": 0,
"audio": {
"encoding": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"channels": 1,
"bit_depth": 16,
"frame_size": 0
main/xiaozhi-server/core/utils/util.py
+33 -16
View File
@@ -227,7 +227,7 @@ def extract_json_from_string(input_string):
def audio_to_data_stream(
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None
) -> None:
# 获取文件后缀名
file_type = os.path.splitext(audio_file_path)[1]
@@ -238,12 +238,12 @@ def audio_to_data_stream(
audio_file_path, format=file_type, parameters=["-nostdin"]
)
# 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
# 转换为单声道/指定采样率/16位小端编码（确保与编码器匹配）
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
# 获取原始PCM数据（16位小端）
raw_data = audio.raw_data
pcm_to_data_stream(raw_data, is_opus, callback)
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
async def audio_to_data(
@@ -325,7 +325,7 @@ async def audio_to_data(
def audio_bytes_to_data_stream(
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any], sample_rate=16000, opus_encoder=None
) -> None:
"""
直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3
@@ -338,18 +338,30 @@ def audio_bytes_to_data_stream(
audio = AudioSegment.from_file(
BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"]
)
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
raw_data = audio.raw_data
pcm_to_data_stream(raw_data, is_opus, callback)
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None):
# 初始化Opus编码器
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None):
"""
将PCM数据流式编码为Opus或直接输出PCM
Args:
raw_data: PCM原始数据
is_opus: 是否编码为Opus
callback: 回调函数
sample_rate: 采样率
opus_encoder: OpusEncoderUtils对象(推荐提供以保持编码器状态连续)
"""
using_temp_encoder = False
if is_opus and opus_encoder is None:
encoder = opuslib_next.Encoder(sample_rate, 1, opuslib_next.APPLICATION_AUDIO)
using_temp_encoder = True
# 编码参数
frame_duration = 60 # 60ms per frame
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
frame_size = int(sample_rate * frame_duration / 1000) # samples/frame
# 按帧处理所有音频数据（包括最后一帧可能补零）
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
@@ -361,12 +373,17 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] =
chunk += b"\x00" * (frame_size * 2 - len(chunk))
if is_opus:
# 转换为numpy数组处理
np_frame = np.frombuffer(chunk, dtype=np.int16)
# 编码Opus数据
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
callback(frame_data)
if using_temp_encoder:
# 使用临时编码器（仅用于独立音频场景）
np_frame = np.frombuffer(chunk, dtype=np.int16)
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
callback(frame_data)
else:
# 使用外部编码器（TTS流式场景,保持状态连续）
is_last = (i + frame_size * 2 >= len(raw_data))
opus_encoder.encode_pcm_to_opus_stream(chunk, end_of_stream=is_last, callback=callback)
else:
# PCM模式,直接输出
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
callback(frame_data)