mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-28 01:53:53 +08:00
update: 自定义配置输出音频采样率
This commit is contained in:
@@ -0,0 +1,31 @@
|
|||||||
|
-- 批量清理 ai_model_provider 中的 sample_rate 字段定义
|
||||||
|
UPDATE `ai_model_provider` ap
|
||||||
|
JOIN (
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
JSON_ARRAYAGG(
|
||||||
|
JSON_OBJECT('key', jt.k, 'label', jt.l, 'type', jt.t)
|
||||||
|
) AS new_fields
|
||||||
|
FROM `ai_model_provider`,
|
||||||
|
JSON_TABLE(`fields`, '$[*]' COLUMNS (
|
||||||
|
k VARCHAR(50) PATH '$.key',
|
||||||
|
l VARCHAR(100) PATH '$.label',
|
||||||
|
t VARCHAR(20) PATH '$.type'
|
||||||
|
)) AS jt
|
||||||
|
WHERE `model_type` = 'TTS'
|
||||||
|
AND jt.k != 'sample_rate'
|
||||||
|
GROUP BY id
|
||||||
|
) filtered ON ap.id = filtered.id
|
||||||
|
SET ap.fields = filtered.new_fields;
|
||||||
|
|
||||||
|
-- 清理 config_json 顶层的 sample_rate
|
||||||
|
UPDATE `ai_model_config`
|
||||||
|
SET `config_json` = JSON_REMOVE(`config_json`, '$.sample_rate')
|
||||||
|
WHERE `model_type` = 'TTS'
|
||||||
|
AND JSON_EXTRACT(`config_json`, '$.sample_rate') IS NOT NULL;
|
||||||
|
|
||||||
|
-- 清理Minimax流式TTS的sample_rate参数(位于audio_setting内部)
|
||||||
|
UPDATE `ai_model_config` SET
|
||||||
|
`config_json` = JSON_SET(`config_json`, '$.audio_setting', JSON_REMOVE(JSON_EXTRACT(`config_json`, '$.audio_setting'), '$.sample_rate'))
|
||||||
|
WHERE `id` = 'TTS_MinimaxStreamTTS'
|
||||||
|
AND JSON_EXTRACT(`config_json`, '$.audio_setting.sample_rate') IS NOT NULL;
|
||||||
@@ -487,3 +487,10 @@ databaseChangeLog:
|
|||||||
- sqlFile:
|
- sqlFile:
|
||||||
encoding: utf8
|
encoding: utf8
|
||||||
path: classpath:db/changelog/202601051433.sql
|
path: classpath:db/changelog/202601051433.sql
|
||||||
|
- changeSet:
|
||||||
|
id: 202601141645
|
||||||
|
author: RanChen
|
||||||
|
changes:
|
||||||
|
- sqlFile:
|
||||||
|
encoding: utf8
|
||||||
|
path: classpath:db/changelog/202601141645.sql
|
||||||
|
|||||||
@@ -89,7 +89,8 @@ xiaozhi:
|
|||||||
transport: websocket
|
transport: websocket
|
||||||
audio_params:
|
audio_params:
|
||||||
format: opus
|
format: opus
|
||||||
sample_rate: 16000
|
# Opus支持的采样率范围为[8000, 12000, 16000, 24000, 48000]
|
||||||
|
sample_rate: 24000
|
||||||
channels: 1
|
channels: 1
|
||||||
frame_duration: 60
|
frame_duration: 60
|
||||||
|
|
||||||
@@ -837,7 +838,6 @@ TTS:
|
|||||||
# - "处理/(chu3)(li3)"
|
# - "处理/(chu3)(li3)"
|
||||||
# - "危险/dangerous"
|
# - "危险/dangerous"
|
||||||
# audio_setting:
|
# audio_setting:
|
||||||
# sample_rate: 24000
|
|
||||||
# bitrate: 128000
|
# bitrate: 128000
|
||||||
# format: "mp3"
|
# format: "mp3"
|
||||||
# channel: 1
|
# channel: 1
|
||||||
@@ -865,7 +865,6 @@ TTS:
|
|||||||
|
|
||||||
# 以下可不用设置,使用默认设置
|
# 以下可不用设置,使用默认设置
|
||||||
# format: wav
|
# format: wav
|
||||||
# sample_rate: 16000
|
|
||||||
# volume: 50
|
# volume: 50
|
||||||
# speech_rate: 0
|
# speech_rate: 0
|
||||||
# pitch_rate: 0
|
# pitch_rate: 0
|
||||||
@@ -889,7 +888,6 @@ TTS:
|
|||||||
host: nls-gateway-cn-beijing.aliyuncs.com
|
host: nls-gateway-cn-beijing.aliyuncs.com
|
||||||
# 以下可不用设置,使用默认设置
|
# 以下可不用设置,使用默认设置
|
||||||
# format: pcm # 音频格式:pcm、wav、mp3
|
# format: pcm # 音频格式:pcm、wav、mp3
|
||||||
# sample_rate: 16000 # 采样率:8000、16000、24000
|
|
||||||
# volume: 50 # 音量:0-100
|
# volume: 50 # 音量:0-100
|
||||||
# speech_rate: 0 # 语速:-500到500
|
# speech_rate: 0 # 语速:-500到500
|
||||||
# pitch_rate: 0 # 语调:-500到500
|
# pitch_rate: 0 # 语调:-500到500
|
||||||
@@ -1004,7 +1002,6 @@ TTS:
|
|||||||
protocol: websocket # protocol choices = ['websocket', 'http']
|
protocol: websocket # protocol choices = ['websocket', 'http']
|
||||||
url: ws://127.0.0.1:8092/paddlespeech/tts/streaming # TTS 服务的 URL 地址,指向本地服务器 [websocket默认ws://127.0.0.1:8092/paddlespeech/tts/streaming,http默认http://127.0.0.1:8090/paddlespeech/tts]
|
url: ws://127.0.0.1:8092/paddlespeech/tts/streaming # TTS 服务的 URL 地址,指向本地服务器 [websocket默认ws://127.0.0.1:8092/paddlespeech/tts/streaming,http默认http://127.0.0.1:8090/paddlespeech/tts]
|
||||||
spk_id: 0 # 发音人 ID,0 通常表示默认的发音人
|
spk_id: 0 # 发音人 ID,0 通常表示默认的发音人
|
||||||
sample_rate: 24000 # 采样率 [websocket默认24000,http默认0 自动选择]
|
|
||||||
speed: 1.0 # 语速,1.0 表示正常语速,>1 表示加快,<1 表示减慢
|
speed: 1.0 # 语速,1.0 表示正常语速,>1 表示加快,<1 表示减慢
|
||||||
volume: 1.0 # 音量,1.0 表示正常音量,>1 表示增大,<1 表示减小
|
volume: 1.0 # 音量,1.0 表示正常音量,>1 表示增大,<1 表示减小
|
||||||
save_path: # 保存路径
|
save_path: # 保存路径
|
||||||
@@ -1028,7 +1025,6 @@ TTS:
|
|||||||
output_dir: tmp/
|
output_dir: tmp/
|
||||||
# 以下可不用设置,使用默认设置
|
# 以下可不用设置,使用默认设置
|
||||||
# format: pcm # 音频格式:pcm、wav、mp3、opus
|
# format: pcm # 音频格式:pcm、wav、mp3、opus
|
||||||
# sample_rate: 24000 # 采样率:16000, 24000, 48000
|
|
||||||
# volume: 50 # 音量:0-100
|
# volume: 50 # 音量:0-100
|
||||||
# rate: 1 # 语速:0.5~2
|
# rate: 1 # 语速:0.5~2
|
||||||
# pitch: 1 # 语调:0.5~2
|
# pitch: 1 # 语调:0.5~2
|
||||||
@@ -1050,7 +1046,6 @@ TTS:
|
|||||||
# stop_split: 0 # 关闭服务端拆句 不关闭:0,关闭:1
|
# stop_split: 0 # 关闭服务端拆句 不关闭:0,关闭:1
|
||||||
# remain: 0 # 是否保留原书面语的样子 保留:1, 不保留:0
|
# remain: 0 # 是否保留原书面语的样子 保留:1, 不保留:0
|
||||||
# format: raw # 音频格式:raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb
|
# format: raw # 音频格式:raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb
|
||||||
# sample_rate: 24000 # 采样率:16000, 8000, 24000
|
|
||||||
# volume: 50 # 音量:0-100
|
# volume: 50 # 音量:0-100
|
||||||
# speed: 50 # 语速:0-100
|
# speed: 50 # 语速:0-100
|
||||||
# pitch: 50 # 语调:0-100
|
# pitch: 50 # 语调:0-100
|
||||||
|
|||||||
@@ -85,6 +85,7 @@ class ConnectionHandler:
|
|||||||
self.max_output_size = 0
|
self.max_output_size = 0
|
||||||
self.chat_history_conf = 0
|
self.chat_history_conf = 0
|
||||||
self.audio_format = "opus"
|
self.audio_format = "opus"
|
||||||
|
self.sample_rate = 16000 # 默认采样率,从客户端 hello 消息中动态更新
|
||||||
|
|
||||||
# 客户端状态相关
|
# 客户端状态相关
|
||||||
self.client_abort = False
|
self.client_abort = False
|
||||||
@@ -206,6 +207,10 @@ class ConnectionHandler:
|
|||||||
self.welcome_msg = self.config["xiaozhi"]
|
self.welcome_msg = self.config["xiaozhi"]
|
||||||
self.welcome_msg["session_id"] = self.session_id
|
self.welcome_msg["session_id"] = self.session_id
|
||||||
|
|
||||||
|
# 从配置中读取采样率
|
||||||
|
self.sample_rate = self.welcome_msg["audio_params"]["sample_rate"]
|
||||||
|
self.logger.bind(tag=TAG).info(f"配置输出音频采样率为: {self.sample_rate}")
|
||||||
|
|
||||||
# 在后台初始化配置和组件(完全不阻塞主循环)
|
# 在后台初始化配置和组件(完全不阻塞主循环)
|
||||||
asyncio.create_task(self._background_initialize())
|
asyncio.create_task(self._background_initialize())
|
||||||
|
|
||||||
|
|||||||
@@ -145,7 +145,8 @@ async def wakeupWordsResponse(conn):
|
|||||||
# 获取当前音色
|
# 获取当前音色
|
||||||
voice = getattr(conn.tts, "voice", "default")
|
voice = getattr(conn.tts, "voice", "default")
|
||||||
|
|
||||||
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
|
# 使用链接的sample_rate
|
||||||
|
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=conn.sample_rate)
|
||||||
file_path = wakeup_words_config.generate_file_path(voice)
|
file_path = wakeup_words_config.generate_file_path(voice)
|
||||||
with open(file_path, "wb") as f:
|
with open(file_path, "wb") as f:
|
||||||
f.write(wav_bytes)
|
f.write(wav_bytes)
|
||||||
|
|||||||
@@ -41,8 +41,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
|
|
||||||
# 音频参数配置
|
# 音频参数配置
|
||||||
self.format = config.get("format", "pcm")
|
self.format = config.get("format", "pcm")
|
||||||
sample_rate = config.get("sample_rate", "24000")
|
|
||||||
self.sample_rate = int(sample_rate) if sample_rate else 24000
|
|
||||||
|
|
||||||
volume = config.get("volume", "50")
|
volume = config.get("volume", "50")
|
||||||
self.volume = int(volume) if volume else 50
|
self.volume = int(volume) if volume else 50
|
||||||
@@ -60,11 +58,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
"X-DashScope-DataInspection": "enable",
|
"X-DashScope-DataInspection": "enable",
|
||||||
}
|
}
|
||||||
|
|
||||||
# 创建Opus编码器
|
|
||||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
|
||||||
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
|
|
||||||
)
|
|
||||||
|
|
||||||
async def _ensure_connection(self):
|
async def _ensure_connection(self):
|
||||||
"""确保WebSocket连接可用,支持60秒内连接复用"""
|
"""确保WebSocket连接可用,支持60秒内连接复用"""
|
||||||
try:
|
try:
|
||||||
@@ -245,7 +238,7 @@ class TTSProvider(TTSProviderBase):
|
|||||||
"text_type": "PlainText",
|
"text_type": "PlainText",
|
||||||
"voice": self.voice,
|
"voice": self.voice,
|
||||||
"format": self.format,
|
"format": self.format,
|
||||||
"sample_rate": self.sample_rate,
|
"sample_rate": self.conn.sample_rate,
|
||||||
"volume": self.volume,
|
"volume": self.volume,
|
||||||
"rate": self.rate,
|
"rate": self.rate,
|
||||||
"pitch": self.pitch,
|
"pitch": self.pitch,
|
||||||
@@ -429,7 +422,7 @@ class TTSProvider(TTSProviderBase):
|
|||||||
"text_type": "PlainText",
|
"text_type": "PlainText",
|
||||||
"voice": self.voice,
|
"voice": self.voice,
|
||||||
"format": self.format,
|
"format": self.format,
|
||||||
"sample_rate": self.sample_rate,
|
"sample_rate": self.conn.sample_rate,
|
||||||
"volume": self.volume,
|
"volume": self.volume,
|
||||||
"rate": self.rate,
|
"rate": self.rate,
|
||||||
"pitch": self.pitch,
|
"pitch": self.pitch,
|
||||||
|
|||||||
@@ -95,8 +95,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
self.appkey = config.get("appkey")
|
self.appkey = config.get("appkey")
|
||||||
self.format = config.get("format", "wav")
|
self.format = config.get("format", "wav")
|
||||||
self.audio_file_type = config.get("format", "wav")
|
self.audio_file_type = config.get("format", "wav")
|
||||||
sample_rate = config.get("sample_rate", "16000")
|
|
||||||
self.sample_rate = int(sample_rate) if sample_rate else 16000
|
|
||||||
|
|
||||||
if config.get("private_voice"):
|
if config.get("private_voice"):
|
||||||
self.voice = config.get("private_voice")
|
self.voice = config.get("private_voice")
|
||||||
@@ -172,7 +170,7 @@ class TTSProvider(TTSProviderBase):
|
|||||||
"token": self.token,
|
"token": self.token,
|
||||||
"text": text,
|
"text": text,
|
||||||
"format": self.format,
|
"format": self.format,
|
||||||
"sample_rate": self.sample_rate,
|
"sample_rate": self.conn.sample_rate,
|
||||||
"voice": self.voice,
|
"voice": self.voice,
|
||||||
"volume": self.volume,
|
"volume": self.volume,
|
||||||
"speech_rate": self.speech_rate,
|
"speech_rate": self.speech_rate,
|
||||||
|
|||||||
@@ -99,10 +99,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
self.format = config.get("format", "pcm")
|
self.format = config.get("format", "pcm")
|
||||||
self.audio_file_type = config.get("format", "pcm")
|
self.audio_file_type = config.get("format", "pcm")
|
||||||
|
|
||||||
# 采样率配置
|
|
||||||
sample_rate = config.get("sample_rate", "16000")
|
|
||||||
self.sample_rate = int(sample_rate) if sample_rate else 16000
|
|
||||||
|
|
||||||
# 音色配置 - CosyVoice大模型音色
|
# 音色配置 - CosyVoice大模型音色
|
||||||
if config.get("private_voice"):
|
if config.get("private_voice"):
|
||||||
self.voice = config.get("private_voice")
|
self.voice = config.get("private_voice")
|
||||||
@@ -134,11 +130,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
# 专属tts设置
|
# 专属tts设置
|
||||||
self.task_id = uuid.uuid4().hex
|
self.task_id = uuid.uuid4().hex
|
||||||
|
|
||||||
# 创建Opus编码器
|
|
||||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
|
||||||
sample_rate=16000, channels=1, frame_size_ms=60
|
|
||||||
)
|
|
||||||
|
|
||||||
# Token管理
|
# Token管理
|
||||||
if self.access_key_id and self.access_key_secret:
|
if self.access_key_id and self.access_key_secret:
|
||||||
self._refresh_token()
|
self._refresh_token()
|
||||||
@@ -344,7 +335,7 @@ class TTSProvider(TTSProviderBase):
|
|||||||
"payload": {
|
"payload": {
|
||||||
"voice": self.voice,
|
"voice": self.voice,
|
||||||
"format": self.format,
|
"format": self.format,
|
||||||
"sample_rate": self.sample_rate,
|
"sample_rate": self.conn.sample_rate,
|
||||||
"volume": self.volume,
|
"volume": self.volume,
|
||||||
"speech_rate": self.speech_rate,
|
"speech_rate": self.speech_rate,
|
||||||
"pitch_rate": self.pitch_rate,
|
"pitch_rate": self.pitch_rate,
|
||||||
@@ -508,7 +499,7 @@ class TTSProvider(TTSProviderBase):
|
|||||||
"payload": {
|
"payload": {
|
||||||
"voice": self.voice,
|
"voice": self.voice,
|
||||||
"format": self.format,
|
"format": self.format,
|
||||||
"sample_rate": self.sample_rate,
|
"sample_rate": self.conn.sample_rate,
|
||||||
"volume": self.volume,
|
"volume": self.volume,
|
||||||
"speech_rate": self.speech_rate,
|
"speech_rate": self.speech_rate,
|
||||||
"pitch_rate": self.pitch_rate,
|
"pitch_rate": self.pitch_rate,
|
||||||
|
|||||||
@@ -1,17 +1,18 @@
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import time
|
|
||||||
import uuid
|
import uuid
|
||||||
import queue
|
import queue
|
||||||
import asyncio
|
import asyncio
|
||||||
import threading
|
import threading
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
from core.utils import p3
|
from core.utils import p3
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from core.utils import textUtils
|
from core.utils import textUtils
|
||||||
from typing import Callable, Any
|
from typing import Callable, Any
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from config.logger import setup_logging
|
from config.logger import setup_logging
|
||||||
|
from core.utils import opus_encoder_utils
|
||||||
from core.utils.tts import MarkdownCleaner
|
from core.utils.tts import MarkdownCleaner
|
||||||
from core.utils.output_counter import add_device_output
|
from core.utils.output_counter import add_device_output
|
||||||
from core.handle.reportHandle import enqueue_tts_report
|
from core.handle.reportHandle import enqueue_tts_report
|
||||||
@@ -97,6 +98,8 @@ class TTSProviderBase(ABC):
|
|||||||
file_type=self.audio_file_type,
|
file_type=self.audio_file_type,
|
||||||
is_opus=True,
|
is_opus=True,
|
||||||
callback=opus_handler,
|
callback=opus_handler,
|
||||||
|
sample_rate=self.conn.sample_rate,
|
||||||
|
opus_encoder=self.opus_encoder,
|
||||||
)
|
)
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
@@ -158,7 +161,8 @@ class TTSProviderBase(ABC):
|
|||||||
audio_bytes,
|
audio_bytes,
|
||||||
file_type=self.audio_file_type,
|
file_type=self.audio_file_type,
|
||||||
is_opus=True,
|
is_opus=True,
|
||||||
callback=lambda data: audio_datas.append(data)
|
callback=lambda data: audio_datas.append(data),
|
||||||
|
sample_rate=self.conn.sample_rate,
|
||||||
)
|
)
|
||||||
return audio_datas
|
return audio_datas
|
||||||
else:
|
else:
|
||||||
@@ -214,13 +218,13 @@ class TTSProviderBase(ABC):
|
|||||||
self, audio_file_path, callback: Callable[[Any], Any] = None
|
self, audio_file_path, callback: Callable[[Any], Any] = None
|
||||||
):
|
):
|
||||||
"""音频文件转换为PCM编码"""
|
"""音频文件转换为PCM编码"""
|
||||||
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback)
|
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=None)
|
||||||
|
|
||||||
def audio_to_opus_data_stream(
|
def audio_to_opus_data_stream(
|
||||||
self, audio_file_path, callback: Callable[[Any], Any] = None
|
self, audio_file_path, callback: Callable[[Any], Any] = None
|
||||||
):
|
):
|
||||||
"""音频文件转换为Opus编码"""
|
"""音频文件转换为Opus编码"""
|
||||||
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback)
|
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=self.opus_encoder)
|
||||||
|
|
||||||
def tts_one_sentence(
|
def tts_one_sentence(
|
||||||
self,
|
self,
|
||||||
@@ -252,6 +256,13 @@ class TTSProviderBase(ABC):
|
|||||||
|
|
||||||
async def open_audio_channels(self, conn):
|
async def open_audio_channels(self, conn):
|
||||||
self.conn = conn
|
self.conn = conn
|
||||||
|
|
||||||
|
# 根据conn的sample_rate创建编码器,如果子类已经创建则不覆盖(IndexTTS接口返回为24kHZ-待重采样处理)
|
||||||
|
if not hasattr(self, 'opus_encoder') or self.opus_encoder is None:
|
||||||
|
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||||
|
sample_rate=conn.sample_rate, channels=1, frame_size_ms=60
|
||||||
|
)
|
||||||
|
|
||||||
# tts 消化线程
|
# tts 消化线程
|
||||||
self.tts_priority_thread = threading.Thread(
|
self.tts_priority_thread = threading.Thread(
|
||||||
target=self.tts_text_priority_thread, daemon=True
|
target=self.tts_text_priority_thread, daemon=True
|
||||||
|
|||||||
@@ -171,9 +171,7 @@ class TTSProvider(TTSProviderBase):
|
|||||||
enable_ws_reuse_value = config.get("enable_ws_reuse", True)
|
enable_ws_reuse_value = config.get("enable_ws_reuse", True)
|
||||||
self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True
|
self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True
|
||||||
self.tts_text = ""
|
self.tts_text = ""
|
||||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
|
||||||
sample_rate=16000, channels=1, frame_size_ms=60
|
|
||||||
)
|
|
||||||
model_key_msg = check_model_key("TTS", self.access_token)
|
model_key_msg = check_model_key("TTS", self.access_token)
|
||||||
if model_key_msg:
|
if model_key_msg:
|
||||||
logger.bind(tag=TAG).error(model_key_msg)
|
logger.bind(tag=TAG).error(model_key_msg)
|
||||||
@@ -646,11 +644,10 @@ class TTSProvider(TTSProviderBase):
|
|||||||
text="",
|
text="",
|
||||||
speaker="",
|
speaker="",
|
||||||
audio_format="pcm",
|
audio_format="pcm",
|
||||||
audio_sample_rate=16000,
|
|
||||||
):
|
):
|
||||||
audio_params = {
|
audio_params = {
|
||||||
"format": audio_format,
|
"format": audio_format,
|
||||||
"sample_rate": audio_sample_rate,
|
"sample_rate": self.conn.sample_rate,
|
||||||
"speech_rate": self.speech_rate,
|
"speech_rate": self.speech_rate,
|
||||||
"loudness_rate": self.loudness_rate
|
"loudness_rate": self.loudness_rate
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -174,6 +174,20 @@ class TTSProvider(TTSProviderBase):
|
|||||||
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
|
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
|
||||||
self.tts_audio_queue.put((SentenceType.LAST, [], None))
|
self.tts_audio_queue.put((SentenceType.LAST, [], None))
|
||||||
|
|
||||||
|
def audio_to_pcm_data_stream(
|
||||||
|
self, audio_file_path, callback=None
|
||||||
|
):
|
||||||
|
"""音频文件转换为PCM编码,使用24kHz采样率"""
|
||||||
|
from core.utils.util import audio_to_data_stream
|
||||||
|
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=24000, opus_encoder=None)
|
||||||
|
|
||||||
|
def audio_to_opus_data_stream(
|
||||||
|
self, audio_file_path, callback=None
|
||||||
|
):
|
||||||
|
"""音频文件转换为Opus编码,使用24kHz采样率和自己的编码器"""
|
||||||
|
from core.utils.util import audio_to_data_stream
|
||||||
|
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=24000, opus_encoder=self.opus_encoder)
|
||||||
|
|
||||||
async def close(self):
|
async def close(self):
|
||||||
"""资源清理"""
|
"""资源清理"""
|
||||||
await super().close()
|
await super().close()
|
||||||
|
|||||||
@@ -25,11 +25,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
self.audio_format = "pcm"
|
self.audio_format = "pcm"
|
||||||
self.before_stop_play_files = []
|
self.before_stop_play_files = []
|
||||||
|
|
||||||
# 创建Opus编码器
|
|
||||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
|
||||||
sample_rate=16000, channels=1, frame_size_ms=60
|
|
||||||
)
|
|
||||||
|
|
||||||
# PCM缓冲区
|
# PCM缓冲区
|
||||||
self.pcm_buffer = bytearray()
|
self.pcm_buffer = bytearray()
|
||||||
|
|
||||||
@@ -127,7 +122,7 @@ class TTSProvider(TTSProviderBase):
|
|||||||
"spk_id": self.voice,
|
"spk_id": self.voice,
|
||||||
"frame_durition": 60,
|
"frame_durition": 60,
|
||||||
"stream": "true",
|
"stream": "true",
|
||||||
"target_sr": 16000,
|
"target_sr": self.conn.sample_rate,
|
||||||
"audio_format": "pcm",
|
"audio_format": "pcm",
|
||||||
"instruct_text": "请生成一段自然流畅的语音",
|
"instruct_text": "请生成一段自然流畅的语音",
|
||||||
}
|
}
|
||||||
@@ -136,7 +131,7 @@ class TTSProvider(TTSProviderBase):
|
|||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
}
|
}
|
||||||
|
|
||||||
# 一帧 PCM 所需字节数:60 ms × 16 kHz × 1 ch × 2 B = 1 920
|
# 一帧 PCM 所需字节数:60 ms × sample_rate × 1 ch × 2 B
|
||||||
frame_bytes = int(
|
frame_bytes = int(
|
||||||
self.opus_encoder.sample_rate
|
self.opus_encoder.sample_rate
|
||||||
* self.opus_encoder.channels # 1
|
* self.opus_encoder.channels # 1
|
||||||
@@ -213,7 +208,7 @@ class TTSProvider(TTSProviderBase):
|
|||||||
"spk_id": self.voice,
|
"spk_id": self.voice,
|
||||||
"frame_duration": 60,
|
"frame_duration": 60,
|
||||||
"stream": False,
|
"stream": False,
|
||||||
"target_sr": 16000,
|
"target_sr": self.conn.sample_rate,
|
||||||
"audio_format": self.audio_format,
|
"audio_format": self.audio_format,
|
||||||
"instruct_text": "请生成一段自然流畅的语音",
|
"instruct_text": "请生成一段自然流畅的语音",
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -64,13 +64,17 @@ class TTSProvider(TTSProviderBase):
|
|||||||
}
|
}
|
||||||
self.audio_file_type = defult_audio_setting.get("format", "pcm")
|
self.audio_file_type = defult_audio_setting.get("format", "pcm")
|
||||||
|
|
||||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
|
||||||
sample_rate=24000, channels=1, frame_size_ms=60
|
|
||||||
)
|
|
||||||
|
|
||||||
# PCM缓冲区
|
# PCM缓冲区
|
||||||
self.pcm_buffer = bytearray()
|
self.pcm_buffer = bytearray()
|
||||||
|
|
||||||
|
async def open_audio_channels(self, conn):
|
||||||
|
"""初始化音频通道,并根据conn.sample_rate更新配置"""
|
||||||
|
# 调用父类方法
|
||||||
|
await super().open_audio_channels(conn)
|
||||||
|
|
||||||
|
# 更新audio_setting中的采样率为实际的conn.sample_rate
|
||||||
|
self.audio_setting["sample_rate"] = conn.sample_rate
|
||||||
|
|
||||||
def tts_text_priority_thread(self):
|
def tts_text_priority_thread(self):
|
||||||
"""流式文本处理线程"""
|
"""流式文本处理线程"""
|
||||||
while not self.conn.stop_event.is_set():
|
while not self.conn.stop_event.is_set():
|
||||||
|
|||||||
@@ -25,10 +25,7 @@ class TTSProvider(TTSProviderBase):
|
|||||||
self.spk_id = int(config.get("private_voice"))
|
self.spk_id = int(config.get("private_voice"))
|
||||||
else:
|
else:
|
||||||
self.spk_id = int(config.get("spk_id", "0"))
|
self.spk_id = int(config.get("spk_id", "0"))
|
||||||
|
|
||||||
sample_rate = config.get("sample_rate", 24000)
|
|
||||||
self.sample_rate = float(sample_rate) if sample_rate else 24000
|
|
||||||
|
|
||||||
speed = config.get("speed", 1.0)
|
speed = config.get("speed", 1.0)
|
||||||
self.speed = float(speed) if speed else 1.0
|
self.speed = float(speed) if speed else 1.0
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
self.voice = config.get("voice")
|
self.voice = config.get("voice")
|
||||||
self.response_format = config.get("response_format", "mp3")
|
self.response_format = config.get("response_format", "mp3")
|
||||||
self.audio_file_type = config.get("response_format", "mp3")
|
self.audio_file_type = config.get("response_format", "mp3")
|
||||||
self.sample_rate = config.get("sample_rate")
|
|
||||||
self.speed = float(config.get("speed", 1.0))
|
self.speed = float(config.get("speed", 1.0))
|
||||||
self.gain = config.get("gain")
|
self.gain = config.get("gain")
|
||||||
|
|
||||||
|
|||||||
@@ -91,9 +91,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
# 音频编码配置
|
# 音频编码配置
|
||||||
self.format = config.get("format", "raw")
|
self.format = config.get("format", "raw")
|
||||||
|
|
||||||
sample_rate = config.get("sample_rate", "24000")
|
|
||||||
self.sample_rate = int(sample_rate) if sample_rate else 24000
|
|
||||||
|
|
||||||
# 口语化配置
|
# 口语化配置
|
||||||
self.oral_level = config.get("oral_level", "mid")
|
self.oral_level = config.get("oral_level", "mid")
|
||||||
|
|
||||||
@@ -113,11 +110,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
# 序列号管理
|
# 序列号管理
|
||||||
self.text_seq = 0
|
self.text_seq = 0
|
||||||
|
|
||||||
# 创建Opus编码器
|
|
||||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
|
||||||
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
|
|
||||||
)
|
|
||||||
|
|
||||||
# 验证必需参数
|
# 验证必需参数
|
||||||
if not all([self.app_id, self.api_key, self.api_secret]):
|
if not all([self.app_id, self.api_key, self.api_secret]):
|
||||||
raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
|
raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
|
||||||
@@ -507,7 +499,7 @@ class TTSProvider(TTSProviderBase):
|
|||||||
"rhy": 0,
|
"rhy": 0,
|
||||||
"audio": {
|
"audio": {
|
||||||
"encoding": self.format,
|
"encoding": self.format,
|
||||||
"sample_rate": self.sample_rate,
|
"sample_rate": self.conn.sample_rate,
|
||||||
"channels": 1,
|
"channels": 1,
|
||||||
"bit_depth": 16,
|
"bit_depth": 16,
|
||||||
"frame_size": 0
|
"frame_size": 0
|
||||||
|
|||||||
@@ -227,7 +227,7 @@ def extract_json_from_string(input_string):
|
|||||||
|
|
||||||
|
|
||||||
def audio_to_data_stream(
|
def audio_to_data_stream(
|
||||||
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None
|
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None
|
||||||
) -> None:
|
) -> None:
|
||||||
# 获取文件后缀名
|
# 获取文件后缀名
|
||||||
file_type = os.path.splitext(audio_file_path)[1]
|
file_type = os.path.splitext(audio_file_path)[1]
|
||||||
@@ -238,12 +238,12 @@ def audio_to_data_stream(
|
|||||||
audio_file_path, format=file_type, parameters=["-nostdin"]
|
audio_file_path, format=file_type, parameters=["-nostdin"]
|
||||||
)
|
)
|
||||||
|
|
||||||
# 转换为单声道/16kHz采样率/16位小端编码(确保与编码器匹配)
|
# 转换为单声道/指定采样率/16位小端编码(确保与编码器匹配)
|
||||||
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
|
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
|
||||||
|
|
||||||
# 获取原始PCM数据(16位小端)
|
# 获取原始PCM数据(16位小端)
|
||||||
raw_data = audio.raw_data
|
raw_data = audio.raw_data
|
||||||
pcm_to_data_stream(raw_data, is_opus, callback)
|
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
|
||||||
|
|
||||||
|
|
||||||
async def audio_to_data(
|
async def audio_to_data(
|
||||||
@@ -325,7 +325,7 @@ async def audio_to_data(
|
|||||||
|
|
||||||
|
|
||||||
def audio_bytes_to_data_stream(
|
def audio_bytes_to_data_stream(
|
||||||
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]
|
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any], sample_rate=16000, opus_encoder=None
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
直接用音频二进制数据转为opus/pcm数据,支持wav、mp3、p3
|
直接用音频二进制数据转为opus/pcm数据,支持wav、mp3、p3
|
||||||
@@ -338,18 +338,30 @@ def audio_bytes_to_data_stream(
|
|||||||
audio = AudioSegment.from_file(
|
audio = AudioSegment.from_file(
|
||||||
BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"]
|
BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"]
|
||||||
)
|
)
|
||||||
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
|
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
|
||||||
raw_data = audio.raw_data
|
raw_data = audio.raw_data
|
||||||
pcm_to_data_stream(raw_data, is_opus, callback)
|
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
|
||||||
|
|
||||||
|
|
||||||
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None):
|
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None):
|
||||||
# 初始化Opus编码器
|
"""
|
||||||
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
|
将PCM数据流式编码为Opus或直接输出PCM
|
||||||
|
|
||||||
|
Args:
|
||||||
|
raw_data: PCM原始数据
|
||||||
|
is_opus: 是否编码为Opus
|
||||||
|
callback: 回调函数
|
||||||
|
sample_rate: 采样率
|
||||||
|
opus_encoder: OpusEncoderUtils对象(推荐提供以保持编码器状态连续)
|
||||||
|
"""
|
||||||
|
using_temp_encoder = False
|
||||||
|
if is_opus and opus_encoder is None:
|
||||||
|
encoder = opuslib_next.Encoder(sample_rate, 1, opuslib_next.APPLICATION_AUDIO)
|
||||||
|
using_temp_encoder = True
|
||||||
|
|
||||||
# 编码参数
|
# 编码参数
|
||||||
frame_duration = 60 # 60ms per frame
|
frame_duration = 60 # 60ms per frame
|
||||||
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
|
frame_size = int(sample_rate * frame_duration / 1000) # samples/frame
|
||||||
|
|
||||||
# 按帧处理所有音频数据(包括最后一帧可能补零)
|
# 按帧处理所有音频数据(包括最后一帧可能补零)
|
||||||
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
|
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
|
||||||
@@ -361,12 +373,17 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] =
|
|||||||
chunk += b"\x00" * (frame_size * 2 - len(chunk))
|
chunk += b"\x00" * (frame_size * 2 - len(chunk))
|
||||||
|
|
||||||
if is_opus:
|
if is_opus:
|
||||||
# 转换为numpy数组处理
|
if using_temp_encoder:
|
||||||
np_frame = np.frombuffer(chunk, dtype=np.int16)
|
# 使用临时编码器(仅用于独立音频场景)
|
||||||
# 编码Opus数据
|
np_frame = np.frombuffer(chunk, dtype=np.int16)
|
||||||
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
|
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
|
||||||
callback(frame_data)
|
callback(frame_data)
|
||||||
|
else:
|
||||||
|
# 使用外部编码器(TTS流式场景,保持状态连续)
|
||||||
|
is_last = (i + frame_size * 2 >= len(raw_data))
|
||||||
|
opus_encoder.encode_pcm_to_opus_stream(chunk, end_of_stream=is_last, callback=callback)
|
||||||
else:
|
else:
|
||||||
|
# PCM模式,直接输出
|
||||||
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
|
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
|
||||||
callback(frame_data)
|
callback(frame_data)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user