mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-29 03:03:56 +08:00
update:优化
This commit is contained in:
hrz
@@ -239,7 +239,7 @@ class ASRProvider(ASRProviderBase):
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
if self._is_token_expired():
|
||||
logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...")
|
||||
logger.warning("Token已过期，正在自动刷新...")
|
||||
self._refresh_token()
|
||||
|
||||
file_path = None
|
||||
|
||||
@@ -30,15 +30,25 @@ class ASRProviderBase(ABC):
|
||||
@staticmethod
|
||||
def decode_opus(opus_data: List[bytes]) -> bytes:
|
||||
"""将Opus音频数据解码为PCM数据"""
|
||||
try:
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
buffer_size = 960 # 每次处理960个采样点
|
||||
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
# 使用较小的缓冲区大小进行处理
|
||||
pcm_frame = decoder.decode(opus_packet, buffer_size)
|
||||
if pcm_frame:
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).warning(f"Opus解码错误，跳过当前数据包: {e}")
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"音频处理错误: {e}", exc_info=True)
|
||||
continue
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
return pcm_data
|
||||
return pcm_data
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}", exc_info=True)
|
||||
return []
|
||||
|
||||
@@ -9,10 +9,14 @@ import uuid
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from funasr import AutoModel
|
||||
from funasr.utils.postprocess_utils import rich_transcription_postprocess
|
||||
import shutil
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
MAX_RETRIES = 2
|
||||
RETRY_DELAY = 1 # 重试延迟（秒）
|
||||
|
||||
|
||||
# 捕获标准输出
|
||||
class CaptureOutput:
|
||||
@@ -68,46 +72,69 @@ class ASRProvider(ASRProviderBase):
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""语音转文本主处理逻辑"""
|
||||
file_path = None
|
||||
try:
|
||||
# 合并所有opus数据包
|
||||
if self.audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
retry_count = 0
|
||||
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
while retry_count < MAX_RETRIES:
|
||||
try:
|
||||
# 合并所有opus数据包
|
||||
if self.audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
|
||||
# 判断是否保存为WAV文件
|
||||
if self.delete_audio_file:
|
||||
pass
|
||||
else:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
|
||||
# 语音识别
|
||||
start_time = time.time()
|
||||
result = self.model.generate(
|
||||
input=combined_pcm_data,
|
||||
cache={},
|
||||
language="auto",
|
||||
use_itn=True,
|
||||
batch_size_s=60,
|
||||
)
|
||||
text = rich_transcription_postprocess(result[0]["text"])
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
|
||||
)
|
||||
# 检查磁盘空间
|
||||
if not self.delete_audio_file:
|
||||
free_space = shutil.disk_usage(self.output_dir).free
|
||||
if free_space < len(combined_pcm_data) * 2: # 预留2倍空间
|
||||
raise OSError("磁盘空间不足")
|
||||
|
||||
return text, file_path
|
||||
# 判断是否保存为WAV文件
|
||||
if self.delete_audio_file:
|
||||
pass
|
||||
else:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", file_path
|
||||
# 语音识别
|
||||
start_time = time.time()
|
||||
result = self.model.generate(
|
||||
input=combined_pcm_data,
|
||||
cache={},
|
||||
language="auto",
|
||||
use_itn=True,
|
||||
batch_size_s=60,
|
||||
)
|
||||
text = rich_transcription_postprocess(result[0]["text"])
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
|
||||
)
|
||||
|
||||
# finally:
|
||||
# # 文件清理逻辑
|
||||
# if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
# try:
|
||||
# os.remove(file_path)
|
||||
# logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
# except Exception as e:
|
||||
# logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
|
||||
return text, file_path
|
||||
|
||||
except OSError as e:
|
||||
retry_count += 1
|
||||
if retry_count >= MAX_RETRIES:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"语音识别失败（已重试{retry_count}次）: {e}", exc_info=True
|
||||
)
|
||||
return "", file_path
|
||||
logger.bind(tag=TAG).warning(
|
||||
f"语音识别失败，正在重试（{retry_count}/{MAX_RETRIES}）: {e}"
|
||||
)
|
||||
time.sleep(RETRY_DELAY)
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", file_path
|
||||
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"文件删除失败: {file_path} | 错误: {e}"
|
||||
)
|
||||
|
||||
@@ -72,6 +72,10 @@ class IntentProvider(IntentProviderBase):
|
||||
'返回: {"function_call": {"name": "get_time"}}\n'
|
||||
"```\n"
|
||||
"```\n"
|
||||
"用户: 当前电池电量是多少？\n"
|
||||
'返回: {"function_call": {"name": "get_battery_level", "arguments": {"response_success": "当前电池电量为{value}%", "response_failure": "无法获取Battery的当前电量百分比"}}}\n'
|
||||
"```\n"
|
||||
"```\n"
|
||||
"用户: 我想结束对话\n"
|
||||
'返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
|
||||
"```\n"
|
||||
@@ -224,7 +228,8 @@ class IntentProvider(IntentProviderBase):
|
||||
if function_name == "continue_chat":
|
||||
# 保留非工具相关的消息
|
||||
clean_history = [
|
||||
msg for msg in conn.dialogue.dialogue
|
||||
msg
|
||||
for msg in conn.dialogue.dialogue
|
||||
if msg.role not in ["tool", "function"]
|
||||
]
|
||||
conn.dialogue.dialogue = clean_history
|
||||
|
||||
@@ -8,81 +8,61 @@ import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
import http.client
|
||||
import urllib.parse
|
||||
import time
|
||||
import uuid
|
||||
from urllib import parse
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AccessToken:
|
||||
@staticmethod
|
||||
def _encode_text(text):
|
||||
encoded_text = parse.quote_plus(text)
|
||||
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
|
||||
|
||||
return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
|
||||
|
||||
@staticmethod
|
||||
def _encode_dict(dic):
|
||||
keys = dic.keys()
|
||||
dic_sorted = [(key, dic[key]) for key in sorted(keys)]
|
||||
encoded_text = parse.urlencode(dic_sorted)
|
||||
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
|
||||
|
||||
return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
|
||||
|
||||
@staticmethod
|
||||
def create_token(access_key_id, access_key_secret):
|
||||
parameters = {
|
||||
"AccessKeyId": access_key_id,
|
||||
"Action": "CreateToken",
|
||||
"Format": "JSON",
|
||||
"RegionId": "cn-shanghai",
|
||||
"SignatureMethod": "HMAC-SHA1",
|
||||
"SignatureNonce": str(uuid.uuid1()),
|
||||
"SignatureVersion": "1.0",
|
||||
"Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"Version": "2019-02-28",
|
||||
}
|
||||
parameters = {'AccessKeyId': access_key_id,
|
||||
'Action': 'CreateToken',
|
||||
'Format': 'JSON',
|
||||
'RegionId': 'cn-shanghai',
|
||||
'SignatureMethod': 'HMAC-SHA1',
|
||||
'SignatureNonce': str(uuid.uuid1()),
|
||||
'SignatureVersion': '1.0',
|
||||
'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
'Version': '2019-02-28'}
|
||||
# 构造规范化的请求字符串
|
||||
query_string = AccessToken._encode_dict(parameters)
|
||||
# print('规范化的请求字符串: %s' % query_string)
|
||||
# 构造待签名字符串
|
||||
string_to_sign = (
|
||||
"GET"
|
||||
+ "&"
|
||||
+ AccessToken._encode_text("/")
|
||||
+ "&"
|
||||
+ AccessToken._encode_text(query_string)
|
||||
)
|
||||
string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string)
|
||||
# print('待签名的字符串: %s' % string_to_sign)
|
||||
# 计算签名
|
||||
secreted_string = hmac.new(
|
||||
bytes(access_key_secret + "&", encoding="utf-8"),
|
||||
bytes(string_to_sign, encoding="utf-8"),
|
||||
hashlib.sha1,
|
||||
).digest()
|
||||
secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'),
|
||||
bytes(string_to_sign, encoding='utf-8'),
|
||||
hashlib.sha1).digest()
|
||||
signature = base64.b64encode(secreted_string)
|
||||
# print('签名: %s' % signature)
|
||||
# 进行URL编码
|
||||
signature = AccessToken._encode_text(signature)
|
||||
# print('URL编码后的签名: %s' % signature)
|
||||
# 调用服务
|
||||
full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (
|
||||
signature,
|
||||
query_string,
|
||||
)
|
||||
full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string)
|
||||
# print('url: %s' % full_url)
|
||||
# 提交HTTP GET请求
|
||||
response = requests.get(full_url)
|
||||
if response.ok:
|
||||
root_obj = response.json()
|
||||
key = "Token"
|
||||
key = 'Token'
|
||||
if key in root_obj:
|
||||
token = root_obj[key]["Id"]
|
||||
expire_time = root_obj[key]["ExpireTime"]
|
||||
token = root_obj[key]['Id']
|
||||
expire_time = root_obj[key]['ExpireTime']
|
||||
return token, expire_time
|
||||
# print(response.text)
|
||||
return None, None
|
||||
@@ -90,9 +70,10 @@ class AccessToken:
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
|
||||
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
|
||||
|
||||
# 新增空值判断逻辑
|
||||
self.access_key_id = config.get("access_key_id")
|
||||
self.access_key_secret = config.get("access_key_secret")
|
||||
@@ -106,7 +87,9 @@ class TTSProvider(TTSProviderBase):
|
||||
self.pitch_rate = config.get("pitch_rate", 0)
|
||||
self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
|
||||
self.api_url = f"https://{self.host}/stream/v1/tts"
|
||||
self.header = {"Content-Type": "application/json"}
|
||||
self.header = {
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
if self.access_key_id and self.access_key_secret:
|
||||
# 使用密钥对生成临时token
|
||||
@@ -116,30 +99,35 @@ class TTSProvider(TTSProviderBase):
|
||||
self.token = config.get("token")
|
||||
self.expire_time = None
|
||||
|
||||
|
||||
def _refresh_token(self):
|
||||
"""刷新Token并记录过期时间"""
|
||||
if self.access_key_id and self.access_key_secret:
|
||||
self.token, expire_time_str = AccessToken.create_token(
|
||||
self.access_key_id, self.access_key_secret
|
||||
self.access_key_id,
|
||||
self.access_key_secret
|
||||
)
|
||||
if not expire_time_str:
|
||||
raise ValueError("无法获取有效的Token过期时间")
|
||||
|
||||
try:
|
||||
# 统一转换为字符串处理
|
||||
#统一转换为字符串处理
|
||||
expire_str = str(expire_time_str).strip()
|
||||
|
||||
if expire_str.isdigit():
|
||||
expire_time = datetime.fromtimestamp(int(expire_str))
|
||||
else:
|
||||
expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ")
|
||||
expire_time = datetime.strptime(
|
||||
expire_str,
|
||||
"%Y-%m-%dT%H:%M:%SZ"
|
||||
)
|
||||
self.expire_time = expire_time.timestamp() - 60
|
||||
except Exception as e:
|
||||
raise ValueError(f"无效的过期时间格式: {expire_str}") from e
|
||||
|
||||
|
||||
else:
|
||||
self.expire_time = None
|
||||
|
||||
|
||||
if not self.token:
|
||||
raise ValueError("无法获取有效的访问Token")
|
||||
|
||||
@@ -154,16 +142,12 @@ class TTSProvider(TTSProviderBase):
|
||||
# f"过期时间 {datetime.fromtimestamp(self.expire_time)} | "
|
||||
# f"剩余 {remaining:.2f}秒")
|
||||
return time.time() > self.expire_time
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(
|
||||
self.output_file,
|
||||
f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
async def text_to_speak(self, text, output_file):
|
||||
if self._is_token_expired():
|
||||
logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...")
|
||||
logger.warning("Token已过期，正在自动刷新...")
|
||||
self._refresh_token()
|
||||
request_json = {
|
||||
"appkey": self.appkey,
|
||||
@@ -174,45 +158,21 @@ class TTSProvider(TTSProviderBase):
|
||||
"voice": self.voice,
|
||||
"volume": self.volume,
|
||||
"speech_rate": self.speech_rate,
|
||||
"pitch_rate": self.pitch_rate,
|
||||
"pitch_rate": self.pitch_rate
|
||||
}
|
||||
|
||||
print(self.api_url, json.dumps(request_json, ensure_ascii=False))
|
||||
# print(self.api_url, json.dumps(request_json, ensure_ascii=False))
|
||||
try:
|
||||
resp = requests.post(
|
||||
self.api_url, json.dumps(request_json), headers=self.header
|
||||
)
|
||||
resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
|
||||
if resp.status_code == 401: # Token过期特殊处理
|
||||
self._refresh_token()
|
||||
resp = requests.post(
|
||||
self.api_url, json.dumps(request_json), headers=self.header
|
||||
)
|
||||
resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
|
||||
# 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的
|
||||
tmp_file = self.generate_filename()
|
||||
if resp.headers["Content-Type"].startswith("audio/"):
|
||||
with open(tmp_file, "wb") as f:
|
||||
if resp.headers['Content-Type'].startswith('audio/'):
|
||||
with open(output_file, 'wb') as f:
|
||||
f.write(resp.content)
|
||||
return output_file
|
||||
else:
|
||||
raise Exception(
|
||||
f"{__name__} status_code: {resp.status_code} response: {resp.content}"
|
||||
)
|
||||
# 使用 pydub 读取临时文件
|
||||
audio = AudioSegment.from_file(tmp_file, format="wav")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
# 用完后删除临时文件
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except FileNotFoundError:
|
||||
# 若文件不存在，忽略该异常
|
||||
pass
|
||||
|
||||
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
|
||||
@@ -1,24 +1,15 @@
|
||||
import asyncio
|
||||
import gc
|
||||
import io
|
||||
import threading
|
||||
import traceback
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import torch
|
||||
import torchaudio
|
||||
|
||||
from config.logger import setup_logging
|
||||
import numpy as np
|
||||
from pydub import AudioSegment
|
||||
from abc import ABC, abstractmethod
|
||||
from core.utils import textUtils
|
||||
from core.utils.util import audio_to_data
|
||||
from core.opus import opus_encoder_utils
|
||||
import queue
|
||||
|
||||
from core.providers.tts.dto.dto import MsgType, TTSMessageDTO, SentenceType
|
||||
import os
|
||||
import json
|
||||
import threading
|
||||
from core.utils import p3
|
||||
from core.handle.sendAudioHandle import sendAudioMessage
|
||||
from core.handle.reportHandle import enqueue_tts_report
|
||||
from abc import ABC, abstractmethod
|
||||
from core.utils.tts import MarkdownCleaner
|
||||
from core.utils.util import audio_to_data
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
@@ -26,273 +17,51 @@ logger = setup_logging()
|
||||
|
||||
class TTSProviderBase(ABC):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
self.config = config
|
||||
self.conn = None
|
||||
self.tts_timeout = 10
|
||||
self.delete_audio_file = delete_audio_file
|
||||
self.output_file = config.get("output_dir")
|
||||
self.tts_text_queue = queue.Queue()
|
||||
self.tts_audio_queue = queue.Queue()
|
||||
self.enable_two_way = False
|
||||
self.stop_event = threading.Event()
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=16000, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
self.tts_text_buff = []
|
||||
self.punctuations = (
|
||||
"。",
|
||||
"？",
|
||||
"！",
|
||||
"；",
|
||||
"：",
|
||||
".",
|
||||
"?",
|
||||
"!",
|
||||
";",
|
||||
":",
|
||||
" ",
|
||||
",",
|
||||
"，",
|
||||
)
|
||||
self.tts_request = False
|
||||
self.tts_stop_request = False
|
||||
self.processed_chars = 0
|
||||
self.stream = False
|
||||
self.last_to_opus_raw = b""
|
||||
|
||||
# 启动tts_text_queue监听线程
|
||||
# 线程任务相关
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.process_tasks_loop = asyncio.get_event_loop()
|
||||
self.max_workers = self.config.get("TTS_SET", {}).get("MAX_WORKERS", 3)
|
||||
self.active_tasks = set() # 追踪当前运行的任务
|
||||
self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
|
||||
|
||||
async def open_audio_channels(self):
|
||||
# 启动tts_text_queue监听线程
|
||||
tts_priority = threading.Thread(
|
||||
target=self._tts_text_priority_thread, daemon=True
|
||||
)
|
||||
tts_priority.start()
|
||||
|
||||
async def close(self):
|
||||
self.stop_event
|
||||
|
||||
def _get_segment_text(self):
|
||||
# 合并当前全部文本并处理未分割部分
|
||||
full_text = "".join(self.tts_text_buff)
|
||||
current_text = full_text[self.processed_chars :] # 从未处理的位置开始
|
||||
last_punct_pos = -1
|
||||
for punct in self.punctuations:
|
||||
pos = current_text.rfind(punct)
|
||||
if (pos != -1 and last_punct_pos == -1) or (
|
||||
pos != -1 and pos < last_punct_pos
|
||||
):
|
||||
last_punct_pos = pos
|
||||
if last_punct_pos != -1:
|
||||
segment_text_raw = current_text[: last_punct_pos + 1]
|
||||
segment_text = textUtils.get_string_no_punctuation_or_emoji(
|
||||
segment_text_raw
|
||||
)
|
||||
self.processed_chars += len(segment_text_raw) # 更新已处理字符位置
|
||||
return segment_text
|
||||
elif self.tts_stop_request and current_text:
|
||||
segment_text = current_text
|
||||
return segment_text
|
||||
else:
|
||||
return None
|
||||
|
||||
async def process_generator(self, generator):
|
||||
async for tts_data in generator:
|
||||
self.tts_audio_queue.put(tts_data)
|
||||
|
||||
def _tts_text_priority_thread(self):
|
||||
logger.bind(tag=TAG).info("开始监听tts文本")
|
||||
if self.enable_two_way:
|
||||
self._enable_two_way_tts()
|
||||
else:
|
||||
self._no_enable_two_way_tts()
|
||||
|
||||
async def start_session(self, session_id):
|
||||
pass
|
||||
|
||||
async def finish_session(self, session_id):
|
||||
pass
|
||||
|
||||
def tts_one_sentence(self, conn, text, u_id=None):
|
||||
if not u_id:
|
||||
if conn.u_id:
|
||||
u_id = conn.u_id
|
||||
else:
|
||||
u_id = str(uuid.uuid4()).replace("-", "")
|
||||
conn.u_id = u_id
|
||||
self.tts_text_queue.put(
|
||||
TTSMessageDTO(u_id=u_id, msg_type=MsgType.START_TTS_REQUEST, content="")
|
||||
)
|
||||
self.tts_text_queue.put(
|
||||
TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_REQUEST, content=text)
|
||||
)
|
||||
self.tts_text_queue.put(
|
||||
TTSMessageDTO(u_id=u_id, msg_type=MsgType.STOP_TTS_REQUEST, content="")
|
||||
)
|
||||
|
||||
def _enable_two_way_tts(self):
|
||||
while not self.stop_event.is_set():
|
||||
try:
|
||||
ttsMessageDTO = self.tts_text_queue.get()
|
||||
msg_type = ttsMessageDTO.msg_type
|
||||
if msg_type == MsgType.START_TTS_REQUEST:
|
||||
# 开始传输tts文本
|
||||
self.tts_request = True
|
||||
self.tts_stop_request = False
|
||||
self.u_id = ttsMessageDTO.u_id
|
||||
# 开启session
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.start_session(ttsMessageDTO.u_id), loop=self.loop
|
||||
)
|
||||
future.result()
|
||||
# await self.start_session(ttsMessageDTO.u_id)
|
||||
elif self.tts_request and msg_type == MsgType.TTS_TEXT_REQUEST:
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.text_to_speak(
|
||||
u_id=ttsMessageDTO.u_id, text=ttsMessageDTO.content
|
||||
),
|
||||
loop=self.loop,
|
||||
)
|
||||
future.result()
|
||||
elif msg_type == MsgType.STOP_TTS_REQUEST:
|
||||
self.tts_request = False
|
||||
self.tts_stop_request = True
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.finish_session(ttsMessageDTO.u_id), loop=self.loop
|
||||
)
|
||||
future.result()
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Failed to process TTS text: {e}")
|
||||
# 报错了。要关闭说话
|
||||
self.tts_audio_queue.put(
|
||||
TTSMessageDTO(
|
||||
u_id=self.u_id,
|
||||
msg_type=MsgType.STOP_TTS_RESPONSE,
|
||||
content=[],
|
||||
tts_finish_text="",
|
||||
sentence_type=None,
|
||||
)
|
||||
)
|
||||
traceback.print_exc()
|
||||
|
||||
def _no_enable_two_way_tts(self):
|
||||
# 为这个线程创建一个新的事件循环
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
while not self.stop_event.is_set():
|
||||
try:
|
||||
ttsMessageDTO = self.tts_text_queue.get()
|
||||
msg_type = ttsMessageDTO.msg_type
|
||||
if not self.enable_two_way:
|
||||
if msg_type == MsgType.START_TTS_REQUEST:
|
||||
# 开始传输tts文本
|
||||
self.tts_request = True
|
||||
self.tts_stop_request = False
|
||||
self.processed_chars = 0
|
||||
self.tts_text_buff = []
|
||||
elif self.tts_request and msg_type == MsgType.TTS_TEXT_REQUEST:
|
||||
self.tts_text_buff.append(ttsMessageDTO.content)
|
||||
elif msg_type == MsgType.STOP_TTS_REQUEST:
|
||||
# 结束传输tts文本,处理最尾巴的数据
|
||||
self.tts_request = False
|
||||
self.tts_stop_request = True
|
||||
segment_text = self._get_segment_text()
|
||||
if segment_text:
|
||||
# 修改部分：创建协程对象
|
||||
# 修改部分：创建协程对象
|
||||
tts_generator = self.text_to_speak(
|
||||
ttsMessageDTO.u_id,
|
||||
segment_text,
|
||||
True if msg_type == MsgType.STOP_TTS_REQUEST else False,
|
||||
(
|
||||
True
|
||||
if msg_type == MsgType.START_TTS_REQUEST
|
||||
else False
|
||||
),
|
||||
)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.process_generator(tts_generator), self.loop
|
||||
)
|
||||
self.active_tasks.add(future)
|
||||
if self.active_tasks:
|
||||
|
||||
async def wrap_future(future):
|
||||
return await asyncio.wrap_future(future)
|
||||
|
||||
wrapped_tasks = [
|
||||
wrap_future(task) for task in self.active_tasks
|
||||
]
|
||||
done, _ = loop.run_until_complete(
|
||||
asyncio.wait(wrapped_tasks)
|
||||
)
|
||||
self.active_tasks -= done
|
||||
|
||||
# 发送合成结束
|
||||
self.tts_audio_queue.put(
|
||||
TTSMessageDTO(
|
||||
u_id=ttsMessageDTO.u_id,
|
||||
msg_type=MsgType.STOP_TTS_RESPONSE,
|
||||
content=[],
|
||||
tts_finish_text="",
|
||||
sentence_type=SentenceType.SENTENCE_END,
|
||||
)
|
||||
)
|
||||
|
||||
segment_text = self._get_segment_text()
|
||||
if segment_text:
|
||||
# 确保这里得到的是协程对象
|
||||
tts_generator = self.text_to_speak(
|
||||
ttsMessageDTO.u_id,
|
||||
segment_text,
|
||||
msg_type == MsgType.STOP_TTS_REQUEST,
|
||||
msg_type == MsgType.START_TTS_REQUEST,
|
||||
)
|
||||
# 提交协程到事件循环
|
||||
tts_generator_future = asyncio.run_coroutine_threadsafe(
|
||||
self.process_generator(tts_generator), loop
|
||||
)
|
||||
self.active_tasks.add(tts_generator_future)
|
||||
if len(self.active_tasks) >= self.max_workers:
|
||||
# 等待所有任务完成
|
||||
try:
|
||||
|
||||
async def wrap_future(future):
|
||||
return await asyncio.wrap_future(future)
|
||||
|
||||
wrapped_tasks = [
|
||||
wrap_future(task) for task in self.active_tasks
|
||||
]
|
||||
done, _ = loop.run_until_complete(
|
||||
asyncio.wait(wrapped_tasks)
|
||||
)
|
||||
self.active_tasks -= done
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"Failed to process TTS text: {e}"
|
||||
)
|
||||
traceback.print_exc()
|
||||
else:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Failed to process TTS text: {e}")
|
||||
traceback.print_exc()
|
||||
self.tts_queue = queue.Queue()
|
||||
self.audio_play_queue = queue.Queue()
|
||||
|
||||
@abstractmethod
|
||||
def generate_filename(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
pass
|
||||
def to_tts(self, text):
|
||||
tmp_file = self.generate_filename()
|
||||
try:
|
||||
max_repeat_time = 5
|
||||
text = MarkdownCleaner.clean_markdown(text)
|
||||
while not os.path.exists(tmp_file) and max_repeat_time > 0:
|
||||
try:
|
||||
asyncio.run(self.text_to_speak(text, tmp_file))
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(
|
||||
f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}"
|
||||
)
|
||||
# 未执行成功，删除文件
|
||||
if os.path.exists(tmp_file):
|
||||
os.remove(tmp_file)
|
||||
max_repeat_time -= 1
|
||||
|
||||
async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
|
||||
raise Exception("该TTS还没有实现stream模式")
|
||||
if max_repeat_time > 0:
|
||||
logger.bind(tag=TAG).info(
|
||||
f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次"
|
||||
)
|
||||
else:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"语音生成失败: {text}，请检查网络或服务是否正常"
|
||||
)
|
||||
|
||||
return tmp_file
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
async def text_to_speak(self, text, output_file):
|
||||
pass
|
||||
|
||||
def audio_to_pcm_data(self, audio_file_path):
|
||||
"""音频文件转换为PCM编码"""
|
||||
@@ -302,16 +71,109 @@ class TTSProviderBase(ABC):
|
||||
"""音频文件转换为Opus编码"""
|
||||
return audio_to_data(audio_file_path, is_opus=True)
|
||||
|
||||
def get_audio_from_tts(self, data_bytes, src_rate, to_rate=16000):
|
||||
tts_speech = torch.from_numpy(
|
||||
np.array(np.frombuffer(data_bytes, dtype=np.int16))
|
||||
).unsqueeze(dim=0)
|
||||
with io.BytesIO() as bf:
|
||||
torchaudio.save(bf, tts_speech, src_rate, format="wav")
|
||||
audio = AudioSegment.from_file(bf, format="wav")
|
||||
audio = audio.set_channels(1).set_frame_rate(to_rate)
|
||||
return audio
|
||||
def startSession(self, conn):
|
||||
self.conn = conn
|
||||
self.tts_timeout = conn.config.get("tts_timeout", 10)
|
||||
# tts 消化线程
|
||||
self.tts_priority_thread = threading.Thread(
|
||||
target=self._tts_priority_thread, daemon=True
|
||||
)
|
||||
self.tts_priority_thread.start()
|
||||
|
||||
def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False):
|
||||
opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end)
|
||||
return opus_datas
|
||||
# 音频播放 消化线程
|
||||
self.audio_play_priority_thread = threading.Thread(
|
||||
target=self._audio_play_priority_thread, daemon=True
|
||||
)
|
||||
self.audio_play_priority_thread.start()
|
||||
|
||||
def _tts_priority_thread(self):
|
||||
while not self.conn.stop_event.is_set():
|
||||
text = None
|
||||
try:
|
||||
try:
|
||||
item = self.tts_queue.get(timeout=1)
|
||||
if item is None:
|
||||
continue
|
||||
future, text_index = item # 解包获取 Future 和 text_index
|
||||
except queue.Empty:
|
||||
if self.conn.stop_event.is_set():
|
||||
break
|
||||
continue
|
||||
if future is None:
|
||||
continue
|
||||
text = None
|
||||
audio_datas, tts_file = [], None
|
||||
try:
|
||||
logger.bind(tag=TAG).debug("正在处理TTS任务...")
|
||||
tts_file, text, _ = future.result(timeout=self.tts_timeout)
|
||||
if tts_file is None:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"TTS出错： file is empty: {text_index}: {text}"
|
||||
)
|
||||
else:
|
||||
logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
|
||||
if os.path.exists(tts_file):
|
||||
if tts_file.endswith(".p3"):
|
||||
audio_datas, _ = p3.decode_opus_from_file(tts_file)
|
||||
elif self.conn.audio_format == "pcm":
|
||||
audio_datas, _ = self.audio_to_pcm_data(tts_file)
|
||||
else:
|
||||
audio_datas, _ = self.audio_to_opus_data(tts_file)
|
||||
# 在这里上报TTS数据
|
||||
enqueue_tts_report(
|
||||
self.conn,
|
||||
tts_file if text is None else text,
|
||||
audio_datas,
|
||||
)
|
||||
else:
|
||||
logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
|
||||
except TimeoutError:
|
||||
logger.bind(tag=TAG).error("TTS超时")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"TTS出错: {e}")
|
||||
if not self.conn.client_abort:
|
||||
# 如果没有中途打断就发送语音
|
||||
self.audio_play_queue.put((audio_datas, text, text_index))
|
||||
if (
|
||||
self.delete_audio_file
|
||||
and tts_file is not None
|
||||
and os.path.exists(tts_file)
|
||||
and tts_file.startswith(self.output_file)
|
||||
):
|
||||
os.remove(tts_file)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
|
||||
self.conn.clearSpeakStatus()
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "tts",
|
||||
"state": "stop",
|
||||
"session_id": self.session_id,
|
||||
}
|
||||
)
|
||||
),
|
||||
self.conn.loop,
|
||||
)
|
||||
logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
|
||||
|
||||
def _audio_play_priority_thread(self):
|
||||
while not self.conn.stop_event.is_set():
|
||||
text = None
|
||||
try:
|
||||
try:
|
||||
audio_datas, text, text_index = self.audio_play_queue.get(timeout=1)
|
||||
except queue.Empty:
|
||||
if self.conn.stop_event.is_set():
|
||||
break
|
||||
continue
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
sendAudioMessage(self.conn, audio_datas, text, text_index),
|
||||
self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"audio_play_priority priority_thread: {text} {e}"
|
||||
)
|
||||
|
||||
@@ -4,11 +4,7 @@ import json
|
||||
import base64
|
||||
import requests
|
||||
from datetime import datetime
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
@@ -20,7 +16,6 @@ class TTSProvider(TTSProviderBase):
|
||||
self.voice = config.get("private_voice")
|
||||
else:
|
||||
self.voice = config.get("voice")
|
||||
|
||||
self.response_format = config.get("response_format")
|
||||
|
||||
self.host = "api.coze.cn"
|
||||
@@ -32,7 +27,7 @@ class TTSProvider(TTSProviderBase):
|
||||
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"model": self.model,
|
||||
"input": text,
|
||||
@@ -43,27 +38,13 @@ class TTSProvider(TTSProviderBase):
|
||||
"Authorization": f"Bearer {self.access_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
response = requests.request(
|
||||
"POST", self.api_url, json=request_json, headers=headers
|
||||
)
|
||||
data = response.content
|
||||
tmp_file = self.generate_filename()
|
||||
file_to_save = open(tmp_file, "wb")
|
||||
file_to_save.write(data)
|
||||
# 使用 pydub 读取临时文件
|
||||
audio = AudioSegment.from_file(tmp_file, format="wav")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
# 用完后删除临时文件
|
||||
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except FileNotFoundError:
|
||||
# 若文件不存在，忽略该异常
|
||||
pass
|
||||
response = requests.request(
|
||||
"POST", self.api_url, json=request_json, headers=headers
|
||||
)
|
||||
data = response.content
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(data)
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
@@ -2,17 +2,13 @@ import os
|
||||
import json
|
||||
import uuid
|
||||
import requests
|
||||
from pydub import AudioSegment
|
||||
|
||||
from config.logger import setup_logging
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
@@ -33,14 +29,10 @@ class TTSProvider(TTSProviderBase):
|
||||
raise TypeError("Custom TTS配置参数出错, 请参考配置说明")
|
||||
|
||||
def generate_filename(self):
|
||||
return os.path.join(
|
||||
self.output_file,
|
||||
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}.{self.format}",
|
||||
)
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}.{self.format}")
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_params = {}
|
||||
tmp_file = self.generate_filename()
|
||||
for k, v in self.params.items():
|
||||
if isinstance(v, str) and "{prompt_text}" in v:
|
||||
v = v.replace("{prompt_text}", text)
|
||||
@@ -51,26 +43,9 @@ class TTSProvider(TTSProviderBase):
|
||||
else:
|
||||
resp = requests.get(self.url, params=request_params, headers=self.headers)
|
||||
if resp.status_code == 200:
|
||||
with open(tmp_file, "wb") as file:
|
||||
with open(output_file, "wb") as file:
|
||||
file.write(resp.content)
|
||||
else:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"Custom TTS请求失败: {resp.status_code} - {resp.text}"
|
||||
)
|
||||
# 使用 pydub 读取临时文件
|
||||
audio = AudioSegment.from_file(tmp_file, format=self.format)
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
# 用完后删除临时文件
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except FileNotFoundError:
|
||||
# 若文件不存在，忽略该异常
|
||||
pass
|
||||
error_msg = f"Custom TTS请求失败: {resp.status_code} - {resp.text}"
|
||||
logger.bind(tag=TAG).error(error_msg)
|
||||
raise Exception(error_msg) # 抛出异常，让调用方捕获
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import os
|
||||
import asyncio
|
||||
from config.logger import setup_logging
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class DefaultTTS(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file=True):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.output_dir = config.get("output_dir", "output")
|
||||
if not os.path.exists(self.output_dir):
|
||||
os.makedirs(self.output_dir)
|
||||
|
||||
def generate_filename(self):
|
||||
"""生成唯一的音频文件名"""
|
||||
import uuid
|
||||
|
||||
return os.path.join(self.output_dir, f"{uuid.uuid4()}.wav")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
logger.bind(tag=TAG).error(f"无法实例化 TTS 服务，请检查配置")
|
||||
@@ -4,10 +4,6 @@ import json
|
||||
import base64
|
||||
import requests
|
||||
from datetime import datetime
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
from core.utils.util import check_model_key
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from config.logger import setup_logging
|
||||
@@ -51,8 +47,7 @@ class TTSProvider(TTSProviderBase):
|
||||
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
tmp_file = self.generate_filename()
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"app": {
|
||||
"appid": f"{self.appid}",
|
||||
@@ -83,7 +78,7 @@ class TTSProvider(TTSProviderBase):
|
||||
)
|
||||
if "data" in resp.json():
|
||||
data = resp.json()["data"]
|
||||
file_to_save = open(tmp_file, "wb")
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(base64.b64decode(data))
|
||||
else:
|
||||
raise Exception(
|
||||
@@ -91,15 +86,3 @@ class TTSProvider(TTSProviderBase):
|
||||
)
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
# 使用 pydub 读取临时文件
|
||||
audio = AudioSegment.from_file(tmp_file, format="wav")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
|
||||
yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START)
|
||||
# 用完后删除临时文件
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except FileNotFoundError:
|
||||
# 若文件不存在，忽略该异常
|
||||
pass
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
from enum import Enum
|
||||
from typing import Union
|
||||
|
||||
|
||||
class MsgType(Enum):
|
||||
# 请求类型
|
||||
START_TTS_REQUEST = "START_TTS_REQUEST"
|
||||
TTS_TEXT_REQUEST = "TTS_TEXT_REQUEST"
|
||||
STOP_TTS_REQUEST = "STOP_TTS_REQUEST"
|
||||
|
||||
# 返回类型
|
||||
START_TTS_RESPONSE = "START_TTS_RESPONSE"
|
||||
TTS_TEXT_RESPONSE = "TTS_TEXT_RESPONSE"
|
||||
STOP_TTS_RESPONSE = "STOP_TTS_RESPONSE"
|
||||
|
||||
|
||||
class SentenceType(Enum):
|
||||
# 句子开始
|
||||
SENTENCE_START = "SENTENCE_START"
|
||||
# 句子结束
|
||||
SENTENCE_END = "SENTENCE_END"
|
||||
|
||||
|
||||
class TTSMessageDTO:
|
||||
def __init__(self, u_id: str, msg_type: MsgType, content: Union[str, bytes], tts_finish_text=None,
|
||||
sentence_type: SentenceType = None, duration=0):
|
||||
if not isinstance(msg_type, MsgType):
|
||||
raise ValueError("msg_type must be an instance of MsgType Enum")
|
||||
if not isinstance(content, (str, list, bytes)):
|
||||
raise ValueError("content must be of type str or bytes")
|
||||
|
||||
# 唯一id，每个合成到合成结束，使用同一个id
|
||||
self.u_id = u_id
|
||||
self.msg_type = msg_type
|
||||
self.sentence_type = sentence_type
|
||||
self.content = content
|
||||
self.tts_finish_text = tts_finish_text
|
||||
self.duration = duration
|
||||
|
||||
def __repr__(self):
|
||||
content_preview = self.content if isinstance(self.content, str) else "<binary data>"
|
||||
return f"MessageDTO(msg_type={self.msg_type}, content={content_preview})"
|
||||
@@ -1,17 +1,8 @@
|
||||
import io
|
||||
import os
|
||||
import uuid
|
||||
import edge_tts
|
||||
from datetime import datetime
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
from config.logger import setup_logging
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
@@ -28,37 +19,19 @@ class TTSProvider(TTSProviderBase):
|
||||
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
async def text_to_speak(self, text, output_file):
|
||||
try:
|
||||
communicate = edge_tts.Communicate(
|
||||
text, voice=self.voice
|
||||
) # Use your preferred voice
|
||||
tmp_file = self.generate_filename()
|
||||
await communicate.save(tmp_file)
|
||||
|
||||
# 使用 pydub 读取临时文件
|
||||
audio = AudioSegment.from_file(tmp_file, format="mp3")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
# 用完后删除临时文件
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except FileNotFoundError:
|
||||
# 若文件不存在，忽略该异常
|
||||
communicate = edge_tts.Communicate(text, voice=self.voice)
|
||||
# 确保目录存在并创建空文件
|
||||
os.makedirs(os.path.dirname(output_file), exist_ok=True)
|
||||
with open(output_file, "wb") as f:
|
||||
pass
|
||||
|
||||
# 流式写入音频数据
|
||||
with open(output_file, "ab") as f: # 改为追加模式避免覆盖
|
||||
async for chunk in communicate.stream():
|
||||
if chunk["type"] == "audio": # 只处理音频数据块
|
||||
f.write(chunk["data"])
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"TTSProvider text_to_speak error: {e}")
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=[],
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
error_msg = f"Edge TTS请求失败: {e}"
|
||||
raise Exception(error_msg) # 抛出异常，让调用方捕获
|
||||
@@ -1,25 +1,14 @@
|
||||
import base64
|
||||
import os
|
||||
import traceback
|
||||
import uuid
|
||||
import queue
|
||||
import io
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
import ormsgpack
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import torchaudio
|
||||
from pydantic import BaseModel, Field, conint, model_validator
|
||||
from pydub import AudioSegment
|
||||
from typing_extensions import Annotated
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
from core.utils.util import check_model_key
|
||||
from core.utils.util import check_model_key, parse_string_to_list
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from config.logger import setup_logging
|
||||
|
||||
@@ -150,121 +139,51 @@ class TTSProvider(TTSProviderBase):
|
||||
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
def _get_audio_from_tts(self, data_bytes):
|
||||
tts_speech = torch.from_numpy(
|
||||
np.array(np.frombuffer(data_bytes, dtype=np.int16))
|
||||
).unsqueeze(dim=0)
|
||||
with io.BytesIO() as bf:
|
||||
torchaudio.save(bf, tts_speech, 44100, format="wav")
|
||||
audio = AudioSegment.from_file(bf, format="wav")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
return audio
|
||||
async def text_to_speak(self, text, output_file):
|
||||
# Prepare reference data
|
||||
byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
|
||||
ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
try:
|
||||
data = {
|
||||
"text": text,
|
||||
"reference_id": self.reference_id,
|
||||
"normalize": self.normalize,
|
||||
"format": self.format,
|
||||
"max_new_tokens": self.max_new_tokens,
|
||||
"chunk_length": self.chunk_length,
|
||||
"top_p": self.top_p,
|
||||
"repetition_penalty": self.repetition_penalty,
|
||||
"temperature": self.temperature,
|
||||
"streaming": self.streaming,
|
||||
"use_memory_cache": self.use_memory_cache,
|
||||
"seed": self.seed,
|
||||
}
|
||||
data = {
|
||||
"text": text,
|
||||
"references": [
|
||||
ServeReferenceAudio(audio=audio if audio else b"", text=text)
|
||||
for text, audio in zip(ref_texts, byte_audios)
|
||||
],
|
||||
"reference_id": self.reference_id,
|
||||
"normalize": self.normalize,
|
||||
"format": self.format,
|
||||
"max_new_tokens": self.max_new_tokens,
|
||||
"chunk_length": self.chunk_length,
|
||||
"top_p": self.top_p,
|
||||
"repetition_penalty": self.repetition_penalty,
|
||||
"temperature": self.temperature,
|
||||
"streaming": self.streaming,
|
||||
"use_memory_cache": self.use_memory_cache,
|
||||
"seed": self.seed,
|
||||
}
|
||||
|
||||
# Prepare reference data
|
||||
if self.reference_audio and self.reference_text:
|
||||
byte_audios = [
|
||||
audio_to_bytes(ref_audio) for ref_audio in self.reference_audio
|
||||
]
|
||||
ref_texts = [
|
||||
read_ref_text(ref_text) for ref_text in self.reference_text
|
||||
]
|
||||
data["references"] = (
|
||||
[
|
||||
ServeReferenceAudio(audio=audio if audio else b"", text=text)
|
||||
for text, audio in zip(ref_texts, byte_audios)
|
||||
],
|
||||
)
|
||||
data["reference_id"] = None
|
||||
pydantic_data = ServeTTSRequest(**data)
|
||||
|
||||
pydantic_data = ServeTTSRequest(**data)
|
||||
audio_buff = None
|
||||
chunk_total = b""
|
||||
last_raw = b""
|
||||
audio_raw = b""
|
||||
print("请求tts")
|
||||
with requests.post(
|
||||
self.api_url,
|
||||
data=ormsgpack.packb(
|
||||
pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC
|
||||
),
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/msgpack",
|
||||
},
|
||||
) as response:
|
||||
if response.status_code == 200:
|
||||
index = 0
|
||||
for chunk in response.iter_content():
|
||||
# 拼接当前块和上一块数据
|
||||
chunk_total += chunk
|
||||
# 最后一个是静音，说明是一个完整的音频
|
||||
if (
|
||||
len(chunk_total) % 2 == 0
|
||||
and chunk_total[-2:] == b"\x00\x00"
|
||||
):
|
||||
audio = self._get_audio_from_tts(chunk_total)
|
||||
audio_raw = audio_raw + audio.raw_data
|
||||
# 长度凑够2贞开始发送，60ms*2=120ms
|
||||
if len(audio_raw) >= 3840:
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
|
||||
if index == 0:
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
else:
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=None,
|
||||
)
|
||||
audio_raw = b""
|
||||
chunk_total = b""
|
||||
if len(chunk_total) > 0:
|
||||
audio = self._get_audio_from_tts(chunk_total)
|
||||
audio_raw = audio_raw + audio.raw_data
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_END,
|
||||
)
|
||||
else:
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=[],
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_END,
|
||||
)
|
||||
response = requests.post(
|
||||
self.api_url,
|
||||
data=ormsgpack.packb(
|
||||
pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC
|
||||
),
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/msgpack",
|
||||
},
|
||||
)
|
||||
|
||||
else:
|
||||
print("请求失败:", response.status_code, response.text)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error("tts发生错误")
|
||||
traceback.print_exc()
|
||||
raise e
|
||||
if response.status_code == 200:
|
||||
audio_content = response.content
|
||||
|
||||
with open(output_file, "wb") as audio_file:
|
||||
audio_file.write(audio_content)
|
||||
|
||||
else:
|
||||
error_msg = f"Request failed with status code {response.status_code}"
|
||||
print(error_msg)
|
||||
print(response.json())
|
||||
raise Exception(error_msg)
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import base64
|
||||
import requests
|
||||
from pydub import AudioSegment
|
||||
from core.utils.util import parse_string_to_list
|
||||
from config.logger import setup_logging
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
from core.utils.util import parse_string_to_list
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
@@ -77,8 +77,7 @@ class TTSProvider(TTSProviderBase):
|
||||
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
tmp_file = self.generate_filename()
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"text": text,
|
||||
"text_lang": self.text_lang,
|
||||
@@ -103,26 +102,9 @@ class TTSProvider(TTSProviderBase):
|
||||
|
||||
resp = requests.post(self.url, json=request_json)
|
||||
if resp.status_code == 200:
|
||||
with open(tmp_file, "wb") as file:
|
||||
with open(output_file, "wb") as file:
|
||||
file.write(resp.content)
|
||||
else:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}"
|
||||
)
|
||||
# 使用 pydub 读取临时文件
|
||||
audio = AudioSegment.from_file(tmp_file, format="wav")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
# 用完后删除临时文件
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except FileNotFoundError:
|
||||
# 若文件不存在，忽略该异常
|
||||
pass
|
||||
error_msg = f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}"
|
||||
logger.bind(tag=TAG).error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
import os
|
||||
import uuid
|
||||
import requests
|
||||
from pydub import AudioSegment
|
||||
from core.utils.util import parse_string_to_list
|
||||
from config.logger import setup_logging
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
from core.utils.util import parse_string_to_list
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
@@ -44,8 +42,7 @@ class TTSProvider(TTSProviderBase):
|
||||
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
tmp_file = self.generate_filename()
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_params = {
|
||||
"refer_wav_path": self.refer_wav_path,
|
||||
"prompt_text": self.prompt_text,
|
||||
@@ -64,26 +61,10 @@ class TTSProvider(TTSProviderBase):
|
||||
|
||||
resp = requests.get(self.url, params=request_params)
|
||||
if resp.status_code == 200:
|
||||
with open(tmp_file, "wb") as file:
|
||||
with open(output_file, "wb") as file:
|
||||
file.write(resp.content)
|
||||
else:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"GPT_SoVITS_V3 TTS请求失败: {resp.status_code} - {resp.text}"
|
||||
)
|
||||
# 使用 pydub 读取临时文件
|
||||
audio = AudioSegment.from_file(tmp_file, format="wav")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
# 用完后删除临时文件
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except FileNotFoundError:
|
||||
# 若文件不存在，忽略该异常
|
||||
pass
|
||||
error_msg = f"GPT_SoVITS_V3 TTS请求失败: {resp.status_code} - {resp.text}"
|
||||
logger.bind(tag=TAG).error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
|
||||
@@ -1,453 +0,0 @@
|
||||
import asyncio
|
||||
import io
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import traceback
|
||||
import uuid
|
||||
import json
|
||||
import base64
|
||||
import requests
|
||||
from datetime import datetime
|
||||
|
||||
import websockets
|
||||
|
||||
from config.logger import setup_logging
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
from core.utils.util import check_model_key
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
PROTOCOL_VERSION = 0b0001
|
||||
DEFAULT_HEADER_SIZE = 0b0001
|
||||
|
||||
# Message Type:
|
||||
FULL_CLIENT_REQUEST = 0b0001
|
||||
AUDIO_ONLY_RESPONSE = 0b1011
|
||||
FULL_SERVER_RESPONSE = 0b1001
|
||||
ERROR_INFORMATION = 0b1111
|
||||
|
||||
# Message Type Specific Flags
|
||||
MsgTypeFlagNoSeq = 0b0000 # Non-terminal packet with no sequence
|
||||
MsgTypeFlagPositiveSeq = 0b1 # Non-terminal packet with sequence > 0
|
||||
MsgTypeFlagLastNoSeq = 0b10 # last packet with no sequence
|
||||
MsgTypeFlagNegativeSeq = 0b11 # Payload contains event number (int32)
|
||||
MsgTypeFlagWithEvent = 0b100
|
||||
# Message Serialization
|
||||
NO_SERIALIZATION = 0b0000
|
||||
JSON = 0b0001
|
||||
# Message Compression
|
||||
COMPRESSION_NO = 0b0000
|
||||
COMPRESSION_GZIP = 0b0001
|
||||
|
||||
EVENT_NONE = 0
|
||||
EVENT_Start_Connection = 1
|
||||
|
||||
EVENT_FinishConnection = 2
|
||||
|
||||
EVENT_ConnectionStarted = 50 # 成功建连
|
||||
|
||||
EVENT_ConnectionFailed = 51 # 建连失败（可能是无法通过权限认证）
|
||||
|
||||
EVENT_ConnectionFinished = 52 # 连接结束
|
||||
|
||||
# 上行Session事件
|
||||
EVENT_StartSession = 100
|
||||
|
||||
EVENT_FinishSession = 102
|
||||
# 下行Session事件
|
||||
EVENT_SessionStarted = 150
|
||||
EVENT_SessionFinished = 152
|
||||
|
||||
EVENT_SessionFailed = 153
|
||||
|
||||
# 上行通用事件
|
||||
EVENT_TaskRequest = 200
|
||||
|
||||
# 下行TTS事件
|
||||
EVENT_TTSSentenceStart = 350
|
||||
|
||||
EVENT_TTSSentenceEnd = 351
|
||||
|
||||
EVENT_TTSResponse = 352
|
||||
|
||||
|
||||
class Header:
|
||||
def __init__(
|
||||
self,
|
||||
protocol_version=PROTOCOL_VERSION,
|
||||
header_size=DEFAULT_HEADER_SIZE,
|
||||
message_type: int = 0,
|
||||
message_type_specific_flags: int = 0,
|
||||
serial_method: int = NO_SERIALIZATION,
|
||||
compression_type: int = COMPRESSION_NO,
|
||||
reserved_data=0,
|
||||
):
|
||||
self.header_size = header_size
|
||||
self.protocol_version = protocol_version
|
||||
self.message_type = message_type
|
||||
self.message_type_specific_flags = message_type_specific_flags
|
||||
self.serial_method = serial_method
|
||||
self.compression_type = compression_type
|
||||
self.reserved_data = reserved_data
|
||||
|
||||
def as_bytes(self) -> bytes:
|
||||
return bytes(
|
||||
[
|
||||
(self.protocol_version << 4) | self.header_size,
|
||||
(self.message_type << 4) | self.message_type_specific_flags,
|
||||
(self.serial_method << 4) | self.compression_type,
|
||||
self.reserved_data,
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class Optional:
|
||||
def __init__(
|
||||
self, event: int = EVENT_NONE, sessionId: str = None, sequence: int = None
|
||||
):
|
||||
self.event = event
|
||||
self.sessionId = sessionId
|
||||
self.errorCode: int = 0
|
||||
self.connectionId: str | None = None
|
||||
self.response_meta_json: str | None = None
|
||||
self.sequence = sequence
|
||||
|
||||
# 转成 byte 序列
|
||||
def as_bytes(self) -> bytes:
|
||||
option_bytes = bytearray()
|
||||
if self.event != EVENT_NONE:
|
||||
option_bytes.extend(self.event.to_bytes(4, "big", signed=True))
|
||||
if self.sessionId is not None:
|
||||
session_id_bytes = str.encode(self.sessionId)
|
||||
size = len(session_id_bytes).to_bytes(4, "big", signed=True)
|
||||
option_bytes.extend(size)
|
||||
option_bytes.extend(session_id_bytes)
|
||||
if self.sequence is not None:
|
||||
option_bytes.extend(self.sequence.to_bytes(4, "big", signed=True))
|
||||
return option_bytes
|
||||
|
||||
|
||||
class Response:
|
||||
def __init__(self, header: Header, optional: Optional):
|
||||
self.optional = optional
|
||||
self.header = header
|
||||
self.payload: bytes | None = None
|
||||
|
||||
def __str__(self):
|
||||
return super().__str__()
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.appId = config.get("appid")
|
||||
self.access_token = config.get("access_token")
|
||||
self.cluster = config.get("cluster")
|
||||
self.resource_id = config.get("resource_id")
|
||||
self.voice = config.get("voice")
|
||||
self.ws_url = config.get("ws_url")
|
||||
self.authorization = config.get("authorization")
|
||||
self.speaker = config.get("speaker")
|
||||
self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
|
||||
self.stop_event_response = threading.Event()
|
||||
self.enable_two_way = True
|
||||
self.start_connection_flag = False
|
||||
self.tts_text = ""
|
||||
|
||||
async def open_audio_channels(self):
|
||||
await super().open_audio_channels()
|
||||
ws_header = {
|
||||
"X-Api-App-Key": self.appId,
|
||||
"X-Api-Access-Key": self.access_token,
|
||||
"X-Api-Resource-Id": self.resource_id,
|
||||
"X-Api-Connect-Id": uuid.uuid4(),
|
||||
}
|
||||
self.ws = await websockets.connect(
|
||||
self.ws_url, additional_headers=ws_header, max_size=1000000000
|
||||
)
|
||||
tts_priority = threading.Thread(
|
||||
target=self._start_monitor_tts_response_thread(), daemon=True
|
||||
)
|
||||
tts_priority.start()
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(
|
||||
self.output_file,
|
||||
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def send_event(
|
||||
self, header: bytes, optional: bytes | None = None, payload: bytes = None
|
||||
):
|
||||
full_client_request = bytearray(header)
|
||||
if optional is not None:
|
||||
full_client_request.extend(optional)
|
||||
if payload is not None:
|
||||
payload_size = len(payload).to_bytes(4, "big", signed=True)
|
||||
full_client_request.extend(payload_size)
|
||||
full_client_request.extend(payload)
|
||||
await self.ws.send(full_client_request)
|
||||
|
||||
async def send_text(self, speaker: str, text: str, session_id):
|
||||
header = Header(
|
||||
message_type=FULL_CLIENT_REQUEST,
|
||||
message_type_specific_flags=MsgTypeFlagWithEvent,
|
||||
serial_method=JSON,
|
||||
).as_bytes()
|
||||
optional = Optional(event=EVENT_TaskRequest, sessionId=session_id).as_bytes()
|
||||
payload = self.get_payload_bytes(
|
||||
event=EVENT_TaskRequest, text=text, speaker=speaker
|
||||
)
|
||||
return await self.send_event(header, optional, payload)
|
||||
|
||||
# 读取 res 数组某段 字符串内容
|
||||
def read_res_content(self, res: bytes, offset: int):
|
||||
content_size = int.from_bytes(res[offset : offset + 4], "big", signed=True)
|
||||
offset += 4
|
||||
content = str(res[offset : offset + content_size])
|
||||
offset += content_size
|
||||
return content, offset
|
||||
|
||||
# 读取 payload
|
||||
def read_res_payload(self, res: bytes, offset: int):
|
||||
payload_size = int.from_bytes(res[offset : offset + 4], "big", signed=True)
|
||||
offset += 4
|
||||
payload = res[offset : offset + payload_size]
|
||||
offset += payload_size
|
||||
return payload, offset
|
||||
|
||||
def parser_response(self, res) -> Response:
|
||||
if isinstance(res, str):
|
||||
raise RuntimeError(res)
|
||||
response = Response(Header(), Optional())
|
||||
# 解析结果
|
||||
# header
|
||||
header = response.header
|
||||
num = 0b00001111
|
||||
header.protocol_version = res[0] >> 4 & num
|
||||
header.header_size = res[0] & 0x0F
|
||||
header.message_type = (res[1] >> 4) & num
|
||||
header.message_type_specific_flags = res[1] & 0x0F
|
||||
header.serialization_method = res[2] >> num
|
||||
header.message_compression = res[2] & 0x0F
|
||||
header.reserved = res[3]
|
||||
#
|
||||
offset = 4
|
||||
optional = response.optional
|
||||
if header.message_type == FULL_SERVER_RESPONSE or AUDIO_ONLY_RESPONSE:
|
||||
# read event
|
||||
if header.message_type_specific_flags == MsgTypeFlagWithEvent:
|
||||
optional.event = int.from_bytes(res[offset:8], "big", signed=True)
|
||||
offset += 4
|
||||
if optional.event == EVENT_NONE:
|
||||
return response
|
||||
# read connectionId
|
||||
elif optional.event == EVENT_ConnectionStarted:
|
||||
optional.connectionId, offset = self.read_res_content(res, offset)
|
||||
elif optional.event == EVENT_ConnectionFailed:
|
||||
optional.response_meta_json, offset = self.read_res_content(
|
||||
res, offset
|
||||
)
|
||||
elif (
|
||||
optional.event == EVENT_SessionStarted
|
||||
or optional.event == EVENT_SessionFailed
|
||||
or optional.event == EVENT_SessionFinished
|
||||
):
|
||||
optional.sessionId, offset = self.read_res_content(res, offset)
|
||||
optional.response_meta_json, offset = self.read_res_content(
|
||||
res, offset
|
||||
)
|
||||
else:
|
||||
optional.sessionId, offset = self.read_res_content(res, offset)
|
||||
response.payload, offset = self.read_res_payload(res, offset)
|
||||
|
||||
elif header.message_type == ERROR_INFORMATION:
|
||||
optional.errorCode = int.from_bytes(
|
||||
res[offset : offset + 4], "big", signed=True
|
||||
)
|
||||
offset += 4
|
||||
response.payload, offset = self.read_res_payload(res, offset)
|
||||
return response
|
||||
|
||||
async def start_connection(self):
|
||||
header = Header(
|
||||
message_type=FULL_CLIENT_REQUEST,
|
||||
message_type_specific_flags=MsgTypeFlagWithEvent,
|
||||
).as_bytes()
|
||||
optional = Optional(event=EVENT_Start_Connection).as_bytes()
|
||||
payload = str.encode("{}")
|
||||
return await self.send_event(header, optional, payload)
|
||||
|
||||
def print_response(self, res, tag_msg: str):
|
||||
logger.bind(tag=TAG).info(f"===>{tag_msg} header:{res.header.__dict__}")
|
||||
logger.bind(tag=TAG).info(f"===>{tag_msg} optional:{res.optional.__dict__}")
|
||||
|
||||
def get_payload_bytes(
|
||||
self,
|
||||
uid="1234",
|
||||
event=EVENT_NONE,
|
||||
text="",
|
||||
speaker="",
|
||||
audio_format="pcm",
|
||||
audio_sample_rate=16000,
|
||||
):
|
||||
return str.encode(
|
||||
json.dumps(
|
||||
{
|
||||
"user": {"uid": uid},
|
||||
"event": event,
|
||||
"namespace": "BidirectionalTTS",
|
||||
"req_params": {
|
||||
"text": text,
|
||||
"speaker": speaker,
|
||||
"audio_params": {
|
||||
"format": audio_format,
|
||||
"sample_rate": audio_sample_rate,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def finish_connection(self):
|
||||
header = Header(
|
||||
message_type=FULL_CLIENT_REQUEST,
|
||||
message_type_specific_flags=MsgTypeFlagWithEvent,
|
||||
serial_method=JSON,
|
||||
).as_bytes()
|
||||
optional = Optional(event=EVENT_FinishConnection).as_bytes()
|
||||
payload = str.encode("{}")
|
||||
await self.send_event(header, optional, payload)
|
||||
return
|
||||
|
||||
async def start_session(self, session_id):
|
||||
self.stop_event_response.clear()
|
||||
header = Header(
|
||||
message_type=FULL_CLIENT_REQUEST,
|
||||
message_type_specific_flags=MsgTypeFlagWithEvent,
|
||||
serial_method=JSON,
|
||||
).as_bytes()
|
||||
optional = Optional(event=EVENT_StartSession, sessionId=session_id).as_bytes()
|
||||
payload = self.get_payload_bytes(event=EVENT_StartSession, speaker=self.speaker)
|
||||
await self.send_event(header, optional, payload)
|
||||
|
||||
async def finish_session(self, session_id):
|
||||
self.stop_event_response.set()
|
||||
header = Header(
|
||||
message_type=FULL_CLIENT_REQUEST,
|
||||
message_type_specific_flags=MsgTypeFlagWithEvent,
|
||||
serial_method=JSON,
|
||||
).as_bytes()
|
||||
optional = Optional(event=EVENT_FinishSession, sessionId=session_id).as_bytes()
|
||||
payload = str.encode("{}")
|
||||
await self.send_event(header, optional, payload)
|
||||
return
|
||||
|
||||
async def reset(self):
|
||||
# 关闭之前的对话
|
||||
if self.start_connection_flag:
|
||||
await self.finish_connection()
|
||||
self.start_connection_flag = False
|
||||
await self.start_connection()
|
||||
self.start_connection_flag = True
|
||||
await super().reset()
|
||||
|
||||
async def close(self):
|
||||
super().close()
|
||||
"""资源清理方法"""
|
||||
await self.finish_connection()
|
||||
await self.ws.close()
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
# 发送文本
|
||||
await self.send_text(self.speaker, text, u_id)
|
||||
return
|
||||
|
||||
def _start_monitor_tts_response_thread(self):
|
||||
# 初始化链接
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self._start_monitor_tts_response(), loop=self.loop
|
||||
)
|
||||
|
||||
async def _start_monitor_tts_response(self):
|
||||
chunk_total = b""
|
||||
while not self.stop_event.is_set():
|
||||
try:
|
||||
msg = await self.ws.recv() # 确保 `recv()` 运行在同一个 event loop
|
||||
res = self.parser_response(msg)
|
||||
self.print_response(res, "send_text res:")
|
||||
|
||||
if (
|
||||
res.optional.event == EVENT_TTSResponse
|
||||
and res.header.message_type == AUDIO_ONLY_RESPONSE
|
||||
):
|
||||
logger.bind(tag=TAG).info(f"推送数据到队列里面～～")
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(res.payload)
|
||||
logger.bind(tag=TAG).info(f"推送数据到队列里面帧数～～{len(opus_datas)}")
|
||||
self.tts_audio_queue.put(
|
||||
TTSMessageDTO(
|
||||
u_id=self.u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text="",
|
||||
sentence_type=None,
|
||||
duration=0,
|
||||
)
|
||||
)
|
||||
elif res.optional.event == EVENT_TTSSentenceStart:
|
||||
json_data = json.loads(res.payload.decode("utf-8"))
|
||||
self.tts_text = json_data.get("text", "")
|
||||
logger.bind(tag=TAG).info(f"句子开始～～{self.tts_text}")
|
||||
self.tts_audio_queue.put(
|
||||
TTSMessageDTO(
|
||||
u_id=self.u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=[],
|
||||
tts_finish_text=self.tts_text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
)
|
||||
elif res.optional.event == EVENT_TTSSentenceEnd:
|
||||
logger.bind(tag=TAG).info(f"句子结束～～{self.tts_text}")
|
||||
self.tts_audio_queue.put(
|
||||
TTSMessageDTO(
|
||||
u_id=self.u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=[],
|
||||
tts_finish_text=self.tts_text,
|
||||
sentence_type=SentenceType.SENTENCE_END,
|
||||
)
|
||||
)
|
||||
elif res.optional.event == EVENT_SessionFinished:
|
||||
logger.bind(tag=TAG).info(f"会话结束～～,最后一句补零")
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(b"", is_end=True)
|
||||
self.tts_audio_queue.put(
|
||||
TTSMessageDTO(
|
||||
u_id=self.u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text="",
|
||||
sentence_type=None,
|
||||
duration=0,
|
||||
)
|
||||
)
|
||||
self.tts_audio_queue.put(
|
||||
TTSMessageDTO(
|
||||
u_id=self.u_id,
|
||||
msg_type=MsgType.STOP_TTS_RESPONSE,
|
||||
content=[],
|
||||
tts_finish_text=self.tts_text,
|
||||
sentence_type=SentenceType.SENTENCE_END,
|
||||
)
|
||||
)
|
||||
else:
|
||||
continue
|
||||
except websockets.ConnectionClosed:
|
||||
break # 连接关闭时退出监听
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in _start_monitor_tts_response: {e}")
|
||||
traceback.print_exc()
|
||||
continue
|
||||
@@ -2,11 +2,9 @@ import os
|
||||
import uuid
|
||||
import json
|
||||
import requests
|
||||
from core.utils.util import parse_string_to_list
|
||||
from datetime import datetime
|
||||
from pydub import AudioSegment
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
from core.utils.util import parse_string_to_list
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
@@ -61,8 +59,7 @@ class TTSProvider(TTSProviderBase):
|
||||
f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
tmp_file = self.generate_filename()
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"model": self.model,
|
||||
"text": text,
|
||||
@@ -83,7 +80,7 @@ class TTSProvider(TTSProviderBase):
|
||||
# 检查返回请求数据的status_code是否为0
|
||||
if resp.json()["base_resp"]["status_code"] == 0:
|
||||
data = resp.json()["data"]["audio"]
|
||||
file_to_save = open(tmp_file, "wb")
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(bytes.fromhex(data))
|
||||
else:
|
||||
raise Exception(
|
||||
@@ -91,20 +88,3 @@ class TTSProvider(TTSProviderBase):
|
||||
)
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
# 使用 pydub 读取临时文件
|
||||
audio = AudioSegment.from_file(tmp_file, format="mp3")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
# 用完后删除临时文件
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except FileNotFoundError:
|
||||
# 若文件不存在，忽略该异常
|
||||
pass
|
||||
|
||||
@@ -2,12 +2,12 @@ import os
|
||||
import uuid
|
||||
import requests
|
||||
from datetime import datetime
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
from core.utils.util import check_model_key
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
@@ -35,8 +35,7 @@ class TTSProvider(TTSProviderBase):
|
||||
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
tmp_file = self.generate_filename()
|
||||
async def text_to_speak(self, text, output_file):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
@@ -50,26 +49,9 @@ class TTSProvider(TTSProviderBase):
|
||||
}
|
||||
response = requests.post(self.api_url, json=data, headers=headers)
|
||||
if response.status_code == 200:
|
||||
with open(tmp_file, "wb") as audio_file:
|
||||
with open(output_file, "wb") as audio_file:
|
||||
audio_file.write(response.content)
|
||||
else:
|
||||
raise Exception(
|
||||
f"OpenAI TTS请求失败: {response.status_code} - {response.text}"
|
||||
)
|
||||
# 使用 pydub 读取临时文件
|
||||
audio = AudioSegment.from_file(tmp_file, format="wav")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
# 用完后删除临时文件
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except FileNotFoundError:
|
||||
# 若文件不存在，忽略该异常
|
||||
pass
|
||||
|
||||
@@ -2,11 +2,7 @@ import os
|
||||
import uuid
|
||||
import requests
|
||||
from datetime import datetime
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
@@ -32,8 +28,7 @@ class TTSProvider(TTSProviderBase):
|
||||
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
tmp_file = self.generate_filename()
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"model": self.model,
|
||||
"input": text,
|
||||
@@ -44,26 +39,12 @@ class TTSProvider(TTSProviderBase):
|
||||
"Authorization": f"Bearer {self.access_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
response = requests.request(
|
||||
"POST", self.api_url, json=request_json, headers=headers
|
||||
)
|
||||
data = response.content
|
||||
file_to_save = open(tmp_file, "wb")
|
||||
file_to_save.write(data)
|
||||
# 使用 pydub 读取临时文件
|
||||
audio = AudioSegment.from_file(tmp_file, format="wav")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
# 用完后删除临时文件
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except FileNotFoundError:
|
||||
# 若文件不存在，忽略该异常
|
||||
pass
|
||||
response = requests.request(
|
||||
"POST", self.api_url, json=request_json, headers=headers
|
||||
)
|
||||
data = response.content
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(data)
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
@@ -4,11 +4,11 @@ import json
|
||||
import requests
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
|
||||
from pydub import AudioSegment
|
||||
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
@@ -39,8 +39,7 @@ class TTSProvider(TTSProviderBase):
|
||||
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
|
||||
tmp_file = self.generate_filename()
|
||||
async def text_to_speak(self, text, output_file):
|
||||
url = f"{self.url}{self.token}"
|
||||
result = "firefly"
|
||||
payload = json.dumps(
|
||||
@@ -59,7 +58,8 @@ class TTSProvider(TTSProviderBase):
|
||||
|
||||
resp = requests.request("POST", url, data=payload)
|
||||
if resp.status_code != 200:
|
||||
return
|
||||
logger.bind(tag=TAG).error(f"TTSON 请求失败: {resp.text}")
|
||||
raise Exception(f"{__name__}: TTS请求失败")
|
||||
resp_json = resp.json()
|
||||
try:
|
||||
result = (
|
||||
@@ -71,29 +71,15 @@ class TTSProvider(TTSProviderBase):
|
||||
+ "&voice_audio_path="
|
||||
+ resp_json["voice_path"]
|
||||
)
|
||||
|
||||
audio_content = requests.get(result)
|
||||
with open(output_file, "wb") as f:
|
||||
f.write(audio_content.content)
|
||||
return True
|
||||
voice_path = resp_json.get("voice_path")
|
||||
des_path = output_file
|
||||
shutil.move(voice_path, des_path)
|
||||
|
||||
except Exception as e:
|
||||
print("error:", e)
|
||||
|
||||
audio_content = requests.get(result)
|
||||
with open(tmp_file, "wb") as f:
|
||||
f.write(audio_content.content)
|
||||
# 使用 pydub 读取临时文件
|
||||
audio = AudioSegment.from_file(tmp_file, format="mp3")
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
|
||||
yield TTSMessageDTO(
|
||||
u_id=u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_datas,
|
||||
tts_finish_text=text,
|
||||
sentence_type=SentenceType.SENTENCE_START,
|
||||
)
|
||||
# 用完后删除临时文件
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except FileNotFoundError:
|
||||
# 若文件不存在，忽略该异常
|
||||
pass
|
||||
voice_path = resp_json.get("voice_path")
|
||||
des_path = tmp_file
|
||||
shutil.move(voice_path, des_path)
|
||||
raise Exception(f"{__name__}: TTS请求失败")
|
||||
Reference in New Issue
Block a user