mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-25 08:33:53 +08:00
@@ -710,6 +710,26 @@ TTS:
|
|||||||
# voice_id: female-shaonv
|
# voice_id: female-shaonv
|
||||||
# weight: 1
|
# weight: 1
|
||||||
# language_boost: auto
|
# language_boost: auto
|
||||||
|
|
||||||
|
# MinimaxTTSHTTPStream和MinimaxTTSWebSocketStream还在测试,测试完再开放
|
||||||
|
#
|
||||||
|
# MinimaxTTSHTTPStream:
|
||||||
|
# # Minimax流式语音合成服务
|
||||||
|
# type: minimax_httpstream
|
||||||
|
# output_dir: tmp/
|
||||||
|
# group_id: 你的minimax平台groupID
|
||||||
|
# api_key: 你的minimax平台接口密钥
|
||||||
|
# model: "speech-01-turbo"
|
||||||
|
# voice_id: "female-shaonv"
|
||||||
|
#
|
||||||
|
# MinimaxTTSWebSocketStream:
|
||||||
|
# type: minimax_webSocket
|
||||||
|
# output_dir: tmp/
|
||||||
|
# group_id: 你的minimax平台groupID
|
||||||
|
# api_key: 你的minimax平台接口密钥
|
||||||
|
# model: "speech-01-turbo"
|
||||||
|
# voice_id: "female-shaonv"
|
||||||
|
|
||||||
AliyunTTS:
|
AliyunTTS:
|
||||||
# 阿里云智能语音交互服务,需要先在阿里云平台开通服务,然后获取验证信息
|
# 阿里云智能语音交互服务,需要先在阿里云平台开通服务,然后获取验证信息
|
||||||
# 平台地址:https://nls-portal.console.aliyun.com/
|
# 平台地址:https://nls-portal.console.aliyun.com/
|
||||||
|
|||||||
@@ -132,6 +132,8 @@ class ConnectionHandler:
|
|||||||
|
|
||||||
# tts相关变量
|
# tts相关变量
|
||||||
self.sentence_id = None
|
self.sentence_id = None
|
||||||
|
# 处理TTS响应没有文本返回
|
||||||
|
self.tts_MessageText = ""
|
||||||
|
|
||||||
# iot相关变量
|
# iot相关变量
|
||||||
self.iot_descriptors = {}
|
self.iot_descriptors = {}
|
||||||
@@ -182,8 +184,13 @@ class ConnectionHandler:
|
|||||||
await ws.send("端口正常,如需测试连接,请使用test_page.html")
|
await ws.send("端口正常,如需测试连接,请使用test_page.html")
|
||||||
await self.close(ws)
|
await self.close(ws)
|
||||||
return
|
return
|
||||||
# 获取客户端ip地址
|
real_ip = self.headers.get("x-real-ip") or self.headers.get(
|
||||||
self.client_ip = ws.remote_address[0]
|
"x-forwarded-for"
|
||||||
|
)
|
||||||
|
if real_ip:
|
||||||
|
self.client_ip = real_ip.split(",")[0].strip()
|
||||||
|
else:
|
||||||
|
self.client_ip = ws.remote_address[0]
|
||||||
self.logger.bind(tag=TAG).info(
|
self.logger.bind(tag=TAG).info(
|
||||||
f"{self.client_ip} conn - Headers: {self.headers}"
|
f"{self.client_ip} conn - Headers: {self.headers}"
|
||||||
)
|
)
|
||||||
@@ -335,7 +342,7 @@ class ConnectionHandler:
|
|||||||
self.config.get("selected_module", {})
|
self.config.get("selected_module", {})
|
||||||
)
|
)
|
||||||
self.logger = create_connection_logger(self.selected_module_str)
|
self.logger = create_connection_logger(self.selected_module_str)
|
||||||
|
|
||||||
"""初始化组件"""
|
"""初始化组件"""
|
||||||
if self.config.get("prompt") is not None:
|
if self.config.get("prompt") is not None:
|
||||||
user_prompt = self.config["prompt"]
|
user_prompt = self.config["prompt"]
|
||||||
@@ -351,10 +358,10 @@ class ConnectionHandler:
|
|||||||
self.vad = self._vad
|
self.vad = self._vad
|
||||||
if self.asr is None:
|
if self.asr is None:
|
||||||
self.asr = self._initialize_asr()
|
self.asr = self._initialize_asr()
|
||||||
|
|
||||||
# 初始化声纹识别
|
# 初始化声纹识别
|
||||||
self._initialize_voiceprint()
|
self._initialize_voiceprint()
|
||||||
|
|
||||||
# 打开语音识别通道
|
# 打开语音识别通道
|
||||||
asyncio.run_coroutine_threadsafe(
|
asyncio.run_coroutine_threadsafe(
|
||||||
self.asr.open_audio_channels(self), self.loop
|
self.asr.open_audio_channels(self), self.loop
|
||||||
@@ -790,9 +797,9 @@ class ConnectionHandler:
|
|||||||
if not bHasError:
|
if not bHasError:
|
||||||
# 如需要大模型先处理一轮,添加相关处理后的日志情况
|
# 如需要大模型先处理一轮,添加相关处理后的日志情况
|
||||||
if len(response_message) > 0:
|
if len(response_message) > 0:
|
||||||
self.dialogue.put(
|
text_buff = "".join(response_message)
|
||||||
Message(role="assistant", content="".join(response_message))
|
self.tts_MessageText = text_buff
|
||||||
)
|
self.dialogue.put(Message(role="assistant", content=text_buff))
|
||||||
response_message.clear()
|
response_message.clear()
|
||||||
self.logger.bind(tag=TAG).debug(
|
self.logger.bind(tag=TAG).debug(
|
||||||
f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}"
|
f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}"
|
||||||
@@ -814,9 +821,9 @@ class ConnectionHandler:
|
|||||||
|
|
||||||
# 存储对话内容
|
# 存储对话内容
|
||||||
if len(response_message) > 0:
|
if len(response_message) > 0:
|
||||||
self.dialogue.put(
|
text_buff = "".join(response_message)
|
||||||
Message(role="assistant", content="".join(response_message))
|
self.tts_MessageText = text_buff
|
||||||
)
|
self.dialogue.put(Message(role="assistant", content=text_buff))
|
||||||
if depth == 0:
|
if depth == 0:
|
||||||
self.tts.tts_text_queue.put(
|
self.tts.tts_text_queue.put(
|
||||||
TTSMessageDTO(
|
TTSMessageDTO(
|
||||||
@@ -893,9 +900,7 @@ class ConnectionHandler:
|
|||||||
if self.executor is None:
|
if self.executor is None:
|
||||||
continue
|
continue
|
||||||
# 提交任务到线程池
|
# 提交任务到线程池
|
||||||
self.executor.submit(
|
self.executor.submit(self._process_report, *item)
|
||||||
self._process_report, *item
|
|
||||||
)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.bind(tag=TAG).error(f"聊天记录上报线程异常: {e}")
|
self.logger.bind(tag=TAG).error(f"聊天记录上报线程异常: {e}")
|
||||||
except queue.Empty:
|
except queue.Empty:
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ async def sendAudioMessage(conn, sentenceType, audios, text):
|
|||||||
conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}")
|
conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}")
|
||||||
|
|
||||||
pre_buffer = False
|
pre_buffer = False
|
||||||
if conn.tts.tts_audio_first_sentence and text is not None:
|
if conn.tts.tts_audio_first_sentence:
|
||||||
conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
|
conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
|
||||||
conn.tts.tts_audio_first_sentence = False
|
conn.tts.tts_audio_first_sentence = False
|
||||||
pre_buffer = True
|
pre_buffer = True
|
||||||
@@ -73,7 +73,7 @@ async def send_tts_message(conn, state, text=None):
|
|||||||
"""发送 TTS 状态消息"""
|
"""发送 TTS 状态消息"""
|
||||||
message = {"type": "tts", "state": state, "session_id": conn.session_id}
|
message = {"type": "tts", "state": state, "session_id": conn.session_id}
|
||||||
if text is not None:
|
if text is not None:
|
||||||
message["text"] = text
|
message["text"] = textUtils.check_emoji(text)
|
||||||
|
|
||||||
# TTS播放结束
|
# TTS播放结束
|
||||||
if state == "stop":
|
if state == "stop":
|
||||||
|
|||||||
@@ -127,8 +127,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
|
|
||||||
# 专属tts设置
|
# 专属tts设置
|
||||||
self.message_id = ""
|
self.message_id = ""
|
||||||
self.tts_text = ""
|
|
||||||
self.text_buffer = []
|
|
||||||
|
|
||||||
# 创建Opus编码器
|
# 创建Opus编码器
|
||||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||||
@@ -229,7 +227,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
|
|
||||||
# aliyunStream独有的参数生成
|
# aliyunStream独有的参数生成
|
||||||
self.message_id = str(uuid.uuid4().hex)
|
self.message_id = str(uuid.uuid4().hex)
|
||||||
self.text_buffer = []
|
|
||||||
|
|
||||||
logger.bind(tag=TAG).info("开始启动TTS会话...")
|
logger.bind(tag=TAG).info("开始启动TTS会话...")
|
||||||
future = asyncio.run_coroutine_threadsafe(
|
future = asyncio.run_coroutine_threadsafe(
|
||||||
@@ -250,7 +247,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
logger.bind(tag=TAG).debug(
|
logger.bind(tag=TAG).debug(
|
||||||
f"开始发送TTS文本: {message.content_detail}"
|
f"开始发送TTS文本: {message.content_detail}"
|
||||||
)
|
)
|
||||||
self.text_buffer.append(message.content_detail)
|
|
||||||
future = asyncio.run_coroutine_threadsafe(
|
future = asyncio.run_coroutine_threadsafe(
|
||||||
self.text_to_speak(message.content_detail, None),
|
self.text_to_speak(message.content_detail, None),
|
||||||
loop=self.conn.loop,
|
loop=self.conn.loop,
|
||||||
@@ -275,9 +271,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
if message.sentence_type == SentenceType.LAST:
|
if message.sentence_type == SentenceType.LAST:
|
||||||
try:
|
try:
|
||||||
logger.bind(tag=TAG).info("开始结束TTS会话...")
|
logger.bind(tag=TAG).info("开始结束TTS会话...")
|
||||||
self.tts_text = textUtils.get_string_no_punctuation_or_emoji(
|
|
||||||
"".join(self.text_buffer).replace("\n", "")
|
|
||||||
)
|
|
||||||
future = asyncio.run_coroutine_threadsafe(
|
future = asyncio.run_coroutine_threadsafe(
|
||||||
self.finish_session(self.conn.sentence_id),
|
self.finish_session(self.conn.sentence_id),
|
||||||
loop=self.conn.loop,
|
loop=self.conn.loop,
|
||||||
@@ -444,34 +437,35 @@ class TTSProvider(TTSProviderBase):
|
|||||||
event_name = header.get("name")
|
event_name = header.get("name")
|
||||||
if event_name == "SynthesisStarted":
|
if event_name == "SynthesisStarted":
|
||||||
logger.bind(tag=TAG).debug("TTS合成已启动")
|
logger.bind(tag=TAG).debug("TTS合成已启动")
|
||||||
elif event_name == "SentenceBegin":
|
|
||||||
logger.bind(tag=TAG).debug(
|
|
||||||
f"句子语音生成开始: {self.tts_text}"
|
|
||||||
)
|
|
||||||
opus_datas_cache = []
|
|
||||||
self.tts_audio_queue.put(
|
self.tts_audio_queue.put(
|
||||||
(SentenceType.FIRST, [], self.tts_text)
|
(SentenceType.FIRST, [], None)
|
||||||
)
|
)
|
||||||
|
elif event_name == "SentenceBegin":
|
||||||
|
opus_datas_cache = []
|
||||||
elif event_name == "SentenceEnd":
|
elif event_name == "SentenceEnd":
|
||||||
logger.bind(tag=TAG).info(
|
|
||||||
f"句子语音生成成功: {self.tts_text}"
|
|
||||||
)
|
|
||||||
if (
|
if (
|
||||||
not is_first_sentence
|
not is_first_sentence
|
||||||
or first_sentence_segment_count > 10
|
or first_sentence_segment_count > 10
|
||||||
):
|
):
|
||||||
# 发送缓存的数据
|
# 发送缓存的数据
|
||||||
self.tts_audio_queue.put(
|
if self.conn.tts_MessageText:
|
||||||
(SentenceType.MIDDLE, opus_datas_cache, None)
|
logger.bind(tag=TAG).info(
|
||||||
)
|
f"句子语音生成成功: {self.conn.tts_MessageText}"
|
||||||
|
)
|
||||||
|
self.tts_audio_queue.put(
|
||||||
|
(SentenceType.MIDDLE, opus_datas_cache, self.conn.tts_MessageText)
|
||||||
|
)
|
||||||
|
self.conn.tts_MessageText = None
|
||||||
|
else:
|
||||||
|
self.tts_audio_queue.put(
|
||||||
|
(SentenceType.MIDDLE, opus_datas_cache, None)
|
||||||
|
)
|
||||||
# 第一句话结束后,将标志设置为False
|
# 第一句话结束后,将标志设置为False
|
||||||
is_first_sentence = False
|
is_first_sentence = False
|
||||||
elif event_name == "SynthesisCompleted":
|
elif event_name == "SynthesisCompleted":
|
||||||
logger.bind(tag=TAG).debug(f"会话结束~~")
|
logger.bind(tag=TAG).debug(f"会话结束~~")
|
||||||
self._process_before_stop_play_files()
|
self._process_before_stop_play_files()
|
||||||
session_finished = True
|
session_finished = True
|
||||||
self.reuse_judgment = time.time()
|
|
||||||
self.tts_text = ""
|
|
||||||
break
|
break
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
logger.bind(tag=TAG).warning("收到无效的JSON消息")
|
logger.bind(tag=TAG).warning("收到无效的JSON消息")
|
||||||
|
|||||||
@@ -232,7 +232,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
loop=self.conn.loop,
|
loop=self.conn.loop,
|
||||||
)
|
)
|
||||||
future.result()
|
future.result()
|
||||||
self.tts_audio_first_sentence = True
|
|
||||||
self.before_stop_play_files.clear()
|
self.before_stop_play_files.clear()
|
||||||
logger.bind(tag=TAG).info("TTS会话启动成功")
|
logger.bind(tag=TAG).info("TTS会话启动成功")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -52,7 +52,6 @@ class TTSProvider(TTSProviderBase):
|
|||||||
self.processed_chars = 0
|
self.processed_chars = 0
|
||||||
self.tts_text_buff = []
|
self.tts_text_buff = []
|
||||||
self.segment_count = 0
|
self.segment_count = 0
|
||||||
self.tts_audio_first_sentence = True
|
|
||||||
self.before_stop_play_files.clear()
|
self.before_stop_play_files.clear()
|
||||||
elif ContentType.TEXT == message.content_type:
|
elif ContentType.TEXT == message.content_type:
|
||||||
self.tts_text_buff.append(message.content_detail)
|
self.tts_text_buff.append(message.content_detail)
|
||||||
|
|||||||
@@ -0,0 +1,230 @@
|
|||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
import json
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Iterator, Optional, Union
|
||||||
|
from core.providers.tts.base import TTSProviderBase
|
||||||
|
from core.utils.util import parse_string_to_list
|
||||||
|
|
||||||
|
|
||||||
|
class TTSProvider(TTSProviderBase):
|
||||||
|
def __init__(self, config, delete_audio_file):
|
||||||
|
super().__init__(config, delete_audio_file)
|
||||||
|
self.group_id = config.get("group_id")
|
||||||
|
self.api_key = config.get("api_key")
|
||||||
|
self.model = config.get("model")
|
||||||
|
if config.get("private_voice"):
|
||||||
|
self.voice = config.get("private_voice")
|
||||||
|
else:
|
||||||
|
self.voice = config.get("voice_id")
|
||||||
|
|
||||||
|
default_voice_setting = {
|
||||||
|
"voice_id": "female-shaonv",
|
||||||
|
"speed": 1,
|
||||||
|
"vol": 1,
|
||||||
|
"pitch": 0,
|
||||||
|
"emotion": "happy",
|
||||||
|
}
|
||||||
|
default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}
|
||||||
|
defult_audio_setting = {
|
||||||
|
"sample_rate": 32000,
|
||||||
|
"bitrate": 128000,
|
||||||
|
"format": "mp3",
|
||||||
|
"channel": 1,
|
||||||
|
}
|
||||||
|
self.voice_setting = {
|
||||||
|
**default_voice_setting,
|
||||||
|
**config.get("voice_setting", {}),
|
||||||
|
}
|
||||||
|
self.pronunciation_dict = {
|
||||||
|
**default_pronunciation_dict,
|
||||||
|
**config.get("pronunciation_dict", {}),
|
||||||
|
}
|
||||||
|
self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
|
||||||
|
self.timber_weights = parse_string_to_list(config.get("timber_weights"))
|
||||||
|
|
||||||
|
if self.voice:
|
||||||
|
self.voice_setting["voice_id"] = self.voice
|
||||||
|
|
||||||
|
self.host = "api.minimax.chat"
|
||||||
|
self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
|
||||||
|
self.header = {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Authorization": f"Bearer {self.api_key}",
|
||||||
|
}
|
||||||
|
self.audio_file_type = defult_audio_setting.get("format", "mp3")
|
||||||
|
|
||||||
|
def generate_filename(self, extension=".mp3"):
|
||||||
|
return os.path.join(
|
||||||
|
self.output_file,
|
||||||
|
f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||||
|
)
|
||||||
|
|
||||||
|
async def text_to_speak(self, text, output_file):
|
||||||
|
"""非流式语音合成(保留原有实现)"""
|
||||||
|
request_json = {
|
||||||
|
"model": self.model,
|
||||||
|
"text": text,
|
||||||
|
"stream": False,
|
||||||
|
"voice_setting": self.voice_setting,
|
||||||
|
"pronunciation_dict": self.pronunciation_dict,
|
||||||
|
"audio_setting": self.audio_setting,
|
||||||
|
}
|
||||||
|
|
||||||
|
if type(self.timber_weights) is list and len(self.timber_weights) > 0:
|
||||||
|
request_json["timber_weights"] = self.timber_weights
|
||||||
|
request_json["voice_setting"]["voice_id"] = ""
|
||||||
|
|
||||||
|
try:
|
||||||
|
resp = requests.post(
|
||||||
|
self.api_url, json.dumps(request_json), headers=self.header
|
||||||
|
)
|
||||||
|
if resp.json()["base_resp"]["status_code"] == 0:
|
||||||
|
data = resp.json()["data"]["audio"]
|
||||||
|
audio_bytes = bytes.fromhex(data)
|
||||||
|
if output_file:
|
||||||
|
with open(output_file, "wb") as file_to_save:
|
||||||
|
file_to_save.write(audio_bytes)
|
||||||
|
else:
|
||||||
|
return audio_bytes
|
||||||
|
else:
|
||||||
|
raise Exception(
|
||||||
|
f"{__name__} status_code: {resp.status_code} response: {resp.content}"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"{__name__} error: {e}")
|
||||||
|
|
||||||
|
def text_to_speak_stream(
|
||||||
|
self,
|
||||||
|
text: str,
|
||||||
|
chunk_callback: Optional[callable] = None
|
||||||
|
) -> Iterator[bytes]:
|
||||||
|
"""
|
||||||
|
流式语音合成方法
|
||||||
|
:param text: 要合成的文本
|
||||||
|
:param chunk_callback: 可选的回调函数,用于处理每个音频块
|
||||||
|
:return: 生成器,每次产生一个音频数据块(bytes)
|
||||||
|
"""
|
||||||
|
request_json = {
|
||||||
|
"model": self.model,
|
||||||
|
"text": text,
|
||||||
|
"stream": True,
|
||||||
|
"voice_setting": self.voice_setting,
|
||||||
|
"pronunciation_dict": self.pronunciation_dict,
|
||||||
|
"audio_setting": self.audio_setting,
|
||||||
|
}
|
||||||
|
|
||||||
|
if isinstance(self.timber_weights, list) and len(self.timber_weights) > 0:
|
||||||
|
request_json["timber_weights"] = self.timber_weights
|
||||||
|
request_json["voice_setting"]["voice_id"] = ""
|
||||||
|
|
||||||
|
try:
|
||||||
|
with requests.post(
|
||||||
|
self.api_url,
|
||||||
|
data=json.dumps(request_json),
|
||||||
|
headers=self.header,
|
||||||
|
stream=True
|
||||||
|
) as response:
|
||||||
|
|
||||||
|
# 检查HTTP状态码
|
||||||
|
if response.status_code != 200:
|
||||||
|
raise Exception(
|
||||||
|
f"HTTP error: {response.status_code}, response: {response.text}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# 处理流式响应
|
||||||
|
for line in response.iter_lines():
|
||||||
|
if line: # 过滤空行
|
||||||
|
# 检查是否为数据行 (SSE格式)
|
||||||
|
if line.startswith(b'data:'):
|
||||||
|
try:
|
||||||
|
data = json.loads(line[5:].strip()) # 去掉"data:"前缀
|
||||||
|
|
||||||
|
# 检查API状态码
|
||||||
|
if data.get("base_resp", {}).get("status_code", -1) != 0:
|
||||||
|
raise Exception(
|
||||||
|
f"API error: {data.get('base_resp', {}).get('status_msg')}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# 跳过非音频数据块
|
||||||
|
if "extra_info" in data:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 提取音频数据
|
||||||
|
audio_hex = data.get("data", {}).get("audio")
|
||||||
|
if audio_hex:
|
||||||
|
audio_chunk = bytes.fromhex(audio_hex)
|
||||||
|
if chunk_callback:
|
||||||
|
chunk_callback(audio_chunk)
|
||||||
|
yield audio_chunk
|
||||||
|
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
# 忽略JSON解析错误(可能是心跳包等)
|
||||||
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
raise e
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"{__name__} stream error: {e}")
|
||||||
|
|
||||||
|
def save_stream_to_file(
|
||||||
|
self,
|
||||||
|
text: str,
|
||||||
|
output_file: Optional[str] = None,
|
||||||
|
progress_callback: Optional[callable] = None
|
||||||
|
) -> str:
|
||||||
|
"""
|
||||||
|
流式合成并保存到文件
|
||||||
|
:param text: 要合成的文本
|
||||||
|
:param output_file: 输出文件路径,如果为None则自动生成
|
||||||
|
:param progress_callback: 可选的回调函数,接收已写入的字节数
|
||||||
|
:return: 保存的文件路径
|
||||||
|
"""
|
||||||
|
if not output_file:
|
||||||
|
output_file = self.generate_filename(extension=f".{self.audio_file_type}")
|
||||||
|
|
||||||
|
os.makedirs(os.path.dirname(output_file), exist_ok=True)
|
||||||
|
|
||||||
|
total_bytes = 0
|
||||||
|
try:
|
||||||
|
with open(output_file, "wb") as audio_file:
|
||||||
|
for audio_chunk in self.text_to_speak_stream(text):
|
||||||
|
audio_file.write(audio_chunk)
|
||||||
|
audio_file.flush()
|
||||||
|
total_bytes += len(audio_chunk)
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback(total_bytes)
|
||||||
|
return output_file
|
||||||
|
except Exception as e:
|
||||||
|
# 清理可能创建的不完整文件
|
||||||
|
if os.path.exists(output_file):
|
||||||
|
os.remove(output_file)
|
||||||
|
raise e
|
||||||
|
|
||||||
|
def stream_to_audio_player(self, text: str, player_command: list = None):
|
||||||
|
"""
|
||||||
|
流式合成并直接播放音频
|
||||||
|
:param text: 要合成的文本
|
||||||
|
:param player_command: 音频播放器命令,默认使用mpv
|
||||||
|
"""
|
||||||
|
if player_command is None:
|
||||||
|
player_command = ["mpv", "--no-cache", "--no-terminal", "--", "fd://0"]
|
||||||
|
|
||||||
|
try:
|
||||||
|
import subprocess
|
||||||
|
player_process = subprocess.Popen(
|
||||||
|
player_command,
|
||||||
|
stdin=subprocess.PIPE,
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
|
||||||
|
for audio_chunk in self.text_to_speak_stream(text):
|
||||||
|
player_process.stdin.write(audio_chunk)
|
||||||
|
player_process.stdin.flush()
|
||||||
|
|
||||||
|
player_process.stdin.close()
|
||||||
|
player_process.wait()
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(f"Audio player error: {e}")
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
import json
|
||||||
|
import asyncio
|
||||||
|
import websockets
|
||||||
|
import ssl
|
||||||
|
from datetime import datetime
|
||||||
|
from core.providers.tts.base import TTSProviderBase
|
||||||
|
from core.utils.util import parse_string_to_list
|
||||||
|
|
||||||
|
|
||||||
|
class TTSProvider(TTSProviderBase):
|
||||||
|
def __init__(self, config, delete_audio_file):
|
||||||
|
super().__init__(config, delete_audio_file)
|
||||||
|
self.group_id = config.get("group_id")
|
||||||
|
self.api_key = config.get("api_key")
|
||||||
|
self.model = config.get("model")
|
||||||
|
|
||||||
|
# 初始化语音设置
|
||||||
|
default_voice_setting = {
|
||||||
|
"voice_id": "female-shaonv",
|
||||||
|
"speed": 1,
|
||||||
|
"vol": 1,
|
||||||
|
"pitch": 0,
|
||||||
|
"emotion": "happy",
|
||||||
|
}
|
||||||
|
default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}
|
||||||
|
default_audio_setting = {
|
||||||
|
"sample_rate": 32000,
|
||||||
|
"bitrate": 128000,
|
||||||
|
"format": "mp3",
|
||||||
|
"channel": 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
# 合并配置
|
||||||
|
self.voice_setting = {
|
||||||
|
**default_voice_setting,
|
||||||
|
**config.get("voice_setting", {}),
|
||||||
|
}
|
||||||
|
self.pronunciation_dict = {
|
||||||
|
**default_pronunciation_dict,
|
||||||
|
**config.get("pronunciation_dict", {}),
|
||||||
|
}
|
||||||
|
self.audio_setting = {
|
||||||
|
**default_audio_setting,
|
||||||
|
**config.get("audio_setting", {})
|
||||||
|
}
|
||||||
|
self.timber_weights = parse_string_to_list(config.get("timber_weights"))
|
||||||
|
|
||||||
|
# 设置语音ID
|
||||||
|
if config.get("private_voice"):
|
||||||
|
self.voice_setting["voice_id"] = config.get("private_voice")
|
||||||
|
elif config.get("voice_id"):
|
||||||
|
self.voice_setting["voice_id"] = config.get("voice_id")
|
||||||
|
|
||||||
|
# WebSocket配置
|
||||||
|
self.ws_url = "wss://api.minimaxi.com/ws/v1/t2a_v2"
|
||||||
|
self.headers = {
|
||||||
|
"Authorization": f"Bearer {self.api_key}",
|
||||||
|
"GroupId": self.group_id
|
||||||
|
}
|
||||||
|
self.audio_file_type = self.audio_setting.get("format", "mp3")
|
||||||
|
|
||||||
|
def generate_filename(self, extension=".mp3"):
|
||||||
|
"""生成唯一的音频文件名"""
|
||||||
|
return os.path.join(
|
||||||
|
self.output_file,
|
||||||
|
f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _establish_connection(self):
|
||||||
|
"""建立WebSocket连接"""
|
||||||
|
ssl_context = ssl.create_default_context()
|
||||||
|
ssl_context.check_hostname = False
|
||||||
|
ssl_context.verify_mode = ssl.CERT_NONE
|
||||||
|
|
||||||
|
try:
|
||||||
|
ws = await websockets.connect(
|
||||||
|
self.ws_url,
|
||||||
|
additional_headers=self.headers,
|
||||||
|
ssl=ssl_context
|
||||||
|
)
|
||||||
|
connected = json.loads(await ws.recv())
|
||||||
|
if connected.get("event") == "connected_success":
|
||||||
|
print("连接成功")
|
||||||
|
return ws
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
print(f"连接失败: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def _start_task(self, websocket):
|
||||||
|
"""发送任务开始请求"""
|
||||||
|
start_msg = {
|
||||||
|
"event": "task_start",
|
||||||
|
"model": self.model,
|
||||||
|
"voice_setting": self.voice_setting,
|
||||||
|
"pronunciation_dict": self.pronunciation_dict,
|
||||||
|
"audio_setting": self.audio_setting
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.timber_weights and len(self.timber_weights) > 0:
|
||||||
|
start_msg["timber_weights"] = self.timber_weights
|
||||||
|
start_msg["voice_setting"]["voice_id"] = ""
|
||||||
|
|
||||||
|
await websocket.send(json.dumps(start_msg))
|
||||||
|
response = json.loads(await websocket.recv())
|
||||||
|
return response.get("event") == "task_started"
|
||||||
|
|
||||||
|
async def _continue_task(self, websocket, text):
|
||||||
|
"""发送继续请求并收集音频数据"""
|
||||||
|
await websocket.send(json.dumps({
|
||||||
|
"event": "task_continue",
|
||||||
|
"text": text
|
||||||
|
}))
|
||||||
|
|
||||||
|
audio_chunks = []
|
||||||
|
while True:
|
||||||
|
response = json.loads(await websocket.recv())
|
||||||
|
if "data" in response and "audio" in response["data"]:
|
||||||
|
audio_chunks.append(response["data"]["audio"])
|
||||||
|
if response.get("is_final"):
|
||||||
|
break
|
||||||
|
return "".join(audio_chunks)
|
||||||
|
|
||||||
|
async def _close_connection(self, websocket):
|
||||||
|
"""关闭连接"""
|
||||||
|
if websocket:
|
||||||
|
await websocket.send(json.dumps({"event": "task_finish"}))
|
||||||
|
await websocket.close()
|
||||||
|
print("连接已关闭")
|
||||||
|
|
||||||
|
async def text_to_speak(self, text, output_file=None):
|
||||||
|
"""主方法:文本转语音"""
|
||||||
|
ws = await self._establish_connection()
|
||||||
|
if not ws:
|
||||||
|
raise Exception("无法建立WebSocket连接")
|
||||||
|
|
||||||
|
try:
|
||||||
|
if not await self._start_task(ws):
|
||||||
|
raise Exception("任务启动失败")
|
||||||
|
|
||||||
|
hex_audio = await self._continue_task(ws, text)
|
||||||
|
audio_bytes = bytes.fromhex(hex_audio)
|
||||||
|
|
||||||
|
# 保存到文件或返回二进制数据
|
||||||
|
if output_file:
|
||||||
|
with open(output_file, "wb") as f:
|
||||||
|
f.write(audio_bytes)
|
||||||
|
print(f"音频已保存为{output_file}")
|
||||||
|
return output_file
|
||||||
|
else:
|
||||||
|
# 返回音频二进制数据(不播放)
|
||||||
|
return audio_bytes
|
||||||
|
|
||||||
|
finally:
|
||||||
|
await self._close_connection(ws)
|
||||||
|
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
"""测试用主函数"""
|
||||||
|
# 示例配置
|
||||||
|
config = {
|
||||||
|
"group_id": "YOUR_GROUP_ID", # 替换为实际的group_id
|
||||||
|
"api_key": "YOUR_API_KEY", # 替换为实际的api_key
|
||||||
|
"model": "your-model", # 替换为实际的模型名称
|
||||||
|
"voice_id": "male-qn-qingse",
|
||||||
|
"voice_setting": {
|
||||||
|
"speed": 1.2,
|
||||||
|
"emotion": "happy"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tts = TTSProvider(config, delete_audio_file=True)
|
||||||
|
output_file = tts.generate_filename()
|
||||||
|
await tts.text_to_speak("这是一个测试文本,用于验证流式语音合成功能", output_file)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(main())
|
||||||
@@ -24,6 +24,15 @@ EMOJI_MAP = {
|
|||||||
"😘": "kissy",
|
"😘": "kissy",
|
||||||
"😏": "confident",
|
"😏": "confident",
|
||||||
}
|
}
|
||||||
|
EMOJI_RANGES = [
|
||||||
|
(0x1F600, 0x1F64F),
|
||||||
|
(0x1F300, 0x1F5FF),
|
||||||
|
(0x1F680, 0x1F6FF),
|
||||||
|
(0x1F900, 0x1F9FF),
|
||||||
|
(0x1FA70, 0x1FAFF),
|
||||||
|
(0x2600, 0x26FF),
|
||||||
|
(0x2700, 0x27BF),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def get_string_no_punctuation_or_emoji(s):
|
def get_string_no_punctuation_or_emoji(s):
|
||||||
@@ -65,18 +74,7 @@ def is_punctuation_or_emoji(char):
|
|||||||
}
|
}
|
||||||
if char.isspace() or char in punctuation_set:
|
if char.isspace() or char in punctuation_set:
|
||||||
return True
|
return True
|
||||||
# 检查表情符号(保留原有逻辑)
|
return is_emoji(char)
|
||||||
code_point = ord(char)
|
|
||||||
emoji_ranges = [
|
|
||||||
(0x1F600, 0x1F64F),
|
|
||||||
(0x1F300, 0x1F5FF),
|
|
||||||
(0x1F680, 0x1F6FF),
|
|
||||||
(0x1F900, 0x1F9FF),
|
|
||||||
(0x1FA70, 0x1FAFF),
|
|
||||||
(0x2600, 0x26FF),
|
|
||||||
(0x2700, 0x27BF),
|
|
||||||
]
|
|
||||||
return any(start <= code_point <= end for start, end in emoji_ranges)
|
|
||||||
|
|
||||||
|
|
||||||
async def get_emotion(conn, text):
|
async def get_emotion(conn, text):
|
||||||
@@ -102,3 +100,14 @@ async def get_emotion(conn, text):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
conn.logger.bind(tag=TAG).warning(f"发送情绪表情失败,错误:{e}")
|
conn.logger.bind(tag=TAG).warning(f"发送情绪表情失败,错误:{e}")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def is_emoji(char):
|
||||||
|
"""检查字符是否为emoji表情"""
|
||||||
|
code_point = ord(char)
|
||||||
|
return any(start <= code_point <= end for start, end in EMOJI_RANGES)
|
||||||
|
|
||||||
|
|
||||||
|
def check_emoji(text):
|
||||||
|
"""去除文本中的所有emoji表情"""
|
||||||
|
return ''.join(char for char in text if not is_emoji(char) and char != "\n")
|
||||||
|
|||||||
Reference in New Issue
Block a user