mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-31 03:43:57 +08:00
update:旧非流失兼容改造
This commit is contained in:
hrz
@@ -2,12 +2,12 @@ import asyncio
|
||||
from config.logger import setup_logging
|
||||
import queue
|
||||
import os
|
||||
import json
|
||||
import uuid
|
||||
import threading
|
||||
from enum import Enum
|
||||
from core.utils import p3
|
||||
from core.utils import textUtils
|
||||
from core.handle.sendAudioHandle import sendAudioMessage
|
||||
from core.handle.reportHandle import enqueue_tts_report
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType, ContentType
|
||||
from abc import ABC, abstractmethod
|
||||
from core.utils.tts import MarkdownCleaner
|
||||
from core.utils.util import audio_to_data
|
||||
@@ -16,30 +16,39 @@ TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TTSImplementationType(Enum):
|
||||
"""TTS实现类型枚举"""
|
||||
|
||||
NON_STREAMING = "non_streaming" # 非流式实现
|
||||
SINGLE_STREAMING = "single_streaming" # 单流式实现
|
||||
DOUBLE_STREAMING = "double_streaming" # 双流式实现
|
||||
|
||||
|
||||
class TTSProviderBase(ABC):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
self.conn = None
|
||||
self.tts_timeout = 10
|
||||
self.delete_audio_file = delete_audio_file
|
||||
self.output_file = config.get("output_dir")
|
||||
self.tts_queue = queue.Queue()
|
||||
self.audio_play_queue = queue.Queue()
|
||||
# 添加实现类型属性，默认为非流式
|
||||
self.interface_type = TTSImplementationType.NON_STREAMING
|
||||
self.tts_text_queue = queue.Queue()
|
||||
self.tts_audio_queue = queue.Queue()
|
||||
self.stop_event = threading.Event()
|
||||
self.loop = asyncio.get_event_loop()
|
||||
|
||||
self.tts_text_buff = []
|
||||
self.punctuations = (
|
||||
"。",
|
||||
".",
|
||||
"？",
|
||||
"?",
|
||||
"！",
|
||||
"!",
|
||||
"；",
|
||||
";",
|
||||
"：",
|
||||
)
|
||||
self.first_sentence_punctuations = ("，", ",")
|
||||
self.tts_stop_request = False
|
||||
self.processed_chars = 0
|
||||
self.is_first_sentence = True
|
||||
|
||||
@abstractmethod
|
||||
def generate_filename(self):
|
||||
pass
|
||||
|
||||
def to_tts(self, text, index):
|
||||
def to_tts(self, text):
|
||||
tmp_file = self.generate_filename()
|
||||
try:
|
||||
max_repeat_time = 5
|
||||
@@ -82,12 +91,51 @@ class TTSProviderBase(ABC):
|
||||
"""音频文件转换为Opus编码"""
|
||||
return audio_to_data(audio_file_path, is_opus=True)
|
||||
|
||||
def tts_one_sentence(
|
||||
self,
|
||||
conn,
|
||||
content_type,
|
||||
content_detail=None,
|
||||
content_file=None,
|
||||
sentence_id=None,
|
||||
):
|
||||
"""发送一句话"""
|
||||
if not sentence_id:
|
||||
if conn.sentence_id:
|
||||
sentence_id = conn.sentence_id
|
||||
else:
|
||||
sentence_id = str(uuid.uuid4()).replace("-", "")
|
||||
conn.sentence_id = sentence_id
|
||||
self.tts_text_queue.put(
|
||||
TTSMessageDTO(
|
||||
sentence_id=sentence_id,
|
||||
sentence_type=SentenceType.FIRST,
|
||||
content_type=ContentType.ACTION,
|
||||
)
|
||||
)
|
||||
self.tts_text_queue.put(
|
||||
TTSMessageDTO(
|
||||
sentence_id=sentence_id,
|
||||
sentence_type=SentenceType.MIDDLE,
|
||||
content_type=content_type,
|
||||
content_detail=content_detail,
|
||||
content_file=content_file,
|
||||
)
|
||||
)
|
||||
self.tts_text_queue.put(
|
||||
TTSMessageDTO(
|
||||
sentence_id=sentence_id,
|
||||
sentence_type=SentenceType.LAST,
|
||||
content_type=ContentType.ACTION,
|
||||
)
|
||||
)
|
||||
|
||||
async def open_audio_channels(self, conn):
|
||||
self.conn = conn
|
||||
self.tts_timeout = conn.config.get("tts_timeout", 10)
|
||||
# tts 消化线程
|
||||
self.tts_priority_thread = threading.Thread(
|
||||
target=self._tts_priority_thread, daemon=True
|
||||
target=self._tts_text_priority_thread, daemon=True
|
||||
)
|
||||
self.tts_priority_thread.start()
|
||||
|
||||
@@ -97,113 +145,60 @@ class TTSProviderBase(ABC):
|
||||
)
|
||||
self.audio_play_priority_thread.start()
|
||||
|
||||
def _tts_priority_thread(self):
|
||||
while not self.conn.stop_event.is_set():
|
||||
text = None
|
||||
# 这里默认是非流式的处理方式
|
||||
# 流式处理方式请在子类中重写
|
||||
def _tts_text_priority_thread(self):
|
||||
while not self.stop_event.is_set():
|
||||
try:
|
||||
try:
|
||||
item = self.tts_queue.get(timeout=1)
|
||||
if item is None:
|
||||
continue
|
||||
future, text_index = item # 解包获取 Future 和 text_index
|
||||
except queue.Empty:
|
||||
if self.conn.stop_event.is_set():
|
||||
break
|
||||
continue
|
||||
if future is None:
|
||||
continue
|
||||
text = None
|
||||
audio_datas, tts_file = [], None
|
||||
try:
|
||||
logger.bind(tag=TAG).debug("正在处理TTS任务...")
|
||||
tts_file, text, _ = future.result(timeout=self.tts_timeout)
|
||||
|
||||
# 如果tts_file返回流式标识，则不继续处理
|
||||
if tts_file is None:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"TTS出错： file is empty: {text_index}: {text}"
|
||||
message = self.tts_text_queue.get()
|
||||
if message.sentence_type == SentenceType.FIRST:
|
||||
# 初始化参数
|
||||
self.tts_stop_request = False
|
||||
self.processed_chars = 0
|
||||
self.tts_text_buff = []
|
||||
self.is_first_sentence = True
|
||||
elif ContentType.TEXT == message.content_type:
|
||||
self.tts_text_buff.append(message.content_detail)
|
||||
segment_text = self._get_segment_text()
|
||||
if segment_text:
|
||||
tts_file = self.to_tts(segment_text)
|
||||
audio_datas = self._process_audio_file(tts_file)
|
||||
self.tts_audio_queue.put(
|
||||
(message.sentence_type, audio_datas, segment_text)
|
||||
)
|
||||
elif (
|
||||
tts_file == TTSImplementationType.SINGLE_STREAMING.value
|
||||
or tts_file == TTSImplementationType.DOUBLE_STREAMING.value
|
||||
):
|
||||
logger.bind(tag=TAG).debug(f"TTS生成：流式标识: {tts_file}")
|
||||
continue
|
||||
else:
|
||||
logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
|
||||
if os.path.exists(tts_file):
|
||||
if tts_file.endswith(".p3"):
|
||||
audio_datas, _ = p3.decode_opus_from_file(tts_file)
|
||||
elif self.conn.audio_format == "pcm":
|
||||
audio_datas, _ = self.audio_to_pcm_data(tts_file)
|
||||
else:
|
||||
audio_datas, _ = self.audio_to_opus_data(tts_file)
|
||||
# 在这里上报TTS数据
|
||||
enqueue_tts_report(
|
||||
self.conn,
|
||||
tts_file if text is None else text,
|
||||
audio_datas,
|
||||
)
|
||||
else:
|
||||
logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
|
||||
except TimeoutError:
|
||||
logger.bind(tag=TAG).error("TTS超时")
|
||||
except Exception as e:
|
||||
import traceback
|
||||
elif ContentType.FILE == message.content_type:
|
||||
self._process_remaining_text()
|
||||
|
||||
error_info = {
|
||||
"error_type": type(e).__name__,
|
||||
"error_message": str(e),
|
||||
"stack_trace": traceback.format_exc(),
|
||||
"text_index": text_index,
|
||||
"text": text,
|
||||
"tts_file": tts_file,
|
||||
"audio_format": getattr(self.conn, "audio_format", None),
|
||||
"interface_type": self.interface_type.value,
|
||||
}
|
||||
logger.bind(tag=TAG).error(
|
||||
f"TTS处理出错: {json.dumps(error_info, ensure_ascii=False)}"
|
||||
tts_file = message.content_file
|
||||
audio_datas = self._process_audio_file(tts_file)
|
||||
self.tts_audio_queue.put(
|
||||
(message.sentence_type, audio_datas, message.content_detail)
|
||||
)
|
||||
if not self.conn.client_abort:
|
||||
# 如果没有中途打断就发送语音
|
||||
self.audio_play_queue.put((audio_datas, text, text_index))
|
||||
if (
|
||||
self.delete_audio_file
|
||||
and tts_file is not None
|
||||
and os.path.exists(tts_file)
|
||||
and tts_file.startswith(self.output_file)
|
||||
):
|
||||
os.remove(tts_file)
|
||||
if message.sentence_type == SentenceType.LAST:
|
||||
self._process_remaining_text()
|
||||
|
||||
self.tts_audio_queue.put(
|
||||
(message.sentence_type, [], message.content_detail)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
|
||||
self.conn.clearSpeakStatus()
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "tts",
|
||||
"state": "stop",
|
||||
"session_id": self.conn.session_id,
|
||||
}
|
||||
)
|
||||
),
|
||||
self.conn.loop,
|
||||
)
|
||||
logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
|
||||
logger.bind(tag=TAG).error(f"Failed to process TTS text: {e}")
|
||||
|
||||
def _audio_play_priority_thread(self):
|
||||
while not self.conn.stop_event.is_set():
|
||||
while not self.stop_event.is_set():
|
||||
text = None
|
||||
try:
|
||||
try:
|
||||
audio_datas, text, text_index = self.audio_play_queue.get(timeout=1)
|
||||
sentence_type, audio_datas, text = self.tts_audio_queue.get(
|
||||
timeout=1
|
||||
)
|
||||
except queue.Empty:
|
||||
if self.conn.stop_event.is_set():
|
||||
if self.stop_event.is_set():
|
||||
break
|
||||
continue
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
sendAudioMessage(self.conn, audio_datas, text, text_index),
|
||||
self.conn.loop,
|
||||
sendAudioMessage(self.conn, sentence_type, audio_datas, text),
|
||||
self.loop,
|
||||
)
|
||||
future.result()
|
||||
except Exception as e:
|
||||
@@ -221,3 +216,81 @@ class TTSProviderBase(ABC):
|
||||
"""资源清理方法"""
|
||||
if hasattr(self, "ws") and self.ws:
|
||||
await self.ws.close()
|
||||
|
||||
def _get_segment_text(self):
|
||||
# 合并当前全部文本并处理未分割部分
|
||||
full_text = "".join(self.tts_text_buff)
|
||||
current_text = full_text[self.processed_chars :] # 从未处理的位置开始
|
||||
last_punct_pos = -1
|
||||
|
||||
# 根据是否是第一句话选择不同的标点符号集合
|
||||
punctuations_to_use = (
|
||||
self.first_sentence_punctuations
|
||||
if self.is_first_sentence
|
||||
else self.punctuations
|
||||
)
|
||||
|
||||
for punct in punctuations_to_use:
|
||||
pos = current_text.rfind(punct)
|
||||
if (pos != -1 and last_punct_pos == -1) or (
|
||||
pos != -1 and pos < last_punct_pos
|
||||
):
|
||||
last_punct_pos = pos
|
||||
|
||||
if last_punct_pos != -1:
|
||||
segment_text_raw = current_text[: last_punct_pos + 1]
|
||||
segment_text = textUtils.get_string_no_punctuation_or_emoji(
|
||||
segment_text_raw
|
||||
)
|
||||
self.processed_chars += len(segment_text_raw) # 更新已处理字符位置
|
||||
|
||||
# 如果是第一句话，在找到第一个逗号后，将标志设置为False
|
||||
if self.is_first_sentence:
|
||||
self.is_first_sentence = False
|
||||
|
||||
return segment_text
|
||||
elif self.tts_stop_request and current_text:
|
||||
segment_text = current_text
|
||||
self.is_first_sentence = True # 重置标志
|
||||
return segment_text
|
||||
else:
|
||||
return None
|
||||
|
||||
def _process_audio_file(self, tts_file):
|
||||
"""处理音频文件并转换为指定格式
|
||||
|
||||
Args:
|
||||
tts_file: 音频文件路径
|
||||
content_detail: 内容详情
|
||||
|
||||
Returns:
|
||||
tuple: (sentence_type, audio_datas, content_detail)
|
||||
"""
|
||||
audio_datas = []
|
||||
if tts_file.endswith(".p3"):
|
||||
audio_datas, _ = p3.decode_opus_from_file(tts_file)
|
||||
elif self.conn.audio_format == "pcm":
|
||||
audio_datas, _ = self.audio_to_pcm_data(tts_file)
|
||||
else:
|
||||
audio_datas, _ = self.audio_to_opus_data(tts_file)
|
||||
return audio_datas
|
||||
|
||||
def _process_remaining_text(self):
|
||||
"""处理剩余的文本并生成语音
|
||||
|
||||
Returns:
|
||||
bool: 是否成功处理了文本
|
||||
"""
|
||||
full_text = "".join(self.tts_text_buff)
|
||||
remaining_text = full_text[self.processed_chars :]
|
||||
if remaining_text:
|
||||
segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text)
|
||||
if segment_text:
|
||||
tts_file = self.to_tts(segment_text)
|
||||
audio_datas = self._process_audio_file(tts_file)
|
||||
self.tts_audio_queue.put(
|
||||
(SentenceType.MIDDLE, audio_datas, segment_text)
|
||||
)
|
||||
self.processed_chars += len(full_text)
|
||||
return True
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user