add：火山双向tts语音流式输入输出

main/xiaozhi-server/core/providers/tts/base.py

@@ -1,20 +1,231 @@
 import asyncio
+import gc
+import io
+import threading
+import traceback
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+
+import torch
+import torchaudio
+
 from config.logger import setup_logging
 import os
 import numpy as np
 import opuslib_next
 from pydub import AudioSegment
 from abc import ABC, abstractmethod
+from core.utils import textUtils
 import queue
 
+from core.providers.tts.dto.dto import MsgType, TTSMessageDTO, SentenceType
+
 TAG = __name__
 logger = setup_logging()
 
 
 class TTSProviderBase(ABC):
     def __init__(self, config, delete_audio_file):
+        self.config = config
         self.delete_audio_file = delete_audio_file
         self.output_file = config.get("output_dir")
+        self.tts_text_queue = queue.Queue()
+        self.tts_audio_queue = queue.Queue()
+        self.enable_two_way = False
+        self.stop_event = threading.Event()
+
+        self.tts_text_buff = []
+        self.punctuations = ("。", "？", "！", "；", "：", ".", "?", "!", ";", ":", " ", ",", "，")
+        self.tts_request = False
+        self.processed_chars = 0
+        self.stream = False
+        self.last_to_opus_raw = b''
+
+        # 启动tts_text_queue监听线程
+        # 线程任务相关
+        self.loop = asyncio.get_event_loop()
+        self.process_tasks_loop = asyncio.get_event_loop()
+        self.max_workers = self.config.get("TTS_SET", {}).get("MAX_WORKERS", 3)
+        self.active_tasks = set()  # 追踪当前运行的任务
+        self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
+
+    async def open_audio_channels(self):
+        pass
+
+    async def reset(self):
+        try:
+            logger.bind(tag=TAG).info("说明开始了新的对话，重建tts监听")
+            await self.stop_listen_resource()
+            self.tts_text_queue = queue.Queue()
+            self.tts_audio_queue = queue.Queue()
+            # 启动tts_text_queue监听线程
+            self.stop_event.clear()
+            tts_priority = threading.Thread(target=self._tts_text_priority_thread, daemon=True)
+            tts_priority.start()
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Failed to process TTS text: {e}")
+            traceback.print_exc()
+
+    async def stop_listen_resource(self):
+        """资源清理方法"""
+        self.stop_event.set()
+        self.tts_text_queue = None
+        self.tts_audio_queue = None
+        gc.collect()  # 强制执行垃圾回收
+
+    async def close(self):
+        pass
+
+    def _get_segment_text(self):
+        # 合并当前全部文本并处理未分割部分
+        full_text = "".join(self.tts_text_buff)
+        current_text = full_text[self.processed_chars:]  # 从未处理的位置开始
+        last_punct_pos = -1
+        for punct in self.punctuations:
+            pos = current_text.rfind(punct)
+            if (pos != -1 and last_punct_pos == -1) or (pos != -1 and pos < last_punct_pos):
+                last_punct_pos = pos
+        if last_punct_pos != -1:
+            segment_text_raw = current_text[:last_punct_pos + 1]
+            segment_text = textUtils.get_string_no_punctuation_or_emoji(segment_text_raw)
+            self.processed_chars += len(segment_text_raw)  # 更新已处理字符位置
+            return segment_text
+        else:
+            return None
+
+    async def process_generator(self, generator):
+        async for tts_data in generator:
+            self.tts_audio_queue.put(tts_data)
+
+    def _tts_text_priority_thread(self):
+        logger.bind(tag=TAG).info("开始监听tts文本")
+        if self.enable_two_way:
+            self._enable_two_way_tts()
+        else:
+            self._no_enable_two_way_tts()
+
+    async def start_session(self, session_id):
+        pass
+
+    async def finish_session(self, session_id):
+        pass
+
+    async def tts_one_sentence(self,text):
+        uuid_str = str(uuid.uuid4()).replace("-", "")
+        self.tts.tts_text_queue.put(TTSMessageDTO(u_id=uuid_str, msg_type=MsgType.START_TTS_REQUEST, content=''))
+        self.tts.tts_text_queue.put(TTSMessageDTO(u_id=uuid_str, msg_type=MsgType.TTS_TEXT_REQUEST, content=text))
+        self.tts.tts_text_queue.put(TTSMessageDTO(u_id=uuid_str, msg_type=MsgType.STOP_TTS_REQUEST, content=text))
+
+    def _enable_two_way_tts(self):
+        while not self.stop_event.is_set():
+            try:
+                ttsMessageDTO = self.tts_text_queue.get()
+                msg_type = ttsMessageDTO.msg_type
+                if msg_type == MsgType.START_TTS_REQUEST:
+                    # 开始传输tts文本
+                    self.tts_request = True
+                    self.u_id = ttsMessageDTO.u_id
+                    # 开启session
+                    future = asyncio.run_coroutine_threadsafe(self.start_session(ttsMessageDTO.u_id), loop=self.loop)
+                    future.result()
+                    # await self.start_session(ttsMessageDTO.u_id)
+                elif self.tts_request and msg_type == MsgType.TTS_TEXT_REQUEST:
+                    future = asyncio.run_coroutine_threadsafe(
+                        self.text_to_speak(u_id=ttsMessageDTO.u_id, text=ttsMessageDTO.content), loop=self.loop)
+                    future.result()
+                elif msg_type == MsgType.STOP_TTS_REQUEST:
+                    self.tts_request = False
+                    future = asyncio.run_coroutine_threadsafe(self.finish_session(ttsMessageDTO.u_id), loop=self.loop)
+                    future.result()
+
+
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"Failed to process TTS text: {e}")
+                # 报错了。要关闭说话
+                self.tts_audio_queue.put(
+                    TTSMessageDTO(
+                        u_id=self.u_id, msg_type=MsgType.STOP_TTS_RESPONSE, content=[],
+                        tts_finish_text='',
+                        sentence_type=None
+                    )
+                )
+                traceback.print_exc()
+
+    def _no_enable_two_way_tts(self):
+        # 为这个线程创建一个新的事件循环
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        while not self.stop_event.is_set():
+            try:
+                ttsMessageDTO = self.tts_text_queue.get()
+                msg_type = ttsMessageDTO.msg_type
+                if not self.enable_two_way:
+                    if msg_type == MsgType.START_TTS_REQUEST:
+                        # 开始传输tts文本
+                        self.tts_request = True
+                        self.processed_chars = 0
+                        self.tts_text_buff = []
+                    elif self.tts_request and msg_type == MsgType.TTS_TEXT_REQUEST:
+                        self.tts_text_buff.append(ttsMessageDTO.content)
+                    elif msg_type == MsgType.STOP_TTS_REQUEST:
+                        # 结束传输tts文本,处理最尾巴的数据
+                        self.tts_request = False
+                        segment_text = self._get_segment_text()
+                        if segment_text:
+                            # 修改部分：创建协程对象
+                            # 修改部分：创建协程对象
+                            tts_generator = self.text_to_speak(ttsMessageDTO.u_id, segment_text,
+                                                               True if msg_type == MsgType.STOP_TTS_REQUEST else False,
+                                                               True if msg_type == MsgType.START_TTS_REQUEST else False)
+                            future = asyncio.run_coroutine_threadsafe(self.process_generator(tts_generator), self.loop)
+                            self.active_tasks.add(future)
+                        if self.active_tasks:
+                            async def wrap_future(future):
+                                return await asyncio.wrap_future(future)
+
+                            wrapped_tasks = [wrap_future(task) for task in self.active_tasks]
+                            done, _ = loop.run_until_complete(asyncio.wait(wrapped_tasks))
+                            self.active_tasks -= done
+
+                        # 发送合成结束
+                        self.tts_audio_queue.put(TTSMessageDTO(u_id=ttsMessageDTO.u_id,
+                                                               msg_type=MsgType.STOP_TTS_RESPONSE,
+                                                               content=[],
+                                                               tts_finish_text='',
+                                                               sentence_type=SentenceType.SENTENCE_END))
+
+                    segment_text = self._get_segment_text()
+                    if segment_text:
+                        # 确保这里得到的是协程对象
+                        tts_generator = self.text_to_speak(
+                            ttsMessageDTO.u_id,
+                            segment_text,
+                            msg_type == MsgType.STOP_TTS_REQUEST,
+                            msg_type == MsgType.START_TTS_REQUEST
+                        )
+                        # 提交协程到事件循环
+                        tts_generator_future = asyncio.run_coroutine_threadsafe(
+                            self.process_generator(tts_generator),
+                            loop
+                        )
+                        self.active_tasks.add(tts_generator_future)
+                        if len(self.active_tasks) >= self.max_workers:
+                            # 等待所有任务完成
+                            try:
+                                async def wrap_future(future):
+                                    return await asyncio.wrap_future(future)
+
+                                wrapped_tasks = [wrap_future(task) for task in self.active_tasks]
+                                done, _ = loop.run_until_complete(asyncio.wait(wrapped_tasks))
+                                self.active_tasks -= done
+                            except Exception as e:
+                                logger.bind(tag=TAG).error(f"Failed to process TTS text: {e}")
+                                traceback.print_exc()
+                else:
+                    pass
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"Failed to process TTS text: {e}")
+                traceback.print_exc()
 
     @abstractmethod
     def generate_filename(self):
@@ -38,7 +249,7 @@ class TTSProviderBase(ABC):
             logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
             return None
 
-    def to_tts_stream(self, text, queue: queue.Queue, text_index=0):
+    def to_tts_stream(self, u_id, text, queue: queue.Queue, text_index=0):
         try:
             asyncio.run(self.text_to_speak_stream(text, queue, text_index))
         except Exception as e:
@@ -46,7 +257,7 @@ class TTSProviderBase(ABC):
             return None
 
     @abstractmethod
-    async def text_to_speak(self, text, output_file):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
         pass
 
     async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
@@ -95,7 +306,17 @@ class TTSProviderBase(ABC):
 
         return opus_datas, duration
 
-    def wav_to_opus_data_audio_raw(self, raw_data):
+    def get_audio_from_tts(self, data_bytes, src_rate, to_rate=16000):
+        tts_speech = torch.from_numpy(np.array(np.frombuffer(data_bytes, dtype=np.int16))).unsqueeze(dim=0)
+        with io.BytesIO() as bf:
+            torchaudio.save(bf, tts_speech, src_rate, format="wav")
+            audio = AudioSegment.from_file(bf, format="wav")
+        audio = audio.set_channels(1).set_frame_rate(to_rate)
+        return audio
+
+    def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False):
+        raw_data = self.last_to_opus_raw + raw_data_var
+        self.last_to_opus_raw = b''
         # 初始化Opus编码器
         encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
 
@@ -110,8 +331,13 @@ class TTSProviderBase(ABC):
             chunk = raw_data[i:i + frame_size * 2]
 
             # 如果最后一帧不足，补零
-            if len(chunk) < frame_size * 2:
-                # logger.bind(tag=TAG).info("开始补0")
+            # 缓存记录一下
+            if len(chunk) < frame_size * 2 and not is_end:
+                logger.bind(tag=TAG).info("如果最后一帧不足,缓存记录一下")
+                self.last_to_opus_raw = chunk
+                break
+            if len(chunk) < frame_size * 2 and is_end:
+                logger.bind(tag=TAG).info("是最后一句了，补零")
                 chunk += b'\x00' * (frame_size * 2 - len(chunk))
 
             # 转换为numpy数组处理

main/xiaozhi-server/core/providers/tts/dto/dto.py

@@ -0,0 +1,42 @@
+from enum import Enum
+from typing import Union
+
+
+class MsgType(Enum):
+    # 请求类型
+    START_TTS_REQUEST = "START_TTS_REQUEST"
+    TTS_TEXT_REQUEST = "TTS_TEXT_REQUEST"
+    STOP_TTS_REQUEST = "STOP_TTS_REQUEST"
+
+    # 返回类型
+    START_TTS_RESPONSE = "START_TTS_RESPONSE"
+    TTS_TEXT_RESPONSE = "TTS_TEXT_RESPONSE"
+    STOP_TTS_RESPONSE = "STOP_TTS_RESPONSE"
+
+
+class SentenceType(Enum):
+    # 句子开始
+    SENTENCE_START = "SENTENCE_START"
+    # 句子结束
+    SENTENCE_END = "SENTENCE_END"
+
+
+class TTSMessageDTO:
+    def __init__(self, u_id: str, msg_type: MsgType, content: Union[str, bytes], tts_finish_text=None,
+                 sentence_type: SentenceType = None, duration=0):
+        if not isinstance(msg_type, MsgType):
+            raise ValueError("msg_type must be an instance of MsgType Enum")
+        if not isinstance(content, (str, list, bytes)):
+            raise ValueError("content must be of type str or bytes")
+
+        # 唯一id，每个合成到合成结束，使用同一个id
+        self.u_id = u_id
+        self.msg_type = msg_type
+        self.sentence_type = sentence_type
+        self.content = content
+        self.tts_finish_text = tts_finish_text
+        self.duration = duration
+
+    def __repr__(self):
+        content_preview = self.content if isinstance(self.content, str) else "<binary data>"
+        return f"MessageDTO(msg_type={self.msg_type}, content={content_preview})"

main/xiaozhi-server/core/providers/tts/edge.py

@@ -1,9 +1,17 @@
+import io
 import os
 import uuid
 import edge_tts
 from datetime import datetime
-from core.providers.tts.base import TTSProviderBase
 
+from pydub import AudioSegment
+
+from config.logger import setup_logging
+from core.providers.tts.base import TTSProviderBase
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
+
+TAG = __name__
+logger = setup_logging()
 
 class TTSProvider(TTSProviderBase):
     def __init__(self, config, delete_audio_file):
@@ -13,6 +21,25 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self, extension=".mp3"):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
-    async def text_to_speak(self, text, output_file):
-        communicate = edge_tts.Communicate(text, voice=self.voice)  # Use your preferred voice
-        await communicate.save(output_file)
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
+        try:
+            communicate = edge_tts.Communicate(text, voice=self.voice)  # Use your preferred voice
+            tmp_file = self.generate_filename()
+            await communicate.save(tmp_file)
+
+            # 使用 pydub 读取临时文件
+            audio = AudioSegment.from_file(tmp_file, format="mp3")
+            audio = audio.set_channels(1).set_frame_rate(16000)
+            opus_datas = self.wav_to_opus_data_audio_raw(audio.raw_data)
+            yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas, tts_finish_text=text,sentence_type=SentenceType.SENTENCE_START)
+            # 用完后删除临时文件
+            try:
+                os.remove(tmp_file)
+            except FileNotFoundError:
+                # 若文件不存在，忽略该异常
+                pass
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"TTSProvider text_to_speak error: {e}")
+            yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=[], tts_finish_text=text,sentence_type=SentenceType.SENTENCE_START)
+
+

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -17,6 +17,8 @@ from pydub import AudioSegment
 from typing_extensions import Annotated
 from datetime import datetime
 from typing import Literal
+
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
 from core.utils.util import check_model_key
 from core.providers.tts.base import TTSProviderBase
 from config.logger import setup_logging
@@ -118,55 +120,6 @@ class TTSProvider(TTSProviderBase):
     def generate_filename(self, extension=".wav"):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
 
-    async def text_to_speak(self, text, output_file):
-        # Prepare reference data
-        byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
-        ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
-
-        data = {
-            "text": text,
-            "references": [
-                ServeReferenceAudio(
-                    audio=audio if audio else b"", text=text
-                )
-                for text, audio in zip(ref_texts, byte_audios)
-            ],
-            "reference_id": self.reference_id,
-            "normalize": self.normalize,
-            "format": self.format,
-            "max_new_tokens": self.max_new_tokens,
-            "chunk_length": self.chunk_length,
-            "top_p": self.top_p,
-            "repetition_penalty": self.repetition_penalty,
-            "temperature": self.temperature,
-            "streaming": self.streaming,
-            "use_memory_cache": self.use_memory_cache,
-            "seed": self.seed,
-        }
-
-        pydantic_data = ServeTTSRequest(**data)
-
-        response = requests.post(
-            self.api_url,
-            data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
-            headers={
-                "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/msgpack",
-            },
-        )
-
-        if response.status_code == 200:
-            audio_content = response.content
-
-            with open(output_file, "wb") as audio_file:
-                audio_file.write(audio_content)
-
-
-
-        else:
-            print(f"Request failed with status code {response.status_code}")
-            print(response.json())
-
     def _get_audio_from_tts(self, data_bytes):
         tts_speech = torch.from_numpy(np.array(np.frombuffer(data_bytes, dtype=np.int16))).unsqueeze(dim=0)
         with io.BytesIO() as bf:
@@ -175,7 +128,7 @@ class TTSProvider(TTSProviderBase):
         audio = audio.set_channels(1).set_frame_rate(16000)
         return audio
 
-    async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
         try:
             data = {
                 "text": text,
@@ -219,6 +172,7 @@ class TTSProvider(TTSProviderBase):
                     },
             ) as response:
                 if response.status_code == 200:
+                    index = 0
                     for chunk in response.iter_content():
                         # 拼接当前块和上一块数据
                         chunk_total += chunk
@@ -228,42 +182,29 @@ class TTSProvider(TTSProviderBase):
                             audio_raw = audio_raw + audio.raw_data
                             # 长度凑够2贞开始发送，60ms*4=240ms
                             if len(audio_raw) >= 7680:
-                                duration = 60 * len(audio_raw) // 1920
-                                if (len(audio_raw) % 1920) > 0:
-                                    duration += 60
-                                duration = duration / 1000.0
-                                # logger.bind(tag=TAG).info(f'发送数据长度：{len(audio_raw)}')
                                 opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
-                                queue.put({
-                                    "data": opus_datas,
-                                    "duration": duration,
-                                    "end": False,
-                                    "text_index": text_index
-                                })
+                                if index == 0:
+                                    yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE,
+                                                        content=opus_datas,
+                                                        tts_finish_text=text, sentence_type=SentenceType.SENTENCE_START)
+                                else:
+                                    yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE,
+                                                        content=opus_datas,
+                                                        tts_finish_text=text, sentence_type=None)
                                 audio_raw = b''
                             chunk_total = b''
                     if len(chunk_total) > 0:
                         audio = self._get_audio_from_tts(chunk_total)
                         audio_raw = audio_raw + audio.raw_data
-                        duration = 60 * len(audio_raw) // 1920
-                        if (len(audio_raw) % 1920) > 0:
-                            duration += 60
-                        duration = duration / 1000.0
-                        # 把 audio 转成 opus
-                        # logger.bind(tag=TAG).info(f'发送数据长度：{len(audio_raw)}')
                         opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
-                        queue.put({
-                            "data": opus_datas,
-                            "duration": duration,
-                            "end": False
-                        })
+                        yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas,
+                                            tts_finish_text=text, sentence_type=SentenceType.SENTENCE_END)
+                    else:
+                        yield TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=[],
+                                            tts_finish_text=text, sentence_type=SentenceType.SENTENCE_END)
 
                 else:
                     print('请求失败:', response.status_code, response.text)
-                queue.put({
-                    "data": None,
-                    "end": True
-                })
         except Exception as e:
             logger.bind(tag=TAG).error("tts发生错误")
             traceback.print_exc()

main/xiaozhi-server/core/providers/tts/huoshan.py

@@ -0,0 +1,390 @@
+import asyncio
+import io
+import os
+import subprocess
+import threading
+import traceback
+import uuid
+import json
+import base64
+import requests
+from datetime import datetime
+from mutagen.oggopus import OggOpus
+
+import websockets
+
+from config.logger import setup_logging
+from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
+from core.utils.util import check_model_key
+from core.providers.tts.base import TTSProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+PROTOCOL_VERSION = 0b0001
+DEFAULT_HEADER_SIZE = 0b0001
+
+# Message Type:
+FULL_CLIENT_REQUEST = 0b0001
+AUDIO_ONLY_RESPONSE = 0b1011
+FULL_SERVER_RESPONSE = 0b1001
+ERROR_INFORMATION = 0b1111
+
+# Message Type Specific Flags
+MsgTypeFlagNoSeq = 0b0000  # Non-terminal packet with no sequence
+MsgTypeFlagPositiveSeq = 0b1  # Non-terminal packet with sequence > 0
+MsgTypeFlagLastNoSeq = 0b10  # last packet with no sequence
+MsgTypeFlagNegativeSeq = 0b11  # Payload contains event number (int32)
+MsgTypeFlagWithEvent = 0b100
+# Message Serialization
+NO_SERIALIZATION = 0b0000
+JSON = 0b0001
+# Message Compression
+COMPRESSION_NO = 0b0000
+COMPRESSION_GZIP = 0b0001
+
+EVENT_NONE = 0
+EVENT_Start_Connection = 1
+
+EVENT_FinishConnection = 2
+
+EVENT_ConnectionStarted = 50  # 成功建连
+
+EVENT_ConnectionFailed = 51  # 建连失败（可能是无法通过权限认证）
+
+EVENT_ConnectionFinished = 52  # 连接结束
+
+# 上行Session事件
+EVENT_StartSession = 100
+
+EVENT_FinishSession = 102
+# 下行Session事件
+EVENT_SessionStarted = 150
+EVENT_SessionFinished = 152
+
+EVENT_SessionFailed = 153
+
+# 上行通用事件
+EVENT_TaskRequest = 200
+
+# 下行TTS事件
+EVENT_TTSSentenceStart = 350
+
+EVENT_TTSSentenceEnd = 351
+
+EVENT_TTSResponse = 352
+
+
+class Header:
+    def __init__(self,
+                 protocol_version=PROTOCOL_VERSION,
+                 header_size=DEFAULT_HEADER_SIZE,
+                 message_type: int = 0,
+                 message_type_specific_flags: int = 0,
+                 serial_method: int = NO_SERIALIZATION,
+                 compression_type: int = COMPRESSION_NO,
+                 reserved_data=0):
+        self.header_size = header_size
+        self.protocol_version = protocol_version
+        self.message_type = message_type
+        self.message_type_specific_flags = message_type_specific_flags
+        self.serial_method = serial_method
+        self.compression_type = compression_type
+        self.reserved_data = reserved_data
+
+    def as_bytes(self) -> bytes:
+        return bytes([
+            (self.protocol_version << 4) | self.header_size,
+            (self.message_type << 4) | self.message_type_specific_flags,
+            (self.serial_method << 4) | self.compression_type,
+            self.reserved_data
+        ])
+
+
+class Optional:
+    def __init__(self, event: int = EVENT_NONE, sessionId: str = None, sequence: int = None):
+        self.event = event
+        self.sessionId = sessionId
+        self.errorCode: int = 0
+        self.connectionId: str | None = None
+        self.response_meta_json: str | None = None
+        self.sequence = sequence
+
+    # 转成 byte 序列
+    def as_bytes(self) -> bytes:
+        option_bytes = bytearray()
+        if self.event != EVENT_NONE:
+            option_bytes.extend(self.event.to_bytes(4, "big", signed=True))
+        if self.sessionId is not None:
+            session_id_bytes = str.encode(self.sessionId)
+            size = len(session_id_bytes).to_bytes(4, "big", signed=True)
+            option_bytes.extend(size)
+            option_bytes.extend(session_id_bytes)
+        if self.sequence is not None:
+            option_bytes.extend(self.sequence.to_bytes(4, "big", signed=True))
+        return option_bytes
+
+
+class Response:
+    def __init__(self, header: Header, optional: Optional):
+        self.optional = optional
+        self.header = header
+        self.payload: bytes | None = None
+
+    def __str__(self):
+        return super().__str__()
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.appId = config.get("appid")
+        self.access_token = config.get("access_token")
+        self.cluster = config.get("cluster")
+        self.voice = config.get("voice")
+        self.ws_url = config.get("ws_url")
+        self.authorization = config.get("authorization")
+        self.speaker = config.get("speaker")
+        self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
+        self.stop_event_response = threading.Event()
+        self.enable_two_way = True
+        self.start_connection_flag = False
+        self.tts_text = ""
+
+    async def open_audio_channels(self):
+        self.loop_tts = asyncio.new_event_loop()
+        ws_header = {
+            "X-Api-App-Key": self.appId,
+            "X-Api-Access-Key": self.access_token,
+            "X-Api-Resource-Id": 'volc.service_type.10029',
+            "X-Api-Connect-Id": uuid.uuid4(),
+        }
+        self.ws = await websockets.connect(self.ws_url, additional_headers=ws_header, max_size=1000000000)
+        tts_priority = threading.Thread(target=self._start_monitor_tts_response_thread(), daemon=True)
+        tts_priority.start()
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def send_event(self, header: bytes, optional: bytes | None = None,
+                         payload: bytes = None):
+        full_client_request = bytearray(header)
+        if optional is not None:
+            full_client_request.extend(optional)
+        if payload is not None:
+            payload_size = len(payload).to_bytes(4, 'big', signed=True)
+            full_client_request.extend(payload_size)
+            full_client_request.extend(payload)
+        await self.ws.send(full_client_request)
+
+    async def send_text(self, speaker: str, text: str, session_id):
+        header = Header(message_type=FULL_CLIENT_REQUEST,
+                        message_type_specific_flags=MsgTypeFlagWithEvent,
+                        serial_method=JSON).as_bytes()
+        optional = Optional(event=EVENT_TaskRequest, sessionId=session_id).as_bytes()
+        payload = self.get_payload_bytes(event=EVENT_TaskRequest, text=text, speaker=speaker)
+        return await self.send_event(header, optional, payload)
+
+    # 读取 res 数组某段 字符串内容
+    def read_res_content(self, res: bytes, offset: int):
+        content_size = int.from_bytes(res[offset: offset + 4], "big", signed=True)
+        offset += 4
+        content = str(res[offset: offset + content_size])
+        offset += content_size
+        return content, offset
+
+    # 读取 payload
+    def read_res_payload(self, res: bytes, offset: int):
+        payload_size = int.from_bytes(res[offset: offset + 4], "big", signed=True)
+        offset += 4
+        payload = res[offset: offset + payload_size]
+        offset += payload_size
+        return payload, offset
+
+    def parser_response(self, res) -> Response:
+        if isinstance(res, str):
+            raise RuntimeError(res)
+        response = Response(Header(), Optional())
+        # 解析结果
+        # header
+        header = response.header
+        num = 0b00001111
+        header.protocol_version = res[0] >> 4 & num
+        header.header_size = res[0] & 0x0f
+        header.message_type = (res[1] >> 4) & num
+        header.message_type_specific_flags = res[1] & 0x0f
+        header.serialization_method = res[2] >> num
+        header.message_compression = res[2] & 0x0f
+        header.reserved = res[3]
+        #
+        offset = 4
+        optional = response.optional
+        if header.message_type == FULL_SERVER_RESPONSE or AUDIO_ONLY_RESPONSE:
+            # read event
+            if header.message_type_specific_flags == MsgTypeFlagWithEvent:
+                optional.event = int.from_bytes(res[offset:8], "big", signed=True)
+                offset += 4
+                if optional.event == EVENT_NONE:
+                    return response
+                # read connectionId
+                elif optional.event == EVENT_ConnectionStarted:
+                    optional.connectionId, offset = self.read_res_content(res, offset)
+                elif optional.event == EVENT_ConnectionFailed:
+                    optional.response_meta_json, offset = self.read_res_content(res, offset)
+                elif (optional.event == EVENT_SessionStarted
+                      or optional.event == EVENT_SessionFailed
+                      or optional.event == EVENT_SessionFinished):
+                    optional.sessionId, offset = self.read_res_content(res, offset)
+                    optional.response_meta_json, offset = self.read_res_content(res, offset)
+                else:
+                    optional.sessionId, offset = self.read_res_content(res, offset)
+                    response.payload, offset = self.read_res_payload(res, offset)
+
+        elif header.message_type == ERROR_INFORMATION:
+            optional.errorCode = int.from_bytes(res[offset:offset + 4], "big", signed=True)
+            offset += 4
+            response.payload, offset = self.read_res_payload(res, offset)
+        return response
+
+    async def start_connection(self):
+        header = Header(message_type=FULL_CLIENT_REQUEST, message_type_specific_flags=MsgTypeFlagWithEvent).as_bytes()
+        optional = Optional(event=EVENT_Start_Connection).as_bytes()
+        payload = str.encode("{}")
+        return await self.send_event(header, optional, payload)
+
+    def print_response(self, res, tag_msg: str):
+        logger.bind(tag=TAG).info(f'===>{tag_msg} header:{res.header.__dict__}')
+        logger.bind(tag=TAG).info(f'===>{tag_msg} optional:{res.optional.__dict__}')
+
+    def get_payload_bytes(self, uid='1234', event=EVENT_NONE, text='', speaker='', audio_format='pcm',
+                          audio_sample_rate=16000):
+        return str.encode(json.dumps(
+            {
+                "user": {"uid": uid},
+                "event": event,
+                "namespace": "BidirectionalTTS",
+                "req_params": {
+                    "text": text,
+                    "speaker": speaker,
+                    "audio_params": {
+                        "format": audio_format,
+                        "sample_rate": audio_sample_rate
+                    }
+                }
+            }
+        ))
+
+    async def finish_connection(self):
+        header = Header(message_type=FULL_CLIENT_REQUEST,
+                        message_type_specific_flags=MsgTypeFlagWithEvent,
+                        serial_method=JSON
+                        ).as_bytes()
+        optional = Optional(event=EVENT_FinishConnection).as_bytes()
+        payload = str.encode('{}')
+        await self.send_event(header, optional, payload)
+        return
+
+    async def start_session(self, session_id):
+        self.stop_event_response.clear()
+        header = Header(message_type=FULL_CLIENT_REQUEST,
+                        message_type_specific_flags=MsgTypeFlagWithEvent,
+                        serial_method=JSON
+                        ).as_bytes()
+        optional = Optional(event=EVENT_StartSession, sessionId=session_id).as_bytes()
+        payload = self.get_payload_bytes(event=EVENT_StartSession, speaker=self.speaker)
+        await self.send_event(header, optional, payload)
+
+    async def finish_session(self, session_id):
+        self.stop_event_response.set()
+        header = Header(message_type=FULL_CLIENT_REQUEST,
+                        message_type_specific_flags=MsgTypeFlagWithEvent,
+                        serial_method=JSON
+                        ).as_bytes()
+        optional = Optional(event=EVENT_FinishSession, sessionId=session_id).as_bytes()
+        payload = str.encode('{}')
+        await self.send_event(header, optional, payload)
+        return
+
+    async def reset(self):
+        # 关闭之前的对话
+        if self.start_connection_flag:
+            await self.finish_connection()
+            self.start_connection_flag = False
+        await self.start_connection()
+        self.start_connection_flag = True
+        await super().reset()
+
+    async def close(self):
+        """资源清理方法"""
+        await self.ws.close()
+
+    async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
+        # 发送文本
+        await self.send_text(self.speaker, text, u_id)
+        return
+
+    def _start_monitor_tts_response_thread(self):
+        # 初始化链接
+        asyncio.run_coroutine_threadsafe(self._start_monitor_tts_response(), loop=self.loop)
+
+    async def _start_monitor_tts_response(self):
+        chunk_total = b''
+        while True:
+            try:
+                msg = await self.ws.recv()  # 确保 `recv()` 运行在同一个 event loop
+                res = self.parser_response(msg)
+                self.print_response(res, 'send_text res:')
+
+                if res.optional.event == EVENT_TTSResponse and res.header.message_type == AUDIO_ONLY_RESPONSE:
+                    logger.bind(tag=TAG).info(f'推送数据到队列里面～～')
+                    opus_datas = self.wav_to_opus_data_audio_raw(res.payload)
+                    self.tts_audio_queue.put(
+                        TTSMessageDTO(
+                            u_id=self.u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas,
+                            tts_finish_text="", sentence_type=None, duration=0
+                        )
+                    )
+                elif res.optional.event == EVENT_TTSSentenceStart:
+                    json_data = json.loads(res.payload.decode('utf-8'))
+                    self.tts_text = json_data.get("text", "")
+                    logger.bind(tag=TAG).info(f'句子开始～～{self.tts_text}')
+                    self.tts_audio_queue.put(
+                        TTSMessageDTO(
+                            u_id=self.u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=[],
+                            tts_finish_text=self.tts_text,
+                            sentence_type=SentenceType.SENTENCE_START
+                        )
+                    )
+                elif res.optional.event == EVENT_TTSSentenceEnd:
+                    logger.bind(tag=TAG).info(f'句子结束～～{self.tts_text}')
+                    self.tts_audio_queue.put(
+                        TTSMessageDTO(
+                            u_id=self.u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=[],
+                            tts_finish_text=self.tts_text,
+                            sentence_type=SentenceType.SENTENCE_END
+                        )
+                    )
+                elif res.optional.event == EVENT_SessionFinished:
+                    logger.bind(tag=TAG).info(f'会话结束～～,最后一句补零')
+                    opus_datas = self.wav_to_opus_data_audio_raw(b'', is_end=True)
+                    self.tts_audio_queue.put(
+                        TTSMessageDTO(
+                            u_id=self.u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas,
+                            tts_finish_text="", sentence_type=None, duration=0
+                        )
+                    )
+                    self.tts_audio_queue.put(
+                        TTSMessageDTO(
+                            u_id=self.u_id, msg_type=MsgType.STOP_TTS_RESPONSE, content=[],
+                            tts_finish_text=self.tts_text,
+                            sentence_type=SentenceType.SENTENCE_END
+                        )
+                    )
+                else:
+                    continue
+            except websockets.ConnectionClosed:
+                break  # 连接关闭时退出监听
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"Error in _start_monitor_tts_response: {e}")
+                traceback.print_exc()
+                continue