mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-29 14:53:56 +08:00
410 lines
15 KiB
Python
410 lines
15 KiB
Python
import asyncio
|
|
import os
|
|
import threading
|
|
import traceback
|
|
import uuid
|
|
import json
|
|
from datetime import datetime
|
|
|
|
import websockets
|
|
|
|
from config.logger import setup_logging
|
|
from core.providers.tts.base import TTSProviderBase, TTSImplementationType
|
|
|
|
TAG = __name__
|
|
logger = setup_logging()
|
|
|
|
PROTOCOL_VERSION = 0b0001
|
|
DEFAULT_HEADER_SIZE = 0b0001
|
|
|
|
# Message Type:
|
|
FULL_CLIENT_REQUEST = 0b0001
|
|
AUDIO_ONLY_RESPONSE = 0b1011
|
|
FULL_SERVER_RESPONSE = 0b1001
|
|
ERROR_INFORMATION = 0b1111
|
|
|
|
# Message Type Specific Flags
|
|
MsgTypeFlagNoSeq = 0b0000 # Non-terminal packet with no sequence
|
|
MsgTypeFlagPositiveSeq = 0b1 # Non-terminal packet with sequence > 0
|
|
MsgTypeFlagLastNoSeq = 0b10 # last packet with no sequence
|
|
MsgTypeFlagNegativeSeq = 0b11 # Payload contains event number (int32)
|
|
MsgTypeFlagWithEvent = 0b100
|
|
# Message Serialization
|
|
NO_SERIALIZATION = 0b0000
|
|
JSON = 0b0001
|
|
# Message Compression
|
|
COMPRESSION_NO = 0b0000
|
|
COMPRESSION_GZIP = 0b0001
|
|
|
|
EVENT_NONE = 0
|
|
EVENT_Start_Connection = 1
|
|
|
|
EVENT_FinishConnection = 2
|
|
|
|
EVENT_ConnectionStarted = 50 # 成功建连
|
|
|
|
EVENT_ConnectionFailed = 51 # 建连失败（可能是无法通过权限认证）
|
|
|
|
EVENT_ConnectionFinished = 52 # 连接结束
|
|
|
|
# 上行Session事件
|
|
EVENT_StartSession = 100
|
|
|
|
EVENT_FinishSession = 102
|
|
# 下行Session事件
|
|
EVENT_SessionStarted = 150
|
|
EVENT_SessionFinished = 152
|
|
|
|
EVENT_SessionFailed = 153
|
|
|
|
# 上行通用事件
|
|
EVENT_TaskRequest = 200
|
|
|
|
# 下行TTS事件
|
|
EVENT_TTSSentenceStart = 350
|
|
|
|
EVENT_TTSSentenceEnd = 351
|
|
|
|
EVENT_TTSResponse = 352
|
|
|
|
|
|
class Header:
|
|
def __init__(
|
|
self,
|
|
protocol_version=PROTOCOL_VERSION,
|
|
header_size=DEFAULT_HEADER_SIZE,
|
|
message_type: int = 0,
|
|
message_type_specific_flags: int = 0,
|
|
serial_method: int = NO_SERIALIZATION,
|
|
compression_type: int = COMPRESSION_NO,
|
|
reserved_data=0,
|
|
):
|
|
self.header_size = header_size
|
|
self.protocol_version = protocol_version
|
|
self.message_type = message_type
|
|
self.message_type_specific_flags = message_type_specific_flags
|
|
self.serial_method = serial_method
|
|
self.compression_type = compression_type
|
|
self.reserved_data = reserved_data
|
|
|
|
def as_bytes(self) -> bytes:
|
|
return bytes(
|
|
[
|
|
(self.protocol_version << 4) | self.header_size,
|
|
(self.message_type << 4) | self.message_type_specific_flags,
|
|
(self.serial_method << 4) | self.compression_type,
|
|
self.reserved_data,
|
|
]
|
|
)
|
|
|
|
|
|
class Optional:
|
|
def __init__(
|
|
self, event: int = EVENT_NONE, sessionId: str = None, sequence: int = None
|
|
):
|
|
self.event = event
|
|
self.sessionId = sessionId
|
|
self.errorCode: int = 0
|
|
self.connectionId: str | None = None
|
|
self.response_meta_json: str | None = None
|
|
self.sequence = sequence
|
|
|
|
# 转成 byte 序列
|
|
def as_bytes(self) -> bytes:
|
|
option_bytes = bytearray()
|
|
if self.event != EVENT_NONE:
|
|
option_bytes.extend(self.event.to_bytes(4, "big", signed=True))
|
|
if self.sessionId is not None:
|
|
session_id_bytes = str.encode(self.sessionId)
|
|
size = len(session_id_bytes).to_bytes(4, "big", signed=True)
|
|
option_bytes.extend(size)
|
|
option_bytes.extend(session_id_bytes)
|
|
if self.sequence is not None:
|
|
option_bytes.extend(self.sequence.to_bytes(4, "big", signed=True))
|
|
return option_bytes
|
|
|
|
|
|
class Response:
|
|
def __init__(self, header: Header, optional: Optional):
|
|
self.optional = optional
|
|
self.header = header
|
|
self.payload: bytes | None = None
|
|
|
|
def __str__(self):
|
|
return super().__str__()
|
|
|
|
|
|
class TTSProvider(TTSProviderBase):
|
|
def __init__(self, config, delete_audio_file):
|
|
super().__init__(config, delete_audio_file)
|
|
self.appId = config.get("appid")
|
|
self.access_token = config.get("access_token")
|
|
self.cluster = config.get("cluster")
|
|
self.resource_id = config.get("resource_id")
|
|
self.voice = config.get("voice")
|
|
self.ws_url = config.get("ws_url")
|
|
self.authorization = config.get("authorization")
|
|
self.speaker = config.get("speaker")
|
|
self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
|
|
self.stop_event_response = threading.Event()
|
|
self.enable_two_way = True
|
|
self.start_connection_flag = False
|
|
self.tts_text = ""
|
|
self.interface_type = TTSImplementationType.DOUBLE_STREAMING
|
|
|
|
async def open_audio_channels(self, conn):
|
|
await super().open_audio_channels(conn)
|
|
ws_header = {
|
|
"X-Api-App-Key": self.appId,
|
|
"X-Api-Access-Key": self.access_token,
|
|
"X-Api-Resource-Id": self.resource_id,
|
|
"X-Api-Connect-Id": uuid.uuid4(),
|
|
}
|
|
self.ws = await websockets.connect(
|
|
self.ws_url, additional_headers=ws_header, max_size=1000000000
|
|
)
|
|
tts_priority = threading.Thread(
|
|
target=self._start_monitor_tts_response_thread(), daemon=True
|
|
)
|
|
tts_priority.start()
|
|
await self.start_session(conn.session_id)
|
|
|
|
def generate_filename(self, extension=".wav"):
|
|
return os.path.join(
|
|
self.output_file,
|
|
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
|
)
|
|
|
|
async def send_event(
|
|
self, header: bytes, optional: bytes | None = None, payload: bytes = None
|
|
):
|
|
full_client_request = bytearray(header)
|
|
if optional is not None:
|
|
full_client_request.extend(optional)
|
|
if payload is not None:
|
|
payload_size = len(payload).to_bytes(4, "big", signed=True)
|
|
full_client_request.extend(payload_size)
|
|
full_client_request.extend(payload)
|
|
await self.ws.send(full_client_request)
|
|
|
|
async def send_text(self, speaker: str, text: str, session_id):
|
|
header = Header(
|
|
message_type=FULL_CLIENT_REQUEST,
|
|
message_type_specific_flags=MsgTypeFlagWithEvent,
|
|
serial_method=JSON,
|
|
).as_bytes()
|
|
optional = Optional(event=EVENT_TaskRequest, sessionId=session_id).as_bytes()
|
|
payload = self.get_payload_bytes(
|
|
event=EVENT_TaskRequest, text=text, speaker=speaker
|
|
)
|
|
return await self.send_event(header, optional, payload)
|
|
|
|
# 读取 res 数组某段 字符串内容
|
|
def read_res_content(self, res: bytes, offset: int):
|
|
content_size = int.from_bytes(res[offset : offset + 4], "big", signed=True)
|
|
offset += 4
|
|
content = str(res[offset : offset + content_size])
|
|
offset += content_size
|
|
return content, offset
|
|
|
|
# 读取 payload
|
|
def read_res_payload(self, res: bytes, offset: int):
|
|
payload_size = int.from_bytes(res[offset : offset + 4], "big", signed=True)
|
|
offset += 4
|
|
payload = res[offset : offset + payload_size]
|
|
offset += payload_size
|
|
return payload, offset
|
|
|
|
def parser_response(self, res) -> Response:
|
|
if isinstance(res, str):
|
|
raise RuntimeError(res)
|
|
response = Response(Header(), Optional())
|
|
# 解析结果
|
|
# header
|
|
header = response.header
|
|
num = 0b00001111
|
|
header.protocol_version = res[0] >> 4 & num
|
|
header.header_size = res[0] & 0x0F
|
|
header.message_type = (res[1] >> 4) & num
|
|
header.message_type_specific_flags = res[1] & 0x0F
|
|
header.serialization_method = res[2] >> num
|
|
header.message_compression = res[2] & 0x0F
|
|
header.reserved = res[3]
|
|
#
|
|
offset = 4
|
|
optional = response.optional
|
|
if header.message_type == FULL_SERVER_RESPONSE or AUDIO_ONLY_RESPONSE:
|
|
# read event
|
|
if header.message_type_specific_flags == MsgTypeFlagWithEvent:
|
|
optional.event = int.from_bytes(res[offset:8], "big", signed=True)
|
|
offset += 4
|
|
if optional.event == EVENT_NONE:
|
|
return response
|
|
# read connectionId
|
|
elif optional.event == EVENT_ConnectionStarted:
|
|
optional.connectionId, offset = self.read_res_content(res, offset)
|
|
elif optional.event == EVENT_ConnectionFailed:
|
|
optional.response_meta_json, offset = self.read_res_content(
|
|
res, offset
|
|
)
|
|
elif (
|
|
optional.event == EVENT_SessionStarted
|
|
or optional.event == EVENT_SessionFailed
|
|
or optional.event == EVENT_SessionFinished
|
|
):
|
|
optional.sessionId, offset = self.read_res_content(res, offset)
|
|
optional.response_meta_json, offset = self.read_res_content(
|
|
res, offset
|
|
)
|
|
else:
|
|
optional.sessionId, offset = self.read_res_content(res, offset)
|
|
response.payload, offset = self.read_res_payload(res, offset)
|
|
|
|
elif header.message_type == ERROR_INFORMATION:
|
|
optional.errorCode = int.from_bytes(
|
|
res[offset : offset + 4], "big", signed=True
|
|
)
|
|
offset += 4
|
|
response.payload, offset = self.read_res_payload(res, offset)
|
|
return response
|
|
|
|
async def start_connection(self):
|
|
header = Header(
|
|
message_type=FULL_CLIENT_REQUEST,
|
|
message_type_specific_flags=MsgTypeFlagWithEvent,
|
|
).as_bytes()
|
|
optional = Optional(event=EVENT_Start_Connection).as_bytes()
|
|
payload = str.encode("{}")
|
|
return await self.send_event(header, optional, payload)
|
|
|
|
def print_response(self, res, tag_msg: str):
|
|
logger.bind(tag=TAG).info(f"===>{tag_msg} header:{res.header.__dict__}")
|
|
logger.bind(tag=TAG).info(f"===>{tag_msg} optional:{res.optional.__dict__}")
|
|
|
|
def get_payload_bytes(
|
|
self,
|
|
uid="1234",
|
|
event=EVENT_NONE,
|
|
text="",
|
|
speaker="",
|
|
audio_format="pcm",
|
|
audio_sample_rate=16000,
|
|
):
|
|
return str.encode(
|
|
json.dumps(
|
|
{
|
|
"user": {"uid": uid},
|
|
"event": event,
|
|
"namespace": "BidirectionalTTS",
|
|
"req_params": {
|
|
"text": text,
|
|
"speaker": speaker,
|
|
"audio_params": {
|
|
"format": audio_format,
|
|
"sample_rate": audio_sample_rate,
|
|
},
|
|
},
|
|
}
|
|
)
|
|
)
|
|
|
|
async def finish_connection(self):
|
|
header = Header(
|
|
message_type=FULL_CLIENT_REQUEST,
|
|
message_type_specific_flags=MsgTypeFlagWithEvent,
|
|
serial_method=JSON,
|
|
).as_bytes()
|
|
optional = Optional(event=EVENT_FinishConnection).as_bytes()
|
|
payload = str.encode("{}")
|
|
await self.send_event(header, optional, payload)
|
|
return
|
|
|
|
async def start_session(self, session_id):
|
|
self.stop_event_response.clear()
|
|
header = Header(
|
|
message_type=FULL_CLIENT_REQUEST,
|
|
message_type_specific_flags=MsgTypeFlagWithEvent,
|
|
serial_method=JSON,
|
|
).as_bytes()
|
|
optional = Optional(event=EVENT_StartSession, sessionId=session_id).as_bytes()
|
|
payload = self.get_payload_bytes(event=EVENT_StartSession, speaker=self.speaker)
|
|
await self.send_event(header, optional, payload)
|
|
|
|
async def finish_session(self, session_id):
|
|
self.stop_event_response.set()
|
|
header = Header(
|
|
message_type=FULL_CLIENT_REQUEST,
|
|
message_type_specific_flags=MsgTypeFlagWithEvent,
|
|
serial_method=JSON,
|
|
).as_bytes()
|
|
optional = Optional(event=EVENT_FinishSession, sessionId=session_id).as_bytes()
|
|
payload = str.encode("{}")
|
|
await self.send_event(header, optional, payload)
|
|
return
|
|
|
|
async def reset(self):
|
|
# 关闭之前的对话
|
|
if self.start_connection_flag:
|
|
await self.finish_connection()
|
|
self.start_connection_flag = False
|
|
await self.start_connection()
|
|
self.start_connection_flag = True
|
|
await super().reset()
|
|
|
|
async def close(self):
|
|
super().close()
|
|
"""资源清理方法"""
|
|
await self.finish_connection()
|
|
await self.ws.close()
|
|
|
|
async def text_to_speak(self, text, _):
|
|
# 发送文本
|
|
await self.send_text(self.speaker, text, self.conn.session_id)
|
|
return
|
|
|
|
def _start_monitor_tts_response_thread(self):
|
|
# 初始化链接
|
|
asyncio.run_coroutine_threadsafe(
|
|
self._start_monitor_tts_response(), loop=self.conn.loop
|
|
)
|
|
|
|
async def _start_monitor_tts_response(self):
|
|
chunk_total = b""
|
|
while not self.conn.stop_event.is_set():
|
|
try:
|
|
msg = await self.ws.recv() # 确保 `recv()` 运行在同一个 event loop
|
|
res = self.parser_response(msg)
|
|
self.print_response(res, "send_text res:")
|
|
|
|
if (
|
|
res.optional.event == EVENT_TTSResponse
|
|
and res.header.message_type == AUDIO_ONLY_RESPONSE
|
|
):
|
|
logger.bind(tag=TAG).info(f"推送数据到队列里面～～")
|
|
opus_datas = self.wav_to_opus_data_audio_raw(res.payload)
|
|
logger.bind(tag=TAG).info(
|
|
f"推送数据到队列里面帧数～～{len(opus_datas)}"
|
|
)
|
|
self.audio_play_queue.put((opus_datas, None, 0))
|
|
elif res.optional.event == EVENT_TTSSentenceStart:
|
|
json_data = json.loads(res.payload.decode("utf-8"))
|
|
self.tts_text = json_data.get("text", "")
|
|
logger.bind(tag=TAG).info(f"句子开始～～{self.tts_text}")
|
|
self.audio_play_queue.put((None, self.tts_text, 0))
|
|
|
|
elif res.optional.event == EVENT_TTSSentenceEnd:
|
|
logger.bind(tag=TAG).info(f"句子结束～～{self.tts_text}")
|
|
self.audio_play_queue.put((None, self.tts_text, 0))
|
|
elif res.optional.event == EVENT_SessionFinished:
|
|
logger.bind(tag=TAG).info(f"会话结束～～,最后一句补零")
|
|
# opus_datas = self.wav_to_opus_data_audio_raw(b"", is_end=True)
|
|
self.audio_play_queue.put((None, self.tts_text, 0))
|
|
else:
|
|
continue
|
|
except websockets.ConnectionClosed:
|
|
break # 连接关闭时退出监听
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(f"Error in _start_monitor_tts_response: {e}")
|
|
traceback.print_exc()
|
|
continue
|