Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-29 11:13:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'xinnan-tech:main' into dev

Browse Source
This commit is contained in:
myifeng
2025-05-29 16:13:13 +08:00
committed by GitHub
parent 69520391f2 625d079168
commit 3db46a84b2
3 changed files with 163 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/connection.py
+34 -25
View File
@@ -606,7 +606,7 @@ class ConnectionHandler:
text_index = 0 text_index = 0
for response in llm_responses: for response in llm_responses:
if self.intent_type == "function_call": if functions is not None:
content, tools_call = response content, tools_call = response
if "content" in response: if "content" in response:
content = response["content"] content = response["content"]
@@ -822,6 +822,9 @@ class ConnectionHandler:
type, text, audio_data, report_time = item type, text, audio_data, report_time = item
try: try:
# 检查线程池状态
if self.executor is None:
continue
# 提交任务到线程池 # 提交任务到线程池
self.executor.submit( self.executor.submit(
self._process_report, type, text, audio_data, report_time self._process_report, type, text, audio_data, report_time
@@ -852,35 +855,37 @@ class ConnectionHandler:
async def close(self, ws=None): async def close(self, ws=None):
"""资源清理方法""" """资源清理方法"""
try:
# 取消超时任务
if self.timeout_task:
self.timeout_task.cancel()
self.timeout_task = None
# 取消超时任务 # 清理MCP资源
if self.timeout_task: if hasattr(self, "mcp_manager") and self.mcp_manager:
self.timeout_task.cancel() await self.mcp_manager.cleanup_all()
self.timeout_task = None
# 清理MCP资源 # 触发停止事件
if hasattr(self, "mcp_manager") and self.mcp_manager: if self.stop_event:
await self.mcp_manager.cleanup_all() self.stop_event.set()
# 触发停止事件 # 清空任务队列
if self.stop_event: self.clear_queues()
self.stop_event.set()
# 清空任务队列 # 关闭WebSocket连接
self.clear_queues() if ws:
await ws.close()
elif self.websocket:
await self.websocket.close()
# 关闭WebSocket连接 # 最后关闭线程池（避免阻塞）
if ws: if self.executor:
await ws.close() self.executor.shutdown(wait=False)
elif self.websocket: self.executor = None
await self.websocket.close()
# 最后关闭线程池（避免阻塞） self.logger.bind(tag=TAG).info("连接资源已释放")
if self.executor: except Exception as e:
self.executor.shutdown(wait=False) self.logger.bind(tag=TAG).error(f"关闭连接时出错: {e}")
self.executor = None
self.logger.bind(tag=TAG).info("连接资源已释放")
def clear_queues(self): def clear_queues(self):
"""清空所有任务队列""" """清空所有任务队列"""
@@ -890,7 +895,11 @@ class ConnectionHandler:
) )
# 使用非阻塞方式清空队列 # 使用非阻塞方式清空队列
for q in [self.tts.tts_text_queue, self.tts.tts_audio_queue]: for q in [
self.tts.tts_text_queue,
self.tts.tts_audio_queue,
self.report_queue,
]:
if not q: if not q:
continue continue
while True: while True:
main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py
+88 -21
View File
@@ -161,6 +161,12 @@ class TTSProvider(TTSProviderBase):
) )
check_model_key("TTS", self.access_token) check_model_key("TTS", self.access_token)
# 添加会话状态控制
self._session_lock = asyncio.Lock() # 会话操作的并发锁
self._current_session_id = None # 当前会话ID
self._session_started = False # 会话是否已开始
self._session_finished = False # 会话是否已结束
################################################################################### ###################################################################################
# 火山双流式TTS重写父类的方法--开始 # 火山双流式TTS重写父类的方法--开始
################################################################################### ###################################################################################
@@ -236,17 +242,22 @@ class TTSProvider(TTSProviderBase):
) )
async def _start_monitor_tts_response(self): async def _start_monitor_tts_response(self):
opus_datas_cache = []
# 添加标志来区分是否是第一句话
is_first_sentence = True
while not self.conn.stop_event.is_set(): while not self.conn.stop_event.is_set():
try: try:
msg = await self.ws.recv() # 确保 `recv()` 运行在同一个 event loop # 确保 `recv()` 运行在同一个 event loop
msg = await self.ws.recv()
res = self.parser_response(msg) res = self.parser_response(msg)
self.print_response(res, "send_text res:") self.print_response(res, "send_text res:")
if res.optional.event == EVENT_TTSSentenceStart: if res.optional.event == EVENT_TTSSentenceStart:
json_data = json.loads(res.payload.decode("utf-8")) json_data = json.loads(res.payload.decode("utf-8"))
self.tts_text = json_data.get("text", "") self.tts_text = json_data.get("text", "")
logger.bind(tag=TAG).info(f"语音生成成功: {self.tts_text}") logger.bind(tag=TAG).debug(f"句子语音生成开始: {self.tts_text}")
self.tts_audio_queue.put((SentenceType.FIRST, [], self.tts_text)) self.tts_audio_queue.put((SentenceType.FIRST, [], self.tts_text))
opus_datas_cache = []
elif ( elif (
res.optional.event == EVENT_TTSResponse res.optional.event == EVENT_TTSResponse
and res.header.message_type == AUDIO_ONLY_RESPONSE and res.header.message_type == AUDIO_ONLY_RESPONSE
@@ -256,9 +267,23 @@ class TTSProvider(TTSProviderBase):
logger.bind(tag=TAG).debug( logger.bind(tag=TAG).debug(
f"推送数据到队列里面帧数～～{len(opus_datas)}" f"推送数据到队列里面帧数～～{len(opus_datas)}"
) )
self.tts_audio_queue.put((SentenceType.MIDDLE, opus_datas, None)) if is_first_sentence:
# 第一句话直接发送
self.tts_audio_queue.put(
(SentenceType.MIDDLE, opus_datas, self.tts_text)
)
else:
# 后续句子缓存
opus_datas_cache = opus_datas_cache + opus_datas
elif res.optional.event == EVENT_TTSSentenceEnd: elif res.optional.event == EVENT_TTSSentenceEnd:
logger.bind(tag=TAG).debug(f"句子结束～～{self.tts_text}") logger.bind(tag=TAG).info(f"句子语音生成成功：{self.tts_text}")
if not is_first_sentence:
# 只有非第一句话才发送缓存的数据
self.tts_audio_queue.put(
(SentenceType.MIDDLE, opus_datas_cache, self.tts_text)
)
# 第一句话结束后，将标志设置为False
is_first_sentence = False
elif res.optional.event == EVENT_SessionFinished: elif res.optional.event == EVENT_SessionFinished:
logger.bind(tag=TAG).debug(f"会话结束～～") logger.bind(tag=TAG).debug(f"会话结束～～")
for tts_file, text in self.before_stop_play_files: for tts_file, text in self.before_stop_play_files:
@@ -269,6 +294,9 @@ class TTSProvider(TTSProviderBase):
) )
self.before_stop_play_files.clear() self.before_stop_play_files.clear()
self.tts_audio_queue.put((SentenceType.LAST, [], None)) self.tts_audio_queue.put((SentenceType.LAST, [], None))
opus_datas_cache = []
is_first_sentence = True
continue continue
except websockets.ConnectionClosed: except websockets.ConnectionClosed:
break # 连接关闭时退出监听 break # 连接关闭时退出监听
@@ -422,27 +450,66 @@ class TTSProvider(TTSProviderBase):
return return
async def start_session(self, session_id): async def start_session(self, session_id):
header = Header(
message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON,
).as_bytes()
optional = Optional(event=EVENT_StartSession, sessionId=session_id).as_bytes()
payload = self.get_payload_bytes(event=EVENT_StartSession, speaker=self.speaker)
await self.send_event(header, optional, payload)
logger.bind(tag=TAG).debug(f"开始会话～～{session_id}") logger.bind(tag=TAG).debug(f"开始会话～～{session_id}")
async with self._session_lock:
# 如果已有会话未结束，先关闭它
if self._session_started and not self._session_finished:
logger.bind(tag=TAG).warning(
f"发现未关闭的会话 {self._current_session_id}，正在关闭..."
)
await self.finish_session(self._current_session_id)
# 重置会话状态
self._current_session_id = session_id
self._session_started = True
self._session_finished = False
header = Header(
message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON,
).as_bytes()
optional = Optional(
event=EVENT_StartSession, sessionId=session_id
).as_bytes()
payload = self.get_payload_bytes(
event=EVENT_StartSession, speaker=self.speaker
)
await self.send_event(header, optional, payload)
async def finish_session(self, session_id): async def finish_session(self, session_id):
logger.bind(tag=TAG).debug(f"关闭会话～～{session_id}") logger.bind(tag=TAG).debug(f"关闭会话～～{session_id}")
header = Header( async with self._session_lock:
message_type=FULL_CLIENT_REQUEST, # 检查会话状态
message_type_specific_flags=MsgTypeFlagWithEvent, if not self._session_started:
serial_method=JSON, logger.bind(tag=TAG).warning(f"尝试关闭未开始的会话 {session_id}")
).as_bytes() return
optional = Optional(event=EVENT_FinishSession, sessionId=session_id).as_bytes()
payload = str.encode("{}") if self._session_finished:
await self.send_event(header, optional, payload) logger.bind(tag=TAG).warning(f"会话 {session_id} 已经关闭")
return return
if self._current_session_id != session_id:
logger.bind(tag=TAG).warning(
f"尝试关闭错误的会话 {session_id}，当前会话为 {self._current_session_id}"
)
return
header = Header(
message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON,
).as_bytes()
optional = Optional(
event=EVENT_FinishSession, sessionId=session_id
).as_bytes()
payload = str.encode("{}")
await self.send_event(header, optional, payload)
# 更新会话状态
self._session_finished = True
self._session_started = False
self._current_session_id = None
async def reset(self): async def reset(self):
# 关闭之前的对话 # 关闭之前的对话
main/xiaozhi-server/test/test_page.html
+41 -33
View File
@@ -803,7 +803,10 @@
if (frameData && frameData.length > 0) { if (frameData && frameData.length > 0) {
// 转换为Float32 // 转换为Float32
const floatData = convertInt16ToFloat32(frameData); const floatData = convertInt16ToFloat32(frameData);
decodedSamples.push(...floatData); // 使用循环替代展开运算符
for (let i = 0; i < floatData.length; i++) {
decodedSamples.push(floatData[i]);
}
} }
} catch (error) { } catch (error) {
log("Opus解码失败: " + error.message, 'error'); log("Opus解码失败: " + error.message, 'error');
@@ -811,8 +814,10 @@
} }
if (decodedSamples.length > 0) { if (decodedSamples.length > 0) {
// 添加到解码队列 // 使用循环替代展开运算符
this.queue.push(...decodedSamples); for (let i = 0; i < decodedSamples.length; i++) {
this.queue.push(decodedSamples[i]);
}
this.totalSamples += decodedSamples.length; this.totalSamples += decodedSamples.length;
// 如果累积了至少0.2秒的音频，开始播放 // 如果累积了至少0.2秒的音频，开始播放
@@ -868,36 +873,39 @@
this.source = null; this.source = null;
this.playing = false; this.playing = false;
// 如果队列中还有数据或者缓冲区有新数据，继续播放 // 使用setTimeout避免递归调用
if (this.queue.length > 0) { setTimeout(() => {
setTimeout(() => this.startPlaying(), 10); // 如果队列中还有数据，继续播放
} else if (audioBufferQueue.length > 0) { if (this.queue.length > 0) {
// 缓冲区有新数据，进行解码 this.startPlaying();
const frames = [...audioBufferQueue]; } else if (audioBufferQueue.length > 0) {
audioBufferQueue = []; // 缓冲区有新数据，进行解码
this.decodeOpusFrames(frames); const frames = [...audioBufferQueue];
} else if (this.endOfStream) { audioBufferQueue = [];
// 流已结束且没有更多数据 this.decodeOpusFrames(frames);
log("音频播放完成", 'info'); } else if (this.endOfStream) {
isAudioPlaying = false; // 流已结束且没有更多数据
this.endOfStream = false; log("音频播放完成", 'info');
streamingContext = null; isAudioPlaying = false;
} else { this.endOfStream = false;
// 等待更多数据 streamingContext = null;
setTimeout(() => { } else {
// 如果仍然没有新数据，但有更多的包到达 // 等待更多数据
if (this.queue.length === 0 && audioBufferQueue.length > 0) { setTimeout(() => {
const frames = [...audioBufferQueue]; // 如果仍然没有新数据，但有更多的包到达
audioBufferQueue = []; if (this.queue.length === 0 && audioBufferQueue.length > 0) {
this.decodeOpusFrames(frames); const frames = [...audioBufferQueue];
} else if (this.queue.length === 0 && audioBufferQueue.length === 0) { audioBufferQueue = [];
// 真的没有更多数据了 this.decodeOpusFrames(frames);
log("音频播放完成 (超时)", 'info'); } else if (this.queue.length === 0 && audioBufferQueue.length === 0) {
isAudioPlaying = false; // 真的没有更多数据了
streamingContext = null; log("音频播放完成 (超时)", 'info');
} isAudioPlaying = false;
}, 500); // 500ms超时 streamingContext = null;
} }
}, 500); // 500ms超时
}
}, 10); // 10ms延迟，避免立即递归
}; };
this.source.start(); this.source.start();