From 93acdb586138d4fff4a218ba38423e5d83521c75 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 4 Feb 2025 12:20:10 +0800 Subject: [PATCH] =?UTF-8?q?add:=E5=AE=9E=E6=97=B6=E6=89=93=E6=96=AD?= =?UTF-8?q?=E5=AF=B9=E8=AF=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 ++++++-- README_en.md | 9 +++++++-- core/connection.py | 17 ++++++++++++++--- core/handle/abortHandle.py | 16 ++++++++++++++++ core/handle/audioHandle.py | 1 + 5 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 core/handle/abortHandle.py diff --git a/README.md b/README.md index 3456997f..ac695497 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,13 @@ ## 已实现 - `xiaozhi-esp32` 通信 WebSocket 协议 +- 支持实时打断对话，试一下在聊天的时候说一句`你好小智` - 支持国语、粤语、英语、日语、韩语 5 种语言识别（FunASR（默认）） - 自由更换 LLM（支持ChatGLM（默认）、Dify、DeepSeek） - 自由更换 TTS（支持EdgeTTS（默认）、火山引擎豆包TTS） ## 正在实现 -- 打断对话 - 按键手动对话 - 长时间不聊天进入休眠状态 - 对话记忆 @@ -213,7 +213,11 @@ https://espressif.github.io/esp-launchpad/ 建议：大模型和TTS都是依赖接口，如果网络环境不佳，可以考虑换成本地模型。或多尝试切换不同的接口模型。 -## 3、更多问题，可联系我们反馈 +## 3、为啥我的ChatGLMLLM回复有点问题，明明它才是小智，却把我当小智。 + +建议：第一步可以先`配置文件`里的调整提示词。第二步，`配置文件`中用的模型是免费的模型：`glm-4-flash`，可以考虑换成收费版本的。 + +## 4、更多问题，可联系我们反馈 ![图片](docs/images/wechat.jpg) diff --git a/README_en.md b/README_en.md index 58ea7132..ae97cc3e 100644 --- a/README_en.md +++ b/README_en.md @@ -32,13 +32,13 @@ To fully experience this project, follow these steps: ## Implemented - `xiaozhi-esp32` WebSocket communication protocol +- Support interrupt dialogue in real time, try to say a sentence when chatting `你好小智` - Support for 5 languages: Mandarin, Cantonese, English, Japanese, Korean (FunASR - default) - Flexible LLM switching (ChatGLM - default, Dify, DeepSeek) - Flexible TTS switching (EdgeTTS - default, ByteDance Doubao TTS) ## In Progress -- Conversation interruption - Manual button-triggered dialogue - Sleep mode after inactivity - Dialogue memory @@ -223,7 +223,12 @@ both are slow, the network environment may need to be optimized. Suggestions: Both big models and TTS are dependent interfaces. If the network environment is not good, you can consider changing the local model. Or try to switch different interface models. -## 3、For more questions, contact us to feedback +## 3、Why is my ChatGLMLLM replying to a bit? Obviously it is Xiaozhi, but treats me as Xiaozhi. + +Suggestion: The first step can be to adjust the prompt words in the configuration file. The second step, the model used +in the configuration file is the free model: `glm-4-flash`. You might consider switching to the paid version. + +## 4、For more questions, contact us to feedback ![图片](docs/images/wechat.jpg) diff --git a/core/connection.py b/core/connection.py index f17b8d0b..c824bf6e 100644 --- a/core/connection.py +++ b/core/connection.py @@ -12,6 +12,7 @@ from collections import deque from core.utils.util import is_segment from core.utils.dialogue import Message, Dialogue from core.handle.textHandle import handleTextMessage +from core.handle.abortHandle import handleAbortMessage from core.handle.helloHandle import handleHelloMessage from core.utils.util import get_string_no_punctuation_or_emoji from concurrent.futures import ThreadPoolExecutor, TimeoutError @@ -28,6 +29,7 @@ class ConnectionHandler: self.session_id = None self.prompt = None self.welcome_msg = None + self.client_abort = False # 线程任务相关 self.loop = asyncio.get_event_loop() @@ -100,6 +102,8 @@ class ConnectionHandler: msg_json = json.loads(message) if msg_json["type"] == "hello": await handleHelloMessage(self, "你好") + if msg_json["type"] == "abort": + await handleAbortMessage(self) except json.JSONDecodeError: await handleTextMessage(self, message) @@ -126,6 +130,11 @@ class ConnectionHandler: self.llm_finish_task = False for content in llm_responses: response_message.append(content) + # 如果中途被打断，就停止生成 + if self.client_abort: + start = len(response_message) + break + end_time = time.time() # 记录结束时间 self.logger.debug(f"大模型返回时间时间: {end_time - start_time} 秒, 生成token={content}") if is_segment(response_message): @@ -169,9 +178,11 @@ class ConnectionHandler: except Exception as e: self.logger.error(f"TTS 任务出错: {e}") continue - asyncio.run_coroutine_threadsafe( - sendAudioMessage(self, opus_datas, duration, text), self.loop - ) + if not self.client_abort: + # 如果没有中途打断就发送语音 + asyncio.run_coroutine_threadsafe( + sendAudioMessage(self, opus_datas, duration, text), self.loop + ) if self.tts.delete_audio_file and os.path.exists(tts_file): os.remove(tts_file) except Exception as e: diff --git a/core/handle/abortHandle.py b/core/handle/abortHandle.py new file mode 100644 index 00000000..78549797 --- /dev/null +++ b/core/handle/abortHandle.py @@ -0,0 +1,16 @@ +import json +import logging + +logger = logging.getLogger(__name__) + + +async def handleAbortMessage(conn): + logger.info("Abort message received") + # 设置成打断状态，会自动打断llm、tts任务 + conn.client_abort = True + # 打断屏显任务 + conn.stop_all_tasks() + # 打断客户端说话状态 + await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id})) + conn.clearSpeakStatus() + logger.info("Abort message received-end") diff --git a/core/handle/audioHandle.py b/core/handle/audioHandle.py index 59219f56..24b22f27 100644 --- a/core/handle/audioHandle.py +++ b/core/handle/audioHandle.py @@ -20,6 +20,7 @@ async def handleAudioMessage(conn, audio): conn.asr_audio.append(audio) # 如果本段有声音，且已经停止了 if conn.client_voice_stop: + conn.client_abort = False conn.asr_server_receive = False text, file_path = conn.asr.speech_to_text(conn.asr_audio, conn.session_id) logger.info(f"识别文本: {text}")