From 22506019a9bac39190b50d9c1c3e81e59aa0b7ff Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 4 Feb 2025 13:57:05 +0800 Subject: [PATCH] =?UTF-8?q?add:=E6=94=AF=E6=8C=81=E6=89=8B=E5=8A=A8?= =?UTF-8?q?=E6=8C=89=E4=BD=8F=E8=AF=B4=E8=AF=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- README_en.md | 4 ++-- core/connection.py | 19 ++++--------------- core/handle/audioHandle.py | 36 +++++++++++++++++++++--------------- core/handle/textHandle.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 59 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index ac695497..9084b983 100644 --- a/README.md +++ b/README.md @@ -27,16 +27,16 @@ ## 已实现 - `xiaozhi-esp32` 通信 WebSocket 协议 -- 支持实时打断对话，试一下在聊天的时候说一句`你好小智` +- 支持唤醒对话、手动对话、实时打断对话 - 支持国语、粤语、英语、日语、韩语 5 种语言识别（FunASR（默认）） - 自由更换 LLM（支持ChatGLM（默认）、Dify、DeepSeek） - 自由更换 TTS（支持EdgeTTS（默认）、火山引擎豆包TTS） ## 正在实现 -- 按键手动对话 - 长时间不聊天进入休眠状态 - 对话记忆 +- 更换心情模式 ## 本项目依赖服务 diff --git a/README_en.md b/README_en.md index ae97cc3e..543a801a 100644 --- a/README_en.md +++ b/README_en.md @@ -32,16 +32,16 @@ To fully experience this project, follow these steps: ## Implemented - `xiaozhi-esp32` WebSocket communication protocol -- Support interrupt dialogue in real time, try to say a sentence when chatting `你好小智` +- Supports wake-word initiated dialogue, manual dialogue, and real-time interruption of dialogue. - Support for 5 languages: Mandarin, Cantonese, English, Japanese, Korean (FunASR - default) - Flexible LLM switching (ChatGLM - default, Dify, DeepSeek) - Flexible TTS switching (EdgeTTS - default, ByteDance Doubao TTS) ## In Progress -- Manual button-triggered dialogue - Sleep mode after inactivity - Dialogue memory +- Change the mood mode ## Dependencies diff --git a/core/connection.py b/core/connection.py index c824bf6e..65d0bbf6 100644 --- a/core/connection.py +++ b/core/connection.py @@ -12,8 +12,6 @@ from collections import deque from core.utils.util import is_segment from core.utils.dialogue import Message, Dialogue from core.handle.textHandle import handleTextMessage -from core.handle.abortHandle import handleAbortMessage -from core.handle.helloHandle import handleHelloMessage from core.utils.util import get_string_no_punctuation_or_emoji from concurrent.futures import ThreadPoolExecutor, TimeoutError from core.handle.audioHandle import handleAudioMessage, sendAudioMessage @@ -29,7 +27,10 @@ class ConnectionHandler: self.session_id = None self.prompt = None self.welcome_msg = None + + # 客户端状态相关 self.client_abort = False + self.client_listen_mode = "auto" # 线程任务相关 self.loop = asyncio.get_event_loop() @@ -91,22 +92,10 @@ class ConnectionHandler: async def _route_message(self, message): """消息路由""" if isinstance(message, str): - await self._handle_text(message) + await handleTextMessage(self, message) elif isinstance(message, bytes): await handleAudioMessage(self, message) - async def _handle_text(self, message): - """处理文本消息""" - self.logger.info(f"收到文本消息：{message}") - try: - msg_json = json.loads(message) - if msg_json["type"] == "hello": - await handleHelloMessage(self, "你好") - if msg_json["type"] == "abort": - await handleAbortMessage(self) - except json.JSONDecodeError: - await handleTextMessage(self, message) - def _initialize_components(self): self.prompt = self.config["prompt"] # 赋予LLM时间观念 diff --git a/core/handle/audioHandle.py b/core/handle/audioHandle.py index 24b22f27..985a4a6d 100644 --- a/core/handle/audioHandle.py +++ b/core/handle/audioHandle.py @@ -11,7 +11,10 @@ async def handleAudioMessage(conn, audio): if not conn.asr_server_receive: logger.debug(f"前期数据处理中，暂停接收") return - have_voice = conn.vad.is_vad(conn, audio) + if conn.client_listen_mode == "auto": + have_voice = conn.vad.is_vad(conn, audio) + else: + have_voice = conn.client_have_voice # 如果本次没有声音，本段也没声音，就把声音丢弃了 if have_voice == False and conn.client_have_voice == False: @@ -26,25 +29,28 @@ async def handleAudioMessage(conn, audio): logger.info(f"识别文本: {text}") text_len = remove_punctuation_and_length(text) if text_len > 0: - stt_text = get_string_no_punctuation_or_emoji(text) - await conn.websocket.send(json.dumps({ - "type": "stt", - "text": stt_text, - "session_id": conn.session_id} - )) - await conn.websocket.send( - json.dumps({ - "type": "llm", - "text": "😊", - "emotion": "happy", - "session_id": conn.session_id} - )) - conn.executor.submit(conn.chat, text) + await startToChat(conn, text) else: conn.asr_server_receive = True conn.asr_audio.clear() conn.reset_vad_states() +async def startToChat(conn, text): + stt_text = get_string_no_punctuation_or_emoji(text) + await conn.websocket.send(json.dumps({ + "type": "stt", + "text": stt_text, + "session_id": conn.session_id} + )) + await conn.websocket.send( + json.dumps({ + "type": "llm", + "text": "😊", + "emotion": "happy", + "session_id": conn.session_id} + )) + conn.executor.submit(conn.chat, text) + async def sendAudioMessage(conn, audios, duration, text): base_delay = conn.tts_duration diff --git a/core/handle/textHandle.py b/core/handle/textHandle.py index c1f97440..3a85fdc4 100644 --- a/core/handle/textHandle.py +++ b/core/handle/textHandle.py @@ -1,7 +1,37 @@ import logging +import json +from core.handle.abortHandle import handleAbortMessage +from core.handle.helloHandle import handleHelloMessage +from core.handle.audioHandle import startToChat logger = logging.getLogger(__name__) async def handleTextMessage(conn, message): + """处理文本消息""" + logger.info(f"收到文本消息：{message}") + try: + msg_json = json.loads(message) + if msg_json["type"] == "hello": + await handleHelloMessage(conn, "你好") + elif msg_json["type"] == "abort": + await handleAbortMessage(conn) + elif msg_json["type"] == "listen": + if "mode" in msg_json: + conn.client_listen_mode = msg_json["mode"] + logger.info(f"客户端拾音模式：{conn.client_listen_mode}") + if msg_json["state"] == "start": + conn.client_have_voice = True + conn.client_voice_stop = False + elif msg_json["state"] == "stop": + conn.client_have_voice = True + conn.client_voice_stop = True + elif msg_json["state"] == "detect": + conn.asr_server_receive = False + conn.client_have_voice = False + conn.asr_audio.clear() + if "text" in msg_json: + await startToChat(conn, msg_json["text"]) + except json.JSONDecodeError: + await handleTextMessage(conn, message) await conn.websocket.send(message)