From 4c91bffcc70a35a192d2e5b6c15e8368ca9f6308 Mon Sep 17 00:00:00 2001 From: Jad Date: Sun, 30 Mar 2025 12:46:54 +0800 Subject: [PATCH 1/7] =?UTF-8?q?fix(server):=20=E5=8A=A0=E8=BD=BD=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E6=94=B9=E4=B8=BA=E5=88=9D=E5=A7=8B=E5=8C=96=E6=97=B6?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E6=89=A7=E8=A1=8C=20#559?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 0f4e2a70..ff3c816a 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -102,6 +102,19 @@ class ConnectionHandler: if self.config["selected_module"]["Intent"] == 'function_call': self.use_function_call_mode = True + try: + # 加载插件 + if self.use_function_call_mode: + self.func_handler = FunctionHandler(self) + self.logger.bind(tag=TAG).info("初始化FunctionHandler成功") + else: + self.func_handler = None + self.logger.bind(tag=TAG).info("未启用function_call模式，跳过FunctionHandler初始化") + except Exception as e: + self.logger.bind(tag=TAG).error(f"初始化FunctionHandler失败: {str(e)}") + self.func_handler = None + self.use_function_call_mode = False + self.mcp_manager = MCPManager(self) async def handle_connection(self, ws): @@ -191,9 +204,6 @@ class ConnectionHandler: self.prompt = self.private_config.private_config.get("prompt", self.prompt) self.dialogue.put(Message(role="system", content=self.prompt)) - """加载插件""" - self.func_handler = FunctionHandler(self) - """加载记忆""" device_id = self.headers.get("device-id", None) self.memory.init_memory(device_id, self.llm) From b6b6b0310f62749a2c3885f6d878c6b0cba84971 Mon Sep 17 00:00:00 2001 From: Jad Date: Sun, 30 Mar 2025 12:58:27 +0800 Subject: [PATCH 2/7] =?UTF-8?q?fix(server):=20=E9=80=80=E5=87=BA=E6=97=B6?= =?UTF-8?q?=E7=AD=89=E5=BE=85=E4=BF=9D=E5=AD=98=E8=AE=B0=E5=BF=86=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=AB=8B=E5=8D=B3=E9=87=8D=E8=BF=9E=E6=97=B6=E9=98=BB?= =?UTF-8?q?=E5=A1=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index ff3c816a..373c977a 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -181,13 +181,23 @@ class ConnectionHandler: except AuthenticationError as e: self.logger.bind(tag=TAG).error(f"Authentication failed: {str(e)}") + await self.close(ws) return except Exception as e: stack_trace = traceback.format_exc() self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}") + await self.close(ws) return finally: + self._save_and_close(ws) + + async def _save_and_close(self, ws): + """保存记忆并关闭连接""" + try: await self.memory.save_memory(self.dialogue.dialogue) + except Exception as e: + self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}") + finally: await self.close(ws) async def _route_message(self, message): From 1f3654cc013f4cd71364033ca41f026720939dc8 Mon Sep 17 00:00:00 2001 From: Jad Date: Sun, 30 Mar 2025 13:01:04 +0800 Subject: [PATCH 3/7] =?UTF-8?q?feat(server):=20=E5=BC=80=E5=90=AF=E5=94=A4?= =?UTF-8?q?=E9=86=92=E8=AF=8D=E5=8A=A0=E9=80=9F=E6=97=B6=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E5=94=A4=E9=86=92=E8=AF=8D=E6=96=87=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/handle/helloHandle.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 4583a6e3..d765de56 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -11,7 +11,7 @@ import time logger = setup_logging() WAKEUP_CONFIG = {"dir": "config/assets/", "file_name": "wakeup_words", "create_time": time.time(), "refresh_time": 10, - "words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"]} + "words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"], "text": ""} async def handleHelloMessage(conn): @@ -36,7 +36,10 @@ async def checkWakeupWords(conn, text): asyncio.create_task(wakeupWordsResponse(conn)) return False opus_packets, duration = conn.tts.audio_to_opus_data(file) - conn.audio_play_queue.put((opus_packets, text, 0)) + text_hello = WAKEUP_CONFIG["text"] + if not text_hello: + text_hello, _ = await conn.asr.speech_to_text(opus_packets, conn.session_id) + conn.audio_play_queue.put((opus_packets, text_hello, 0)) if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]: asyncio.create_task(wakeupWordsResponse(conn)) return True @@ -62,6 +65,7 @@ async def wakeupWordsResponse(conn): wakeup_word = random.choice(WAKEUP_CONFIG["words"]) result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word) tts_file = await asyncio.to_thread(conn.tts.to_tts, result) + if tts_file is not None and os.path.exists(tts_file): file_type = os.path.splitext(tts_file)[1] if file_type: @@ -72,3 +76,4 @@ async def wakeupWordsResponse(conn): """将文件挪到"wakeup_words.mp3""" shutil.move(tts_file, WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type) WAKEUP_CONFIG["create_time"] = time.time() + WAKEUP_CONFIG["text"] = result From d02c56a0e09bc7bff0b3ee8ab512be66949e69c1 Mon Sep 17 00:00:00 2001 From: Jad Date: Sun, 30 Mar 2025 13:03:01 +0800 Subject: [PATCH 4/7] =?UTF-8?q?feat(server):=20=E4=BD=BF=E7=94=A8=E4=B8=BB?= =?UTF-8?q?=E5=8A=A8=E9=80=80=E5=87=BA=E6=8C=87=E4=BB=A4=E6=97=B6=E6=98=BE?= =?UTF-8?q?=E7=A4=BA=E5=88=B0=E8=AE=BE=E5=A4=87=E4=B8=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/handle/intentHandler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index 5105a4d9..5b840e97 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -37,6 +37,7 @@ async def check_direct_exit(conn, text): for cmd in cmd_exit: if text == cmd: logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}") + await send_stt_message(conn, text) await conn.close() return True return False From 549c2af9884272365f7005b00757c3a7aa8ae551 Mon Sep 17 00:00:00 2001 From: Jad Date: Sun, 30 Mar 2025 13:06:02 +0800 Subject: [PATCH 5/7] =?UTF-8?q?feat(server):=20asr=E4=BF=9D=E7=95=99?= =?UTF-8?q?=E6=9B=B4=E5=A4=9A=E7=9A=84=E9=9F=B3=E9=A2=91=E5=89=8D=E5=AF=BC?= =?UTF-8?q?=E5=B8=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 语速较快或者声音不清晰时，保留更多前导帧可以提升asr识别效果 --- main/xiaozhi-server/core/handle/receiveAudioHandle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index c5b4135c..f97e1e23 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -21,7 +21,7 @@ async def handleAudioMessage(conn, audio): if have_voice == False and conn.client_have_voice == False: await no_voice_close_connect(conn) conn.asr_audio.append(audio) - conn.asr_audio = conn.asr_audio[-5:] # 保留最新的5帧音频内容，解决ASR句首丢字问题 + conn.asr_audio = conn.asr_audio[-10:] # 保留最新的10帧音频内容，解决ASR句首丢字问题 return conn.client_no_voice_last_time = 0.0 conn.asr_audio.append(audio) @@ -30,7 +30,7 @@ async def handleAudioMessage(conn, audio): conn.client_abort = False conn.asr_server_receive = False # 音频太短了，无法识别 - if len(conn.asr_audio) < 10: + if len(conn.asr_audio) < 15: conn.asr_server_receive = True else: text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id) From 8bbd610f12d98f2588a8ca5f313014e81ee68d8f Mon Sep 17 00:00:00 2001 From: Jad Date: Sun, 30 Mar 2025 13:09:18 +0800 Subject: [PATCH 6/7] =?UTF-8?q?fix(server):=20=E9=80=82=E9=85=8D1.5.2?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E5=AE=A2=E6=88=B7=E7=AB=AF=E7=9A=84=E8=AE=BE?= =?UTF-8?q?=E5=A4=87=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 已经在 4c91bff 修复了自动注册的iot方法，和此插件功能重复了，也许可以删掉此插件？ 另外观察到function_call准确率不够高，可能是llm的问题 --- .../plugins_func/functions/handle_device.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/main/xiaozhi-server/plugins_func/functions/handle_device.py b/main/xiaozhi-server/plugins_func/functions/handle_device.py index b215a486..03fc9f5d 100644 --- a/main/xiaozhi-server/plugins_func/functions/handle_device.py +++ b/main/xiaozhi-server/plugins_func/functions/handle_device.py @@ -46,15 +46,17 @@ def _handle_device_action(conn, func, success_message, error_message, *args, **k return ActionResponse(action=Action.RESPONSE, result=None, response=response) # 设备控制 +# TODO: 和自动注册的iot方法重复了，也许可以删掉这里手动定义的实现？ +# 另外观察到function_call准确率不够高，可能是llm的问题 handle_device_function_desc = { "type": "function", "function": { "name": "handle_device", "description": ( "用户想要获取或者设置设备的音量/亮度大小，或者用户觉得声音/亮度过高或过低，或者用户想提高或降低音量/亮度。" - "比如用户说现在亮度多少，参数为：device_type:Backlight,action:get。" + "比如用户说现在亮度多少，参数为：device_type:Screen,action:get。" "比如用户说设置音量为50，参数为：device_type:Speaker,action:set,value:50。" - "比如用户说亮度太高了，参数为：device_type:Backlight,action:lower。" + "比如用户说亮度太高了，参数为：device_type:Screen,action:lower。" "比如用户说调大音量，参数为：device_type:Speaker,action:raise。" ), "parameters": { @@ -62,7 +64,7 @@ handle_device_function_desc = { "properties": { "device_type": { "type": "string", - "description": "设备类型，可选值：Speaker(音量),Backlight(亮度)" + "description": "设备类型，可选值：Speaker(音量),Screen(亮度)" }, "action": { "type": "string", @@ -82,7 +84,7 @@ handle_device_function_desc = { def handle_device(conn, device_type: str, action: str, value: int = None): if device_type == "Speaker": method_name, property_name, device_name = "SetVolume", "volume", "音量" - elif device_type == "Backlight": + elif device_type == "Screen": method_name, property_name, device_name = "SetBrightness", "brightness", "亮度" else: raise Exception(f"未识别的设备类型: {device_type}") From 326342b8347fb79706a7d2b1242edea3f308dbbe Mon Sep 17 00:00:00 2001 From: Jad Date: Sun, 30 Mar 2025 13:11:52 +0800 Subject: [PATCH 7/7] =?UTF-8?q?chore(server):=20=E7=A7=BB=E9=99=A4?= =?UTF-8?q?=E9=AB=98=E9=A2=91=E8=A7=A6=E5=8F=91=E7=9A=84debug=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 4 ++-- main/xiaozhi-server/core/handle/receiveAudioHandle.py | 2 +- main/xiaozhi-server/core/handle/sendAudioHandle.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 373c977a..790e2879 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -314,7 +314,7 @@ class ConnectionHandler: break end_time = time.time() - self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}") + # self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}") # 合并当前全部文本并处理未分割部分 full_text = "".join(response_message) @@ -433,7 +433,7 @@ class ConnectionHandler: break end_time = time.time() - self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}") + # self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}") # 处理文本分段和TTS逻辑 # 合并当前全部文本并处理未分割部分 diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index f97e1e23..a082ada8 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -10,7 +10,7 @@ logger = setup_logging() async def handleAudioMessage(conn, audio): if not conn.asr_server_receive: - logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收") + # logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收") return if conn.client_listen_mode == "auto": have_voice = conn.vad.is_vad(conn, audio) diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index c0a664eb..9dfa5bd2 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -36,7 +36,7 @@ async def sendAudio(conn, audios): pre_buffer = min(3, len(audios)) for i in range(pre_buffer): await conn.websocket.send(audios[i]) - conn.logger.bind(tag=TAG).debug(f"预缓冲帧 {i}, 时间: {(time.perf_counter() - start_time) * 1000:.2f}ms") + # conn.logger.bind(tag=TAG).debug(f"预缓冲帧 {i}, 时间: {(time.perf_counter() - start_time) * 1000:.2f}ms") # 正常播放剩余帧 for opus_packet in audios[pre_buffer:]: @@ -51,7 +51,7 @@ async def sendAudio(conn, audios): await asyncio.sleep(delay) await conn.websocket.send(opus_packet) - conn.logger.bind(tag=TAG).debug(f"发送帧，位置: {play_position}ms, 实际间隔: {(time.perf_counter() - current_time) * 1000:.2f}ms") + # conn.logger.bind(tag=TAG).debug(f"发送帧，位置: {play_position}ms, 实际间隔: {(time.perf_counter() - current_time) * 1000:.2f}ms") play_position += frame_duration