Merge pull request #591 from journey-ad/patch-2

一些功能优化和bug修复

main/xiaozhi-server/core/connection.py

@@ -102,6 +102,19 @@ class ConnectionHandler:
         if self.config["selected_module"]["Intent"] == 'function_call':
             self.use_function_call_mode = True
         
+        try:
+            # 加载插件
+            if self.use_function_call_mode:
+                self.func_handler = FunctionHandler(self)
+                self.logger.bind(tag=TAG).info("初始化FunctionHandler成功")
+            else:
+                self.func_handler = None
+                self.logger.bind(tag=TAG).info("未启用function_call模式，跳过FunctionHandler初始化")
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"初始化FunctionHandler失败: {str(e)}")
+            self.func_handler = None
+            self.use_function_call_mode = False
+
         self.mcp_manager = MCPManager(self)
 
     async def handle_connection(self, ws):
@@ -168,13 +181,23 @@ class ConnectionHandler:
 
         except AuthenticationError as e:
             self.logger.bind(tag=TAG).error(f"Authentication failed: {str(e)}")
+            await self.close(ws)
             return
         except Exception as e:
             stack_trace = traceback.format_exc()
             self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}")
+            await self.close(ws)
             return
         finally:
+            self._save_and_close(ws)
+
+    async def _save_and_close(self, ws):
+        """保存记忆并关闭连接"""
+        try:
             await self.memory.save_memory(self.dialogue.dialogue)
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
+        finally:
             await self.close(ws)
 
     async def _route_message(self, message):
@@ -191,9 +214,6 @@ class ConnectionHandler:
             self.prompt = self.private_config.private_config.get("prompt", self.prompt)
         self.dialogue.put(Message(role="system", content=self.prompt))
 
-        """加载插件"""
-        self.func_handler = FunctionHandler(self)
-
         """加载记忆"""
         device_id = self.headers.get("device-id", None)
         self.memory.init_memory(device_id, self.llm)
@@ -294,7 +314,7 @@ class ConnectionHandler:
                 break
 
             end_time = time.time()
-            self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
+            # self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
 
             # 合并当前全部文本并处理未分割部分
             full_text = "".join(response_message)
@@ -413,7 +433,7 @@ class ConnectionHandler:
                         break
 
                     end_time = time.time()
-                    self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
+                    # self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
 
                     # 处理文本分段和TTS逻辑
                     # 合并当前全部文本并处理未分割部分

main/xiaozhi-server/core/handle/helloHandle.py

@@ -11,7 +11,7 @@ import time
 logger = setup_logging()
 
 WAKEUP_CONFIG = {"dir": "config/assets/", "file_name": "wakeup_words", "create_time": time.time(), "refresh_time": 10,
-                 "words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"]}
+                 "words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"], "text": ""}
 
 
 async def handleHelloMessage(conn):
@@ -36,7 +36,10 @@ async def checkWakeupWords(conn, text):
             asyncio.create_task(wakeupWordsResponse(conn))
             return False
         opus_packets, duration = conn.tts.audio_to_opus_data(file)
-        conn.audio_play_queue.put((opus_packets, text, 0))
+        text_hello = WAKEUP_CONFIG["text"]
+        if not text_hello:
+            text_hello, _ = await conn.asr.speech_to_text(opus_packets, conn.session_id)
+        conn.audio_play_queue.put((opus_packets, text_hello, 0))
         if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
             asyncio.create_task(wakeupWordsResponse(conn))
         return True
@@ -62,6 +65,7 @@ async def wakeupWordsResponse(conn):
     wakeup_word = random.choice(WAKEUP_CONFIG["words"])
     result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
     tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
+
     if tts_file is not None and os.path.exists(tts_file):
         file_type = os.path.splitext(tts_file)[1]
         if file_type:
@@ -72,3 +76,4 @@ async def wakeupWordsResponse(conn):
         """将文件挪到"wakeup_words.mp3"""
         shutil.move(tts_file, WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type)
         WAKEUP_CONFIG["create_time"] = time.time()
+        WAKEUP_CONFIG["text"] = result

main/xiaozhi-server/core/handle/intentHandler.py

@@ -37,6 +37,7 @@ async def check_direct_exit(conn, text):
     for cmd in cmd_exit:
         if text == cmd:
             logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
+            await send_stt_message(conn, text)
             await conn.close()
             return True
     return False

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -10,7 +10,7 @@ logger = setup_logging()
 
 async def handleAudioMessage(conn, audio):
     if not conn.asr_server_receive:
-        logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
+        # logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
         return
     if conn.client_listen_mode == "auto":
         have_voice = conn.vad.is_vad(conn, audio)
@@ -21,7 +21,7 @@ async def handleAudioMessage(conn, audio):
     if have_voice == False and conn.client_have_voice == False:
         await no_voice_close_connect(conn)
         conn.asr_audio.append(audio)
-        conn.asr_audio = conn.asr_audio[-5:]  # 保留最新的5帧音频内容，解决ASR句首丢字问题
+        conn.asr_audio = conn.asr_audio[-10:]  # 保留最新的10帧音频内容，解决ASR句首丢字问题
         return
     conn.client_no_voice_last_time = 0.0
     conn.asr_audio.append(audio)
@@ -30,7 +30,7 @@ async def handleAudioMessage(conn, audio):
         conn.client_abort = False
         conn.asr_server_receive = False
         # 音频太短了，无法识别
-        if len(conn.asr_audio) < 10:
+        if len(conn.asr_audio) < 15:
             conn.asr_server_receive = True
         else:
             text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -36,7 +36,7 @@ async def sendAudio(conn, audios):
     pre_buffer = min(3, len(audios))
     for i in range(pre_buffer):
         await conn.websocket.send(audios[i])
-        conn.logger.bind(tag=TAG).debug(f"预缓冲帧 {i}, 时间: {(time.perf_counter() - start_time) * 1000:.2f}ms")
+        # conn.logger.bind(tag=TAG).debug(f"预缓冲帧 {i}, 时间: {(time.perf_counter() - start_time) * 1000:.2f}ms")
 
     # 正常播放剩余帧
     for opus_packet in audios[pre_buffer:]:
@@ -51,7 +51,7 @@ async def sendAudio(conn, audios):
             await asyncio.sleep(delay)
 
         await conn.websocket.send(opus_packet)
-        conn.logger.bind(tag=TAG).debug(f"发送帧，位置: {play_position}ms, 实际间隔: {(time.perf_counter() - current_time) * 1000:.2f}ms")
+        # conn.logger.bind(tag=TAG).debug(f"发送帧，位置: {play_position}ms, 实际间隔: {(time.perf_counter() - current_time) * 1000:.2f}ms")
 
         play_position += frame_duration
 

main/xiaozhi-server/plugins_func/functions/handle_device.py

@@ -46,15 +46,17 @@ def _handle_device_action(conn, func, success_message, error_message, *args, **k
         return ActionResponse(action=Action.RESPONSE, result=None, response=response)
 
 # 设备控制
+# TODO: 和自动注册的iot方法重复了，也许可以删掉这里手动定义的实现？
+# 另外观察到function_call准确率不够高，可能是llm的问题
 handle_device_function_desc = {
     "type": "function",
     "function": {
         "name": "handle_device",
         "description": (
             "用户想要获取或者设置设备的音量/亮度大小，或者用户觉得声音/亮度过高或过低，或者用户想提高或降低音量/亮度。"
-            "比如用户说现在亮度多少，参数为：device_type:Backlight,action:get。"
+            "比如用户说现在亮度多少，参数为：device_type:Screen,action:get。"
             "比如用户说设置音量为50，参数为：device_type:Speaker,action:set,value:50。"
-            "比如用户说亮度太高了，参数为：device_type:Backlight,action:lower。"
+            "比如用户说亮度太高了，参数为：device_type:Screen,action:lower。"
             "比如用户说调大音量，参数为：device_type:Speaker,action:raise。"
         ),
         "parameters": {
@@ -62,7 +64,7 @@ handle_device_function_desc = {
             "properties": {
                 "device_type": {
                     "type": "string",
-                    "description": "设备类型，可选值：Speaker(音量),Backlight(亮度)"
+                    "description": "设备类型，可选值：Speaker(音量),Screen(亮度)"
                 },
                 "action": {
                     "type": "string",
@@ -82,7 +84,7 @@ handle_device_function_desc = {
 def handle_device(conn, device_type: str, action: str, value: int = None):
     if device_type == "Speaker":
         method_name, property_name, device_name = "SetVolume", "volume", "音量"
-    elif device_type == "Backlight":
+    elif device_type == "Screen":
         method_name, property_name, device_name = "SetBrightness", "brightness", "亮度"
     else:
         raise Exception(f"未识别的设备类型: {device_type}")