From 885b7a0b05bcfcac46ff05bedc6b468adbd70dcc Mon Sep 17 00:00:00 2001 From: goodyhao <865700600@qq.com> Date: Tue, 13 May 2025 15:54:00 +0800 Subject: [PATCH 01/12] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E4=B8=8A?= =?UTF-8?q?=E6=8A=A5=E7=BA=BF=E7=A8=8B=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 63b79736..68964629 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -88,7 +88,8 @@ class ConnectionHandler: self.audio_play_queue = queue.Queue() self.executor = ThreadPoolExecutor(max_workers=10) - # 上报线程 + # 添加上报线程池 + self.report_thread_pool = ThreadPoolExecutor(max_workers=5, thread_name_prefix="report_worker") self.report_queue = queue.Queue() self.report_thread = None # TODO(haotian): 2025/5/12 可以通过修改此处，调节asr的上报和tts的上报 @@ -944,13 +945,10 @@ class ConnectionHandler: type, text, audio_data = item try: - # 执行上报（传入二进制数据） - report(self, type, text, audio_data) + # 提交任务到线程池 + self.report_thread_pool.submit(self._process_report, type, text, audio_data) except Exception as e: self.logger.bind(tag=TAG).error(f"聊天记录上报线程异常: {e}") - finally: - # 标记任务完成 - self.report_queue.task_done() except queue.Empty: continue except Exception as e: @@ -958,6 +956,17 @@ class ConnectionHandler: self.logger.bind(tag=TAG).info("聊天记录上报线程已退出") + def _process_report(self, type, text, audio_data): + """处理上报任务""" + try: + # 执行上报（传入二进制数据） + report(self, type, text, audio_data) + except Exception as e: + self.logger.bind(tag=TAG).error(f"上报处理异常: {e}") + finally: + # 标记任务完成 + self.report_queue.task_done() + def speak_and_play(self, text, text_index=0): if text is None or len(text) <= 0: self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}") @@ -1006,6 +1015,11 @@ class ConnectionHandler: # 添加毒丸对象到上报队列确保线程退出 self.report_queue.put(None) + # 关闭上报线程池 + if hasattr(self, 'report_thread_pool'): + self.report_thread_pool.shutdown(wait=True) + self.logger.bind(tag=TAG).info("上报线程池已关闭") + # 清空任务队列 self.clear_queues() From a5f74d7767fc26d36d623605a7d773bcf9f187d0 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 22 May 2025 22:22:43 +0800 Subject: [PATCH 02/12] =?UTF-8?q?update:=E6=9B=B4=E6=96=B0=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E6=95=99=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4b3215f6..27c8c759 100644 --- a/README.md +++ b/README.md @@ -142,10 +142,11 @@ #### 🚀 部署方式选择 -| 部署方式 | 特点 | 适用场景 | Docker部署文档 | 源码部署文档 | -|---------|------|---------|---------|---------| -| **最简化安装** | 智能对话、IOT功能，数据存储在配置文件 | 低配置环境，无需数据库 | [Docker只运行Server](./docs/Deployment.md#%E6%96%B9%E5%BC%8F%E4%B8%80docker%E5%8F%AA%E8%BF%90%E8%A1%8Cserver) | [本地源码只运行Server](./docs/Deployment.md#%E6%96%B9%E5%BC%8F%E4%BA%8C%E6%9C%AC%E5%9C%B0%E6%BA%90%E7%A0%81%E5%8F%AA%E8%BF%90%E8%A1%8Cserver)| -| **全模块安装** | 智能对话、IOT、OTA、智控台，数据存储在数据库 | 完整功能体验 |[Docker运行全模块](./docs/Deployment_all.md#%E6%96%B9%E5%BC%8F%E4%B8%80docker%E8%BF%90%E8%A1%8C%E5%85%A8%E6%A8%A1%E5%9D%97) | [本地源码运行全模块](./docs/Deployment_all.md#%E6%96%B9%E5%BC%8F%E4%BA%8C%E6%9C%AC%E5%9C%B0%E6%BA%90%E7%A0%81%E8%BF%90%E8%A1%8C%E5%85%A8%E6%A8%A1%E5%9D%97) | +| 部署方式 | 特点 | 适用场景 | Docker部署文档 | 源码部署文档 | 视频教程 | +|---------|------|---------|---------|---------|---------| +| **最简化安装** | 智能对话、IOT功能，数据存储在配置文件 | 低配置环境，无需数据库 | [Docker只运行Server](./docs/Deployment.md#%E6%96%B9%E5%BC%8F%E4%B8%80docker%E5%8F%AA%E8%BF%90%E8%A1%8Cserver) | [本地源码只运行Server](./docs/Deployment.md#%E6%96%B9%E5%BC%8F%E4%BA%8C%E6%9C%AC%E5%9C%B0%E6%BA%90%E7%A0%81%E5%8F%AA%E8%BF%90%E8%A1%8Cserver)| - | +| **全模块安装** | 智能对话、IOT、OTA、智控台，数据存储在数据库 | 完整功能体验 |[Docker运行全模块](./docs/Deployment_all.md#%E6%96%B9%E5%BC%8F%E4%B8%80docker%E8%BF%90%E8%A1%8C%E5%85%A8%E6%A8%A1%E5%9D%97) | [本地源码运行全模块](./docs/Deployment_all.md#%E6%96%B9%E5%BC%8F%E4%BA%8C%E6%9C%AC%E5%9C%B0%E6%BA%90%E7%A0%81%E8%BF%90%E8%A1%8C%E5%85%A8%E6%A8%A1%E5%9D%97) | [本地源码视频教程](https://www.bilibili.com/video/BV1wBJhz4Ewe) | + > 💡 提示：以下是按最新代码部署后的测试平台，有需要可烧录测试，并发为6个，每天会清空数据 From 17fb60b7aec4d903cd1a1107047e0c8e16b94eea Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Fri, 23 May 2025 16:03:15 +0800 Subject: [PATCH 03/12] =?UTF-8?q?update:=E4=BF=AE=E5=A4=8Diotbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/providers/asr/base.py | 30 ++++-- .../core/providers/asr/fun_local.py | 101 +++++++++++------- .../providers/intent/intent_llm/intent_llm.py | 7 +- .../functions/handle_speaker_or_screen.py | 81 ++++++++++---- 4 files changed, 149 insertions(+), 70 deletions(-) diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 227a906d..9d974180 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -30,15 +30,25 @@ class ASRProviderBase(ABC): @staticmethod def decode_opus(opus_data: List[bytes]) -> bytes: """将Opus音频数据解码为PCM数据""" + try: + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + buffer_size = 960 # 每次处理960个采样点 - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] + for opus_packet in opus_data: + try: + # 使用较小的缓冲区大小进行处理 + pcm_frame = decoder.decode(opus_packet, buffer_size) + if pcm_frame: + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).warning(f"Opus解码错误，跳过当前数据包: {e}") + continue + except Exception as e: + logger.bind(tag=TAG).error(f"音频处理错误: {e}", exc_info=True) + continue - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data + return pcm_data + except Exception as e: + logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}", exc_info=True) + return [] diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index ec3df17f..c8446574 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -9,10 +9,14 @@ import uuid from core.providers.asr.base import ASRProviderBase from funasr import AutoModel from funasr.utils.postprocess_utils import rich_transcription_postprocess +import shutil TAG = __name__ logger = setup_logging() +MAX_RETRIES = 2 +RETRY_DELAY = 1 # 重试延迟（秒） + # 捕获标准输出 class CaptureOutput: @@ -68,46 +72,69 @@ class ASRProvider(ASRProviderBase): ) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None - try: - # 合并所有opus数据包 - if self.audio_format == "pcm": - pcm_data = opus_data - else: - pcm_data = self.decode_opus(opus_data) + retry_count = 0 - combined_pcm_data = b"".join(pcm_data) + while retry_count < MAX_RETRIES: + try: + # 合并所有opus数据包 + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) - # 判断是否保存为WAV文件 - if self.delete_audio_file: - pass - else: - file_path = self.save_audio_to_file(pcm_data, session_id) + combined_pcm_data = b"".join(pcm_data) - # 语音识别 - start_time = time.time() - result = self.model.generate( - input=combined_pcm_data, - cache={}, - language="auto", - use_itn=True, - batch_size_s=60, - ) - text = rich_transcription_postprocess(result[0]["text"]) - logger.bind(tag=TAG).debug( - f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" - ) + # 检查磁盘空间 + if not self.delete_audio_file: + free_space = shutil.disk_usage(self.output_dir).free + if free_space < len(combined_pcm_data) * 2: # 预留2倍空间 + raise OSError("磁盘空间不足") - return text, file_path + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + file_path = self.save_audio_to_file(pcm_data, session_id) - except Exception as e: - logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", file_path + # 语音识别 + start_time = time.time() + result = self.model.generate( + input=combined_pcm_data, + cache={}, + language="auto", + use_itn=True, + batch_size_s=60, + ) + text = rich_transcription_postprocess(result[0]["text"]) + logger.bind(tag=TAG).debug( + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" + ) - # finally: - # # 文件清理逻辑 - # if self.delete_audio_file and file_path and os.path.exists(file_path): - # try: - # os.remove(file_path) - # logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") - # except Exception as e: - # logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") + return text, file_path + + except OSError as e: + retry_count += 1 + if retry_count >= MAX_RETRIES: + logger.bind(tag=TAG).error( + f"语音识别失败（已重试{retry_count}次）: {e}", exc_info=True + ) + return "", file_path + logger.bind(tag=TAG).warning( + f"语音识别失败，正在重试（{retry_count}/{MAX_RETRIES}）: {e}" + ) + time.sleep(RETRY_DELAY) + + except Exception as e: + logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) + return "", file_path + + finally: + # 文件清理逻辑 + if self.delete_audio_file and file_path and os.path.exists(file_path): + try: + os.remove(file_path) + logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") + except Exception as e: + logger.bind(tag=TAG).error( + f"文件删除失败: {file_path} | 错误: {e}" + ) diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py index ee657420..51696bb7 100644 --- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py +++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py @@ -72,6 +72,10 @@ class IntentProvider(IntentProviderBase): '返回: {"function_call": {"name": "get_time"}}\n' "```\n" "```\n" + "用户: 当前电池电量是多少？\n" + '返回: {"function_call": {"name": "get_battery_level", "arguments": {"response_success": "当前电池电量为{value}%", "response_failure": "无法获取Battery的当前电量百分比"}}}\n' + "```\n" + "```\n" "用户: 我想结束对话\n" '返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n' "```\n" @@ -224,7 +228,8 @@ class IntentProvider(IntentProviderBase): if function_name == "continue_chat": # 保留非工具相关的消息 clean_history = [ - msg for msg in conn.dialogue.dialogue + msg + for msg in conn.dialogue.dialogue if msg.role not in ["tool", "function"] ] conn.dialogue.dialogue = clean_history diff --git a/main/xiaozhi-server/plugins_func/functions/handle_speaker_or_screen.py b/main/xiaozhi-server/plugins_func/functions/handle_speaker_or_screen.py index fa357ff5..f89f30b0 100644 --- a/main/xiaozhi-server/plugins_func/functions/handle_speaker_or_screen.py +++ b/main/xiaozhi-server/plugins_func/functions/handle_speaker_or_screen.py @@ -15,15 +15,26 @@ async def _get_device_status(conn, device_name, device_type, property_name): return status -async def _set_device_property(conn, device_name, device_type, method_name, property_name, new_value=None, action=None, step=10): +async def _set_device_property( + conn, + device_name, + device_type, + method_name, + property_name, + new_value=None, + action=None, + step=10, +): """设置设备属性""" - current_value = await _get_device_status(conn, device_name, device_type, property_name) + current_value = await _get_device_status( + conn, device_name, device_type, property_name + ) - if action == 'raise': + if action == "raise": current_value += step - elif action == 'lower': + elif action == "lower": current_value -= step - elif action == 'set': + elif action == "set": if new_value is None: raise Exception(f"缺少{property_name}参数") current_value = new_value @@ -37,8 +48,7 @@ async def _set_device_property(conn, device_name, device_type, method_name, prop def _handle_device_action(conn, func, success_message, error_message, *args, **kwargs): """处理设备操作的通用函数""" - future = asyncio.run_coroutine_threadsafe( - func(conn, *args, **kwargs), conn.loop) + future = asyncio.run_coroutine_threadsafe(func(conn, *args, **kwargs), conn.loop) try: result = future.result() logger.bind(tag=TAG).info(f"{success_message}: {result}") @@ -75,26 +85,41 @@ handle_device_function_desc = { "device_type": { "type": "string", "description": "设备类型，**严格限定为Speaker（音量）或Screen（亮度）**，其他设备类型禁止调用此函数", - "enum": ["Speaker", "Screen"] - + "enum": ["Speaker", "Screen"], }, "action": { "type": "string", - "description": "动作名称，可选值：get(获取),set(设置),raise(提高),lower(降低)" + "description": "动作名称，可选值：get(获取),set(设置),raise(提高),lower(降低)", }, "value": { "type": "integer", - "description": "值大小，可选值：0-100之间的整数" - } + "description": "值大小，可选值：0-100之间的整数", + }, }, - "required": ["device_type", "action"] - } - } + "required": ["device_type", "action"], + }, + }, } -@register_function('handle_speaker_volume_or_screen_brightness', handle_device_function_desc, ToolType.IOT_CTL) -def handle_speaker_volume_or_screen_brightness(conn, device_type: str, action: str, value: int = None): +@register_function( + "handle_speaker_volume_or_screen_brightness", + handle_device_function_desc, + ToolType.IOT_CTL, +) +def handle_speaker_volume_or_screen_brightness( + conn, device_type: str, action: str, value: int = None +): + # 检查value是否为中文值 + if ( + value is not None + and isinstance(value, str) + and any("\u4e00" <= char <= "\u9fff" for char in str(value)) + ): + raise Exception( + f"请直接告诉我要将{'音量' if device_type=='Speaker' else '亮度'}调整成多少" + ) + if device_type == "Speaker": method_name, property_name, device_name = "SetVolume", "volume", "音量" elif device_type == "Screen": @@ -108,13 +133,25 @@ def handle_speaker_volume_or_screen_brightness(conn, device_type: str, action: s if action == "get": # get return _handle_device_action( - conn, _get_device_status, f"当前{device_name}", f"获取{device_name}失败", - device_name=device_name, device_type=device_type, property_name=property_name, + conn, + _get_device_status, + f"当前{device_name}", + f"获取{device_name}失败", + device_name=device_name, + device_type=device_type, + property_name=property_name, ) else: # set, raise, lower return _handle_device_action( - conn, _set_device_property, f"{device_name}已调整到", f"{device_name}调整失败", - device_name=device_name, device_type=device_type, method_name=method_name, - property_name=property_name, new_value=value, action=action + conn, + _set_device_property, + f"{device_name}已调整到", + f"{device_name}调整失败", + device_name=device_name, + device_type=device_type, + method_name=method_name, + property_name=property_name, + new_value=value, + action=action, ) From 98e2526f8a09fa58b2d591a587a615a82d60da5b Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sat, 24 May 2025 02:15:06 +0800 Subject: [PATCH 04/12] =?UTF-8?q?update:=E5=90=88=E5=B9=B6chat=E5=92=8Ccha?= =?UTF-8?q?t=5Fwith=5Ffunction=5Fcalling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 153 +++++------------- .../core/handle/receiveAudioHandle.py | 10 +- 2 files changed, 40 insertions(+), 123 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 74b612fe..e73b340d 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -506,106 +506,20 @@ class ConnectionHandler: # 更新系统prompt至上下文 self.dialogue.update_system_message(self.prompt) - def chat(self, query): - - self.dialogue.put(Message(role="user", content=query)) - - response_message = [] - processed_chars = 0 # 跟踪已处理的字符位置 - try: - # 使用带记忆的对话 - memory_str = None - if self.memory is not None: - future = asyncio.run_coroutine_threadsafe( - self.memory.query_memory(query), self.loop - ) - memory_str = future.result() - - self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}") - llm_responses = self.llm.response( - self.session_id, self.dialogue.get_llm_dialogue_with_memory(memory_str) - ) - except Exception as e: - self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}") - return None - - self.llm_finish_task = False - text_index = 0 - for content in llm_responses: - response_message.append(content) - if self.client_abort: - break - - # 合并当前全部文本并处理未分割部分 - full_text = "".join(response_message) - current_text = full_text[processed_chars:] # 从未处理的位置开始 - - # 查找最后一个有效标点 - punctuations = ("。", ".", "？", "?", "！", "!", "；", ";", "：") - last_punct_pos = -1 - number_flag = True - for punct in punctuations: - pos = current_text.rfind(punct) - prev_char = current_text[pos - 1] if pos - 1 >= 0 else "" - # 如果.前面是数字统一判断为小数 - if prev_char.isdigit() and punct == ".": - number_flag = False - if pos > last_punct_pos and number_flag: - last_punct_pos = pos - - # 找到分割点则处理 - if last_punct_pos != -1: - segment_text_raw = current_text[: last_punct_pos + 1] - segment_text = get_string_no_punctuation_or_emoji(segment_text_raw) - if segment_text: - # 强制设置空字符，测试TTS出错返回语音的健壮性 - # if text_index % 2 == 0: - # segment_text = " " - text_index += 1 - self.recode_first_last_text(segment_text, text_index) - future = self.executor.submit( - self.speak_and_play, segment_text, text_index - ) - self.tts_queue.put((future, text_index)) - processed_chars += len(segment_text_raw) # 更新已处理字符位置 - - # 处理最后剩余的文本 - full_text = "".join(response_message) - remaining_text = full_text[processed_chars:] - if remaining_text: - segment_text = get_string_no_punctuation_or_emoji(remaining_text) - if segment_text: - text_index += 1 - self.recode_first_last_text(segment_text, text_index) - future = self.executor.submit( - self.speak_and_play, segment_text, text_index - ) - self.tts_queue.put((future, text_index)) - - self.llm_finish_task = True - self.dialogue.put(Message(role="assistant", content="".join(response_message))) - self.logger.bind(tag=TAG).debug( - json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False) - ) - return True - - def chat_with_function_calling(self, query, tool_call=False): - self.logger.bind(tag=TAG).debug(f"Chat with function calling start: {query}") - """Chat with function calling for intent detection using streaming""" + def chat(self, query, tool_call=False): + self.logger.bind(tag=TAG).debug(f"Chat: {query}") if not tool_call: self.dialogue.put(Message(role="user", content=query)) # Define intent functions functions = None - if hasattr(self, "func_handler"): + if self.intent_type == "function_call" and hasattr(self, "func_handler"): functions = self.func_handler.get_functions() response_message = [] processed_chars = 0 # 跟踪已处理的字符位置 try: - start_time = time.time() - # 使用带记忆的对话 memory_str = None if self.memory is not None: @@ -614,14 +528,18 @@ class ConnectionHandler: ) memory_str = future.result() - # self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}") - - # 使用支持functions的streaming接口 - llm_responses = self.llm.response_with_functions( - self.session_id, - self.dialogue.get_llm_dialogue_with_memory(memory_str), - functions=functions, - ) + if functions is not None: + # 使用支持functions的streaming接口 + llm_responses = self.llm.response_with_functions( + self.session_id, + self.dialogue.get_llm_dialogue_with_memory(memory_str), + functions=functions, + ) + else: + llm_responses = self.llm.response( + self.session_id, + self.dialogue.get_llm_dialogue_with_memory(memory_str), + ) except Exception as e: self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}") return None @@ -637,27 +555,28 @@ class ConnectionHandler: content_arguments = "" for response in llm_responses: - content, tools_call = response + if self.intent_type == "function_call": + content, tools_call = response + if "content" in response: + content = response["content"] + tools_call = None + if content is not None and len(content) > 0: + content_arguments += content - if "content" in response: - content = response["content"] - tools_call = None - if content is not None and len(content) > 0: - content_arguments += content - - if not tool_call_flag and content_arguments.startswith(""): - # print("content_arguments", content_arguments) - tool_call_flag = True - - if tools_call is not None: - tool_call_flag = True - if tools_call[0].id is not None: - function_id = tools_call[0].id - if tools_call[0].function.name is not None: - function_name = tools_call[0].function.name - if tools_call[0].function.arguments is not None: - function_arguments += tools_call[0].function.arguments + if not tool_call_flag and content_arguments.startswith(""): + # print("content_arguments", content_arguments) + tool_call_flag = True + if tools_call is not None: + tool_call_flag = True + if tools_call[0].id is not None: + function_id = tools_call[0].id + if tools_call[0].function.name is not None: + function_name = tools_call[0].function.name + if tools_call[0].function.arguments is not None: + function_arguments += tools_call[0].function.arguments + else: + content = response if content is not None and len(content) > 0: if not tool_call_flag: response_message.append(content) @@ -853,7 +772,7 @@ class ConnectionHandler: content=text, ) ) - self.chat_with_function_calling(text, tool_call=True) + self.chat(text, tool_call=True) elif result.action == Action.NOTFOUND or result.action == Action.ERROR: text = result.result self.recode_first_last_text(text, text_index) diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 9c693751..2584ab83 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -39,7 +39,9 @@ async def handleAudioMessage(conn, audio): if len(conn.asr_audio) < 15: conn.asr_server_receive = True else: - raw_text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id) # 确保ASR模块返回原始文本 + raw_text, _ = await conn.asr.speech_to_text( + conn.asr_audio, conn.session_id + ) # 确保ASR模块返回原始文本 conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}") text_len, _ = remove_punctuation_and_length(raw_text) if text_len > 0: @@ -76,11 +78,7 @@ async def startToChat(conn, text): # 意图未被处理，继续常规聊天流程 await send_stt_message(conn, text) - if conn.intent_type == "function_call": - # 使用支持function calling的聊天方法 - conn.executor.submit(conn.chat_with_function_calling, text) - else: - conn.executor.submit(conn.chat, text) + conn.executor.submit(conn.chat, text) async def no_voice_close_connect(conn): From d97f8b2e9a2b24b474dfb92957f61225980d6589 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sat, 24 May 2025 02:17:49 +0800 Subject: [PATCH 05/12] =?UTF-8?q?update:=E4=BE=9B=E5=BA=94=E5=99=A8?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E6=9B=B4=E5=90=8D=E4=B8=BA=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E7=AE=A1=E7=90=86=EF=BC=8C=E9=98=B2=E6=AD=A2=E6=B7=B7=E6=B7=86?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/manager-web/src/components/HeaderBar.vue | 2 +- main/manager-web/src/views/ProviderManagement.vue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main/manager-web/src/components/HeaderBar.vue b/main/manager-web/src/components/HeaderBar.vue index 70b64f61..d0a36acb 100644 --- a/main/manager-web/src/components/HeaderBar.vue +++ b/main/manager-web/src/components/HeaderBar.vue @@ -51,7 +51,7 @@ 字典管理 - 供应器管理 + 字段管理 服务端管理 diff --git a/main/manager-web/src/views/ProviderManagement.vue b/main/manager-web/src/views/ProviderManagement.vue index 7634b511..5af816f6 100644 --- a/main/manager-web/src/views/ProviderManagement.vue +++ b/main/manager-web/src/views/ProviderManagement.vue @@ -3,7 +3,7 @@
-

供应器管理

+

字段管理

From 472106390dca8f3656153eae396f5b58a7911dd9 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sat, 24 May 2025 09:39:10 +0800 Subject: [PATCH 06/12] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E9=9F=B3?= =?UTF-8?q?=E9=A2=91=E6=92=AD=E6=94=BE=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 46 +++++++++++-------- .../core/handle/intentHandler.py | 12 ++--- .../plugins_func/functions/play_music.py | 20 +++----- 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index e73b340d..ea0be02d 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -14,6 +14,8 @@ import websockets from typing import Dict, Any from plugins_func.loadplugins import auto_import_modules from config.logger import setup_logging +from config.config_loader import get_project_dir +from core.utils import p3 from core.utils.dialogue import Message, Dialogue from core.handle.textHandle import handleTextMessage from core.utils.util import ( @@ -615,7 +617,7 @@ class ConnectionHandler: text_index += 1 self.recode_first_last_text(segment_text, text_index) future = self.executor.submit( - self.speak_and_play, segment_text, text_index + self.speak_and_play, None, segment_text, text_index ) self.tts_queue.put((future, text_index)) # 更新已处理字符位置 @@ -674,7 +676,7 @@ class ConnectionHandler: text_index += 1 self.recode_first_last_text(segment_text, text_index) future = self.executor.submit( - self.speak_and_play, segment_text, text_index + self.speak_and_play, None, segment_text, text_index ) self.tts_queue.put((future, text_index)) @@ -737,7 +739,7 @@ class ConnectionHandler: if result.action == Action.RESPONSE: # 直接回复前端 text = result.response self.recode_first_last_text(text, text_index) - future = self.executor.submit(self.speak_and_play, text, text_index) + future = self.executor.submit(self.speak_and_play, None, text, text_index) self.tts_queue.put((future, text_index)) self.dialogue.put(Message(role="assistant", content=text)) elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 @@ -776,7 +778,7 @@ class ConnectionHandler: elif result.action == Action.NOTFOUND or result.action == Action.ERROR: text = result.result self.recode_first_last_text(text, text_index) - future = self.executor.submit(self.speak_and_play, text, text_index) + future = self.executor.submit(self.speak_and_play, None, text, text_index) self.tts_queue.put((future, text_index)) self.dialogue.put(Message(role="assistant", content=text)) else: @@ -803,11 +805,7 @@ class ConnectionHandler: self.logger.bind(tag=TAG).debug("正在处理TTS任务...") tts_timeout = int(self.config.get("tts_timeout", 10)) tts_file, text, _ = future.result(timeout=tts_timeout) - if text is None or len(text) <= 0: - self.logger.bind(tag=TAG).error( - f"TTS出错：{text_index}: tts text is empty" - ) - elif tts_file is None: + if tts_file is None: self.logger.bind(tag=TAG).error( f"TTS出错： file is empty: {text_index}: {text}" ) @@ -816,12 +814,16 @@ class ConnectionHandler: f"TTS生成：文件路径: {tts_file}" ) if os.path.exists(tts_file): - if self.audio_format == "pcm": + if tts_file.endswith(".p3"): + audio_datas, _ = p3.decode_opus_from_file(tts_file) + elif self.audio_format == "pcm": audio_datas, _ = self.tts.audio_to_pcm_data(tts_file) else: audio_datas, _ = self.tts.audio_to_opus_data(tts_file) # 在这里上报TTS数据 - enqueue_tts_report(self, text, audio_datas) + enqueue_tts_report( + self, tts_file if text is None else text, audio_datas + ) else: self.logger.bind(tag=TAG).error( f"TTS出错：文件不存在{tts_file}" @@ -837,6 +839,7 @@ class ConnectionHandler: self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file) + and tts_file.startswith(self.tts.output_file) ): os.remove(tts_file) except Exception as e: @@ -903,18 +906,21 @@ class ConnectionHandler: self.logger.bind(tag=TAG).info("聊天记录上报线程已退出") - def speak_and_play(self, text, text_index=0): - if text is None or len(text) <= 0: - self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}") - return None, text, text_index - tts_file = self.tts.to_tts(text) + def speak_and_play(self, file_path, content, text_index=0): + if file_path is not None: + self.logger.bind(tag=TAG).info(f"无需tts转换: 从文件播放，{file_path}") + return file_path, content, text_index + if content is None or len(content) <= 0: + self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{content}") + return None, content, text_index + tts_file = self.tts.to_tts(content) if tts_file is None: - self.logger.bind(tag=TAG).error(f"tts转换失败，{text}") - return None, text, text_index + self.logger.bind(tag=TAG).error(f"tts转换失败，{content}") + return None, content, text_index self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}") if self.max_output_size > 0: - add_device_output(self.headers.get("device-id"), len(text)) - return tts_file, text, text_index + add_device_output(self.headers.get("device-id"), len(content)) + return tts_file, content, text_index def clearSpeakStatus(self): self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态") diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index 495da96f..e61b1265 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -109,21 +109,21 @@ async def process_intent_result(conn, intent_result, original_text): if result.action == Action.RESPONSE: # 直接回复前端 text = result.response if text is not None: - speak_and_play(conn, text) + speak_txt(conn, text) elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 text = result.result conn.dialogue.put(Message(role="tool", content=text)) llm_result = conn.intent.replyResult(text, original_text) if llm_result is None: llm_result = text - speak_and_play(conn, llm_result) + speak_txt(conn, llm_result) elif ( result.action == Action.NOTFOUND or result.action == Action.ERROR ): text = result.result if text is not None: - speak_and_play(conn, text) + speak_txt(conn, text) elif function_name != "play_music": # For backward compatibility with original code # 获取当前最新的文本索引 @@ -131,7 +131,7 @@ async def process_intent_result(conn, intent_result, original_text): if text is None: text = result.result if text is not None: - speak_and_play(conn, text) + speak_txt(conn, text) # 将函数执行放在线程池中 conn.executor.submit(process_function_call) @@ -142,12 +142,12 @@ async def process_intent_result(conn, intent_result, original_text): return False -def speak_and_play(conn, text): +def speak_txt(conn, text): text_index = ( conn.tts_last_text_index + 1 if hasattr(conn, "tts_last_text_index") else 0 ) conn.recode_first_last_text(text, text_index) - future = conn.executor.submit(conn.speak_and_play, text, text_index) + future = conn.executor.submit(conn.speak_and_play, None, text, text_index) conn.llm_finish_task = True conn.tts_queue.put((future, text_index)) conn.dialogue.put(Message(role="assistant", content=text)) diff --git a/main/xiaozhi-server/plugins_func/functions/play_music.py b/main/xiaozhi-server/plugins_func/functions/play_music.py index 3de19d76..76465641 100644 --- a/main/xiaozhi-server/plugins_func/functions/play_music.py +++ b/main/xiaozhi-server/plugins_func/functions/play_music.py @@ -216,24 +216,16 @@ async def play_local_music(conn, specific_file=None): text = _get_random_play_prompt(selected_music) await send_stt_message(conn, text) conn.dialogue.put(Message(role="assistant", content=text)) - conn.tts_first_text_index = 0 - conn.tts_last_text_index = 0 - tts_file = await asyncio.to_thread(conn.tts.to_tts, text) - if tts_file is not None and os.path.exists(tts_file): - conn.tts_last_text_index = 1 - opus_packets, _ = conn.tts.audio_to_opus_data(tts_file) - conn.audio_play_queue.put((opus_packets, None, 0)) - os.remove(tts_file) + conn.recode_first_last_text(text, 0) + future = conn.executor.submit(conn.speak_and_play, None, text, 0) + conn.tts_queue.put((future, 0)) + conn.recode_first_last_text(text, 1) + future = conn.executor.submit(conn.speak_and_play, music_path, None, 1) + conn.tts_queue.put((future, 1)) conn.llm_finish_task = True - if music_path.endswith(".p3"): - opus_packets, _ = p3.decode_opus_from_file(music_path) - else: - opus_packets, _ = conn.tts.audio_to_opus_data(music_path) - conn.audio_play_queue.put((opus_packets, None, conn.tts_last_text_index)) - except Exception as e: conn.logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}") conn.logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}") From 562424b74db9d2e25fab3f69267ed99f33058f4c Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Sun, 25 May 2025 00:09:36 +0800 Subject: [PATCH 07/12] =?UTF-8?q?fix=EF=BC=9A=E4=BF=AE=E5=A4=8D=E2=80=9C?= =?UTF-8?q?=E5=B0=8F=E6=99=BA=E6=9C=8D=E5=8A=A1=E5=99=A8=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E6=92=AD=E6=94=BE=E8=AF=AD=E9=9F=B3=E9=97=AE?= =?UTF-8?q?=E9=A2=98=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/test/test_page.html | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/main/xiaozhi-server/test/test_page.html b/main/xiaozhi-server/test/test_page.html index d9b2eff6..1df5cae2 100644 --- a/main/xiaozhi-server/test/test_page.html +++ b/main/xiaozhi-server/test/test_page.html @@ -879,6 +879,7 @@ // 流已结束且没有更多数据 log("音频播放完成", 'info'); isAudioPlaying = false; + this.endOfStream = false; streamingContext = null; } else { // 等待更多数据 @@ -1409,6 +1410,17 @@ }; websocket.onclose = () => { + + // 清理音频缓冲和播放状态 + audioBufferQueue = []; + isAudioBuffering = false; + isAudioPlaying = false; + streamingContext = null; + if (visualizationRequest) { + cancelAnimationFrame(visualizationRequest); + visualizationRequest = null; + } + log('已断开连接', 'info'); connectionStatus.textContent = 'ws已断开'; connectionStatus.style.color = 'red'; @@ -1563,6 +1575,10 @@ const message = messageInput.value.trim(); if (message === '' || !websocket || websocket.readyState !== WebSocket.OPEN) return; + audioBufferQueue = []; + isAudioBuffering = false; + isAudioPlaying = false; + try { // 直接发送listen消息，不需要重复发送hello const listenMessage = { @@ -2382,6 +2398,8 @@ // 如果收到的是第一个音频包，开始缓冲过程 if (audioBufferQueue.length === 1 && !isAudioBuffering && !isAudioPlaying) { startAudioBuffering(); + } else if (isAudioPlaying && streamingContext) { + streamingContext.decodeOpusFrames([opusData]); } } else { log('收到空音频数据帧，可能是结束标志', 'warning'); From e0da59096a23dd67469f393026b56546e136b0b9 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 25 May 2025 09:10:11 +0800 Subject: [PATCH 08/12] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E9=9F=B3=E9=A2=91=E9=87=8D=E5=A4=8D=E6=92=AD?= =?UTF-8?q?=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/test/test_page.html | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/main/xiaozhi-server/test/test_page.html b/main/xiaozhi-server/test/test_page.html index 1df5cae2..d9b2eff6 100644 --- a/main/xiaozhi-server/test/test_page.html +++ b/main/xiaozhi-server/test/test_page.html @@ -879,7 +879,6 @@ // 流已结束且没有更多数据 log("音频播放完成", 'info'); isAudioPlaying = false; - this.endOfStream = false; streamingContext = null; } else { // 等待更多数据 @@ -1410,17 +1409,6 @@ }; websocket.onclose = () => { - - // 清理音频缓冲和播放状态 - audioBufferQueue = []; - isAudioBuffering = false; - isAudioPlaying = false; - streamingContext = null; - if (visualizationRequest) { - cancelAnimationFrame(visualizationRequest); - visualizationRequest = null; - } - log('已断开连接', 'info'); connectionStatus.textContent = 'ws已断开'; connectionStatus.style.color = 'red'; @@ -1575,10 +1563,6 @@ const message = messageInput.value.trim(); if (message === '' || !websocket || websocket.readyState !== WebSocket.OPEN) return; - audioBufferQueue = []; - isAudioBuffering = false; - isAudioPlaying = false; - try { // 直接发送listen消息，不需要重复发送hello const listenMessage = { @@ -2398,8 +2382,6 @@ // 如果收到的是第一个音频包，开始缓冲过程 if (audioBufferQueue.length === 1 && !isAudioBuffering && !isAudioPlaying) { startAudioBuffering(); - } else if (isAudioPlaying && streamingContext) { - streamingContext.decodeOpusFrames([opusData]); } } else { log('收到空音频数据帧，可能是结束标志', 'warning'); From d7564a65f7a15760a36f2c485583f552b6a58448 Mon Sep 17 00:00:00 2001 From: Minamiyama Date: Sun, 25 May 2025 15:24:06 +0800 Subject: [PATCH 09/12] =?UTF-8?q?refactor(test=5Fpage.html):=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96UI=E5=B8=83=E5=B1=80=E5=92=8C=E8=BE=93=E5=85=A5?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=EF=BC=8C=E5=A2=9E=E5=8A=A0=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在OTA和WebSocket服务器地址输入框中添加更详细的占位符提示 - 增加本地存储功能，保存并恢复OTA和WebSocket服务器地址 --- main/xiaozhi-server/test/test_page.html | 40 ++++++++++++++++--------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/main/xiaozhi-server/test/test_page.html b/main/xiaozhi-server/test/test_page.html index d9b2eff6..d2151acc 100644 --- a/main/xiaozhi-server/test/test_page.html +++ b/main/xiaozhi-server/test/test_page.html @@ -14,7 +14,7 @@ } .container { - max-width: 800px; + max-width: 1000px; margin: 0 auto; background-color: white; border-radius: 10px; @@ -482,9 +482,10 @@
- + + placeholder="WebSocket服务器地址，如：ws://127.0.0.1:8000/xiaozhi/v1/" />
@@ -1061,7 +1062,7 @@ } } - // 初始化音频录制和处理 + // 初始化音频录制和处理 async function initAudio() { try { // 请求麦克风权限 @@ -1306,7 +1307,8 @@ // 先检查OTA状态 log('正在检查OTA状态...', 'info'); const otaUrl = document.getElementById('otaUrl').value.trim(); - + localStorage.setItem('otaUrl', otaUrl); + localStorage.setItem('wsUrl', url); try { const otaResponse = await fetch(otaUrl, { method: 'POST', @@ -1633,6 +1635,16 @@ // 初始更新显示值 updateDisplayValues(); + const savedOtaUrl = localStorage.getItem('otaUrl'); + if (savedOtaUrl) { + document.getElementById('otaUrl').value = savedOtaUrl; + } + + const savedWsUrl = localStorage.getItem('wsUrl'); + if (savedWsUrl) { + document.getElementById('serverUrl').value = savedWsUrl; + } + // 切换面板显示 toggleButton.addEventListener('click', () => { const isExpanded = configPanel.classList.contains('expanded'); @@ -1954,7 +1966,7 @@ this.buffer = new Int16Array(this.frameSize); this.bufferIndex = 0; this.isRecording = false; - + // 监听来自主线程的消息 this.port.onmessage = (event) => { if (event.data.command === 'start') { @@ -1962,7 +1974,7 @@ this.port.postMessage({ type: 'status', status: 'started' }); } else if (event.data.command === 'stop') { this.isRecording = false; - + // 发送剩余的缓冲区 if (this.bufferIndex > 0) { const finalBuffer = this.buffer.slice(0, this.bufferIndex); @@ -1972,18 +1984,18 @@ }); this.bufferIndex = 0; } - + this.port.postMessage({ type: 'status', status: 'stopped' }); } }; } - + process(inputs, outputs, parameters) { if (!this.isRecording) return true; - + const input = inputs[0][0]; // 获取第一个输入通道 if (!input) return true; - + // 将浮点采样转换为16位整数并存储 for (let i = 0; i < input.length; i++) { if (this.bufferIndex >= this.frameSize) { @@ -1994,15 +2006,15 @@ }); this.bufferIndex = 0; } - + // 转换为16位整数 (-32768到32767) this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767))); } - + return true; } } - + registerProcessor('audio-recorder-processor', AudioRecorderProcessor); `; From 94c92c5f3884f5547327af8d7a393d3205ffa836 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Mon, 26 May 2025 11:23:42 +0800 Subject: [PATCH 10/12] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8D=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E9=9F=B3=E9=A2=91=E9=87=8D=E5=A4=8D=E6=92=AD?= =?UTF-8?q?=E6=94=BE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/test/test_page.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/main/xiaozhi-server/test/test_page.html b/main/xiaozhi-server/test/test_page.html index d9b2eff6..39c08d5f 100644 --- a/main/xiaozhi-server/test/test_page.html +++ b/main/xiaozhi-server/test/test_page.html @@ -879,6 +879,7 @@ // 流已结束且没有更多数据 log("音频播放完成", 'info'); isAudioPlaying = false; + this.endOfStream = false; streamingContext = null; } else { // 等待更多数据 @@ -1563,6 +1564,10 @@ const message = messageInput.value.trim(); if (message === '' || !websocket || websocket.readyState !== WebSocket.OPEN) return; + audioBufferQueue = []; + isAudioBuffering = false; + isAudioPlaying = false; + try { // 直接发送listen消息，不需要重复发送hello const listenMessage = { From 9a0240ef3e0a72633b905b44b2cea6e4c1280a25 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Mon, 26 May 2025 15:34:21 +0800 Subject: [PATCH 11/12] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E4=BA=86?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E6=A0=B7=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/manager-web/src/views/ModelConfig.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/manager-web/src/views/ModelConfig.vue b/main/manager-web/src/views/ModelConfig.vue index aa788f2f..8abf8bb6 100644 --- a/main/manager-web/src/views/ModelConfig.vue +++ b/main/manager-web/src/views/ModelConfig.vue @@ -467,7 +467,7 @@ export default { .main-wrapper { margin: 5px 22px; border-radius: 15px; - min-height: calc(100vh - 24vh); + min-height: calc(100vh - 26vh); height: auto; max-height: 80vh; box-shadow: 0 2px 12px rgba(0, 0, 0, 0.1); From 761fc05331164d87e984a6b0ef018219a652906c Mon Sep 17 00:00:00 2001 From: goodyhao <865700600@qq.com> Date: Mon, 26 May 2025 20:27:22 +0800 Subject: [PATCH 12/12] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E4=B8=8A?= =?UTF-8?q?=E6=8A=A5=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/manager-api/pom.xml | 2 +- .../agent/dto/AgentChatHistoryReportDTO.java | 2 ++ .../agent/service/AgentChatAudioService.java | 2 +- .../biz/impl/AgentChatHistoryBizServiceImpl.java | 12 +++++++----- .../service/impl/AgentChatAudioServiceImpl.java | 2 +- .../src/main/resources/application-dev.yml | 2 +- main/xiaozhi-server/config/manage_api_client.py | 3 ++- main/xiaozhi-server/core/connection.py | 14 ++++++++++---- main/xiaozhi-server/core/handle/reportHandle.py | 13 ++++++++----- 9 files changed, 33 insertions(+), 19 deletions(-) diff --git a/main/manager-api/pom.xml b/main/manager-api/pom.xml index 883295bf..bc4030c0 100644 --- a/main/manager-api/pom.xml +++ b/main/manager-api/pom.xml @@ -258,4 +258,4 @@ - \ No newline at end of file + diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java index bdaed950..ae58783f 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java @@ -28,4 +28,6 @@ public class AgentChatHistoryReportDTO { private String content; @Schema(description = "base64编码的opus音频数据", example = "") private String audioBase64; + @Schema(description = "上报时间，十位时间戳，空时默认使用当前时间", example = "1745657732") + private Long reportTime; } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatAudioService.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatAudioService.java index e284b69f..04be9f36 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatAudioService.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatAudioService.java @@ -27,4 +27,4 @@ public interface AgentChatAudioService extends IService { * @return 音频数据 */ byte[] getAudio(String audioId); -} \ No newline at end of file +} diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java index bf6c188b..087c1b9d 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java @@ -47,7 +47,8 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic public Boolean report(AgentChatHistoryReportDTO report) { String macAddress = report.getMacAddress(); Byte chatType = report.getChatType(); - log.info("小智设备聊天上报请求: macAddress={}, type={}", macAddress, chatType); + Long reportTimeMillis = null != report.getReportTime() ? report.getReportTime() * 1000 : System.currentTimeMillis(); + log.info("小智设备聊天上报请求: macAddress={}, type={} reportTime={}", macAddress, chatType, reportTimeMillis); // 根据设备MAC地址查询对应的默认智能体，判断是否需要上报 AgentEntity agentEntity = agentService.getDefaultAgentByMacAddress(macAddress); @@ -59,10 +60,10 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic String agentId = agentEntity.getId(); if (Objects.equals(chatHistoryConf, Constant.ChatHistoryConfEnum.RECORD_TEXT.getCode())) { - saveChatText(report, agentId, macAddress, null); + saveChatText(report, agentId, macAddress, null, reportTimeMillis); } else if (Objects.equals(chatHistoryConf, Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode())) { String audioId = saveChatAudio(report); - saveChatText(report, agentId, macAddress, audioId); + saveChatText(report, agentId, macAddress, audioId, reportTimeMillis); } // 更新设备最后对话时间 @@ -92,8 +93,7 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic /** * 组装上报数据 */ - private void saveChatText(AgentChatHistoryReportDTO report, String agentId, String macAddress, String audioId) { - + private void saveChatText(AgentChatHistoryReportDTO report, String agentId, String macAddress, String audioId, Long reportTime) { // 构建聊天记录实体 AgentChatHistoryEntity entity = AgentChatHistoryEntity.builder() .macAddress(macAddress) @@ -102,6 +102,8 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic .chatType(report.getChatType()) .content(report.getContent()) .audioId(audioId) + .createdAt(new Date(reportTime)) + // NOTE(haotian): 2025/5/26 updateAt可以不设置，重点是createAt，而且这样可以看到上报延迟 .build(); // 保存数据 diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatAudioServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatAudioServiceImpl.java index cf804678..efcae485 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatAudioServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatAudioServiceImpl.java @@ -31,4 +31,4 @@ public class AgentChatAudioServiceImpl extends ServiceImpl Optional[Dict]: def report( - mac_address: str, session_id: str, chat_type: int, content: str, audio + mac_address: str, session_id: str, chat_type: int, content: str, audio, report_time ) -> Optional[Dict]: """带熔断的业务方法示例""" if not content or not ManageApiClient._instance: @@ -174,6 +174,7 @@ def report( "sessionId": session_id, "chatType": chat_type, "content": content, + "reportTime": report_time, "audioBase64": ( base64.b64encode(audio).decode("utf-8") if audio else None ), diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 3d9d8e3c..9742ebc3 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -890,11 +890,11 @@ class ConnectionHandler: if item is None: # 检测毒丸对象 break - type, text, audio_data = item + type, text, audio_data, report_time = item try: # 提交任务到线程池 - self.report_thread_pool.submit(self._process_report, type, text, audio_data) + self.report_thread_pool.submit(self._process_report, type, text, audio_data, report_time) except Exception as e: self.logger.bind(tag=TAG).error(f"聊天记录上报线程异常: {e}") except queue.Empty: @@ -904,11 +904,11 @@ class ConnectionHandler: self.logger.bind(tag=TAG).info("聊天记录上报线程已退出") - def _process_report(self, type, text, audio_data): + def _process_report(self, type, text, audio_data, report_time): """处理上报任务""" try: # 执行上报（传入二进制数据） - report(self, type, text, audio_data) + report(self, type, text, audio_data, report_time) except Exception as e: self.logger.bind(tag=TAG).error(f"上报处理异常: {e}") finally: @@ -973,6 +973,12 @@ class ConnectionHandler: self.executor.shutdown(wait=False) self.executor = None + # 关闭上报线程池 + if self.report_thread_pool: + self.report_thread_pool.shutdown(wait=False) + self.report_thread_pool = None + self.logger.bind(tag=TAG).info("上报线程池已关闭") + self.logger.bind(tag=TAG).info("连接资源已释放") def clear_queues(self): diff --git a/main/xiaozhi-server/core/handle/reportHandle.py b/main/xiaozhi-server/core/handle/reportHandle.py index bb1ea066..b1214fd5 100644 --- a/main/xiaozhi-server/core/handle/reportHandle.py +++ b/main/xiaozhi-server/core/handle/reportHandle.py @@ -8,6 +8,7 @@ TTS上报功能已集成到ConnectionHandler类中。 具体实现请参考core/connection.py中的相关代码。 """ +import time import opuslib_next @@ -16,7 +17,7 @@ from config.manage_api_client import report as manage_report TAG = __name__ -def report(conn, type, text, opus_data): +def report(conn, type, text, opus_data, report_time): """执行聊天记录上报操作 Args: @@ -24,6 +25,7 @@ def report(conn, type, text, opus_data): type: 上报类型，1为用户，2为智能体 text: 合成文本 opus_data: opus音频数据 + report_time: 上报时间 """ try: if opus_data: @@ -37,6 +39,7 @@ def report(conn, type, text, opus_data): chat_type=type, content=text, audio=audio_data, + report_time=report_time, ) except Exception as e: conn.logger.bind(tag=TAG).error(f"聊天记录上报失败: {e}") @@ -104,12 +107,12 @@ def enqueue_tts_report(conn, text, opus_data): try: # 使用连接对象的队列，传入文本和二进制数据而非文件路径 if conn.chat_history_conf == 2: - conn.report_queue.put((2, text, opus_data)) + conn.report_queue.put((2, text, opus_data, int(time.time()))) conn.logger.bind(tag=TAG).debug( f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} " ) else: - conn.report_queue.put((2, text, None)) + conn.report_queue.put((2, text, None, int(time.time()))) conn.logger.bind(tag=TAG).debug( f"TTS数据已加入上报队列: {conn.device_id}, 不上报音频" ) @@ -132,12 +135,12 @@ def enqueue_asr_report(conn, text, opus_data): try: # 使用连接对象的队列，传入文本和二进制数据而非文件路径 if conn.chat_history_conf == 2: - conn.report_queue.put((1, text, opus_data)) + conn.report_queue.put((1, text, opus_data, int(time.time()))) conn.logger.bind(tag=TAG).debug( f"ASR数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} " ) else: - conn.report_queue.put((1, text, None)) + conn.report_queue.put((1, text, None, int(time.time()))) conn.logger.bind(tag=TAG).debug( f"ASR数据已加入上报队列: {conn.device_id}, 不上报音频" )