From 885b7a0b05bcfcac46ff05bedc6b468adbd70dcc Mon Sep 17 00:00:00 2001 From: goodyhao <865700600@qq.com> Date: Tue, 13 May 2025 15:54:00 +0800 Subject: [PATCH 1/7] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E4=B8=8A?= =?UTF-8?q?=E6=8A=A5=E7=BA=BF=E7=A8=8B=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 63b79736..68964629 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -88,7 +88,8 @@ class ConnectionHandler: self.audio_play_queue = queue.Queue() self.executor = ThreadPoolExecutor(max_workers=10) - # 上报线程 + # 添加上报线程池 + self.report_thread_pool = ThreadPoolExecutor(max_workers=5, thread_name_prefix="report_worker") self.report_queue = queue.Queue() self.report_thread = None # TODO(haotian): 2025/5/12 可以通过修改此处，调节asr的上报和tts的上报 @@ -944,13 +945,10 @@ class ConnectionHandler: type, text, audio_data = item try: - # 执行上报（传入二进制数据） - report(self, type, text, audio_data) + # 提交任务到线程池 + self.report_thread_pool.submit(self._process_report, type, text, audio_data) except Exception as e: self.logger.bind(tag=TAG).error(f"聊天记录上报线程异常: {e}") - finally: - # 标记任务完成 - self.report_queue.task_done() except queue.Empty: continue except Exception as e: @@ -958,6 +956,17 @@ class ConnectionHandler: self.logger.bind(tag=TAG).info("聊天记录上报线程已退出") + def _process_report(self, type, text, audio_data): + """处理上报任务""" + try: + # 执行上报（传入二进制数据） + report(self, type, text, audio_data) + except Exception as e: + self.logger.bind(tag=TAG).error(f"上报处理异常: {e}") + finally: + # 标记任务完成 + self.report_queue.task_done() + def speak_and_play(self, text, text_index=0): if text is None or len(text) <= 0: self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}") @@ -1006,6 +1015,11 @@ class ConnectionHandler: # 添加毒丸对象到上报队列确保线程退出 self.report_queue.put(None) + # 关闭上报线程池 + if hasattr(self, 'report_thread_pool'): + self.report_thread_pool.shutdown(wait=True) + self.logger.bind(tag=TAG).info("上报线程池已关闭") + # 清空任务队列 self.clear_queues() From 02cb9c35b312af71644d61eff9b83650195d2af3 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 22 May 2025 17:14:58 +0800 Subject: [PATCH 2/7] =?UTF-8?q?update:=20=E8=AE=B0=E5=BF=86=E6=A8=A1?= =?UTF-8?q?=E5=9D=97=E4=BD=BF=E7=94=A8=E7=8B=AC=E7=AB=8BLLM=20openai?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=B6=85=E5=8F=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 4 +++ main/xiaozhi-server/core/connection.py | 28 +++++++++++++++++ .../xiaozhi-server/core/providers/llm/base.py | 6 ++-- .../core/providers/llm/openai/openai.py | 31 +++++++++++++------ .../core/providers/memory/base.py | 8 ++++- .../memory/mem_local_short/mem_local_short.py | 21 +++++++------ 6 files changed, 75 insertions(+), 23 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 44845df8..ebec6ccd 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -220,6 +220,10 @@ Memory: mem_local_short: # 本地记忆功能，通过selected_module的llm总结，数据保存在本地，不会上传到服务器 type: mem_local_short + # 配备记忆存储独立的思考模型 + # 如果这里不填，则会默认使用selected_module.LLM的模型作为意图识别的思考模型 + # 如果你的不想使用selected_module.LLM记忆存储，这里最好使用独立的LLM作为意图识别，例如使用免费的ChatGLMLLM + llm: ChatGLMLLM ASR: FunASR: diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 74b612fe..fd8900ed 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -452,6 +452,34 @@ class ConnectionHandler: save_to_file=not self.read_config_from_api, ) + # 获取记忆总结配置 + memory_config = self.config["Memory"] + memory_type = self.config["Memory"][self.config["selected_module"]["Memory"]][ + "type" + ] + # 如果使用 nomen，直接返回 + if memory_type == "nomem": + return + # 使用 mem_local_short 模式 + elif memory_type == "mem_local_short": + memory_llm_name = memory_config[self.config["selected_module"]["Memory"]]["llm"] + if memory_llm_name and memory_llm_name in self.config["LLM"]: + # 如果配置了专用LLM，则创建独立的LLM实例 + from core.utils import llm as llm_utils + memory_llm_config = self.config["LLM"][memory_llm_name] + memory_llm_type = memory_llm_config.get("type", memory_llm_name) + memory_llm = llm_utils.create_instance( + memory_llm_type, memory_llm_config + ) + self.logger.bind(tag=TAG).info( + f"为记忆总结创建了专用LLM: {memory_llm_name}, 类型: {memory_llm_type}" + ) + self.memory.set_llm(memory_llm) + else: + # 否则使用主LLM + self.memory.set_llm(self.llm) + self.logger.bind(tag=TAG).info("使用主LLM作为意图识别模型") + def _initialize_intent(self): self.intent_type = self.config["Intent"][ self.config["selected_module"]["Intent"] diff --git a/main/xiaozhi-server/core/providers/llm/base.py b/main/xiaozhi-server/core/providers/llm/base.py index 97d0d8e7..77786140 100644 --- a/main/xiaozhi-server/core/providers/llm/base.py +++ b/main/xiaozhi-server/core/providers/llm/base.py @@ -10,7 +10,7 @@ class LLMProviderBase(ABC): """LLM response generator""" pass - def response_no_stream(self, system_prompt, user_prompt): + def response_no_stream(self, system_prompt, user_prompt, **kwargs): try: # 构造对话格式 dialogue = [ @@ -18,7 +18,7 @@ class LLMProviderBase(ABC): {"role": "user", "content": user_prompt} ] result = "" - for part in self.response("", dialogue): + for part in self.response("", dialogue, **kwargs): result += part return result @@ -30,7 +30,7 @@ class LLMProviderBase(ABC): """ Default implementation for function calling (streaming) This should be overridden by providers that support function calls - + Returns: generator that yields either text tokens or a special function call token """ # For providers that don't support functions, just return regular response diff --git a/main/xiaozhi-server/core/providers/llm/openai/openai.py b/main/xiaozhi-server/core/providers/llm/openai/openai.py index a20067af..bc0e7f21 100644 --- a/main/xiaozhi-server/core/providers/llm/openai/openai.py +++ b/main/xiaozhi-server/core/providers/llm/openai/openai.py @@ -16,26 +16,37 @@ class LLMProvider(LLMProviderBase): self.base_url = config.get("base_url") else: self.base_url = config.get("url") - max_tokens = config.get("max_tokens") - if max_tokens is None or max_tokens == "": - max_tokens = 500 - try: - max_tokens = int(max_tokens) - except (ValueError, TypeError): - max_tokens = 500 - self.max_tokens = max_tokens + param_defaults = { + "max_tokens": (500, int), + "temperature": (0.7, lambda x: round(float(x), 1)), + "top_p": (1.0, lambda x: round(float(x), 1)), + "frequency_penalty": (0, lambda x: round(float(x), 1)) + } + + for param, (default, converter) in param_defaults.items(): + value = config.get(param) + try: + setattr(self, param, converter(value) if value not in (None, "") else default) + except (ValueError, TypeError): + setattr(self, param, default) + + logger.debug( + f"意图识别参数初始化: {self.temperature}, {self.max_tokens}, {self.top_p}, {self.frequency_penalty}") check_model_key("LLM", self.api_key) self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url) - def response(self, session_id, dialogue): + def response(self, session_id, dialogue, **kwargs): try: responses = self.client.chat.completions.create( model=self.model_name, messages=dialogue, stream=True, - max_tokens=self.max_tokens, + max_tokens=kwargs.get("max_tokens", self.max_tokens), + temperature=kwargs.get("temperature", self.temperature), + top_p=kwargs.get("top_p", self.top_p), + frequency_penalty=kwargs.get("frequency_penalty", self.frequency_penalty), ) is_active = True diff --git a/main/xiaozhi-server/core/providers/memory/base.py b/main/xiaozhi-server/core/providers/memory/base.py index 19dcabb8..f404f15e 100644 --- a/main/xiaozhi-server/core/providers/memory/base.py +++ b/main/xiaozhi-server/core/providers/memory/base.py @@ -9,7 +9,13 @@ class MemoryProviderBase(ABC): def __init__(self, config): self.config = config self.role_id = None - self.llm = None + + def set_llm(self, llm): + self.llm = llm + # 获取模型名称和类型信息 + model_name = getattr(llm, "model_name", str(llm.__class__.__name__)) + # 记录更详细的日志 + logger.bind(tag=TAG).info(f"记忆总结设置LLM: {model_name}") @abstractmethod async def save_memory(self, msgs): diff --git a/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py b/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py index a916e6c9..844035bd 100644 --- a/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py +++ b/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py @@ -107,7 +107,7 @@ TAG = __name__ class MemoryProvider(MemoryProviderBase): def __init__(self, config, summary_memory): super().__init__(config) - self.short_momery = "" + self.short_memory = "" self.save_to_file = True self.memory_path = get_project_dir() + "data/.memory.yaml" self.load_memory(summary_memory) @@ -122,7 +122,7 @@ class MemoryProvider(MemoryProviderBase): def load_memory(self, summary_memory): # api获取到总结记忆后直接返回 if summary_memory or not self.save_to_file: - self.short_momery = summary_memory + self.short_memory = summary_memory return all_memory = {} @@ -130,18 +130,21 @@ class MemoryProvider(MemoryProviderBase): with open(self.memory_path, "r", encoding="utf-8") as f: all_memory = yaml.safe_load(f) or {} if self.role_id in all_memory: - self.short_momery = all_memory[self.role_id] + self.short_memory = all_memory[self.role_id] def save_memory_to_file(self): all_memory = {} if os.path.exists(self.memory_path): with open(self.memory_path, "r", encoding="utf-8") as f: all_memory = yaml.safe_load(f) or {} - all_memory[self.role_id] = self.short_momery + all_memory[self.role_id] = self.short_memory with open(self.memory_path, "w", encoding="utf-8") as f: yaml.dump(all_memory, f, allow_unicode=True) async def save_memory(self, msgs): + # 打印使用的模型信息 + model_info = getattr(self.llm, "model_name", str(self.llm.__class__.__name__)) + logger.bind(tag=TAG).debug(f"使用记忆保存模型: {model_info}") if self.llm is None: logger.bind(tag=TAG).error("LLM is not set for memory provider") return None @@ -155,9 +158,9 @@ class MemoryProvider(MemoryProviderBase): msgStr += f"User: {msg.content}\n" elif msg.role == "assistant": msgStr += f"Assistant: {msg.content}\n" - if self.short_momery and len(self.short_momery) > 0: + if self.short_memory and len(self.short_memory) > 0: msgStr += "历史记忆：\n" - msgStr += self.short_momery + msgStr += self.short_memory # 当前时间 time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) @@ -168,7 +171,7 @@ class MemoryProvider(MemoryProviderBase): json_str = extract_json_data(result) try: json.loads(json_str) # 检查json格式是否正确 - self.short_momery = json_str + self.short_memory = json_str self.save_memory_to_file() except Exception as e: print("Error:", e) @@ -179,7 +182,7 @@ class MemoryProvider(MemoryProviderBase): save_mem_local_short(self.role_id, result) logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}") - return self.short_momery + return self.short_memory async def query_memory(self, query: str) -> str: - return self.short_momery + return self.short_memory From c30c4649a483ac9059405344ed3f4170c566ae1f Mon Sep 17 00:00:00 2001 From: tiamohummer Date: Fri, 23 May 2025 17:41:22 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=98=BF=E9=87=8C?= =?UTF-8?q?=E4=BA=91TTS=20private=5Fvoice=20=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/providers/tts/aliyun.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun.py b/main/xiaozhi-server/core/providers/tts/aliyun.py index 61ab4364..4d2b1e13 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun.py @@ -81,7 +81,10 @@ class TTSProvider(TTSProviderBase): self.appkey = config.get("appkey") self.format = config.get("format", "wav") self.sample_rate = config.get("sample_rate", 16000) - self.voice = config.get("voice", "xiaoyun") + if config.get("private_voice"): + self.voice = config.get("private_voice") + else: + self.voice = config.get("voice","xiaoyun") self.volume = config.get("volume", 50) self.speech_rate = config.get("speech_rate", 0) self.pitch_rate = config.get("pitch_rate", 0) From 761fc05331164d87e984a6b0ef018219a652906c Mon Sep 17 00:00:00 2001 From: goodyhao <865700600@qq.com> Date: Mon, 26 May 2025 20:27:22 +0800 Subject: [PATCH 4/7] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E4=B8=8A?= =?UTF-8?q?=E6=8A=A5=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/manager-api/pom.xml | 2 +- .../agent/dto/AgentChatHistoryReportDTO.java | 2 ++ .../agent/service/AgentChatAudioService.java | 2 +- .../biz/impl/AgentChatHistoryBizServiceImpl.java | 12 +++++++----- .../service/impl/AgentChatAudioServiceImpl.java | 2 +- .../src/main/resources/application-dev.yml | 2 +- main/xiaozhi-server/config/manage_api_client.py | 3 ++- main/xiaozhi-server/core/connection.py | 14 ++++++++++---- main/xiaozhi-server/core/handle/reportHandle.py | 13 ++++++++----- 9 files changed, 33 insertions(+), 19 deletions(-) diff --git a/main/manager-api/pom.xml b/main/manager-api/pom.xml index 883295bf..bc4030c0 100644 --- a/main/manager-api/pom.xml +++ b/main/manager-api/pom.xml @@ -258,4 +258,4 @@ - \ No newline at end of file + diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java index bdaed950..ae58783f 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java @@ -28,4 +28,6 @@ public class AgentChatHistoryReportDTO { private String content; @Schema(description = "base64编码的opus音频数据", example = "") private String audioBase64; + @Schema(description = "上报时间，十位时间戳，空时默认使用当前时间", example = "1745657732") + private Long reportTime; } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatAudioService.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatAudioService.java index e284b69f..04be9f36 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatAudioService.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatAudioService.java @@ -27,4 +27,4 @@ public interface AgentChatAudioService extends IService { * @return 音频数据 */ byte[] getAudio(String audioId); -} \ No newline at end of file +} diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java index bf6c188b..087c1b9d 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java @@ -47,7 +47,8 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic public Boolean report(AgentChatHistoryReportDTO report) { String macAddress = report.getMacAddress(); Byte chatType = report.getChatType(); - log.info("小智设备聊天上报请求: macAddress={}, type={}", macAddress, chatType); + Long reportTimeMillis = null != report.getReportTime() ? report.getReportTime() * 1000 : System.currentTimeMillis(); + log.info("小智设备聊天上报请求: macAddress={}, type={} reportTime={}", macAddress, chatType, reportTimeMillis); // 根据设备MAC地址查询对应的默认智能体，判断是否需要上报 AgentEntity agentEntity = agentService.getDefaultAgentByMacAddress(macAddress); @@ -59,10 +60,10 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic String agentId = agentEntity.getId(); if (Objects.equals(chatHistoryConf, Constant.ChatHistoryConfEnum.RECORD_TEXT.getCode())) { - saveChatText(report, agentId, macAddress, null); + saveChatText(report, agentId, macAddress, null, reportTimeMillis); } else if (Objects.equals(chatHistoryConf, Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode())) { String audioId = saveChatAudio(report); - saveChatText(report, agentId, macAddress, audioId); + saveChatText(report, agentId, macAddress, audioId, reportTimeMillis); } // 更新设备最后对话时间 @@ -92,8 +93,7 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic /** * 组装上报数据 */ - private void saveChatText(AgentChatHistoryReportDTO report, String agentId, String macAddress, String audioId) { - + private void saveChatText(AgentChatHistoryReportDTO report, String agentId, String macAddress, String audioId, Long reportTime) { // 构建聊天记录实体 AgentChatHistoryEntity entity = AgentChatHistoryEntity.builder() .macAddress(macAddress) @@ -102,6 +102,8 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic .chatType(report.getChatType()) .content(report.getContent()) .audioId(audioId) + .createdAt(new Date(reportTime)) + // NOTE(haotian): 2025/5/26 updateAt可以不设置，重点是createAt，而且这样可以看到上报延迟 .build(); // 保存数据 diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatAudioServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatAudioServiceImpl.java index cf804678..efcae485 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatAudioServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatAudioServiceImpl.java @@ -31,4 +31,4 @@ public class AgentChatAudioServiceImpl extends ServiceImpl Optional[Dict]: def report( - mac_address: str, session_id: str, chat_type: int, content: str, audio + mac_address: str, session_id: str, chat_type: int, content: str, audio, report_time ) -> Optional[Dict]: """带熔断的业务方法示例""" if not content or not ManageApiClient._instance: @@ -174,6 +174,7 @@ def report( "sessionId": session_id, "chatType": chat_type, "content": content, + "reportTime": report_time, "audioBase64": ( base64.b64encode(audio).decode("utf-8") if audio else None ), diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 3d9d8e3c..9742ebc3 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -890,11 +890,11 @@ class ConnectionHandler: if item is None: # 检测毒丸对象 break - type, text, audio_data = item + type, text, audio_data, report_time = item try: # 提交任务到线程池 - self.report_thread_pool.submit(self._process_report, type, text, audio_data) + self.report_thread_pool.submit(self._process_report, type, text, audio_data, report_time) except Exception as e: self.logger.bind(tag=TAG).error(f"聊天记录上报线程异常: {e}") except queue.Empty: @@ -904,11 +904,11 @@ class ConnectionHandler: self.logger.bind(tag=TAG).info("聊天记录上报线程已退出") - def _process_report(self, type, text, audio_data): + def _process_report(self, type, text, audio_data, report_time): """处理上报任务""" try: # 执行上报（传入二进制数据） - report(self, type, text, audio_data) + report(self, type, text, audio_data, report_time) except Exception as e: self.logger.bind(tag=TAG).error(f"上报处理异常: {e}") finally: @@ -973,6 +973,12 @@ class ConnectionHandler: self.executor.shutdown(wait=False) self.executor = None + # 关闭上报线程池 + if self.report_thread_pool: + self.report_thread_pool.shutdown(wait=False) + self.report_thread_pool = None + self.logger.bind(tag=TAG).info("上报线程池已关闭") + self.logger.bind(tag=TAG).info("连接资源已释放") def clear_queues(self): diff --git a/main/xiaozhi-server/core/handle/reportHandle.py b/main/xiaozhi-server/core/handle/reportHandle.py index bb1ea066..b1214fd5 100644 --- a/main/xiaozhi-server/core/handle/reportHandle.py +++ b/main/xiaozhi-server/core/handle/reportHandle.py @@ -8,6 +8,7 @@ TTS上报功能已集成到ConnectionHandler类中。 具体实现请参考core/connection.py中的相关代码。 """ +import time import opuslib_next @@ -16,7 +17,7 @@ from config.manage_api_client import report as manage_report TAG = __name__ -def report(conn, type, text, opus_data): +def report(conn, type, text, opus_data, report_time): """执行聊天记录上报操作 Args: @@ -24,6 +25,7 @@ def report(conn, type, text, opus_data): type: 上报类型，1为用户，2为智能体 text: 合成文本 opus_data: opus音频数据 + report_time: 上报时间 """ try: if opus_data: @@ -37,6 +39,7 @@ def report(conn, type, text, opus_data): chat_type=type, content=text, audio=audio_data, + report_time=report_time, ) except Exception as e: conn.logger.bind(tag=TAG).error(f"聊天记录上报失败: {e}") @@ -104,12 +107,12 @@ def enqueue_tts_report(conn, text, opus_data): try: # 使用连接对象的队列，传入文本和二进制数据而非文件路径 if conn.chat_history_conf == 2: - conn.report_queue.put((2, text, opus_data)) + conn.report_queue.put((2, text, opus_data, int(time.time()))) conn.logger.bind(tag=TAG).debug( f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} " ) else: - conn.report_queue.put((2, text, None)) + conn.report_queue.put((2, text, None, int(time.time()))) conn.logger.bind(tag=TAG).debug( f"TTS数据已加入上报队列: {conn.device_id}, 不上报音频" ) @@ -132,12 +135,12 @@ def enqueue_asr_report(conn, text, opus_data): try: # 使用连接对象的队列，传入文本和二进制数据而非文件路径 if conn.chat_history_conf == 2: - conn.report_queue.put((1, text, opus_data)) + conn.report_queue.put((1, text, opus_data, int(time.time()))) conn.logger.bind(tag=TAG).debug( f"ASR数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} " ) else: - conn.report_queue.put((1, text, None)) + conn.report_queue.put((1, text, None, int(time.time()))) conn.logger.bind(tag=TAG).debug( f"ASR数据已加入上报队列: {conn.device_id}, 不上报音频" ) From 631787a4f12aa769226893f04cacaec180e29056 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 27 May 2025 13:41:53 +0800 Subject: [PATCH 5/7] =?UTF-8?q?update=EF=BC=9A=E8=BF=99=E5=87=A0=E5=A4=A9t?= =?UTF-8?q?ts=E6=B5=81=E5=BC=8F=E6=94=B9=E9=80=A0=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=90=8E=EF=BC=8Cself.executor=E7=9A=84=E4=B8=BB=E8=A6=81?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E5=B0=86=E6=98=AF=E7=94=A8=E6=9D=A5=E4=B8=8A?= =?UTF-8?q?=E6=8A=A5=E8=81=8A=E5=A4=A9=E8=AE=B0=E5=BD=95=EF=BC=8C=E5=9B=A0?= =?UTF-8?q?=E6=AD=A4=E8=BF=99=E9=87=8C=E5=85=B6=E5=AE=9E=E5=8F=AF=E4=BB=A5?= =?UTF-8?q?=E5=90=88=E5=B9=B6=E5=9C=A8self.executor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 2 +- main/xiaozhi-server/core/connection.py | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index ebec6ccd..e7b2a50e 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -218,7 +218,7 @@ Memory: # 不想使用记忆功能，可以使用nomem type: nomem mem_local_short: - # 本地记忆功能，通过selected_module的llm总结，数据保存在本地，不会上传到服务器 + # 本地记忆功能，通过selected_module的llm总结，数据保存在本地服务器，不会上传到外部服务器 type: mem_local_short # 配备记忆存储独立的思考模型 # 如果这里不填，则会默认使用selected_module.LLM的模型作为意图识别的思考模型 diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 1efa95c2..2d78f761 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -89,13 +89,12 @@ class ConnectionHandler: self.stop_event = threading.Event() self.tts_queue = queue.Queue() self.audio_play_queue = queue.Queue() - self.executor = ThreadPoolExecutor(max_workers=10) + self.executor = ThreadPoolExecutor(max_workers=5) # 添加上报线程池 - self.report_thread_pool = ThreadPoolExecutor(max_workers=5, thread_name_prefix="report_worker") self.report_queue = queue.Queue() self.report_thread = None - # TODO(haotian): 2025/5/12 可以通过修改此处，调节asr的上报和tts的上报 + # 未来可以通过修改此处，调节asr的上报和tts的上报，目前默认都开启 self.report_asr_enable = self.read_config_from_api self.report_tts_enable = self.read_config_from_api @@ -465,10 +464,13 @@ class ConnectionHandler: return # 使用 mem_local_short 模式 elif memory_type == "mem_local_short": - memory_llm_name = memory_config[self.config["selected_module"]["Memory"]]["llm"] + memory_llm_name = memory_config[self.config["selected_module"]["Memory"]][ + "llm" + ] if memory_llm_name and memory_llm_name in self.config["LLM"]: # 如果配置了专用LLM，则创建独立的LLM实例 from core.utils import llm as llm_utils + memory_llm_config = self.config["LLM"][memory_llm_name] memory_llm_type = memory_llm_config.get("type", memory_llm_name) memory_llm = llm_utils.create_instance( @@ -922,7 +924,9 @@ class ConnectionHandler: try: # 提交任务到线程池 - self.report_thread_pool.submit(self._process_report, type, text, audio_data, report_time) + self.executor.submit( + self._process_report, type, text, audio_data, report_time + ) except Exception as e: self.logger.bind(tag=TAG).error(f"聊天记录上报线程异常: {e}") except queue.Empty: @@ -1001,12 +1005,6 @@ class ConnectionHandler: self.executor.shutdown(wait=False) self.executor = None - # 关闭上报线程池 - if self.report_thread_pool: - self.report_thread_pool.shutdown(wait=False) - self.report_thread_pool = None - self.logger.bind(tag=TAG).info("上报线程池已关闭") - self.logger.bind(tag=TAG).info("连接资源已释放") def clear_queues(self): From 626692df29b86278f84553ff74c7717dede4f83c Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 27 May 2025 13:47:07 +0800 Subject: [PATCH 6/7] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E9=98=BF=E9=87=8C?= =?UTF-8?q?=E4=BA=91tts=E6=95=B0=E5=AD=97=E5=8F=82=E6=95=B0=E8=AF=BB?= =?UTF-8?q?=E5=8F=96=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun.py | 123 +++++++++++------- 1 file changed, 75 insertions(+), 48 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun.py b/main/xiaozhi-server/core/providers/tts/aliyun.py index 4d2b1e13..98dcdb9f 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun.py @@ -13,56 +13,71 @@ import urllib.parse import time import uuid from urllib import parse + + class AccessToken: @staticmethod def _encode_text(text): encoded_text = parse.quote_plus(text) - return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') - + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + @staticmethod def _encode_dict(dic): keys = dic.keys() dic_sorted = [(key, dic[key]) for key in sorted(keys)] encoded_text = parse.urlencode(dic_sorted) - return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') - + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + @staticmethod def create_token(access_key_id, access_key_secret): - parameters = {'AccessKeyId': access_key_id, - 'Action': 'CreateToken', - 'Format': 'JSON', - 'RegionId': 'cn-shanghai', - 'SignatureMethod': 'HMAC-SHA1', - 'SignatureNonce': str(uuid.uuid1()), - 'SignatureVersion': '1.0', - 'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - 'Version': '2019-02-28'} + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } # 构造规范化的请求字符串 query_string = AccessToken._encode_dict(parameters) # print('规范化的请求字符串: %s' % query_string) # 构造待签名字符串 - string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string) + string_to_sign = ( + "GET" + + "&" + + AccessToken._encode_text("/") + + "&" + + AccessToken._encode_text(query_string) + ) # print('待签名的字符串: %s' % string_to_sign) # 计算签名 - secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'), - bytes(string_to_sign, encoding='utf-8'), - hashlib.sha1).digest() + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() signature = base64.b64encode(secreted_string) # print('签名: %s' % signature) # 进行URL编码 signature = AccessToken._encode_text(signature) # print('URL编码后的签名: %s' % signature) # 调用服务 - full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string) + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % ( + signature, + query_string, + ) # print('url: %s' % full_url) # 提交HTTP GET请求 response = requests.get(full_url) if response.ok: root_obj = response.json() - key = 'Token' + key = "Token" if key in root_obj: - token = root_obj[key]['Id'] - expire_time = root_obj[key]['ExpireTime'] + token = root_obj[key]["Id"] + expire_time = root_obj[key]["ExpireTime"] return token, expire_time # print(response.text) return None, None @@ -70,29 +85,36 @@ class AccessToken: class TTSProvider(TTSProviderBase): - def __init__(self, config, delete_audio_file): super().__init__(config, delete_audio_file) - + # 新增空值判断逻辑 self.access_key_id = config.get("access_key_id") self.access_key_secret = config.get("access_key_secret") self.appkey = config.get("appkey") self.format = config.get("format", "wav") - self.sample_rate = config.get("sample_rate", 16000) + + sample_rate = config.get("sample_rate", "16000") + self.sample_rate = int(sample_rate) if sample_rate else 16000 + if config.get("private_voice"): self.voice = config.get("private_voice") else: - self.voice = config.get("voice","xiaoyun") - self.volume = config.get("volume", 50) - self.speech_rate = config.get("speech_rate", 0) - self.pitch_rate = config.get("pitch_rate", 0) + self.voice = config.get("voice", "xiaoyun") + + volume = config.get("volume", "50") + self.volume = int(volume) if volume else 50 + + speech_rate = config.get("speech_rate", "0") + self.speech_rate = int(speech_rate) if speech_rate else 0 + + pitch_rate = config.get("pitch_rate", "0") + self.pitch_rate = int(pitch_rate) if pitch_rate else 0 + self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com") self.api_url = f"https://{self.host}/stream/v1/tts" - self.header = { - "Content-Type": "application/json" - } + self.header = {"Content-Type": "application/json"} if self.access_key_id and self.access_key_secret: # 使用密钥对生成临时token @@ -102,35 +124,30 @@ class TTSProvider(TTSProviderBase): self.token = config.get("token") self.expire_time = None - def _refresh_token(self): """刷新Token并记录过期时间""" if self.access_key_id and self.access_key_secret: self.token, expire_time_str = AccessToken.create_token( - self.access_key_id, - self.access_key_secret + self.access_key_id, self.access_key_secret ) if not expire_time_str: raise ValueError("无法获取有效的Token过期时间") try: - #统一转换为字符串处理 + # 统一转换为字符串处理 expire_str = str(expire_time_str).strip() if expire_str.isdigit(): expire_time = datetime.fromtimestamp(int(expire_str)) else: - expire_time = datetime.strptime( - expire_str, - "%Y-%m-%dT%H:%M:%SZ" - ) + expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ") self.expire_time = expire_time.timestamp() - 60 except Exception as e: raise ValueError(f"无效的过期时间格式: {expire_str}") from e - + else: self.expire_time = None - + if not self.token: raise ValueError("无法获取有效的访问Token") @@ -145,8 +162,12 @@ class TTSProvider(TTSProviderBase): # f"过期时间 {datetime.fromtimestamp(self.expire_time)} | " # f"剩余 {remaining:.2f}秒") return time.time() > self.expire_time + def generate_filename(self, extension=".wav"): - return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + return os.path.join( + self.output_file, + f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}", + ) async def text_to_speak(self, text, output_file): if self._is_token_expired(): @@ -161,21 +182,27 @@ class TTSProvider(TTSProviderBase): "voice": self.voice, "volume": self.volume, "speech_rate": self.speech_rate, - "pitch_rate": self.pitch_rate + "pitch_rate": self.pitch_rate, } # print(self.api_url, json.dumps(request_json, ensure_ascii=False)) try: - resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header) + resp = requests.post( + self.api_url, json.dumps(request_json), headers=self.header + ) if resp.status_code == 401: # Token过期特殊处理 self._refresh_token() - resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header) + resp = requests.post( + self.api_url, json.dumps(request_json), headers=self.header + ) # 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的 - if resp.headers['Content-Type'].startswith('audio/'): - with open(output_file, 'wb') as f: + if resp.headers["Content-Type"].startswith("audio/"): + with open(output_file, "wb") as f: f.write(resp.content) return output_file else: - raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}") + raise Exception( + f"{__name__} status_code: {resp.status_code} response: {resp.content}" + ) except Exception as e: raise Exception(f"{__name__} error: {e}") From be7ef08f401a60298f639f0862af4f22127b377c Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 27 May 2025 15:28:17 +0800 Subject: [PATCH 7/7] =?UTF-8?q?update:=E6=99=BA=E6=8E=A7=E5=8F=B0=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E7=8B=AC=E7=AB=8B=E8=AE=B0=E5=BF=86=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/ConfigServiceImpl.java | 30 +++++++-- .../resources/db/changelog/202505271412.sql | 4 ++ .../db/changelog/db.changelog-master.yaml | 9 ++- .../core/providers/llm/coze/coze.py | 2 +- .../core/providers/llm/dify/dify.py | 2 +- .../core/providers/llm/fastgpt/fastgpt.py | 2 +- .../core/providers/llm/gemini/gemini.py | 2 +- .../llm/homeassistant/homeassistant.py | 2 +- .../core/providers/llm/ollama/ollama.py | 64 +++++++++++-------- .../providers/llm/xinference/xinference.py | 55 ++++++++++------ .../memory/mem_local_short/mem_local_short.py | 12 +++- 11 files changed, 125 insertions(+), 59 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202505271412.sql diff --git a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java index 592b3c1d..f28c4e55 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java @@ -251,6 +251,7 @@ public class ConfigServiceImpl implements ConfigService { String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM" }; String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId }; String intentLLMModelId = null; + String memLocalShortLLMModelId = null; for (int i = 0; i < modelIds.length; i++) { if (modelIds[i] == null) { @@ -269,7 +270,7 @@ public class ConfigServiceImpl implements ConfigService { Map map = (Map) model.getConfigJson(); if ("intent_llm".equals(map.get("type"))) { intentLLMModelId = (String) map.get("llm"); - if (intentLLMModelId != null && intentLLMModelId.equals(llmModelId)) { + if (StringUtils.isNotBlank(intentLLMModelId) && intentLLMModelId.equals(llmModelId)) { intentLLMModelId = null; } } @@ -281,10 +282,31 @@ public class ConfigServiceImpl implements ConfigService { } } } + if ("Memory".equals(modelTypes[i])) { + Map map = (Map) model.getConfigJson(); + if ("mem_local_short".equals(map.get("type"))) { + memLocalShortLLMModelId = (String) map.get("llm"); + if (StringUtils.isNotBlank(memLocalShortLLMModelId) + && memLocalShortLLMModelId.equals(llmModelId)) { + memLocalShortLLMModelId = null; + } + } + } // 如果是LLM类型，且intentLLMModelId不为空，则添加附加模型 - if ("LLM".equals(modelTypes[i]) && intentLLMModelId != null) { - ModelConfigEntity intentLLM = modelConfigService.getModelById(intentLLMModelId, isCache); - typeConfig.put(intentLLM.getId(), intentLLM.getConfigJson()); + if ("LLM".equals(modelTypes[i])) { + if (StringUtils.isNotBlank(intentLLMModelId)) { + if (!typeConfig.containsKey(intentLLMModelId)) { + ModelConfigEntity intentLLM = modelConfigService.getModelById(intentLLMModelId, isCache); + typeConfig.put(intentLLM.getId(), intentLLM.getConfigJson()); + } + } + if (StringUtils.isNotBlank(memLocalShortLLMModelId)) { + if (!typeConfig.containsKey(memLocalShortLLMModelId)) { + ModelConfigEntity memLocalShortLLM = modelConfigService + .getModelById(memLocalShortLLMModelId, isCache); + typeConfig.put(memLocalShortLLM.getId(), memLocalShortLLM.getConfigJson()); + } + } } } result.put(modelTypes[i], typeConfig); diff --git a/main/manager-api/src/main/resources/db/changelog/202505271412.sql b/main/manager-api/src/main/resources/db/changelog/202505271412.sql new file mode 100644 index 00000000..70cd2e0d --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202505271412.sql @@ -0,0 +1,4 @@ +-- 本地短期记忆配置可以设置独立的LLM + +update `ai_model_provider` set fields = '[{"key":"llm","label":"LLM模型","type":"string"}]' where id = 'SYSTEM_Memory_mem_local_short'; +update `ai_model_config` set config_json = '{\"type\": \"mem_local_short\", \"llm\": \"LLM_ChatGLMLLM\"}' where id = 'Memory_mem_local_short'; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 9d3808da..aadd2235 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -162,4 +162,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202505151451.sql \ No newline at end of file + path: classpath:db/changelog/202505151451.sql + - changeSet: + id: 202505271412 + author: hrz + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202505271412.sql \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/llm/coze/coze.py b/main/xiaozhi-server/core/providers/llm/coze/coze.py index 60a04667..19002ac8 100644 --- a/main/xiaozhi-server/core/providers/llm/coze/coze.py +++ b/main/xiaozhi-server/core/providers/llm/coze/coze.py @@ -25,7 +25,7 @@ class LLMProvider(LLMProviderBase): self.session_conversation_map = {} # 存储session_id和conversation_id的映射 check_model_key("CozeLLM", self.personal_access_token) - def response(self, session_id, dialogue): + def response(self, session_id, dialogue, **kwargs): coze_api_token = self.personal_access_token coze_api_base = COZE_CN_BASE_URL diff --git a/main/xiaozhi-server/core/providers/llm/dify/dify.py b/main/xiaozhi-server/core/providers/llm/dify/dify.py index 52ca9853..8b01261f 100644 --- a/main/xiaozhi-server/core/providers/llm/dify/dify.py +++ b/main/xiaozhi-server/core/providers/llm/dify/dify.py @@ -17,7 +17,7 @@ class LLMProvider(LLMProviderBase): self.session_conversation_map = {} # 存储session_id和conversation_id的映射 check_model_key("DifyLLM", self.api_key) - def response(self, session_id, dialogue): + def response(self, session_id, dialogue, **kwargs): try: # 取最后一条用户消息 last_msg = next(m for m in reversed(dialogue) if m["role"] == "user") diff --git a/main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py b/main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py index b2e9d6b7..a5581541 100644 --- a/main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py +++ b/main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py @@ -16,7 +16,7 @@ class LLMProvider(LLMProviderBase): self.variables = config.get("variables", {}) check_model_key("FastGPTLLM", self.api_key) - def response(self, session_id, dialogue): + def response(self, session_id, dialogue, **kwargs): try: # 取最后一条用户消息 last_msg = next(m for m in reversed(dialogue) if m["role"] == "user") diff --git a/main/xiaozhi-server/core/providers/llm/gemini/gemini.py b/main/xiaozhi-server/core/providers/llm/gemini/gemini.py index d1cf238e..3369aa2d 100644 --- a/main/xiaozhi-server/core/providers/llm/gemini/gemini.py +++ b/main/xiaozhi-server/core/providers/llm/gemini/gemini.py @@ -112,7 +112,7 @@ class LLMProvider(LLMProviderBase): ] # Gemini文档提到，无需维护session-id，直接用dialogue拼接而成 - def response(self, session_id, dialogue): + def response(self, session_id, dialogue, **kwargs): yield from self._generate(dialogue, None) def response_with_functions(self, session_id, dialogue, functions=None): diff --git a/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py b/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py index c436fc9e..f203e5f7 100644 --- a/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py +++ b/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py @@ -14,7 +14,7 @@ class LLMProvider(LLMProviderBase): self.base_url = config.get("base_url", config.get("url")) # 默认使用 base_url self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL - def response(self, session_id, dialogue): + def response(self, session_id, dialogue, **kwargs): try: # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可 diff --git a/main/xiaozhi-server/core/providers/llm/ollama/ollama.py b/main/xiaozhi-server/core/providers/llm/ollama/ollama.py index aad6354f..c973cacd 100644 --- a/main/xiaozhi-server/core/providers/llm/ollama/ollama.py +++ b/main/xiaozhi-server/core/providers/llm/ollama/ollama.py @@ -18,13 +18,13 @@ class LLMProvider(LLMProviderBase): self.client = OpenAI( base_url=self.base_url, - api_key="ollama" # Ollama doesn't need an API key but OpenAI client requires one + api_key="ollama", # Ollama doesn't need an API key but OpenAI client requires one ) # 检查是否是qwen3模型 self.is_qwen3 = self.model_name and self.model_name.lower().startswith("qwen3") - def response(self, session_id, dialogue): + def response(self, session_id, dialogue, **kwargs): try: # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令 if self.is_qwen3: @@ -35,7 +35,9 @@ class LLMProvider(LLMProviderBase): for i in range(len(dialogue_copy) - 1, -1, -1): if dialogue_copy[i]["role"] == "user": # 在用户消息前添加/no_think指令 - dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"] + dialogue_copy[i]["content"] = ( + "/no_think " + dialogue_copy[i]["content"] + ) logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令") break @@ -43,9 +45,7 @@ class LLMProvider(LLMProviderBase): dialogue = dialogue_copy responses = self.client.chat.completions.create( - model=self.model_name, - messages=dialogue, - stream=True + model=self.model_name, messages=dialogue, stream=True ) is_active = True # 用于处理跨chunk的标签 @@ -53,29 +53,33 @@ class LLMProvider(LLMProviderBase): for chunk in responses: try: - delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None - content = delta.content if hasattr(delta, 'content') else '' + delta = ( + chunk.choices[0].delta + if getattr(chunk, "choices", None) + else None + ) + content = delta.content if hasattr(delta, "content") else "" if content: # 将内容添加到缓冲区 buffer += content # 处理缓冲区中的标签 - while '' in buffer and '' in buffer: + while "" in buffer and "" in buffer: # 找到完整的标签并移除 - pre = buffer.split('', 1)[0] - post = buffer.split('', 1)[1] + pre = buffer.split("", 1)[0] + post = buffer.split("", 1)[1] buffer = pre + post # 处理只有开始标签的情况 - if '' in buffer: + if "" in buffer: is_active = False - buffer = buffer.split('', 1)[0] + buffer = buffer.split("", 1)[0] # 处理只有结束标签的情况 - if '' in buffer: + if "" in buffer: is_active = True - buffer = buffer.split('', 1)[1] + buffer = buffer.split("", 1)[1] # 如果当前处于活动状态且缓冲区有内容，则输出 if is_active and buffer: @@ -100,7 +104,9 @@ class LLMProvider(LLMProviderBase): for i in range(len(dialogue_copy) - 1, -1, -1): if dialogue_copy[i]["role"] == "user": # 在用户消息前添加/no_think指令 - dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"] + dialogue_copy[i]["content"] = ( + "/no_think " + dialogue_copy[i]["content"] + ) logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令") break @@ -119,9 +125,15 @@ class LLMProvider(LLMProviderBase): for chunk in stream: try: - delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None - content = delta.content if hasattr(delta, 'content') else None - tool_calls = delta.tool_calls if hasattr(delta, 'tool_calls') else None + delta = ( + chunk.choices[0].delta + if getattr(chunk, "choices", None) + else None + ) + content = delta.content if hasattr(delta, "content") else None + tool_calls = ( + delta.tool_calls if hasattr(delta, "tool_calls") else None + ) # 如果是工具调用，直接传递 if tool_calls: @@ -134,21 +146,21 @@ class LLMProvider(LLMProviderBase): buffer += content # 处理缓冲区中的标签 - while '' in buffer and '' in buffer: + while "" in buffer and "" in buffer: # 找到完整的标签并移除 - pre = buffer.split('', 1)[0] - post = buffer.split('', 1)[1] + pre = buffer.split("", 1)[0] + post = buffer.split("", 1)[1] buffer = pre + post # 处理只有开始标签的情况 - if '' in buffer: + if "" in buffer: is_active = False - buffer = buffer.split('', 1)[0] + buffer = buffer.split("", 1)[0] # 处理只有结束标签的情况 - if '' in buffer: + if "" in buffer: is_active = True - buffer = buffer.split('', 1)[1] + buffer = buffer.split("", 1)[1] # 如果当前处于活动状态且缓冲区有内容，则输出 if is_active and buffer: diff --git a/main/xiaozhi-server/core/providers/llm/xinference/xinference.py b/main/xiaozhi-server/core/providers/llm/xinference/xinference.py index b90b0418..a4e9a5e6 100644 --- a/main/xiaozhi-server/core/providers/llm/xinference/xinference.py +++ b/main/xiaozhi-server/core/providers/llm/xinference/xinference.py @@ -15,39 +15,45 @@ class LLMProvider(LLMProviderBase): # 如果没有v1，增加v1 if not self.base_url.endswith("/v1"): self.base_url = f"{self.base_url}/v1" - - logger.bind(tag=TAG).info(f"Initializing Xinference LLM provider with model: {self.model_name}, base_url: {self.base_url}") + + logger.bind(tag=TAG).info( + f"Initializing Xinference LLM provider with model: {self.model_name}, base_url: {self.base_url}" + ) try: self.client = OpenAI( base_url=self.base_url, - api_key="xinference" # Xinference has a similar setup to Ollama where it doesn't need an actual key + api_key="xinference", # Xinference has a similar setup to Ollama where it doesn't need an actual key ) logger.bind(tag=TAG).info("Xinference client initialized successfully") except Exception as e: logger.bind(tag=TAG).error(f"Error initializing Xinference client: {e}") raise - def response(self, session_id, dialogue): + def response(self, session_id, dialogue, **kwargs): try: - logger.bind(tag=TAG).debug(f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}") - responses = self.client.chat.completions.create( - model=self.model_name, - messages=dialogue, - stream=True + logger.bind(tag=TAG).debug( + f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}" ) - is_active=True + responses = self.client.chat.completions.create( + model=self.model_name, messages=dialogue, stream=True + ) + is_active = True for chunk in responses: try: - delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None - content = delta.content if hasattr(delta, 'content') else '' + delta = ( + chunk.choices[0].delta + if getattr(chunk, "choices", None) + else None + ) + content = delta.content if hasattr(delta, "content") else "" if content: - if '' in content: + if "" in content: is_active = False - content = content.split('')[0] - if '' in content: + content = content.split("")[0] + if "" in content: is_active = True - content = content.split('')[-1] + content = content.split("")[-1] if is_active: yield content except Exception as e: @@ -59,10 +65,14 @@ class LLMProvider(LLMProviderBase): def response_with_functions(self, session_id, dialogue, functions=None): try: - logger.bind(tag=TAG).debug(f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}") + logger.bind(tag=TAG).debug( + f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}" + ) if functions: - logger.bind(tag=TAG).debug(f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}") - + logger.bind(tag=TAG).debug( + f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}" + ) + stream = self.client.chat.completions.create( model=self.model_name, messages=dialogue, @@ -74,7 +84,7 @@ class LLMProvider(LLMProviderBase): delta = chunk.choices[0].delta content = delta.content tool_calls = delta.tool_calls - + if content: yield content, tool_calls elif tool_calls: @@ -82,4 +92,7 @@ class LLMProvider(LLMProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"Error in Xinference function call: {e}") - yield {"type": "content", "content": f"【Xinference服务响应异常: {str(e)}】"} + yield { + "type": "content", + "content": f"【Xinference服务响应异常: {str(e)}】", + } diff --git a/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py b/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py index 844035bd..9f855cc6 100644 --- a/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py +++ b/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py @@ -167,7 +167,12 @@ class MemoryProvider(MemoryProviderBase): msgStr += f"当前时间：{time_str}" if self.save_to_file: - result = self.llm.response_no_stream(short_term_memory_prompt, msgStr) + result = self.llm.response_no_stream( + short_term_memory_prompt, + msgStr, + max_tokens=2000, + temperature=0.2, + ) json_str = extract_json_data(result) try: json.loads(json_str) # 检查json格式是否正确 @@ -177,7 +182,10 @@ class MemoryProvider(MemoryProviderBase): print("Error:", e) else: result = self.llm.response_no_stream( - short_term_memory_prompt_only_content, msgStr + short_term_memory_prompt_only_content, + msgStr, + max_tokens=2000, + temperature=0.2, ) save_mem_local_short(self.role_id, result) logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")