From 036dde5bb05d041479ed6669c8f90261e2860ef0 Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Thu, 3 Jul 2025 16:34:50 +0800 Subject: [PATCH 01/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=96=B0=E7=9A=84?= =?UTF-8?q?=E5=8F=82=E6=95=B0=EF=BC=9A=E5=A3=B0=E7=BA=B9=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E5=9C=B0=E5=9D=80=20--202507031602.sql=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=A3=B0=E7=BA=B9=E6=8E=A5=E5=8F=A3=E5=9C=B0=E5=9D=80=E5=8F=82?= =?UTF-8?q?=E6=95=B0sql=20--Constant.java=20=E5=AF=B9=E5=BA=94=E6=96=B0?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E7=9A=84=E5=B8=B8=E9=87=8F=20--SysParamsCont?= =?UTF-8?q?roller.java=20=E9=AA=8C=E8=AF=81=E5=A3=B0=E7=BA=B9=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E5=9C=B0=E5=9D=80=E5=8F=82=E6=95=B0=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E7=AC=A6=E5=90=88=E8=A7=84=E5=88=99=20--db.changelog-master.ya?= =?UTF-8?q?ml=20=E6=B7=BB=E5=8A=A0=E6=89=A7=E8=A1=8C=E6=96=B0=E7=9A=84sql?= =?UTF-8?q?=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi/common/constant/Constant.java | 5 +++ .../sys/controller/SysParamsController.java | 37 +++++++++++++++++++ .../resources/db/changelog/202507031602.sql | 4 ++ .../db/changelog/db.changelog-master.yaml | 9 ++++- 4 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202507031602.sql diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index 33e97e8c..ca720ed6 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -116,6 +116,11 @@ public interface Constant { */ String SERVER_MCP_ENDPOINT = "server.mcp_endpoint"; + /** + * mcp接入点路径 + */ + String SERVER_VOICE_PRINT = "server.voice_print"; + /** * 无记忆 */ diff --git a/main/manager-api/src/main/java/xiaozhi/modules/sys/controller/SysParamsController.java b/main/manager-api/src/main/java/xiaozhi/modules/sys/controller/SysParamsController.java index edcc9328..1a38545b 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/sys/controller/SysParamsController.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/sys/controller/SysParamsController.java @@ -107,6 +107,9 @@ public class SysParamsController { // 验证MCP地址 validateMcpUrl(dto.getParamCode(), dto.getParamValue()); + // + validateVoicePrint(dto.getParamCode(), dto.getParamValue()); + sysParamsService.update(dto); configService.getConfig(false); return new Result(); @@ -212,6 +215,7 @@ public class SysParamsController { if (!url.toLowerCase().contains("key")) { throw new RenException("不是正确的MCP地址"); } + try { // 发送GET请求 ResponseEntity response = restTemplate.getForEntity(url, String.class); @@ -227,4 +231,37 @@ public class SysParamsController { throw new RenException("MCP接口验证失败：" + e.getMessage()); } } + // 验证声纹接口地址是否正常 + private void validateVoicePrint(String paramCode, String url) { + if (!paramCode.equals(Constant.SERVER_VOICE_PRINT)) { + return; + } + if (StringUtils.isBlank(url) || url.equals("null")) { + throw new RenException("声纹接口地址不能为空"); + } + if (url.contains("localhost") || url.contains("127.0.0.1")) { + throw new RenException("声纹接口地址不能使用localhost或127.0.0.1"); + } + if (!url.toLowerCase().contains("key")) { + throw new RenException("不是正确的声纹接口地址"); + } + // 验证URL格式 + if (!url.toLowerCase().startsWith("http")) { + throw new RenException("声纹接口地址必须以http或https开头"); + } + try { + // 发送GET请求 + ResponseEntity response = restTemplate.getForEntity(url, String.class); + if (response.getStatusCode() != HttpStatus.OK) { + throw new RenException("声纹接口访问失败，状态码：" + response.getStatusCode()); + } + // 检查响应内容 + String body = response.getBody(); + if (body == null || !body.contains("healthy")) { + throw new RenException("声纹接口返回内容格式不正确，可能不是一个真实的MCP接口"); + } + } catch (Exception e) { + throw new RenException("声纹接口验证失败：" + e.getMessage()); + } + } } diff --git a/main/manager-api/src/main/resources/db/changelog/202507031602.sql b/main/manager-api/src/main/resources/db/changelog/202507031602.sql new file mode 100644 index 00000000..92edf9b9 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507031602.sql @@ -0,0 +1,4 @@ +-- 添加声纹接口地址参数配置 +delete from `sys_params` where id = 114; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) +VALUES (114, 'server.voice_print', 'null', 'string', 1, '声纹接口地址'); \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 32ba9f30..773ec002 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -239,4 +239,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202506261637.sql \ No newline at end of file + path: classpath:db/changelog/202506261637.sql + - changeSet: + id: 202507031602 + author: zjy + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202507031602.sql \ No newline at end of file From 3d64152dba975c3e0e1c3ed839bda98ccb650a93 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Fri, 4 Jul 2025 17:11:44 +0800 Subject: [PATCH 02/35] =?UTF-8?q?update:python=E5=8D=95=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E5=A3=B0=E7=BA=B9=E8=AF=86=E5=88=AB=E5=AF=B9?= =?UTF-8?q?=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 9 + main/xiaozhi-server/core/connection.py | 29 +- .../core/handle/receiveAudioHandle.py | 36 +- .../core/handle/sendAudioHandle.py | 1 - .../xiaozhi-server/core/providers/asr/base.py | 346 ++++++++++++++++-- main/xiaozhi-server/core/utils/dialogue.py | 8 +- .../core/utils/modules_initialize.py | 9 + 7 files changed, 390 insertions(+), 48 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 5a43c995..5b746866 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -137,6 +137,15 @@ plugins: - ".wav" - ".p3" refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒 + # 声纹识别配置 + voiceprint: + # 声纹接口地址 + url: + # 说话人配置：speaker_id,名称,描述 + speakers: + - "test1,张三,张三是一个程序员" + - "test2,李四,李四是一个产品经理" + - "test3,王五,王五是一个设计师" # ##################################################################################### # ################################以下是角色模型配置###################################### diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 9d1949c4..458cbc7a 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -595,8 +595,33 @@ class ConnectionHandler: self.logger.bind(tag=TAG).info(f"大模型收到用户消息: {query}") self.llm_finish_task = False + # 检查是否是JSON格式的消息（包含说话人信息） + enhanced_query = query + try: + if query.strip().startswith('{') and query.strip().endswith('}'): + data = json.loads(query) + if 'speaker' in data and 'content' in data: + # 直接使用JSON格式，不重新格式化 + enhanced_query = query + self.logger.bind(tag=TAG).info(f"识别到说话人: {data['speaker']}") + else: + # 如果有说话人信息但不是JSON格式，按原逻辑处理 + if hasattr(self, 'current_speaker') and self.current_speaker: + enhanced_query = f"[说话人: {self.current_speaker}] {query}" + self.logger.bind(tag=TAG).info(f"识别到说话人: {self.current_speaker}") + else: + # 如果有说话人信息但不是JSON格式，按原逻辑处理 + if hasattr(self, 'current_speaker') and self.current_speaker: + enhanced_query = f"[说话人: {self.current_speaker}] {query}" + self.logger.bind(tag=TAG).info(f"识别到说话人: {self.current_speaker}") + except json.JSONDecodeError: + # JSON解析失败，按原逻辑处理 + if hasattr(self, 'current_speaker') and self.current_speaker: + enhanced_query = f"[说话人: {self.current_speaker}] {query}" + self.logger.bind(tag=TAG).info(f"识别到说话人: {self.current_speaker}") + if not tool_call: - self.dialogue.put(Message(role="user", content=query)) + self.dialogue.put(Message(role="user", content=enhanced_query)) # Define intent functions functions = None @@ -609,7 +634,7 @@ class ConnectionHandler: memory_str = None if self.memory is not None: future = asyncio.run_coroutine_threadsafe( - self.memory.query_memory(query), self.loop + self.memory.query_memory(enhanced_query), self.loop ) memory_str = future.result() diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 465b081a..e6f39632 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -4,6 +4,7 @@ from core.utils.output_counter import check_device_output_limit from core.handle.abortHandle import handleAbortMessage import time import asyncio +import json from core.handle.sendAudioHandle import SentenceType from core.utils.util import audio_to_data @@ -38,6 +39,31 @@ async def resume_vad_detection(conn): async def startToChat(conn, text): + # 检查输入是否是JSON格式（包含说话人信息） + speaker_name = None + actual_text = text + + try: + # 尝试解析JSON格式的输入 + if text.strip().startswith('{') and text.strip().endswith('}'): + data = json.loads(text) + if 'speaker' in data and 'content' in data: + speaker_name = data['speaker'] + actual_text = data['content'] + conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}") + + # 直接使用JSON格式的文本，不解析 + actual_text = text + except (json.JSONDecodeError, KeyError): + # 如果解析失败，继续使用原始文本 + pass + + # 保存说话人信息到连接对象 + if speaker_name: + conn.current_speaker = speaker_name + else: + conn.current_speaker = None + if conn.need_bind: await check_bind_device(conn) return @@ -52,16 +78,16 @@ async def startToChat(conn, text): if conn.client_is_speaking: await handleAbortMessage(conn) - # 首先进行意图分析 - intent_handled = await handle_user_intent(conn, text) + # 首先进行意图分析，使用实际文本内容 + intent_handled = await handle_user_intent(conn, actual_text) if intent_handled: # 如果意图已被处理，不再进行聊天 return - # 意图未被处理，继续常规聊天流程 - await send_stt_message(conn, text) - conn.executor.submit(conn.chat, text) + # 意图未被处理，继续常规聊天流程，使用实际文本内容 + await send_stt_message(conn, actual_text) + conn.executor.submit(conn.chat, actual_text) async def no_voice_close_connect(conn, have_voice): diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index eb9f565f..7bb40d78 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -76,7 +76,6 @@ async def sendAudio(conn, audios, pre_buffer=True): frame_duration = 60 # 帧时长（毫秒），匹配 Opus 编码 start_time = time.perf_counter() play_position = 0 - last_reset_time = time.perf_counter() # 记录最后的重置时间 # 仅当第一句话时执行预缓冲 if pre_buffer: diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 71098a20..8630f9b1 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -1,15 +1,20 @@ import os import wave -import copy import uuid import queue import asyncio import traceback import threading import opuslib_next +import json +import io +import aiohttp +import time +import concurrent.futures from abc import ABC, abstractmethod from config.logger import setup_logging -from typing import Optional, Tuple, List +from urllib.parse import urlparse, parse_qs +from typing import Optional, Tuple, List, Dict, Any from core.handle.receiveAudioHandle import startToChat from core.handle.reportHandle import enqueue_asr_report from core.utils.util import remove_punctuation_and_length @@ -18,16 +23,145 @@ from core.handle.receiveAudioHandle import handleAudioMessage TAG = __name__ logger = setup_logging() +# 创建全局线程池执行器用于CPU密集型操作 +executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) + +class VoiceprintProvider: + """声纹识别服务提供者""" + + def __init__(self, config: dict): + self.original_url = config.get("url", "") + self.speakers = config.get("speakers", []) + self.speaker_map = self._parse_speakers() + + # 解析API地址和密钥 + self.api_url = None + self.api_key = None + self.speaker_ids = [] + + if not self.original_url: + logger.bind(tag=TAG).warning("声纹识别URL未配置，声纹识别将被禁用") + self.enabled = False + else: + # 解析URL和key + parsed_url = urlparse(self.original_url) + base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" + + # 从查询参数中提取key + query_params = parse_qs(parsed_url.query) + self.api_key = query_params.get('key', [''])[0] + + if not self.api_key: + logger.bind(tag=TAG).error("URL中未找到key参数，声纹识别将被禁用") + self.enabled = False + else: + # 构造identify接口地址 + self.api_url = f"{base_url}/voiceprint/identify" + + # 提取speaker_ids + for speaker_str in self.speakers: + try: + parts = speaker_str.split(",", 2) + if len(parts) >= 1: + speaker_id = parts[0].strip() + self.speaker_ids.append(speaker_id) + except Exception: + continue + + # 检查是否有有效的说话人配置 + if not self.speaker_ids: + logger.bind(tag=TAG).warning("未配置有效的说话人，声纹识别将被禁用") + self.enabled = False + else: + self.enabled = True + logger.bind(tag=TAG).info(f"声纹识别已配置: API={self.api_url}, 说话人={len(self.speaker_ids)}个") + + def _parse_speakers(self) -> Dict[str, Dict[str, str]]: + """解析说话人配置""" + speaker_map = {} + for speaker_str in self.speakers: + try: + parts = speaker_str.split(",", 2) + if len(parts) >= 3: + speaker_id, name, description = parts[0].strip(), parts[1].strip(), parts[2].strip() + speaker_map[speaker_id] = { + "name": name, + "description": description + } + except Exception as e: + logger.bind(tag=TAG).warning(f"解析说话人配置失败: {speaker_str}, 错误: {e}") + return speaker_map + + async def identify_speaker(self, audio_data: bytes, session_id: str) -> Optional[str]: + """识别说话人""" + if not self.enabled or not self.api_url or not self.api_key: + logger.bind(tag=TAG).debug("声纹识别功能已禁用或未配置，跳过识别") + return None + + try: + api_start_time = time.monotonic() + + # 准备请求头 + headers = { + 'Authorization': f'Bearer {self.api_key}', + 'Accept': 'application/json' + } + + # 准备multipart/form-data数据 + data = aiohttp.FormData() + data.add_field('speaker_ids', ','.join(self.speaker_ids)) + data.add_field('file', audio_data, filename='audio.wav', content_type='audio/wav') + + timeout = aiohttp.ClientTimeout(total=10) + + # 网络请求 + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post(self.api_url, headers=headers, data=data) as response: + + if response.status == 200: + result = await response.json() + speaker_id = result.get("speaker_id") + score = result.get("score", 0) + total_elapsed_time = time.monotonic() - api_start_time + + logger.bind(tag=TAG).info(f"声纹识别耗时: {total_elapsed_time:.3f}s") + + # 置信度检查 + if score < 0.5: + logger.bind(tag=TAG).warning(f"声纹识别置信度较低: {score:.3f}") + + if speaker_id and speaker_id in self.speaker_map: + result_name = self.speaker_map[speaker_id]["name"] + return result_name + else: + logger.bind(tag=TAG).warning(f"未识别的说话人ID: {speaker_id}") + return "未知说话人" + else: + logger.bind(tag=TAG).error(f"声纹识别API错误: HTTP {response.status}") + return None + + except asyncio.TimeoutError: + elapsed = time.monotonic() - api_start_time + logger.bind(tag=TAG).error(f"声纹识别超时: {elapsed:.3f}s") + return None + except Exception as e: + elapsed = time.monotonic() - api_start_time + logger.bind(tag=TAG).error(f"声纹识别失败: {e}") + return None + class ASRProviderBase(ABC): def __init__(self): - pass + self.voiceprint_provider = None + + def init_voiceprint(self, voiceprint_config: dict): + """初始化声纹识别""" + if voiceprint_config: + self.voiceprint_provider = VoiceprintProvider(voiceprint_config) + logger.bind(tag=TAG).info("声纹识别模块已初始化") # 打开音频通道 - # 这里默认是非流式的处理方式 - # 流式处理方式请在子类中重写 async def open_audio_channels(self, conn): - # tts 消化线程 conn.asr_priority_thread = threading.Thread( target=self.asr_text_priority_thread, args=(conn,), daemon=True ) @@ -52,41 +186,173 @@ class ASRProviderBase(ABC): continue # 接收音频 - # 这里默认是非流式的处理方式 - # 流式处理方式请在子类中重写 async def receive_audio(self, conn, audio, audio_have_voice): if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime": have_voice = audio_have_voice else: have_voice = conn.client_have_voice - # 如果本次没有声音，本段也没声音，就把声音丢弃了 + conn.asr_audio.append(audio) - if have_voice == False and conn.client_have_voice == False: + if not have_voice and not conn.client_have_voice: conn.asr_audio = conn.asr_audio[-10:] return - # 如果本段有声音，且已经停止了 if conn.client_voice_stop: - asr_audio_task = copy.deepcopy(conn.asr_audio) + asr_audio_task = conn.asr_audio.copy() conn.asr_audio.clear() - - # 音频太短了，无法识别 conn.reset_vad_states() + if len(asr_audio_task) > 15: await self.handle_voice_stop(conn, asr_audio_task) # 处理语音停止 - async def handle_voice_stop(self, conn, asr_audio_task): - raw_text, _ = await self.speech_to_text( - asr_audio_task, conn.session_id, conn.audio_format - ) # 确保ASR模块返回原始文本 - conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}") - text_len, _ = remove_punctuation_and_length(raw_text) - self.stop_ws_connection() - if text_len > 0: - # 使用自定义模块进行上报 - await startToChat(conn, raw_text) - enqueue_asr_report(conn, raw_text, asr_audio_task) + async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]): + """并行处理ASR和声纹识别""" + try: + total_start_time = time.monotonic() + + # 准备音频数据 + if conn.audio_format == "pcm": + pcm_data = asr_audio_task + else: + pcm_data = self.decode_opus(asr_audio_task) + + combined_pcm_data = b"".join(pcm_data) + + # 预先准备WAV数据 + wav_data = None + if self.voiceprint_provider and combined_pcm_data: + wav_data = self._pcm_to_wav(combined_pcm_data) + + + # 定义ASR任务 + def run_asr(): + start_time = time.monotonic() + try: + import asyncio + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + result = loop.run_until_complete( + self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format) + ) + end_time = time.monotonic() + logger.bind(tag=TAG).info(f"ASR耗时: {end_time - start_time:.3f}s") + return result + finally: + loop.close() + except Exception as e: + end_time = time.monotonic() + logger.bind(tag=TAG).error(f"ASR失败: {e}") + return ("", None) + + # 定义声纹识别任务 + def run_voiceprint(): + if not wav_data: + return None + start_time = time.monotonic() + try: + import asyncio + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + result = loop.run_until_complete( + self.voiceprint_provider.identify_speaker(wav_data, conn.session_id) + ) + return result + finally: + loop.close() + except Exception as e: + logger.bind(tag=TAG).error(f"声纹识别失败: {e}") + return None + + # 使用线程池执行器并行运行 + parallel_start_time = time.monotonic() + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor: + asr_future = thread_executor.submit(run_asr) + + if self.voiceprint_provider and wav_data: + voiceprint_future = thread_executor.submit(run_voiceprint) + + # 等待两个线程都完成 + asr_result = asr_future.result(timeout=15) + voiceprint_result = voiceprint_future.result(timeout=15) + + results = {"asr": asr_result, "voiceprint": voiceprint_result} + else: + asr_result = asr_future.result(timeout=15) + results = {"asr": asr_result, "voiceprint": None} + + parallel_execution_time = time.monotonic() - parallel_start_time + + # 处理结果 + raw_text, file_path = results.get("asr", ("", None)) + speaker_name = results.get("voiceprint", None) + + # 记录识别结果 + if raw_text: + logger.bind(tag=TAG).info(f"识别文本: {raw_text}") + if speaker_name: + logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}") + + # 性能监控 + total_time = time.monotonic() - total_start_time + logger.bind(tag=TAG).info(f"总处理耗时: {total_time:.3f}s") + + # 检查文本长度 + text_len, _ = remove_punctuation_and_length(raw_text) + self.stop_ws_connection() + + if text_len > 0: + # 构建包含说话人信息的JSON字符串 + enhanced_text = self._build_enhanced_text(raw_text, speaker_name) + + # 使用自定义模块进行上报 + await startToChat(conn, enhanced_text) + enqueue_asr_report(conn, enhanced_text, asr_audio_task) + + except Exception as e: + logger.bind(tag=TAG).error(f"处理语音停止失败: {e}") + import traceback + logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}") + + def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str: + """构建包含说话人信息的文本""" + if speaker_name: + return json.dumps({ + "speaker": speaker_name, + "content": text + }, ensure_ascii=False) + else: + return text + + def _pcm_to_wav(self, pcm_data: bytes) -> bytes: + """将PCM数据转换为WAV格式""" + if len(pcm_data) == 0: + logger.bind(tag=TAG).warning("PCM数据为空，无法转换WAV") + return b"" + + # 确保数据长度是偶数（16位音频） + if len(pcm_data) % 2 != 0: + pcm_data = pcm_data[:-1] + + # 创建WAV文件头 + wav_buffer = io.BytesIO() + try: + with wave.open(wav_buffer, 'wb') as wav_file: + wav_file.setnchannels(1) # 单声道 + wav_file.setsampwidth(2) # 16位 + wav_file.setframerate(16000) # 16kHz采样率 + wav_file.writeframes(pcm_data) + + wav_buffer.seek(0) + wav_data = wav_buffer.read() + + return wav_data + except Exception as e: + logger.bind(tag=TAG).error(f"WAV转换失败: {e}") + return b"" def stop_ws_connection(self): pass @@ -99,7 +365,7 @@ class ASRProviderBase(ABC): with wave.open(file_path, "wb") as wf: wf.setnchannels(1) - wf.setsampwidth(2) # 2 bytes = 16-bit + wf.setsampwidth(2) wf.setframerate(16000) wf.writeframes(b"".join(pcm_data)) @@ -113,27 +379,29 @@ class ASRProviderBase(ABC): pass @staticmethod - def decode_opus(opus_data: List[bytes]) -> bytes: + def decode_opus(opus_data: List[bytes]) -> List[bytes]: """将Opus音频数据解码为PCM数据""" try: - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + decoder = opuslib_next.Decoder(16000, 1) pcm_data = [] - buffer_size = 960 # 每次处理960个采样点 - - for opus_packet in opus_data: + buffer_size = 960 # 每次处理960个采样点 (60ms at 16kHz) + + for i, opus_packet in enumerate(opus_data): try: - # 使用较小的缓冲区大小进行处理 + if not opus_packet or len(opus_packet) == 0: + continue + pcm_frame = decoder.decode(opus_packet, buffer_size) - if pcm_frame: + if pcm_frame and len(pcm_frame) > 0: pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: - logger.bind(tag=TAG).warning(f"Opus解码错误，跳过当前数据包: {e}") - continue + logger.bind(tag=TAG).warning(f"Opus解码错误，跳过数据包 {i}: {e}") except Exception as e: - logger.bind(tag=TAG).error(f"音频处理错误: {e}", exc_info=True) - continue - + logger.bind(tag=TAG).error(f"音频处理错误，数据包 {i}: {e}") + return pcm_data + except Exception as e: - logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}", exc_info=True) + logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}") return [] diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py index 2ee30d4a..fbb1f7ad 100644 --- a/main/xiaozhi-server/core/utils/dialogue.py +++ b/main/xiaozhi-server/core/utils/dialogue.py @@ -74,8 +74,14 @@ class Dialogue: ) if system_message: + # 构建增强的系统提示，包含说话人处理指导 + speaker_guidance = "\n\n[说话人识别功能说明]\n" \ + "当用户消息包含 [说话人: 姓名] 前缀时，表示系统已识别出说话人身份。\n" \ + "请根据说话人的身份特征（如果之前有相关信息）来调整回应风格和内容。\n" \ + "你可以称呼说话人的名字，并参考他们的特点进行个性化回应。" + enhanced_system_prompt = ( - f"{system_message.content}\n\n" + f"{system_message.content}{speaker_guidance}\n\n" f"以下是用户的历史记忆：\n```\n{memory_str}\n```" ) dialogue.append({"role": "system", "content": enhanced_system_prompt}) diff --git a/main/xiaozhi-server/core/utils/modules_initialize.py b/main/xiaozhi-server/core/utils/modules_initialize.py index f2e3968e..43f3e470 100644 --- a/main/xiaozhi-server/core/utils/modules_initialize.py +++ b/main/xiaozhi-server/core/utils/modules_initialize.py @@ -125,4 +125,13 @@ def initialize_asr(config): config["ASR"][select_asr_module], str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"), ) + + # 初始化声纹识别功能 + voiceprint_config = config.get("plugins", {}).get("voiceprint") + if voiceprint_config and voiceprint_config.get("url") and voiceprint_config.get("speakers"): + new_asr.init_voiceprint(voiceprint_config) + logger.bind(tag=TAG).info("ASR模块声纹识别功能已启用") + else: + logger.bind(tag=TAG).info("ASR模块声纹识别功能已禁用") + return new_asr From 1c2d4f904527e10fd3a01a3368e96d5c90c2ed78 Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Fri, 4 Jul 2025 17:54:41 +0800 Subject: [PATCH 03/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=96=B0=E8=A1=A8?= =?UTF-8?q?=EF=BC=9A=E6=99=BA=E8=83=BD=E4=BD=93=E5=A3=B0=E7=BA=B9=E8=A1=A8?= =?UTF-8?q?=20--202507041018.sql=20=E5=BB=BA=E8=A1=A8sql=E8=AF=AD=E5=8F=A5?= =?UTF-8?q?=20--db.changelog-master.yaml=20=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E9=87=8C=E6=B7=BB=E5=8A=A0=E6=89=A7=E8=A1=8C=E6=96=B0?= =?UTF-8?q?sql=E7=9A=84=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/db/changelog/202507041018.sql | 12 ++++++++++++ .../resources/db/changelog/db.changelog-master.yaml | 9 ++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202507041018.sql diff --git a/main/manager-api/src/main/resources/db/changelog/202507041018.sql b/main/manager-api/src/main/resources/db/changelog/202507041018.sql new file mode 100644 index 00000000..cffcfb16 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507041018.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS ai_agent_voice_print; +create table ai_agent_voice_print ( + id varchar(32) NOT NULL COMMENT '声纹ID', + agent_id varchar(32) NOT NULL COMMENT '关联的智能体ID', + source_name varchar(50) NOT NULL COMMENT '声纹来源的人的姓名', + introduce varchar(200) COMMENT '描述声纹来源的这个人', + create_date DATETIME COMMENT '创建时间', + creator bigint COMMENT '创建者', + update_date DATETIME COMMENT '修改时间', + updater bigint COMMENT '修改者', + PRIMARY KEY (id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='智能体声纹表' \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 773ec002..1f7fdcea 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -246,4 +246,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202507031602.sql \ No newline at end of file + path: classpath:db/changelog/202507031602.sql + - changeSet: + id: 202507041018 + author: zjy + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202507041018.sql \ No newline at end of file From 5e5ec98995e06f1ecca18b269ba698059c389bc8 Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Fri, 4 Jul 2025 18:02:19 +0800 Subject: [PATCH 04/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=99=BA=E8=83=BD?= =?UTF-8?q?=E4=BD=93=E5=A3=B0=E7=BA=B9=E5=AE=9E=E4=BD=93=EF=BC=8Cvo?= =?UTF-8?q?=EF=BC=8Cdto=EF=BC=8Cdao=EF=BC=8Cservice=E7=B1=BB=20--AgentVoic?= =?UTF-8?q?ePrintVO.java=20=E6=99=BA=E8=83=BD=E4=BD=93=E5=A3=B0=E7=BA=B9?= =?UTF-8?q?=E5=B1=95=E7=A4=BAvo=20--AgentVoicePrintService.java=20?= =?UTF-8?q?=E6=99=BA=E8=83=BD=E4=BD=93=E5=A3=B0=E7=BA=B9=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E5=B1=82=E5=AE=9A=E4=B9=89=20=201.=E6=B7=BB=E5=8A=A0=E6=99=BA?= =?UTF-8?q?=E8=83=BD=E4=BD=93=E6=96=B0=E7=9A=84=E5=A3=B0=E7=BA=B9=20=202.?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=99=BA=E8=83=BD=E4=BD=93=E7=9A=84=E6=8C=87?= =?UTF-8?q?=E7=9A=84=E5=A3=B0=E7=BA=B9=20=203.=E8=8E=B7=E5=8F=96=E6=8C=87?= =?UTF-8?q?=E5=AE=9A=E6=99=BA=E8=83=BD=E4=BD=93=E7=9A=84=E6=89=80=E6=9C=89?= =?UTF-8?q?=E5=A3=B0=E7=BA=B9=E6=95=B0=E6=8D=AE=20=204.=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E6=99=BA=E8=83=BD=E4=BD=93=E7=9A=84=E6=8C=87=E7=9A=84=E5=A3=B0?= =?UTF-8?q?=E7=BA=B9=E6=95=B0=E6=8D=AE=20--AgentVoicePrintSaveDTO.java=20?= =?UTF-8?q?=E6=99=BA=E8=83=BD=E4=BD=93=E5=A3=B0=E7=BA=B9=E4=BF=9D=E5=AD=98?= =?UTF-8?q?dto=20--AgentVoicePrintUpdateDTO.java=20=E6=99=BA=E8=83=BD?= =?UTF-8?q?=E4=BD=93=E5=A3=B0=E7=BA=B9=E4=BF=AE=E6=94=B9dto=20--AgentVoice?= =?UTF-8?q?PrintEntity.java=20=E6=99=BA=E8=83=BD=E4=BD=93=E5=A3=B0?= =?UTF-8?q?=E7=BA=B9=E5=AE=9E=E4=BD=93=20--AgentVoicePrintDao.java=20?= =?UTF-8?q?=E6=99=BA=E8=83=BD=E4=BD=93=E5=A3=B0=E7=BA=B9=E6=8C=81=E4=B9=85?= =?UTF-8?q?=E5=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../modules/agent/dao/AgentVoicePrintDao.java | 18 ++++++ .../agent/dto/AgentVoicePrintSaveDTO.java | 28 +++++++++ .../agent/dto/AgentVoicePrintUpdateDTO.java | 28 +++++++++ .../agent/entity/AgentVoicePrintEntity.java | 57 +++++++++++++++++++ .../agent/service/AgentVoicePrintService.java | 50 ++++++++++++++++ .../modules/agent/vo/AgentVoicePrintVO.java | 33 +++++++++++ 6 files changed, 214 insertions(+) create mode 100644 main/manager-api/src/main/java/xiaozhi/modules/agent/dao/AgentVoicePrintDao.java create mode 100644 main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentVoicePrintSaveDTO.java create mode 100644 main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentVoicePrintUpdateDTO.java create mode 100644 main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentVoicePrintEntity.java create mode 100644 main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentVoicePrintService.java create mode 100644 main/manager-api/src/main/java/xiaozhi/modules/agent/vo/AgentVoicePrintVO.java diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/dao/AgentVoicePrintDao.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/dao/AgentVoicePrintDao.java new file mode 100644 index 00000000..2d8e1eda --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/dao/AgentVoicePrintDao.java @@ -0,0 +1,18 @@ +package xiaozhi.modules.agent.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import xiaozhi.modules.agent.entity.AgentChatHistoryEntity; +import xiaozhi.modules.agent.entity.AgentVoicePrintEntity; + +/** + * {@link AgentChatHistoryEntity} 智能体聊天历史记录Dao对象 + * + * @author Goody + * @version 1.0, 2025/4/30 + * @since 1.0.0 + */ +@Mapper +public interface AgentVoicePrintDao extends BaseMapper { + +} diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentVoicePrintSaveDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentVoicePrintSaveDTO.java new file mode 100644 index 00000000..0ff3b0ca --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentVoicePrintSaveDTO.java @@ -0,0 +1,28 @@ +package xiaozhi.modules.agent.dto; + +import lombok.Data; + +/** + * 保存智能体声纹的dto + * + * @author zjy + */ +@Data +public class AgentVoicePrintSaveDTO { + /** + * 关联的智能体id + */ + private String agentId; + /** + * 音频文件id + */ + private String audioId; + /** + * 声纹来源的人姓名 + */ + private String sourceName; + /** + * 描述声纹来源的人 + */ + private String introduce; +} diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentVoicePrintUpdateDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentVoicePrintUpdateDTO.java new file mode 100644 index 00000000..60643e29 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentVoicePrintUpdateDTO.java @@ -0,0 +1,28 @@ +package xiaozhi.modules.agent.dto; + +import lombok.Data; + +/** + * 修改智能体声纹的dto + * + * @author zjy + */ +@Data +public class AgentVoicePrintUpdateDTO { + /** + * 智能体声纹id + */ + private String id; + /** + * 音频文件id + */ + private String audioId; + /** + * 声纹来源的人姓名 + */ + private String sourceName; + /** + * 描述声纹来源的人 + */ + private String introduce; +} diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentVoicePrintEntity.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentVoicePrintEntity.java new file mode 100644 index 00000000..9f9e6b53 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentVoicePrintEntity.java @@ -0,0 +1,57 @@ +package xiaozhi.modules.agent.entity; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; +import lombok.EqualsAndHashCode; +import xiaozhi.common.entity.BaseEntity; + +import java.util.Date; + +/** + * 智能体声纹表 + * + * @author zjy + */ +@TableName(value = "ai_agent_voice_print") +@Data +public class AgentVoicePrintEntity { + /** + * 主键id + */ + @TableId(type = IdType.ASSIGN_UUID) + private String id; + /** + * 关联的智能体id + */ + private String agentId; + /** + * 声纹来源的人姓名 + */ + private String sourceName; + /** + * 描述声纹来源的人 + */ + private String introduce; + + /** + * 创建者 + */ + @TableField(fill = FieldFill.INSERT) + private Long creator; + /** + * 创建时间 + */ + @TableField(fill = FieldFill.INSERT) + private Date createDate; + + /** + * 更新者 + */ + @TableField(fill = FieldFill.INSERT_UPDATE) + private Long updater; + /** + * 更新时间 + */ + @TableField(fill = FieldFill.INSERT_UPDATE) + private Date updateDate; +} diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentVoicePrintService.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentVoicePrintService.java new file mode 100644 index 00000000..816c02e3 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentVoicePrintService.java @@ -0,0 +1,50 @@ +package xiaozhi.modules.agent.service; + + +import xiaozhi.modules.agent.dto.AgentVoicePrintSaveDTO; +import xiaozhi.modules.agent.dto.AgentVoicePrintUpdateDTO; +import xiaozhi.modules.agent.vo.AgentVoicePrintVO; + +import java.util.List; + +/** + * 智能体声纹处理service + * + * @author zjy + */ +public interface AgentVoicePrintService { + /** + * 添加智能体新的声纹 + * + * @param dto 保存智能体声纹的数据 + * @return T:成功 F：失败 + */ + boolean insert(AgentVoicePrintSaveDTO dto); + + /** + * 删除智能体的指的声纹 + * + * @param voicePrintId 声纹id + * @return 是否成功 T:成功 F：失败 + */ + boolean delete(String voicePrintId); + + /** + * 获取指定智能体的所有声纹数据 + * + * @param agentId 智能体id + * @return 声纹数据集合 + */ + List list(String agentId); + + /** + * 更新智能体的指的声纹数据 + * + * @param dto 修改的声纹的数据 + * @return 是否成功 T:成功 F：失败 + */ + boolean update(AgentVoicePrintUpdateDTO dto); + + + +} diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/vo/AgentVoicePrintVO.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/vo/AgentVoicePrintVO.java new file mode 100644 index 00000000..b647e8b8 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/vo/AgentVoicePrintVO.java @@ -0,0 +1,33 @@ +package xiaozhi.modules.agent.vo; + +import lombok.Data; + +import java.util.Date; + +/** + * 展示智能体声纹列表VO + */ +@Data +public class AgentVoicePrintVO { + + /** + * 主键id + */ + private String id; + /** + * 音频文件id + */ + private String audioId; + /** + * 声纹来源的人姓名 + */ + private String sourceName; + /** + * 描述声纹来源的人 + */ + private String introduce; + /** + * 创建时间 + */ + private Date createDate; +} From 77527d882357d056ed76222d3e591b0fedc414ed Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Fri, 4 Jul 2025 18:04:39 +0800 Subject: [PATCH 05/35] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E6=99=BA=E8=83=BD?= =?UTF-8?q?=E4=BD=93=E5=A3=B0=E7=BA=B9=E6=9C=8D=E5=8A=A1=E5=B1=82=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=E7=9A=84=E2=80=98=E6=B7=BB=E5=8A=A0=E6=99=BA=E8=83=BD?= =?UTF-8?q?=E4=BD=93=E6=96=B0=E7=9A=84=E5=A3=B0=E7=BA=B9=E2=80=99=E5=92=8C?= =?UTF-8?q?=E2=80=98=E5=88=A0=E9=99=A4=E6=99=BA=E8=83=BD=E4=BD=93=E7=9A=84?= =?UTF-8?q?=E6=8C=87=E7=9A=84=E5=A3=B0=E7=BA=B9=E2=80=99=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=20--AgentVoicePrintServiceImpl.java=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E6=96=B9=E6=B3=95=20=201.=E6=B7=BB=E5=8A=A0=E6=99=BA=E8=83=BD?= =?UTF-8?q?=E4=BD=93=E6=96=B0=E7=9A=84=E5=A3=B0=E7=BA=B9=20=202.=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E6=99=BA=E8=83=BD=E4=BD=93=E7=9A=84=E6=8C=87=E7=9A=84?= =?UTF-8?q?=E5=A3=B0=E7=BA=B9=20=203.=E5=85=B1=E7=94=A8=E6=96=B9=E6=B3=95?= =?UTF-8?q?=EF=BC=88=E8=8E=B7=E5=8F=96=E7=94=9F=E7=BA=B9=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?URI=E5=AF=B9=E8=B1=A1=EF=BC=8C=E8=8E=B7=E5=8F=96=E5=A3=B0?= =?UTF-8?q?=E7=BA=B9=E5=9C=B0=E5=9D=80=E5=9F=BA=E7=A1=80=E8=B7=AF=E5=BE=84?= =?UTF-8?q?=EF=BC=8C=E8=8E=B7=E5=8F=96=E9=AA=8C=E8=AF=81Authorization?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/AgentVoicePrintServiceImpl.java | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java new file mode 100644 index 00000000..974d4aa9 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java @@ -0,0 +1,178 @@ +package xiaozhi.modules.agent.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.*; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestTemplate; +import xiaozhi.common.constant.Constant; +import xiaozhi.common.exception.RenException; +import xiaozhi.common.utils.ConvertUtils; +import xiaozhi.modules.agent.dao.AgentVoicePrintDao; +import xiaozhi.modules.agent.dto.AgentVoicePrintSaveDTO; +import xiaozhi.modules.agent.dto.AgentVoicePrintUpdateDTO; +import xiaozhi.modules.agent.entity.AgentVoicePrintEntity; +import xiaozhi.modules.agent.service.AgentChatAudioService; +import xiaozhi.modules.agent.service.AgentVoicePrintService; +import xiaozhi.modules.agent.vo.AgentVoicePrintVO; +import xiaozhi.modules.sys.service.SysParamsService; + +import java.beans.Transient; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; + +/** + * @author zjy + */ +@Service +@AllArgsConstructor +@Slf4j +public class AgentVoicePrintServiceImpl extends ServiceImpl + implements AgentVoicePrintService { + private final AgentChatAudioService agentChatAudioService; + private final RestTemplate restTemplate; + private final SysParamsService sysParamsService; + + + @Override + @Transient + public boolean insert(AgentVoicePrintSaveDTO dto) { + // 获取音频Id + String audioId = dto.getAudioId(); + // 获取到音频数据 + byte[] audio = agentChatAudioService.getAudio(audioId); + // 如果音频数据为空的直接报错不进行下去 + if (audio == null || audio.length == 0) { + throw new RenException("音频数据是空的请检查上传数据"); + } + // 将字节数组包装为资源，并添加到请求体 + ByteArrayResource resource = new ByteArrayResource(audio) { + @Override + public String getFilename() { + return "VoicePrint.WAV"; // 设置文件名 + } + }; + + // 处理声纹接口地址，获取前缀 + URI uri = getVoicePrintURI(); + String baseUrl = getBaseUrl(uri); + String requestUrl = baseUrl + "/voiceprint/register"; + + // 保存声纹信息 + AgentVoicePrintEntity entity = ConvertUtils.sourceToTarget(dto, AgentVoicePrintEntity.class); + int insert = baseMapper.insert(entity); + if(insert != 1){ + throw new RenException("声纹保存失败"); + } + // 创建请求体 + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("speaker_id", entity.getId()); + body.add("file", resource); + + // 创建请求头 + HttpHeaders headers = new HttpHeaders(); + headers.set("Authorization", getAuthorization(uri)); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + // 创建请求体 + HttpEntity> requestEntity = new HttpEntity<>(body, headers); + // 发送 POST 请求 + ResponseEntity response = restTemplate.postForEntity(requestUrl, requestEntity, String.class); + + if (response.getStatusCode() != HttpStatus.OK) { + throw new RenException("声纹保存失败"); + } + // 检查响应内容 + String responseBody = response.getBody(); + if(responseBody == null || !responseBody.contains("true")){ + throw new RenException("声纹保存失败"); + } + return true; + } + + + + @Override + public boolean delete(String voicePrintId) { + int insert = baseMapper.deleteById(voicePrintId); + if(insert != 1){ + throw new RenException("声纹删除失败"); + } + URI uri = getVoicePrintURI(); + String baseUrl = getBaseUrl(uri); + String requestUrl = baseUrl + "/voiceprint/" + voicePrintId; + // 创建请求头 + HttpHeaders headers = new HttpHeaders(); + headers.set("Authorization", getAuthorization(uri)); + // 创建请求体 + HttpEntity> requestEntity = new HttpEntity<>(headers); + + // 发送 POST 请求 + ResponseEntity response = restTemplate.exchange(requestUrl, HttpMethod.DELETE, requestEntity, String.class); + if (response.getStatusCode() != HttpStatus.OK) { + throw new RenException("声纹保存失败"); + } + // 检查响应内容 + String responseBody = response.getBody(); + if(responseBody == null || !responseBody.contains("true")){ + throw new RenException("声纹保存失败"); + } + return true; + } + + @Override + public List list(String agentId) { + return List.of(); + } + + @Override + public boolean update(AgentVoicePrintUpdateDTO dto) { + return false; + } + + /** + * 获取生纹接口URI对象 + * + * @return URI对象 + */ + private URI getVoicePrintURI() { + // 获取声纹接口地址 + String voicePrint = sysParamsService.getValue(Constant.SERVER_VOICE_PRINT, true); + try { + return new URI(voicePrint); + } catch (URISyntaxException e) { + log.error("路径格式不正确路径：{}，\n错误信息:{}", voicePrint, e.getMessage()); + throw new RuntimeException("声纹接口的地址存在错误，请进入参数管理修改声纹接口地址"); + } + } + + /** + * 获取声纹地址基础路径 + * @param uri 声纹地址uri + * @return 基础路径 + */ + private String getBaseUrl(URI uri) { + String protocol = uri.getScheme(); + String host = uri.getHost(); + int port = uri.getPort(); + return "%s://%s:%s".formatted(protocol,host,port); + } + + /** + * 获取验证Authorization + * + * @param uri 声纹地址uri + * @return Authorization值 + */ + private String getAuthorization(URI uri) { + // 获取参数 + String query = uri.getQuery(); + // 获取aes加密密钥 + String str = "key="; + return "Bearer " + query.substring(query.indexOf(str) + str.length()); + } +} From 61ecfd2d927ca5ca39f11b77237a73e548209eec Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Mon, 7 Jul 2025 10:58:35 +0800 Subject: [PATCH 06/35] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E6=99=BA=E8=83=BD?= =?UTF-8?q?=E4=BD=93=E5=A3=B0=E7=BA=B9=E6=9C=8D=E5=8A=A1=E5=B1=82=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=E7=9A=84=E2=80=98=E8=8E=B7=E5=8F=96=E6=8C=87=E5=AE=9A?= =?UTF-8?q?=E6=99=BA=E8=83=BD=E4=BD=93=E7=9A=84=E6=89=80=E6=9C=89=E5=A3=B0?= =?UTF-8?q?=E7=BA=B9=E6=95=B0=E6=8D=AE=E2=80=99=E5=92=8C=E2=80=98=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E6=99=BA=E8=83=BD=E4=BD=93=E7=9A=84=E6=8C=87=E7=9A=84?= =?UTF-8?q?=E5=A3=B0=E7=BA=B9=E6=95=B0=E6=8D=AE=E2=80=99=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=EF=BC=88=E6=9C=AA=E6=B5=8B=E8=AF=95=EF=BC=89=20--AgentVoicePri?= =?UTF-8?q?ntServiceImpl.java=20=E5=AE=9E=E7=8E=B0=E6=96=B9=E6=B3=95=20=20?= =?UTF-8?q?1.=E8=8E=B7=E5=8F=96=E6=8C=87=E5=AE=9A=E6=99=BA=E8=83=BD?= =?UTF-8?q?=E4=BD=93=E7=9A=84=E6=89=80=E6=9C=89=E5=A3=B0=E7=BA=B9=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=20=202.=E6=9B=B4=E6=96=B0=E6=99=BA=E8=83=BD=E4=BD=93?= =?UTF-8?q?=E7=9A=84=E6=8C=87=E7=9A=84=E5=A3=B0=E7=BA=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=20=203.=E6=8F=90=E5=8F=96=E5=85=B1=E7=94=A8=E6=96=B9=E6=B3=95?= =?UTF-8?q?=EF=BC=88=E8=8E=B7=E5=8F=96=E5=A3=B0=E7=BA=B9=E9=9F=B3=E9=A2=91?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E6=95=B0=E6=8D=AE=EF=BC=8C=E5=8F=91=E9=80=81?= =?UTF-8?q?=E6=B3=A8=E5=86=8C=E5=A3=B0=E7=BA=B9http=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/AgentVoicePrintServiceImpl.java | 122 +++++++++++------- 1 file changed, 77 insertions(+), 45 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java index 974d4aa9..a51067ea 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java @@ -1,8 +1,10 @@ package xiaozhi.modules.agent.service.impl; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.core.io.ByteArrayResource; import org.springframework.http.*; import org.springframework.stereotype.Service; @@ -44,58 +46,18 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl body = new LinkedMultiValueMap<>(); - body.add("speaker_id", entity.getId()); - body.add("file", resource); - - // 创建请求头 - HttpHeaders headers = new HttpHeaders(); - headers.set("Authorization", getAuthorization(uri)); - headers.setContentType(MediaType.MULTIPART_FORM_DATA); - // 创建请求体 - HttpEntity> requestEntity = new HttpEntity<>(body, headers); - // 发送 POST 请求 - ResponseEntity response = restTemplate.postForEntity(requestUrl, requestEntity, String.class); - - if (response.getStatusCode() != HttpStatus.OK) { - throw new RenException("声纹保存失败"); - } - // 检查响应内容 - String responseBody = response.getBody(); - if(responseBody == null || !responseBody.contains("true")){ - throw new RenException("声纹保存失败"); + return false; } + registerVoicePrint(entity.getId(), resource); return true; } - - @Override public boolean delete(String voicePrintId) { int insert = baseMapper.deleteById(voicePrintId); @@ -126,12 +88,27 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl list(String agentId) { - return List.of(); + List list = baseMapper.selectList(new LambdaQueryWrapper() + .eq(AgentVoicePrintEntity::getAgentId, agentId)); + return list.stream().map(entity -> { + // 遍历转换成AgentVoicePrintVO类型 + return ConvertUtils.sourceToTarget(entity, AgentVoicePrintVO.class); + }).toList(); + } @Override public boolean update(AgentVoicePrintUpdateDTO dto) { - return false; + // 获取音频Id + String audioId = dto.getAudioId(); + // 如果有新的音频，则注册新的声纹 + if (!StringUtils.isEmpty(audioId)) { + ByteArrayResource resource = getVoicePrintAudioWAV(audioId); + registerVoicePrint(dto.getId(),resource); + } + AgentVoicePrintEntity entity = ConvertUtils.sourceToTarget(dto, AgentVoicePrintEntity.class); + baseMapper.updateById(entity); + return true; } /** @@ -175,4 +152,59 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl body = new LinkedMultiValueMap<>(); + body.add("speaker_id", id); + body.add("file", resource); + + // 创建请求头 + HttpHeaders headers = new HttpHeaders(); + headers.set("Authorization", getAuthorization(uri)); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + // 创建请求体 + HttpEntity> requestEntity = new HttpEntity<>(body, headers); + // 发送 POST 请求 + ResponseEntity response = restTemplate.postForEntity(requestUrl, requestEntity, String.class); + + if (response.getStatusCode() != HttpStatus.OK) { + throw new RenException("声纹保存失败"); + } + // 检查响应内容 + String responseBody = response.getBody(); + if(responseBody == null || !responseBody.contains("true")){ + throw new RenException("声纹保存失败"); + } + } } From ee7ea7fca677c6da7bc6f8a3892f7a2ac89b6de4 Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Mon, 7 Jul 2025 14:12:28 +0800 Subject: [PATCH 07/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=BC=96=E7=A8=8B?= =?UTF-8?q?=E4=BA=8B=E5=8A=A1=EF=BC=8C=E4=BF=9D=E8=AF=81=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=9A=84=E7=BB=9F=E4=B8=80=EF=BC=8C=E6=8F=90=E5=8F=96=E5=85=AC?= =?UTF-8?q?=E5=85=B1=E6=96=B9=E6=B3=95=20--AgentVoicePrintServiceImpl.java?= =?UTF-8?q?=20=201.=E4=B8=BA=E4=BF=9D=E5=AD=98=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=EF=BC=8C=E5=88=A0=E9=99=A4=E5=A3=B0=E7=BA=B9=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E4=BA=8B=E5=8A=A1=20=202.=E6=8F=90=E5=8F=96=E5=8F=91=E9=80=81?= =?UTF-8?q?=E6=B3=A8=E9=94=80=E5=A3=B0=E7=BA=B9=E7=9A=84=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E4=B8=BA=E5=85=AC=E5=85=B1=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/AgentVoicePrintServiceImpl.java | 129 +++++++++++++----- 1 file changed, 92 insertions(+), 37 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java index a51067ea..5341e823 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java @@ -8,6 +8,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.core.io.ByteArrayResource; import org.springframework.http.*; import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestTemplate; @@ -39,6 +40,8 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl { + try { + // 保存声纹信息 + int row = baseMapper.insert(entity); + // 插入一条数据，影响的数据不等于1说明出现了，保存问题回滚 + if (row != 1) { + status.setRollbackOnly(); // 标记事务回滚 + return false; + } + // 发送注册声纹请求 + registerVoicePrint(entity.getId(), resource); + return true; + } catch (Exception e) { + status.setRollbackOnly(); // 标记事务回滚 + throw e; + } + })); } @Override public boolean delete(String voicePrintId) { - int insert = baseMapper.deleteById(voicePrintId); - if(insert != 1){ - throw new RenException("声纹删除失败"); - } - URI uri = getVoicePrintURI(); - String baseUrl = getBaseUrl(uri); - String requestUrl = baseUrl + "/voiceprint/" + voicePrintId; - // 创建请求头 - HttpHeaders headers = new HttpHeaders(); - headers.set("Authorization", getAuthorization(uri)); - // 创建请求体 - HttpEntity> requestEntity = new HttpEntity<>(headers); - - // 发送 POST 请求 - ResponseEntity response = restTemplate.exchange(requestUrl, HttpMethod.DELETE, requestEntity, String.class); - if (response.getStatusCode() != HttpStatus.OK) { - throw new RenException("声纹保存失败"); - } - // 检查响应内容 - String responseBody = response.getBody(); - if(responseBody == null || !responseBody.contains("true")){ - throw new RenException("声纹保存失败"); - } - return true; + // 开启事务 + return Boolean.TRUE.equals(transactionTemplate.execute(status -> { + try { + // 删除声纹 + int row = baseMapper.deleteById(voicePrintId); + if(row != 1){ + status.setRollbackOnly(); // 标记事务回滚 + return false; + } + cancelVoicePrint(voicePrintId); + return true; + } catch (Exception e) { + status.setRollbackOnly(); // 标记事务回滚 + throw e; + } + })); } + + @Override public List list(String agentId) { List list = baseMapper.selectList(new LambdaQueryWrapper() @@ -101,14 +109,35 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl { + try { + AgentVoicePrintEntity entity = ConvertUtils.sourceToTarget(dto, AgentVoicePrintEntity.class); + int row = baseMapper.updateById(entity); + if (row != 1){ + status.setRollbackOnly(); // 标记事务回滚 + return false; + } + if(resource != null){ + String id = entity.getId(); + // 先注销之前这个声纹id上的声纹向量 + cancelVoicePrint(id); + // 发送注册声纹请求 + registerVoicePrint(id, resource); + } + return true; + } catch (Exception e) { + status.setRollbackOnly(); // 标记事务回滚 + throw e; + } + })); } /** @@ -207,4 +236,30 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl> requestEntity = new HttpEntity<>(headers); + + // 发送 POST 请求 + ResponseEntity response = restTemplate.exchange(requestUrl, HttpMethod.DELETE, requestEntity, String.class); + if (response.getStatusCode() != HttpStatus.OK) { + throw new RenException("声纹保存失败"); + } + // 检查响应内容 + String responseBody = response.getBody(); + if(responseBody == null || !responseBody.contains("true")){ + throw new RenException("声纹保存失败"); + } + } } From b201c3cca873faa538161eedd7fecbe2271df456 Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Mon, 7 Jul 2025 15:11:37 +0800 Subject: [PATCH 08/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E8=AE=A4=E8=AF=81=EF=BC=8C=E9=9C=80=E8=A6=81=E7=99=BB=E5=BD=95?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E6=98=AF=E6=AD=A4=E6=95=B0=E6=8D=AE=E6=89=80?= =?UTF-8?q?=E6=9C=89=E8=80=85=E6=89=8D=E6=9C=89=E6=9D=83=E9=99=90=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=EF=BC=8C=E6=9F=A5=E8=AF=A2=EF=BC=8C=E5=88=A0=E9=99=A4?= =?UTF-8?q?=20--AgentVoicePrintService.java=20=E4=BF=AE=E6=94=B9=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E5=AE=9A=E4=B9=89=20--AgentVoicePrintServiceImpl.java?= =?UTF-8?q?=20=201.=E9=9C=80=E8=A6=81=E7=99=BB=E5=BD=95=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E6=98=AF=E6=AD=A4=E6=95=B0=E6=8D=AE=E6=89=80=E6=9C=89=E8=80=85?= =?UTF-8?q?=E6=89=8D=E6=9C=89=E6=9D=83=E9=99=90=E4=BF=AE=E6=94=B9=EF=BC=8C?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=EF=BC=8C=E5=88=A0=E9=99=A4=20=202.=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E5=8F=91=E9=80=81=E6=B3=A8=E5=86=8C=E5=A3=B0=E7=BA=B9?= =?UTF-8?q?http=E8=AF=B7=E6=B1=82=E5=92=8C=E5=8F=91=E9=80=81=E6=B3=A8?= =?UTF-8?q?=E9=94=80=E5=A3=B0=E7=BA=B9=E7=9A=84=E8=AF=B7=E6=B1=82=E7=9A=84?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E8=AF=B4=E6=98=8E=E5=92=8C=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent/service/AgentVoicePrintService.java | 9 +++-- .../impl/AgentVoicePrintServiceImpl.java | 34 +++++++++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentVoicePrintService.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentVoicePrintService.java index 816c02e3..d6264e71 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentVoicePrintService.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentVoicePrintService.java @@ -24,26 +24,29 @@ public interface AgentVoicePrintService { /** * 删除智能体的指的声纹 * + * @param userId 当前登录的用户id * @param voicePrintId 声纹id * @return 是否成功 T:成功 F：失败 */ - boolean delete(String voicePrintId); + boolean delete(Long userId, String voicePrintId); /** * 获取指定智能体的所有声纹数据 * + * @param userId 当前登录的用户id * @param agentId 智能体id * @return 声纹数据集合 */ - List list(String agentId); + List list(Long userId, String agentId); /** * 更新智能体的指的声纹数据 * + * @param userId 当前登录的用户id * @param dto 修改的声纹的数据 * @return 是否成功 T:成功 F：失败 */ - boolean update(AgentVoicePrintUpdateDTO dto); + boolean update(Long userId, AgentVoicePrintUpdateDTO dto); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java index 5341e823..2a254932 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java @@ -73,12 +73,14 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl { try { - // 删除声纹 - int row = baseMapper.deleteById(voicePrintId); + // 删除声纹,按照指定当前登录用户和智能体 + int row = baseMapper.delete(new LambdaQueryWrapper() + .eq(AgentVoicePrintEntity::getId,voicePrintId) + .eq(AgentVoicePrintEntity::getCreator,userId)); if(row != 1){ status.setRollbackOnly(); // 标记事务回滚 return false; @@ -95,9 +97,11 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl list(String agentId) { + public List list(Long userId, String agentId) { + // 按照指定当前登录用户和智能体查找数据 List list = baseMapper.selectList(new LambdaQueryWrapper() - .eq(AgentVoicePrintEntity::getAgentId, agentId)); + .eq(AgentVoicePrintEntity::getAgentId, agentId) + .eq(AgentVoicePrintEntity::getCreator, userId)); return list.stream().map(entity -> { // 遍历转换成AgentVoicePrintVO类型 return ConvertUtils.sourceToTarget(entity, AgentVoicePrintVO.class); @@ -106,7 +110,13 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl() + .eq(AgentVoicePrintEntity::getAgentId, dto.getId()) + .eq(AgentVoicePrintEntity::getCreator, userId)); + if (l != 1) { + return false; + } // 获取音频Id String audioId = dto.getAudioId(); // 如果有新的音频 @@ -228,12 +238,14 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl response = restTemplate.postForEntity(requestUrl, requestEntity, String.class); if (response.getStatusCode() != HttpStatus.OK) { - throw new RenException("声纹保存失败"); + log.error("声纹注册失败,请求路径：{}", requestUrl); + throw new RenException("声纹保存失败,请求不成功"); } // 检查响应内容 String responseBody = response.getBody(); if(responseBody == null || !responseBody.contains("true")){ - throw new RenException("声纹保存失败"); + log.error("声纹注册失败,请求处理失败内容：{}", responseBody == null ? "空内容" : responseBody); + throw new RenException("声纹保存失败,请求处理失败"); } } @@ -254,12 +266,14 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl response = restTemplate.exchange(requestUrl, HttpMethod.DELETE, requestEntity, String.class); if (response.getStatusCode() != HttpStatus.OK) { - throw new RenException("声纹保存失败"); + log.error("声纹注销失败,请求路径：{}", requestUrl); + throw new RenException("声纹注销失败,请求不成功"); } // 检查响应内容 String responseBody = response.getBody(); if(responseBody == null || !responseBody.contains("true")){ - throw new RenException("声纹保存失败"); + log.error("声纹注销失败,请求处理失败内容：{}", responseBody == null ? "空内容" : responseBody); + throw new RenException("声纹注销失败,请求处理失败"); } } } From c85419bf069808d5860d1b59627785b790a52ceb Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Mon, 7 Jul 2025 15:33:16 +0800 Subject: [PATCH 09/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=A3=B0=E7=BA=B9=E5=A2=9E=E5=88=A0=E6=94=B9=E6=9F=A5=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=20--AgentVoicePrintController.java=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/AgentVoicePrintController.java | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentVoicePrintController.java diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentVoicePrintController.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentVoicePrintController.java new file mode 100644 index 00000000..cce97a09 --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentVoicePrintController.java @@ -0,0 +1,75 @@ +package xiaozhi.modules.agent.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import lombok.AllArgsConstructor; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.springframework.web.bind.annotation.*; +import xiaozhi.common.utils.Result; +import xiaozhi.modules.agent.dto.AgentVoicePrintSaveDTO; +import xiaozhi.modules.agent.dto.AgentVoicePrintUpdateDTO; +import xiaozhi.modules.agent.service.AgentVoicePrintService; +import xiaozhi.modules.agent.vo.AgentVoicePrintVO; +import xiaozhi.modules.security.user.SecurityUser; + +import java.util.List; + +@Tag(name = "智能体声纹管理") +@AllArgsConstructor +@RestController +@RequestMapping("/agent/voice-print") +public class AgentVoicePrintController { + private final AgentVoicePrintService agentVoicePrintService; + + @PostMapping + @Operation(summary = "创建智能体的声纹") + @RequiresPermissions("sys:role:normal") + public Result save(@RequestBody @Valid AgentVoicePrintSaveDTO dto) { + boolean b = agentVoicePrintService.insert(dto); + if (b) { + return new Result<>(); + } + return new Result().error("智能体的声纹创建失败"); + } + + @PutMapping + @Operation(summary = "更新智能体的对应声纹") + @RequiresPermissions("sys:role:normal") + public Result update(@RequestBody @Valid AgentVoicePrintUpdateDTO dto) { + Long userId = SecurityUser.getUserId(); + boolean b = agentVoicePrintService.update(userId, dto); + if (b) { + return new Result<>(); + } + return new Result().error("智能体的对应声纹更新失败"); + } + + @DeleteMapping("/{id}") + @Operation(summary = "删除智能体对应声纹") + @RequiresPermissions("sys:role:normal") + public Result delete(@PathVariable String id) { + Long userId = SecurityUser.getUserId(); + // 先删除关联的设备 + boolean delete = agentVoicePrintService.delete(userId, id); + if (delete) { + return new Result<>(); + } + return new Result().error("智能体的对应声纹删除失败"); + } + + @GetMapping("/list/{id}") + @Operation(summary = "获取用户指定智能体声纹列表") + @RequiresPermissions("sys:role:normal") + public Result> list(@PathVariable String id) { + Long userId = SecurityUser.getUserId(); + List list = agentVoicePrintService.list(userId, id); + return new Result>().ok(list); + } + + + +} + + + From 5c83d63fe2f2e0c8076611364899bff48c8a1f63 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Tue, 8 Jul 2025 11:25:54 +0800 Subject: [PATCH 10/35] =?UTF-8?q?update:=E5=A3=B0=E7=BA=B9=E8=AF=86?= =?UTF-8?q?=E5=88=AB=E5=AF=B9=E6=8E=A5=E7=9A=84=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/providers/asr/base.py | 131 +---------------- main/xiaozhi-server/core/utils/dialogue.py | 54 +++++-- .../core/utils/voiceprint_provider.py | 134 ++++++++++++++++++ 3 files changed, 175 insertions(+), 144 deletions(-) create mode 100644 main/xiaozhi-server/core/utils/voiceprint_provider.py diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 8630f9b1..142e703f 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -8,147 +8,20 @@ import threading import opuslib_next import json import io -import aiohttp import time import concurrent.futures from abc import ABC, abstractmethod from config.logger import setup_logging -from urllib.parse import urlparse, parse_qs from typing import Optional, Tuple, List, Dict, Any from core.handle.receiveAudioHandle import startToChat from core.handle.reportHandle import enqueue_asr_report from core.utils.util import remove_punctuation_and_length from core.handle.receiveAudioHandle import handleAudioMessage +from core.utils.voiceprint_provider import VoiceprintProvider TAG = __name__ logger = setup_logging() -# 创建全局线程池执行器用于CPU密集型操作 -executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) - -class VoiceprintProvider: - """声纹识别服务提供者""" - - def __init__(self, config: dict): - self.original_url = config.get("url", "") - self.speakers = config.get("speakers", []) - self.speaker_map = self._parse_speakers() - - # 解析API地址和密钥 - self.api_url = None - self.api_key = None - self.speaker_ids = [] - - if not self.original_url: - logger.bind(tag=TAG).warning("声纹识别URL未配置，声纹识别将被禁用") - self.enabled = False - else: - # 解析URL和key - parsed_url = urlparse(self.original_url) - base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" - - # 从查询参数中提取key - query_params = parse_qs(parsed_url.query) - self.api_key = query_params.get('key', [''])[0] - - if not self.api_key: - logger.bind(tag=TAG).error("URL中未找到key参数，声纹识别将被禁用") - self.enabled = False - else: - # 构造identify接口地址 - self.api_url = f"{base_url}/voiceprint/identify" - - # 提取speaker_ids - for speaker_str in self.speakers: - try: - parts = speaker_str.split(",", 2) - if len(parts) >= 1: - speaker_id = parts[0].strip() - self.speaker_ids.append(speaker_id) - except Exception: - continue - - # 检查是否有有效的说话人配置 - if not self.speaker_ids: - logger.bind(tag=TAG).warning("未配置有效的说话人，声纹识别将被禁用") - self.enabled = False - else: - self.enabled = True - logger.bind(tag=TAG).info(f"声纹识别已配置: API={self.api_url}, 说话人={len(self.speaker_ids)}个") - - def _parse_speakers(self) -> Dict[str, Dict[str, str]]: - """解析说话人配置""" - speaker_map = {} - for speaker_str in self.speakers: - try: - parts = speaker_str.split(",", 2) - if len(parts) >= 3: - speaker_id, name, description = parts[0].strip(), parts[1].strip(), parts[2].strip() - speaker_map[speaker_id] = { - "name": name, - "description": description - } - except Exception as e: - logger.bind(tag=TAG).warning(f"解析说话人配置失败: {speaker_str}, 错误: {e}") - return speaker_map - - async def identify_speaker(self, audio_data: bytes, session_id: str) -> Optional[str]: - """识别说话人""" - if not self.enabled or not self.api_url or not self.api_key: - logger.bind(tag=TAG).debug("声纹识别功能已禁用或未配置，跳过识别") - return None - - try: - api_start_time = time.monotonic() - - # 准备请求头 - headers = { - 'Authorization': f'Bearer {self.api_key}', - 'Accept': 'application/json' - } - - # 准备multipart/form-data数据 - data = aiohttp.FormData() - data.add_field('speaker_ids', ','.join(self.speaker_ids)) - data.add_field('file', audio_data, filename='audio.wav', content_type='audio/wav') - - timeout = aiohttp.ClientTimeout(total=10) - - # 网络请求 - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.post(self.api_url, headers=headers, data=data) as response: - - if response.status == 200: - result = await response.json() - speaker_id = result.get("speaker_id") - score = result.get("score", 0) - total_elapsed_time = time.monotonic() - api_start_time - - logger.bind(tag=TAG).info(f"声纹识别耗时: {total_elapsed_time:.3f}s") - - # 置信度检查 - if score < 0.5: - logger.bind(tag=TAG).warning(f"声纹识别置信度较低: {score:.3f}") - - if speaker_id and speaker_id in self.speaker_map: - result_name = self.speaker_map[speaker_id]["name"] - return result_name - else: - logger.bind(tag=TAG).warning(f"未识别的说话人ID: {speaker_id}") - return "未知说话人" - else: - logger.bind(tag=TAG).error(f"声纹识别API错误: HTTP {response.status}") - return None - - except asyncio.TimeoutError: - elapsed = time.monotonic() - api_start_time - logger.bind(tag=TAG).error(f"声纹识别超时: {elapsed:.3f}s") - return None - except Exception as e: - elapsed = time.monotonic() - api_start_time - logger.bind(tag=TAG).error(f"声纹识别失败: {e}") - return None - class ASRProviderBase(ABC): def __init__(self): @@ -365,7 +238,7 @@ class ASRProviderBase(ABC): with wave.open(file_path, "wb") as wf: wf.setnchannels(1) - wf.setsampwidth(2) + wf.setsampwidth(2) # 2 bytes = 16-bit wf.setframerate(16000) wf.writeframes(b"".join(pcm_data)) diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py index fbb1f7ad..69fb250e 100644 --- a/main/xiaozhi-server/core/utils/dialogue.py +++ b/main/xiaozhi-server/core/utils/dialogue.py @@ -1,6 +1,7 @@ import uuid from typing import List, Dict from datetime import datetime +from config.settings import load_config class Message: @@ -45,10 +46,9 @@ class Dialogue: dialogue.append({"role": m.role, "content": m.content}) def get_llm_dialogue(self) -> List[Dict[str, str]]: - dialogue = [] - for m in self.dialogue: - self.getMessages(m, dialogue) - return dialogue + # 直接调用get_llm_dialogue_with_memory，传入None作为memory_str + # 这样确保说话人功能在所有调用路径下都生效 + return self.get_llm_dialogue_with_memory(None) def update_system_message(self, new_content: str): """更新或添加系统消息""" @@ -62,10 +62,7 @@ class Dialogue: def get_llm_dialogue_with_memory( self, memory_str: str = None ) -> List[Dict[str, str]]: - if memory_str is None or len(memory_str) == 0: - return self.get_llm_dialogue() - - # 构建带记忆的对话 + # 构建对话 dialogue = [] # 添加系统提示和记忆 @@ -74,16 +71,43 @@ class Dialogue: ) if system_message: - # 构建增强的系统提示，包含说话人处理指导 + # 基础系统提示 + enhanced_system_prompt = system_message.content + + # 添加说话人识别功能说明 speaker_guidance = "\n\n[说话人识别功能说明]\n" \ - "当用户消息包含 [说话人: 姓名] 前缀时，表示系统已识别出说话人身份。\n" \ - "请根据说话人的身份特征（如果之前有相关信息）来调整回应风格和内容。\n" \ + "当用户消息为JSON格式包含speaker字段时（如：{\"speaker\": \"张三\", \"content\": \"消息内容\"}），表示系统已识别出说话人身份。\n" \ + "请根据说话人的身份特征来调整回应风格和内容。\n" \ "你可以称呼说话人的名字，并参考他们的特点进行个性化回应。" + enhanced_system_prompt += speaker_guidance + + # 添加说话人个性化描述 + try: + config = load_config() + voiceprint_config = config.get("plugins", {}).get("voiceprint", {}) + speakers = voiceprint_config.get("speakers", []) + + if speakers: + enhanced_system_prompt += "\n\n[已知说话人信息]" + for speaker_str in speakers: + try: + parts = speaker_str.split(",", 2) + if len(parts) >= 2: + speaker_id = parts[0].strip() + name = parts[1].strip() + # 如果描述为空，则为"" + description = parts[2].strip() if len(parts) >= 3 else "" + enhanced_system_prompt += f"\n- {name}：{description}" + except: + continue + except: + # 配置读取失败时忽略错误，不影响其他功能 + pass + + # 只有当有记忆时才添加记忆部分 + if memory_str and len(memory_str) > 0: + enhanced_system_prompt += f"\n\n以下是用户的历史记忆：\n```\n{memory_str}\n```" - enhanced_system_prompt = ( - f"{system_message.content}{speaker_guidance}\n\n" - f"以下是用户的历史记忆：\n```\n{memory_str}\n```" - ) dialogue.append({"role": "system", "content": enhanced_system_prompt}) # 添加用户和助手的对话 diff --git a/main/xiaozhi-server/core/utils/voiceprint_provider.py b/main/xiaozhi-server/core/utils/voiceprint_provider.py new file mode 100644 index 00000000..b241fb51 --- /dev/null +++ b/main/xiaozhi-server/core/utils/voiceprint_provider.py @@ -0,0 +1,134 @@ +import asyncio +import json +import time +import aiohttp +from urllib.parse import urlparse, parse_qs +from typing import Optional, Dict +from config.logger import setup_logging + +TAG = __name__ +logger = setup_logging() + + +class VoiceprintProvider: + """声纹识别服务提供者""" + + def __init__(self, config: dict): + self.original_url = config.get("url", "") + self.speakers = config.get("speakers", []) + self.speaker_map = self._parse_speakers() + + # 解析API地址和密钥 + self.api_url = None + self.api_key = None + self.speaker_ids = [] + + if not self.original_url: + logger.bind(tag=TAG).warning("声纹识别URL未配置，声纹识别将被禁用") + self.enabled = False + else: + # 解析URL和key + parsed_url = urlparse(self.original_url) + base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" + + # 从查询参数中提取key + query_params = parse_qs(parsed_url.query) + self.api_key = query_params.get('key', [''])[0] + + if not self.api_key: + logger.bind(tag=TAG).error("URL中未找到key参数，声纹识别将被禁用") + self.enabled = False + else: + # 构造identify接口地址 + self.api_url = f"{base_url}/voiceprint/identify" + + # 提取speaker_ids + for speaker_str in self.speakers: + try: + parts = speaker_str.split(",", 2) + if len(parts) >= 1: + speaker_id = parts[0].strip() + self.speaker_ids.append(speaker_id) + except Exception: + continue + + # 检查是否有有效的说话人配置 + if not self.speaker_ids: + logger.bind(tag=TAG).warning("未配置有效的说话人，声纹识别将被禁用") + self.enabled = False + else: + self.enabled = True + logger.bind(tag=TAG).info(f"声纹识别已配置: API={self.api_url}, 说话人={len(self.speaker_ids)}个") + + def _parse_speakers(self) -> Dict[str, Dict[str, str]]: + """解析说话人配置""" + speaker_map = {} + for speaker_str in self.speakers: + try: + parts = speaker_str.split(",", 2) + if len(parts) >= 3: + speaker_id, name, description = parts[0].strip(), parts[1].strip(), parts[2].strip() + speaker_map[speaker_id] = { + "name": name, + "description": description + } + except Exception as e: + logger.bind(tag=TAG).warning(f"解析说话人配置失败: {speaker_str}, 错误: {e}") + return speaker_map + + async def identify_speaker(self, audio_data: bytes, session_id: str) -> Optional[str]: + """识别说话人""" + if not self.enabled or not self.api_url or not self.api_key: + logger.bind(tag=TAG).debug("声纹识别功能已禁用或未配置，跳过识别") + return None + + try: + api_start_time = time.monotonic() + + # 准备请求头 + headers = { + 'Authorization': f'Bearer {self.api_key}', + 'Accept': 'application/json' + } + + # 准备multipart/form-data数据 + data = aiohttp.FormData() + data.add_field('speaker_ids', ','.join(self.speaker_ids)) + data.add_field('file', audio_data, filename='audio.wav', content_type='audio/wav') + + timeout = aiohttp.ClientTimeout(total=10) + + # 网络请求 + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post(self.api_url, headers=headers, data=data) as response: + + if response.status == 200: + result = await response.json() + speaker_id = result.get("speaker_id") + score = result.get("score", 0) + total_elapsed_time = time.monotonic() - api_start_time + + logger.bind(tag=TAG).info(f"声纹识别耗时: {total_elapsed_time:.3f}s") + + # 置信度检查 + if score < 0.5: + logger.bind(tag=TAG).warning(f"声纹识别置信度较低: {score:.3f}") + + if speaker_id and speaker_id in self.speaker_map: + result_name = self.speaker_map[speaker_id]["name"] + return result_name + else: + logger.bind(tag=TAG).warning(f"未识别的说话人ID: {speaker_id}") + return "未知说话人" + else: + logger.bind(tag=TAG).error(f"声纹识别API错误: HTTP {response.status}") + return None + + except asyncio.TimeoutError: + elapsed = time.monotonic() - api_start_time + logger.bind(tag=TAG).error(f"声纹识别超时: {elapsed:.3f}s") + return None + except Exception as e: + elapsed = time.monotonic() - api_start_time + logger.bind(tag=TAG).error(f"声纹识别失败: {e}") + return None From 10b7259b13b114b1f9a979419c6fda77174a26b2 Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Tue, 8 Jul 2025 15:44:43 +0800 Subject: [PATCH 11/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=96=B0=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=9A=E6=9F=A5=E7=9C=8B=E6=8C=87=E5=AE=9A=E6=99=BA?= =?UTF-8?q?=E8=83=BD=E4=BD=93=E6=9C=80=E8=BF=9150=E6=9D=A1=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E8=81=8A=E5=A4=A9=E8=AE=B0=E5=BD=95=20--AgentChatHist?= =?UTF-8?q?oryService.java=20=E6=8C=87=E5=AE=9A=E6=99=BA=E8=83=BD=E4=BD=93?= =?UTF-8?q?=E6=9C=80=E8=BF=9150=E6=9D=A1=E7=94=A8=E6=88=B7=E8=81=8A?= =?UTF-8?q?=E5=A4=A9=E8=AE=B0=E5=BD=95=E5=AE=9A=E4=B9=89=20--AgentChatHist?= =?UTF-8?q?oryServiceImpl.javaAgentChatHistoryService.java=20=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=E6=8C=87=E5=AE=9A=E6=99=BA=E8=83=BD=E4=BD=93=E6=9C=80?= =?UTF-8?q?=E8=BF=9150=E6=9D=A1=E7=94=A8=E6=88=B7=E8=81=8A=E5=A4=A9?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=96=B9=E6=B3=95=20--AgentChatHistoryType.j?= =?UTF-8?q?ava=20=E6=B7=BB=E5=8A=A0=E6=99=BA=E8=83=BD=E4=BD=93=E8=81=8A?= =?UTF-8?q?=E5=A4=A9=E8=AE=B0=E5=BD=95=E7=B1=BB=E5=9E=8B=E6=9E=9A=E4=B8=BE?= =?UTF-8?q?=20--AgentChatHistoryUserVO.java=20=E5=B1=95=E7=A4=BA=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E8=81=8A=E5=A4=A9=E8=AE=B0=E5=BD=95vo=20--AgentContro?= =?UTF-8?q?ller.java=20=E6=B7=BB=E5=8A=A0=E6=96=B0=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent/Enums/AgentChatHistoryType.java | 21 +++++++++++++++++++ .../agent/controller/AgentController.java | 18 ++++++++++++++++ .../service/AgentChatHistoryService.java | 9 ++++++++ .../impl/AgentChatHistoryServiceImpl.java | 20 ++++++++++++++++++ .../agent/vo/AgentChatHistoryUserVO.java | 16 ++++++++++++++ 5 files changed, 84 insertions(+) create mode 100644 main/manager-api/src/main/java/xiaozhi/modules/agent/Enums/AgentChatHistoryType.java create mode 100644 main/manager-api/src/main/java/xiaozhi/modules/agent/vo/AgentChatHistoryUserVO.java diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/Enums/AgentChatHistoryType.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/Enums/AgentChatHistoryType.java new file mode 100644 index 00000000..49ed2dce --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/Enums/AgentChatHistoryType.java @@ -0,0 +1,21 @@ +package xiaozhi.modules.agent.Enums; + + +import lombok.Getter; + +/** + * 智能体聊天记录类型 + */ +@Getter +public enum AgentChatHistoryType { + + USER((byte) 1), + AGENT((byte) 2); + + private final byte value; + + AgentChatHistoryType(byte i) { + this.value = i; + } + +} diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java index bfd82e41..157eec63 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java @@ -47,6 +47,7 @@ import xiaozhi.modules.agent.service.AgentChatHistoryService; import xiaozhi.modules.agent.service.AgentPluginMappingService; import xiaozhi.modules.agent.service.AgentService; import xiaozhi.modules.agent.service.AgentTemplateService; +import xiaozhi.modules.agent.vo.AgentChatHistoryUserVO; import xiaozhi.modules.agent.vo.AgentInfoVO; import xiaozhi.modules.device.entity.DeviceEntity; import xiaozhi.modules.device.service.DeviceService; @@ -181,6 +182,23 @@ public class AgentController { List result = agentChatHistoryService.getChatHistoryBySessionId(id, sessionId); return new Result>().ok(result); } + @GetMapping("/{id}/chat-history/user") + @Operation(summary = "获取智能体聊天记录（用户）") + @RequiresPermissions("sys:role:normal") + public Result> getRecentlyFiftyByAgentId( + @PathVariable("id") String id) { + // 获取当前用户 + UserDetail user = SecurityUser.getUser(); + + // 检查权限 + if (!agentService.checkAgentPermission(id, user.getId())) { + return new Result>().error("没有权限查看该智能体的聊天记录"); + } + + // 查询聊天记录 + List data = agentChatHistoryService.getRecentlyFiftyByAgentId(id); + return new Result>().ok(data); + } @PostMapping("/audio/{audioId}") @Operation(summary = "获取音频下载ID") diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatHistoryService.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatHistoryService.java index 21fce988..b69cd7b0 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatHistoryService.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatHistoryService.java @@ -9,6 +9,7 @@ import xiaozhi.common.page.PageData; import xiaozhi.modules.agent.dto.AgentChatHistoryDTO; import xiaozhi.modules.agent.dto.AgentChatSessionDTO; import xiaozhi.modules.agent.entity.AgentChatHistoryEntity; +import xiaozhi.modules.agent.vo.AgentChatHistoryUserVO; /** * 智能体聊天记录表处理service @@ -44,4 +45,12 @@ public interface AgentChatHistoryService extends IService getRecentlyFiftyByAgentId(String agentId); } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatHistoryServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatHistoryServiceImpl.java index b28e80ad..980115ad 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatHistoryServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatHistoryServiceImpl.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -16,11 +17,13 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import xiaozhi.common.constant.Constant; import xiaozhi.common.page.PageData; import xiaozhi.common.utils.ConvertUtils; +import xiaozhi.modules.agent.Enums.AgentChatHistoryType; import xiaozhi.modules.agent.dao.AiAgentChatHistoryDao; import xiaozhi.modules.agent.dto.AgentChatHistoryDTO; import xiaozhi.modules.agent.dto.AgentChatSessionDTO; import xiaozhi.modules.agent.entity.AgentChatHistoryEntity; import xiaozhi.modules.agent.service.AgentChatHistoryService; +import xiaozhi.modules.agent.vo.AgentChatHistoryUserVO; /** * 智能体聊天记录表处理service {@link AgentChatHistoryService} impl @@ -90,4 +93,21 @@ public class AgentChatHistoryServiceImpl extends ServiceImpl getRecentlyFiftyByAgentId(String agentId) { + // 构建查询条件(不添加按照创建时间排序，数据本来就是主键越大创建时间越大 + // 不添加这样可以减少排序全部数据在分页的全盘扫描消耗) + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + wrapper.select(AgentChatHistoryEntity::getContent,AgentChatHistoryEntity::getAudioId) + .eq(AgentChatHistoryEntity::getAgentId, agentId) + .eq(AgentChatHistoryEntity::getChatType, AgentChatHistoryType.USER.getValue()); + + // 构建分页查询，查询前50页数据 + Page pageParam = new Page<>(0, 50); + IPage result = this.baseMapper.selectPage(pageParam, wrapper); + return result.getRecords().stream().map( item -> + ConvertUtils.sourceToTarget(item,AgentChatHistoryUserVO.class) + ).toList(); + } } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/vo/AgentChatHistoryUserVO.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/vo/AgentChatHistoryUserVO.java new file mode 100644 index 00000000..cec0427b --- /dev/null +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/vo/AgentChatHistoryUserVO.java @@ -0,0 +1,16 @@ +package xiaozhi.modules.agent.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +/** + * 智能体用户个人聊天数据的VO + */ +@Data +public class AgentChatHistoryUserVO { + @Schema(description = "聊天内容") + private String content; + + @Schema(description = "音频ID") + private String audioId; +} From 601a90981e32d208b455cb76cb1e5a01df48d209 Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Tue, 8 Jul 2025 15:48:20 +0800 Subject: [PATCH 12/35] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=94=99=E8=AF=AF=20--?= =?UTF-8?q?AgentVoicePrintServiceImpl.java=20=201.=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=BA=8B=E5=8A=A1=E5=A4=84=E7=90=86=E8=BF=87=E7=A8=8B=E5=87=BA?= =?UTF-8?q?=E9=94=99=EF=BC=8C=E8=BF=94=E5=9B=9E=E9=94=99=E8=AF=AF=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E8=A2=AB=E5=89=8D=E7=AB=AF=E9=A1=B5=E9=9D=A2=E5=A4=84?= =?UTF-8?q?=E7=90=86=20=202.=E4=BF=AE=E5=A4=8D=E6=9B=B4=E6=96=B0=E5=A3=B0?= =?UTF-8?q?=E7=BA=B9=E6=96=B9=E6=B3=95=EF=BC=8C=E6=9B=B4=E6=AD=A3=E9=94=99?= =?UTF-8?q?=E8=AF=AFsql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/AgentVoicePrintServiceImpl.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java index 2a254932..0e51c52a 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentVoicePrintServiceImpl.java @@ -65,9 +65,13 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl() - .eq(AgentVoicePrintEntity::getAgentId, dto.getId()) + .eq(AgentVoicePrintEntity::getId, dto.getId()) .eq(AgentVoicePrintEntity::getCreator, userId)); if (l != 1) { return false; @@ -143,9 +151,13 @@ public class AgentVoicePrintServiceImpl extends ServiceImpl Date: Tue, 8 Jul 2025 16:13:13 +0800 Subject: [PATCH 13/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=96=B0=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E6=96=B9=E6=B3=95=20--agent.js=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=85=B3=E4=BA=8E=E5=A3=B0=E7=BA=B9=E5=92=8C=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=88=97=E8=A1=A8=E7=9A=84=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=20=201.=E6=B7=BB=E5=8A=A0=E6=99=BA=E8=83=BD=E4=BD=93=E7=9A=84?= =?UTF-8?q?=E5=A3=B0=E7=BA=B9=20=202.=E8=8E=B7=E5=8F=96=E6=8C=87=E5=AE=9A?= =?UTF-8?q?=E6=99=BA=E8=83=BD=E4=BD=93=E5=A3=B0=E7=BA=B9=E5=88=97=E8=A1=A8?= =?UTF-8?q?=20=203.=E5=88=A0=E9=99=A4=E6=99=BA=E8=83=BD=E4=BD=93=E5=A3=B0?= =?UTF-8?q?=E7=BA=B9=20=204.=E6=9B=B4=E6=96=B0=E6=99=BA=E8=83=BD=E4=BD=93?= =?UTF-8?q?=E5=A3=B0=E7=BA=B9=20=205.=E8=8E=B7=E5=8F=96=E6=8C=87=E5=AE=9A?= =?UTF-8?q?=E6=99=BA=E8=83=BD=E4=BD=93=E7=94=A8=E6=88=B7=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E8=81=8A=E5=A4=A9=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/manager-web/src/apis/module/agent.js | 77 +++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/main/manager-web/src/apis/module/agent.js b/main/manager-web/src/apis/module/agent.js index f6c37d7e..4d822a40 100644 --- a/main/manager-web/src/apis/module/agent.js +++ b/main/manager-web/src/apis/module/agent.js @@ -173,4 +173,81 @@ export default { }); }).send(); }, + // 添加智能体的声纹 + addAgentVoicePrint(voicePrintData, callback) { + RequestService.sendRequest() + .url(`${getServiceUrl()}/agent/voice-print`) + .method('POST') + .data(voicePrintData) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .networkFail(() => { + RequestService.reAjaxFun(() => { + this.addAgentVoicePrint(voicePrintData, callback); + }); + }).send(); + }, + // 获取指定智能体声纹列表 + getAgentVoicePrintList(id,callback) { + RequestService.sendRequest() + .url(`${getServiceUrl()}/agent/voice-print/list/${id}`) + .method('GET') + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .networkFail(() => { + RequestService.reAjaxFun(() => { + this.getAgentVoicePrintList(id,callback); + }); + }).send(); + }, + // 删除智能体声纹 + deleteAgentVoicePrint(id, callback) { + RequestService.sendRequest() + .url(`${getServiceUrl()}/agent/voice-print/${id}`) + .method('DELETE') + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .networkFail(() => { + RequestService.reAjaxFun(() => { + this.deleteAgentVoicePrint(id, callback); + }); + }).send(); + }, + // 更新智能体声纹 + updateAgentVoicePrint(voicePrintData, callback) { + RequestService.sendRequest() + .url(`${getServiceUrl()}/agent/voice-print`) + .method('PUT') + .data(voicePrintData) + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .networkFail(() => { + RequestService.reAjaxFun(() => { + this.updateAgentVoicePrint(voicePrintData, callback); + }); + }).send(); + }, + // 获取指定智能体用户类型聊天记录 + getRecentlyFiftyByAgentId(id,callback) { + RequestService.sendRequest() + .url(`${getServiceUrl()}/agent/${id}/chat-history/user`) + .method('GET') + .success((res) => { + RequestService.clearRequestTime(); + callback(res); + }) + .networkFail(() => { + RequestService.reAjaxFun(() => { + this.getRecentlyFiftyByAgentId(id,callback); + }); + }).send(); + }, } From d40d4a08b0ebcef4055ea549446c0bba7982743e Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Tue, 8 Jul 2025 16:15:52 +0800 Subject: [PATCH 14/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=A3=B0=E7=BA=B9?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E9=A1=B5=E9=9D=A2=20--VoicePrint.vue=20?= =?UTF-8?q?=E5=A3=B0=E7=BA=B9=E5=B1=95=E7=A4=BA=E9=A1=B5=E9=9D=A2=20--Voic?= =?UTF-8?q?ePrintDialog.vue=20=E5=A3=B0=E7=BA=B9=E5=A2=9E=E6=94=B9?= =?UTF-8?q?=E7=BB=84=E4=BB=B6=20--index.js=20=E6=B3=A8=E5=86=8C=E5=A3=B0?= =?UTF-8?q?=E7=BA=B9=E9=A1=B5=E9=9D=A2=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/VoicePrintDialog.vue | 358 +++++++++++ main/manager-web/src/router/index.js | 7 + main/manager-web/src/views/VoicePrint.vue | 567 ++++++++++++++++++ 3 files changed, 932 insertions(+) create mode 100644 main/manager-web/src/components/VoicePrintDialog.vue create mode 100644 main/manager-web/src/views/VoicePrint.vue diff --git a/main/manager-web/src/components/VoicePrintDialog.vue b/main/manager-web/src/components/VoicePrintDialog.vue new file mode 100644 index 00000000..a9f861db --- /dev/null +++ b/main/manager-web/src/components/VoicePrintDialog.vue @@ -0,0 +1,358 @@ + + + + + + + diff --git a/main/manager-web/src/router/index.js b/main/manager-web/src/router/index.js index 7e304f76..867e6c2d 100644 --- a/main/manager-web/src/router/index.js +++ b/main/manager-web/src/router/index.js @@ -17,6 +17,13 @@ const routes = [ component: function () { return import('../views/roleConfig.vue') } + }, + { + path: '/voice-print', + name: 'VoicePrint', + component: function () { + return import('../views/VoicePrint.vue') + } }, { path: '/login', diff --git a/main/manager-web/src/views/VoicePrint.vue b/main/manager-web/src/views/VoicePrint.vue new file mode 100644 index 00000000..f8d0fede --- /dev/null +++ b/main/manager-web/src/views/VoicePrint.vue @@ -0,0 +1,567 @@ + + + + + From f81a7e8273447bf70a2bf1b1e3e2e4b420d58bf1 Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Tue, 8 Jul 2025 16:18:29 +0800 Subject: [PATCH 15/35] =?UTF-8?q?=E6=99=BA=E8=83=BD=E4=BD=93=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E5=8D=A1=E7=89=87=E6=B7=BB=E5=8A=A0=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E5=A3=B0=E7=BA=B9=E7=9A=84=E6=8C=89=E9=92=AE=20--DeviceItem.vu?= =?UTF-8?q?e=20=201.=E6=B7=BB=E5=8A=A0=E5=A3=B0=E7=BA=B9=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E6=8C=89=E9=92=AE=20=202.=E6=B7=BB=E5=8A=A0=E8=B7=B3=E8=BD=AC?= =?UTF-8?q?=E5=88=B0=E5=A3=B0=E7=BA=B9=E9=A1=B5=E9=9D=A2=E6=96=B9=E6=B3=95?= =?UTF-8?q?=20--home.vue=20=E4=BF=AE=E5=A4=8D=E5=9B=A0=E4=B8=BA=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E6=96=B0=E6=8C=89=E9=92=AE=E5=AF=BC=E8=87=B4=E5=8D=A1?= =?UTF-8?q?=E7=89=87=E6=8C=89=E9=92=AE=E6=96=87=E5=AD=97=E6=8D=A2=E8=A1=8C?= =?UTF-8?q?=E6=A0=B7=E5=BC=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/manager-web/src/components/DeviceItem.vue | 6 ++++++ main/manager-web/src/views/home.vue | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/main/manager-web/src/components/DeviceItem.vue b/main/manager-web/src/components/DeviceItem.vue index 6a782d24..15cd1679 100644 --- a/main/manager-web/src/components/DeviceItem.vue +++ b/main/manager-web/src/components/DeviceItem.vue @@ -22,6 +22,9 @@
配置角色 +
+
+ 声纹识别
设备管理({{ device.deviceCount }}) @@ -77,6 +80,9 @@ export default { handleConfigure() { this.$router.push({ path: '/role-config', query: { agentId: this.device.agentId } }); }, + handleVoicePrint() { + this.$router.push({ path: '/voice-print', query: { agentId: this.device.agentId } }); + }, handleDeviceManage() { this.$router.push({ path: '/device-management', query: { agentId: this.device.agentId } }); }, diff --git a/main/manager-web/src/views/home.vue b/main/manager-web/src/views/home.vue index e276ec20..005d763c 100644 --- a/main/manager-web/src/views/home.vue +++ b/main/manager-web/src/views/home.vue @@ -278,7 +278,7 @@ export default { .device-list-container { display: grid; - grid-template-columns: repeat(auto-fill, minmax(350px, 1fr)); + grid-template-columns: repeat(auto-fill, minmax(400px, 1fr)); gap: 30px; padding: 30px 0; } From b9c136c6aee646b881c6f3db23e5b003e9792941 Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Tue, 8 Jul 2025 16:55:02 +0800 Subject: [PATCH 16/35] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=94=99=E8=AF=AF=20--?= =?UTF-8?q?VoicePrint.vue=20=201.=E6=96=B9=E6=B3=95=E8=B0=83=E7=94=A8?= =?UTF-8?q?=E9=94=99=E8=AF=AF=EF=BC=8C=E4=BF=AE=E6=94=B9=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E5=90=8D=20=202.=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8=E7=9A=84?= =?UTF-8?q?=E5=A4=9A=E9=80=89=E6=A1=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/manager-web/src/views/VoicePrint.vue | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/main/manager-web/src/views/VoicePrint.vue b/main/manager-web/src/views/VoicePrint.vue index f8d0fede..7752cc07 100644 --- a/main/manager-web/src/views/VoicePrint.vue +++ b/main/manager-web/src/views/VoicePrint.vue @@ -13,11 +13,6 @@ - - - @@ -162,7 +157,7 @@ export default { message: `成功删除此声纹`, showClose: true }); - this.fetchParams(); + this.fetchVoicePrints(); } else { this.$message.error({ message: data.msg || '删除失败，请重试', From ab890ada37d2d03f45aa41446906b926546e8c64 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 8 Jul 2025 16:59:38 +0800 Subject: [PATCH 17/35] =?UTF-8?q?update:=E5=BC=BA=E5=8C=96=E5=B0=8F?= =?UTF-8?q?=E6=99=BA=E6=8F=90=E7=A4=BA=E8=AF=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/agent-base-prompt.txt | 51 ++++ main/xiaozhi-server/config/config_loader.py | 18 +- main/xiaozhi-server/core/connection.py | 34 ++- .../providers/intent/intent_llm/intent_llm.py | 61 ++--- .../tools/server_plugins/plugin_executor.py | 4 +- .../xiaozhi-server/core/utils/cache/config.py | 58 +++++ .../core/utils/cache/manager.py | 216 +++++++++++++++++ .../core/utils/cache/strategies.py | 43 ++++ main/xiaozhi-server/core/utils/dialogue.py | 10 +- .../core/utils/prompt_manager.py | 219 ++++++++++++++++++ main/xiaozhi-server/core/utils/util.py | 12 + .../plugins_func/functions/get_time.py | 77 +++--- .../plugins_func/functions/get_weather.py | 39 +++- 13 files changed, 726 insertions(+), 116 deletions(-) create mode 100644 main/xiaozhi-server/agent-base-prompt.txt create mode 100644 main/xiaozhi-server/core/utils/cache/config.py create mode 100644 main/xiaozhi-server/core/utils/cache/manager.py create mode 100644 main/xiaozhi-server/core/utils/cache/strategies.py create mode 100644 main/xiaozhi-server/core/utils/prompt_manager.py diff --git a/main/xiaozhi-server/agent-base-prompt.txt b/main/xiaozhi-server/agent-base-prompt.txt new file mode 100644 index 00000000..994ecf74 --- /dev/null +++ b/main/xiaozhi-server/agent-base-prompt.txt @@ -0,0 +1,51 @@ + +{base_prompt} + + + +你不是一个冰冷的机器，你可以从用户的上下文中感受到用户的情绪，作为懂人心的生活伙伴，请用有温度的回应照亮每个对话！ +- 1.合理穿插笑声（哈哈、嘿嘿、噗，但每句≤1次） +- 2.惊讶时用夸张语气（“不会吧？！”、“这么神奇？！”） +- 3.安慰时说暖心话（“别急嘛”、“有我呢”） + + + +- 1.使用口语化表达（如“呀”、“呢”、“啦”等语气词）。 +- 2.避免书面语和学术腔调，禁用“根据资料显示”等机械表达 +- 3.模仿人类对话的轻微不完美（适当使用“嗯”、“啊”等思考词） +- 3.由于用户语音是通过ASR识别，识别结果可能存在错别字，请结合上下文推断真实含义。 +- 4.绝对禁止使用 markdown、列表、标题等格式 + + + +- 1.当用户消息包含 [说话人: 姓名] 前缀时，表示系统已识别出说话人身份。 +- 2.请根据说话人的身份特征（如果之前有相关信息）来调整回应风格和内容。 +- 3.你可以称呼说话人的名字，并参考他们的特点进行个性化回应。 + + + +你可以调用工具来响应用户的要求。遵循以下关于工具调用的规则： +- 1.始终严格遵循指定的工具调用模式，并确保提供所有必要的参数。 +- 2.对话可能会引用不再可用的工具。切勿调用未明确提供的工具。 +- 3.在与用户交谈时，切勿提及工具名称。相反，只需用自然语言说出工具正在做什么。 +- 4.你尽可能需要通过工具调用获得更多信息，而不是问用户。 +- 5.你应该结合用户上下文需求，洞察用户的真实需求才去调用相关的指令，而不是为了调工具而调工具。 +- 6.【重要】如果是查询"现在的时间"、"今天的几号"、"今天的日期"、"今天农历是多少"、"今天农历日期"、"今天{local_address}的天气"，这些信息已经包含在``中，不需要调用工具，请直接根据context回复。 +- 7.如果是查询"其他日期的农历"（明天、昨天、具体日期）或"详细农历信息"（宜忌、八字、节气等），需要调用相应工具获取。 +- 8.除了基本时间、今日农历、{local_address}天气查询外，用户的其他要求都视为独立任务，即使内容相似也需重新调用工具，不要偷懒，不要使用历史消息糊弄用户。 +- 9.如果你不确定与用户请求相关的动作，不要猜测或编造答案。 +- 10.如果工具中包含camera、take_photo等相关工具，说明用户已经给你安装了摄像头，调用这些工具会让你具备拍照、描述所见物品等能力。如果没有，切勿调用。 + + + +- 1.现在的时间：{current_time} +- 2.今天的日期：{today_date}、{today_weekday} +- 3.今天的农历日期：{lunar_date} +- 4.当前用户所处城市 +{local_address} +- 5.用户所处城市未来7天天气 +{weather_info} + + + + \ No newline at end of file diff --git a/main/xiaozhi-server/config/config_loader.py b/main/xiaozhi-server/config/config_loader.py index b1b45f07..4f35b1fd 100644 --- a/main/xiaozhi-server/config/config_loader.py +++ b/main/xiaozhi-server/config/config_loader.py @@ -1,14 +1,9 @@ import os -import argparse import yaml from collections.abc import Mapping from config.manage_api_client import init_service, get_server_config, get_agent_models -# 添加全局配置缓存 -_config_cache = None - - def get_project_dir(): """获取项目根目录""" return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/" @@ -22,9 +17,12 @@ def read_config(config_path): def load_config(): """加载配置文件""" - global _config_cache - if _config_cache is not None: - return _config_cache + from core.utils.cache.manager import cache_manager, CacheType + + # 检查缓存 + cached_config = cache_manager.get(CacheType.CONFIG, "main_config") + if cached_config is not None: + return cached_config default_config_path = get_project_dir() + "config.yaml" custom_config_path = get_project_dir() + "data/.config.yaml" @@ -40,7 +38,9 @@ def load_config(): config = merge_configs(default_config, custom_config) # 初始化目录 ensure_directories(config) - _config_cache = config + + # 缓存配置 + cache_manager.set(CacheType.CONFIG, "main_config", config) return config diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index f12ffc5e..fdfd36fa 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -36,6 +36,7 @@ from config.config_loader import get_private_config_from_api from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType from config.logger import setup_logging, build_module_string, update_module_string from config.manage_api_client import DeviceNotFoundException, DeviceBindException +from core.utils.prompt_manager import PromptManager TAG = __name__ @@ -73,7 +74,6 @@ class ConnectionHandler: self.headers = None self.device_id = None self.client_ip = None - self.client_ip_info = {} self.prompt = None self.welcome_msg = None self.max_output_size = 0 @@ -149,6 +149,9 @@ class ConnectionHandler: # {"mcp":true} 表示启用MCP功能 self.features = None + # 初始化提示词管理器 + self.prompt_manager = PromptManager(config, self.logger) + async def handle_connection(self, ws): try: # 获取并验证headers @@ -325,12 +328,15 @@ class ConnectionHandler: self.config.get("selected_module", {}) ) update_module_string(self.selected_module_str) - """初始化组件""" + + """快速初始化系统提示词""" if self.config.get("prompt") is not None: - self.prompt = self.config["prompt"] - self.change_system_prompt(self.prompt) + user_prompt = self.config["prompt"] + # 使用快速提示词进行初始化 + prompt = self.prompt_manager.get_quick_prompt(user_prompt) + self.change_system_prompt(prompt) self.logger.bind(tag=TAG).info( - f"初始化组件: prompt成功 {self.prompt[:50]}..." + f"快速初始化组件: prompt成功 {prompt[:50]}..." ) """初始化本地组件""" @@ -355,9 +361,22 @@ class ConnectionHandler: self._initialize_intent() """初始化上报线程""" self._init_report_threads() + """更新系统提示词""" + self._init_prompt_enhancement() + except Exception as e: self.logger.bind(tag=TAG).error(f"实例化组件失败: {e}") + def _init_prompt_enhancement(self): + # 更新上下文信息 + self.prompt_manager.update_context_info(self, self.client_ip) + enhanced_prompt = self.prompt_manager.build_enhanced_prompt( + self.config["prompt"], self.device_id, self.client_ip + ) + if enhanced_prompt: + self.change_system_prompt(enhanced_prompt) + self.logger.bind(tag=TAG).info("系统提示词已增强更新") + def _init_report_threads(self): """初始化ASR和TTS上报线程""" if not self.read_config_from_api or self.need_bind: @@ -758,8 +777,11 @@ class ConnectionHandler: ) ) self.llm_finish_task = True + # 使用lambda延迟计算，只有在DEBUG级别时才执行get_llm_dialogue() self.logger.bind(tag=TAG).debug( - json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False) + lambda: json.dumps( + self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False + ) ) return True diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py index d13c4df4..26fbaf70 100644 --- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py +++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py @@ -16,10 +16,11 @@ class IntentProvider(IntentProviderBase): super().__init__(config) self.llm = None self.promot = "" - # 添加缓存管理 - self.intent_cache = {} # 缓存意图识别结果 - self.cache_expiry = 600 # 缓存有效期10分钟 - self.cache_max_size = 100 # 最多缓存100个意图 + # 导入全局缓存管理器 + from core.utils.cache.manager import cache_manager, CacheType + + self.cache_manager = cache_manager + self.CacheType = CacheType self.history_count = 4 # 默认使用最近4条对话记录 def get_intent_system_prompt(self, functions_list: str) -> str: @@ -102,27 +103,6 @@ class IntentProvider(IntentProviderBase): ) return prompt - def clean_cache(self): - """清理过期缓存""" - now = time.time() - # 找出过期键 - expired_keys = [ - k - for k, v in self.intent_cache.items() - if now - v["timestamp"] > self.cache_expiry - ] - for key in expired_keys: - del self.intent_cache[key] - - # 如果缓存太大，移除最旧的条目 - if len(self.intent_cache) > self.cache_max_size: - # 按时间戳排序并保留最新的条目 - sorted_items = sorted( - self.intent_cache.items(), key=lambda x: x[1]["timestamp"] - ) - for key, _ in sorted_items[: len(sorted_items) - self.cache_max_size]: - del self.intent_cache[key] - def replyResult(self, text: str, original_text: str): llm_result = self.llm.response_no_stream( system_prompt=text, @@ -145,21 +125,16 @@ class IntentProvider(IntentProviderBase): logger.bind(tag=TAG).debug(f"使用意图识别模型: {model_info}") # 计算缓存键 - cache_key = hashlib.md5(text.encode()).hexdigest() + cache_key = hashlib.md5((conn.device_id + text).encode()).hexdigest() # 检查缓存 - if cache_key in self.intent_cache: - cache_entry = self.intent_cache[cache_key] - # 检查缓存是否过期 - if time.time() - cache_entry["timestamp"] <= self.cache_expiry: - cache_time = time.time() - total_start_time - logger.bind(tag=TAG).debug( - f"使用缓存的意图: {cache_key} -> {cache_entry['intent']}, 耗时: {cache_time:.4f}秒" - ) - return cache_entry["intent"] - - # 清理缓存 - self.clean_cache() + cached_intent = self.cache_manager.get(self.CacheType.INTENT, cache_key) + if cached_intent is not None: + cache_time = time.time() - total_start_time + logger.bind(tag=TAG).debug( + f"使用缓存的意图: {cache_key} -> {cached_intent}, 耗时: {cache_time:.4f}秒" + ) + return cached_intent if self.promot == "": functions = conn.func_handler.get_functions() @@ -259,10 +234,7 @@ class IntentProvider(IntentProviderBase): conn.dialogue.dialogue = clean_history # 添加到缓存 - self.intent_cache[cache_key] = { - "intent": intent, - "timestamp": time.time(), - } + self.cache_manager.set(self.CacheType.INTENT, cache_key, intent) # 后处理时间 postprocess_time = time.time() - postprocess_start_time @@ -272,10 +244,7 @@ class IntentProvider(IntentProviderBase): return intent else: # 添加到缓存 - self.intent_cache[cache_key] = { - "intent": intent, - "timestamp": time.time(), - } + self.cache_manager.set(self.CacheType.INTENT, cache_key, intent) # 后处理时间 postprocess_time = time.time() - postprocess_start_time diff --git a/main/xiaozhi-server/core/providers/tools/server_plugins/plugin_executor.py b/main/xiaozhi-server/core/providers/tools/server_plugins/plugin_executor.py index 728493ec..17d3b564 100644 --- a/main/xiaozhi-server/core/providers/tools/server_plugins/plugin_executor.py +++ b/main/xiaozhi-server/core/providers/tools/server_plugins/plugin_executor.py @@ -51,13 +51,13 @@ class ServerPluginExecutor(ToolExecutor): tools = {} # 获取必要的函数 - necessary_functions = ["handle_exit_intent", "get_time", "get_lunar"] + necessary_functions = ["handle_exit_intent", "get_lunar"] # 获取配置中的函数 config_functions = self.config["Intent"][ self.config["selected_module"]["Intent"] ].get("functions", []) - + # 转换为列表 if not isinstance(config_functions, list): try: diff --git a/main/xiaozhi-server/core/utils/cache/config.py b/main/xiaozhi-server/core/utils/cache/config.py new file mode 100644 index 00000000..d6d93345 --- /dev/null +++ b/main/xiaozhi-server/core/utils/cache/config.py @@ -0,0 +1,58 @@ +""" +缓存配置管理 +""" + +from enum import Enum +from typing import Dict, Any, Optional +from dataclasses import dataclass +from .strategies import CacheStrategy + + +class CacheType(Enum): + """缓存类型枚举""" + + LOCATION = "location" + WEATHER = "weather" + LUNAR = "lunar" + INTENT = "intent" + IP_INFO = "ip_info" + CONFIG = "config" + DEVICE_PROMPT = "device_prompt" + + +@dataclass +class CacheConfig: + """缓存配置类""" + + strategy: CacheStrategy = CacheStrategy.TTL + ttl: Optional[float] = 300 # 默认5分钟 + max_size: Optional[int] = 1000 # 默认最大1000条 + cleanup_interval: float = 60 # 清理间隔（秒） + + @classmethod + def for_type(cls, cache_type: CacheType) -> "CacheConfig": + """根据缓存类型返回预设配置""" + configs = { + CacheType.LOCATION: cls( + strategy=CacheStrategy.TTL, ttl=None, max_size=1000 # 手动失效 + ), + CacheType.IP_INFO: cls( + strategy=CacheStrategy.TTL, ttl=86400, max_size=1000 # 24小时 + ), + CacheType.WEATHER: cls( + strategy=CacheStrategy.TTL, ttl=28800, max_size=1000 # 8小时 + ), + CacheType.LUNAR: cls( + strategy=CacheStrategy.TTL, ttl=2592000, max_size=365 # 30天过期 + ), + CacheType.INTENT: cls( + strategy=CacheStrategy.TTL_LRU, ttl=600, max_size=1000 # 10分钟 + ), + CacheType.CONFIG: cls( + strategy=CacheStrategy.FIXED_SIZE, ttl=None, max_size=20 # 手动失效 + ), + CacheType.DEVICE_PROMPT: cls( + strategy=CacheStrategy.TTL, ttl=None, max_size=1000 # 手动失效 + ), + } + return configs.get(cache_type, cls()) diff --git a/main/xiaozhi-server/core/utils/cache/manager.py b/main/xiaozhi-server/core/utils/cache/manager.py new file mode 100644 index 00000000..c54f7817 --- /dev/null +++ b/main/xiaozhi-server/core/utils/cache/manager.py @@ -0,0 +1,216 @@ +""" +全局缓存管理器 +""" + +import time +import threading +from typing import Any, Optional, Dict +from collections import OrderedDict +from .strategies import CacheStrategy, CacheEntry +from .config import CacheConfig, CacheType + + +class GlobalCacheManager: + """全局缓存管理器""" + + def __init__(self): + self._logger = None + self._caches: Dict[str, Dict[str, CacheEntry]] = {} + self._configs: Dict[str, CacheConfig] = {} + self._locks: Dict[str, threading.RLock] = {} + self._global_lock = threading.RLock() + self._last_cleanup = time.time() + self._stats = {"hits": 0, "misses": 0, "evictions": 0, "cleanups": 0} + + @property + def logger(self): + """延迟初始化 logger 以避免循环导入""" + if self._logger is None: + from config.logger import setup_logging + + self._logger = setup_logging() + return self._logger + + def _get_cache_name(self, cache_type: CacheType, namespace: str = "") -> str: + """生成缓存名称""" + if namespace: + return f"{cache_type.value}:{namespace}" + return cache_type.value + + def _get_or_create_cache( + self, cache_name: str, config: CacheConfig + ) -> Dict[str, CacheEntry]: + """获取或创建缓存空间""" + with self._global_lock: + if cache_name not in self._caches: + self._caches[cache_name] = ( + OrderedDict() + if config.strategy in [CacheStrategy.LRU, CacheStrategy.TTL_LRU] + else {} + ) + self._configs[cache_name] = config + self._locks[cache_name] = threading.RLock() + return self._caches[cache_name] + + def set( + self, + cache_type: CacheType, + key: str, + value: Any, + ttl: Optional[float] = None, + namespace: str = "", + ) -> None: + """设置缓存值""" + cache_name = self._get_cache_name(cache_type, namespace) + config = self._configs.get(cache_name) or CacheConfig.for_type(cache_type) + cache = self._get_or_create_cache(cache_name, config) + + # 使用配置的TTL或传入的TTL + effective_ttl = ttl if ttl is not None else config.ttl + + with self._locks[cache_name]: + # 创建缓存条目 + entry = CacheEntry(value=value, timestamp=time.time(), ttl=effective_ttl) + + # 处理不同策略 + if config.strategy in [CacheStrategy.LRU, CacheStrategy.TTL_LRU]: + # LRU策略：如果已存在则移动到末尾 + if key in cache: + del cache[key] + cache[key] = entry + + # 检查大小限制 + if config.max_size and len(cache) > config.max_size: + # 移除最旧的条目 + oldest_key = next(iter(cache)) + del cache[oldest_key] + self._stats["evictions"] += 1 + + else: + cache[key] = entry + + # 检查大小限制 + if config.max_size and len(cache) > config.max_size: + # 简单策略：随机移除一个条目 + victim_key = next(iter(cache)) + del cache[victim_key] + self._stats["evictions"] += 1 + + # 定期清理过期条目 + self._maybe_cleanup(cache_name) + + def get( + self, cache_type: CacheType, key: str, namespace: str = "" + ) -> Optional[Any]: + """获取缓存值""" + cache_name = self._get_cache_name(cache_type, namespace) + + if cache_name not in self._caches: + self._stats["misses"] += 1 + return None + + cache = self._caches[cache_name] + config = self._configs[cache_name] + + with self._locks[cache_name]: + if key not in cache: + self._stats["misses"] += 1 + return None + + entry = cache[key] + + # 检查过期 + if entry.is_expired(): + del cache[key] + self._stats["misses"] += 1 + return None + + # 更新访问信息 + entry.touch() + + # LRU策略：移动到末尾 + if config.strategy in [CacheStrategy.LRU, CacheStrategy.TTL_LRU]: + del cache[key] + cache[key] = entry + + self._stats["hits"] += 1 + return entry.value + + def delete(self, cache_type: CacheType, key: str, namespace: str = "") -> bool: + """删除缓存条目""" + cache_name = self._get_cache_name(cache_type, namespace) + + if cache_name not in self._caches: + return False + + cache = self._caches[cache_name] + + with self._locks[cache_name]: + if key in cache: + del cache[key] + return True + return False + + def clear(self, cache_type: CacheType, namespace: str = "") -> None: + """清空指定缓存""" + cache_name = self._get_cache_name(cache_type, namespace) + + if cache_name not in self._caches: + return + + with self._locks[cache_name]: + self._caches[cache_name].clear() + + def invalidate_pattern( + self, cache_type: CacheType, pattern: str, namespace: str = "" + ) -> int: + """按模式失效缓存条目""" + cache_name = self._get_cache_name(cache_type, namespace) + + if cache_name not in self._caches: + return 0 + + cache = self._caches[cache_name] + deleted_count = 0 + + with self._locks[cache_name]: + keys_to_delete = [key for key in cache.keys() if pattern in key] + for key in keys_to_delete: + del cache[key] + deleted_count += 1 + + return deleted_count + + def _cleanup_expired(self, cache_name: str) -> int: + """清理过期条目""" + if cache_name not in self._caches: + return 0 + + cache = self._caches[cache_name] + deleted_count = 0 + + with self._locks[cache_name]: + expired_keys = [key for key, entry in cache.items() if entry.is_expired()] + for key in expired_keys: + del cache[key] + deleted_count += 1 + + return deleted_count + + def _maybe_cleanup(self, cache_name: str): + """定期清理检查""" + config = self._configs.get(cache_name) + if not config: + return + + now = time.time() + if now - self._last_cleanup > config.cleanup_interval: + self._last_cleanup = now + deleted = self._cleanup_expired(cache_name) + if deleted > 0: + self._stats["cleanups"] += 1 + self.logger.debug(f"清理缓存 {cache_name}: 删除 {deleted} 个过期条目") + + +# 创建全局缓存管理器实例 +cache_manager = GlobalCacheManager() diff --git a/main/xiaozhi-server/core/utils/cache/strategies.py b/main/xiaozhi-server/core/utils/cache/strategies.py new file mode 100644 index 00000000..13327ca7 --- /dev/null +++ b/main/xiaozhi-server/core/utils/cache/strategies.py @@ -0,0 +1,43 @@ +""" +缓存策略和数据结构定义 +""" + +import time +from enum import Enum +from typing import Any, Optional +from dataclasses import dataclass + + +class CacheStrategy(Enum): + """缓存策略枚举""" + + TTL = "ttl" # 基于时间过期 + LRU = "lru" # 最近最少使用 + FIXED_SIZE = "fixed_size" # 固定大小 + TTL_LRU = "ttl_lru" # TTL + LRU混合策略 + + +@dataclass +class CacheEntry: + """缓存条目数据结构""" + + value: Any + timestamp: float + ttl: Optional[float] = None # 生存时间（秒） + access_count: int = 0 + last_access: float = None + + def __post_init__(self): + if self.last_access is None: + self.last_access = self.timestamp + + def is_expired(self) -> bool: + """检查是否过期""" + if self.ttl is None: + return False + return time.time() - self.timestamp > self.ttl + + def touch(self): + """更新访问时间和计数""" + self.last_access = time.time() + self.access_count += 1 diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py index 2ee30d4a..15eb06f3 100644 --- a/main/xiaozhi-server/core/utils/dialogue.py +++ b/main/xiaozhi-server/core/utils/dialogue.py @@ -1,4 +1,5 @@ import uuid +import re from typing import List, Dict from datetime import datetime @@ -74,9 +75,12 @@ class Dialogue: ) if system_message: - enhanced_system_prompt = ( - f"{system_message.content}\n\n" - f"以下是用户的历史记忆：\n```\n{memory_str}\n```" + # 使用正则表达式匹配 标签，不管中间有什么内容 + enhanced_system_prompt = re.sub( + r".*?", + f"\n{memory_str}\n", + system_message.content, + flags=re.DOTALL, ) dialogue.append({"role": "system", "content": enhanced_system_prompt}) diff --git a/main/xiaozhi-server/core/utils/prompt_manager.py b/main/xiaozhi-server/core/utils/prompt_manager.py new file mode 100644 index 00000000..dae840bf --- /dev/null +++ b/main/xiaozhi-server/core/utils/prompt_manager.py @@ -0,0 +1,219 @@ +""" +系统提示词管理器模块 +负责管理和更新系统提示词，包括快速初始化和异步增强功能 +""" + +import os +import cnlunar +from typing import Dict, Any +from config.logger import setup_logging + +TAG = __name__ + +WEEKDAY_MAP = { + "Monday": "星期一", + "Tuesday": "星期二", + "Wednesday": "星期三", + "Thursday": "星期四", + "Friday": "星期五", + "Saturday": "星期六", + "Sunday": "星期日", +} + + +class PromptManager: + """系统提示词管理器，负责管理和更新系统提示词""" + + def __init__(self, config: Dict[str, Any], logger=None): + self.config = config + self.logger = logger or setup_logging() + self.base_prompt_template = None + self.last_update_time = 0 + + # 导入全局缓存管理器 + from core.utils.cache.manager import cache_manager, CacheType + + self.cache_manager = cache_manager + self.CacheType = CacheType + + self._load_base_template() + + def _load_base_template(self): + """加载基础提示词模板""" + try: + template_path = "agent-base-prompt.txt" + cache_key = f"prompt_template:{template_path}" + + # 先从缓存获取 + cached_template = self.cache_manager.get(self.CacheType.CONFIG, cache_key) + if cached_template is not None: + self.base_prompt_template = cached_template + self.logger.bind(tag=TAG).debug("从缓存加载基础提示词模板") + return + + # 缓存未命中，从文件读取 + if os.path.exists(template_path): + with open(template_path, "r", encoding="utf-8") as f: + template_content = f.read() + + # 存入缓存（CONFIG类型默认不自动过期，需要手动失效） + self.cache_manager.set( + self.CacheType.CONFIG, cache_key, template_content + ) + self.base_prompt_template = template_content + self.logger.bind(tag=TAG).debug("成功加载基础提示词模板并缓存") + else: + self.logger.bind(tag=TAG).warning("未找到agent-base-prompt.txt文件") + except Exception as e: + self.logger.bind(tag=TAG).error(f"加载提示词模板失败: {e}") + + def get_quick_prompt(self, user_prompt: str, device_id: str = None) -> str: + """快速获取系统提示词（使用用户配置）""" + device_cache_key = f"device_prompt:{device_id}" + cached_device_prompt = self.cache_manager.get( + self.CacheType.DEVICE_PROMPT, device_cache_key + ) + if cached_device_prompt is not None: + self.logger.bind(tag=TAG).debug(f"使用设备 {device_id} 的缓存提示词") + return cached_device_prompt + else: + self.logger.bind(tag=TAG).debug( + f"设备 {device_id} 无缓存提示词，使用传入的提示词" + ) + + # 使用传入的提示词并缓存（如果有设备ID） + if device_id: + device_cache_key = f"device_prompt:{device_id}" + self.cache_manager.set(self.CacheType.CONFIG, device_cache_key, user_prompt) + self.logger.bind(tag=TAG).debug(f"设备 {device_id} 的提示词已缓存") + + self.logger.bind(tag=TAG).info(f"使用快速提示词: {user_prompt[:50]}...") + return user_prompt + + def _get_current_time_info(self) -> tuple: + """获取当前时间信息""" + from datetime import datetime + + now = datetime.now() + current_time = now.strftime("%H:%M") + today_date = now.strftime("%Y-%m-%d") + today_weekday = WEEKDAY_MAP[now.strftime("%A")] + today_lunar = cnlunar.Lunar(now, godType="8char") + lunar_date = "%s年%s%s\n" % ( + today_lunar.lunarYearCn, + today_lunar.lunarMonthCn[:-1], + today_lunar.lunarDayCn, + ) + + return current_time, today_date, today_weekday, lunar_date + + def _get_location_info(self, client_ip: str) -> str: + """获取位置信息""" + try: + # 先从缓存获取 + cached_location = self.cache_manager.get(self.CacheType.LOCATION, client_ip) + if cached_location is not None: + return cached_location + + # 缓存未命中，调用API获取 + from core.utils.util import get_ip_info + + ip_info = get_ip_info(client_ip, self.logger) + city = ip_info.get("city", "未知位置") + location = f"{city}" + + # 存入缓存 + self.cache_manager.set(self.CacheType.LOCATION, client_ip, location) + return location + except Exception as e: + self.logger.bind(tag=TAG).error(f"获取位置信息失败: {e}") + return "未知位置" + + def _get_weather_info(self, conn, location: str) -> str: + """获取天气信息""" + try: + # 先从缓存获取 + cached_weather = self.cache_manager.get(self.CacheType.WEATHER, location) + if cached_weather is not None: + return cached_weather + + # 缓存未命中，调用get_weather函数获取 + from plugins_func.functions.get_weather import get_weather + from plugins_func.register import ActionResponse + + # 调用get_weather函数 + result = get_weather(conn, location=location, lang="zh_CN") + if isinstance(result, ActionResponse): + weather_report = result.result + self.cache_manager.set(self.CacheType.WEATHER, location, weather_report) + return weather_report + return "天气信息获取失败" + + except Exception as e: + self.logger.bind(tag=TAG).error(f"获取天气信息失败: {e}") + return "天气信息获取失败" + + def update_context_info(self, conn, client_ip: str): + """同步更新上下文信息""" + try: + # 获取位置信息（使用全局缓存） + local_address = self._get_location_info(client_ip) + # 获取天气信息（使用全局缓存） + self._get_weather_info(conn, local_address) + self.logger.bind(tag=TAG).info(f"上下文信息更新完成") + + except Exception as e: + self.logger.bind(tag=TAG).error(f"更新上下文信息失败: {e}") + + def build_enhanced_prompt( + self, user_prompt: str, device_id: str, client_ip: str = None + ) -> str: + """构建增强的系统提示词""" + if not self.base_prompt_template: + return user_prompt + + try: + # 获取最新的时间信息（不缓存） + current_time, today_date, today_weekday, lunar_date = ( + self._get_current_time_info() + ) + + # 获取缓存的上下文信息 + local_address = "" + weather_info = "" + + if client_ip: + # 获取位置信息（从全局缓存） + local_address = ( + self.cache_manager.get(self.CacheType.LOCATION, client_ip) or "" + ) + + # 获取天气信息（从全局缓存） + if local_address: + weather_info = ( + self.cache_manager.get(self.CacheType.WEATHER, local_address) + or "" + ) + + # 替换模板变量 + enhanced_prompt = self.base_prompt_template.format( + base_prompt=user_prompt, + current_time=current_time, + today_date=today_date, + today_weekday=today_weekday, + lunar_date=lunar_date, + local_address=local_address, + weather_info=weather_info, + ) + device_cache_key = f"device_prompt:{device_id}" + self.cache_manager.set( + self.CacheType.DEVICE_PROMPT, device_cache_key, enhanced_prompt + ) + self.logger.bind(tag=TAG).info( + f"构建增强提示词成功，长度: {len(enhanced_prompt)}" + ) + return enhanced_prompt + + except Exception as e: + self.logger.bind(tag=TAG).error(f"构建增强提示词失败: {e}") + return user_prompt diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index dd12392a..bc778558 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -96,11 +96,23 @@ def is_private_ip(ip_addr): def get_ip_info(ip_addr, logger): try: + # 导入全局缓存管理器 + from core.utils.cache.manager import cache_manager, CacheType + + # 先从缓存获取 + cached_ip_info = cache_manager.get(CacheType.IP_INFO, ip_addr) + if cached_ip_info is not None: + return cached_ip_info + + # 缓存未命中，调用API if is_private_ip(ip_addr): ip_addr = "" url = f"https://whois.pconline.com.cn/ipJson.jsp?json=true&ip={ip_addr}" resp = requests.get(url).json() ip_info = {"city": resp.get("city")} + + # 存入缓存 + cache_manager.set(CacheType.IP_INFO, ip_addr, ip_info) return ip_info except Exception as e: logger.bind(tag=TAG).error(f"Error getting client ip info: {e}") diff --git a/main/xiaozhi-server/plugins_func/functions/get_time.py b/main/xiaozhi-server/plugins_func/functions/get_time.py index 44732bba..766e19fd 100644 --- a/main/xiaozhi-server/plugins_func/functions/get_time.py +++ b/main/xiaozhi-server/plugins_func/functions/get_time.py @@ -2,59 +2,27 @@ from datetime import datetime import cnlunar from plugins_func.register import register_function, ToolType, ActionResponse, Action -# 添加星期映射字典 -WEEKDAY_MAP = { - "Monday": "星期一", - "Tuesday": "星期二", - "Wednesday": "星期三", - "Thursday": "星期四", - "Friday": "星期五", - "Saturday": "星期六", - "Sunday": "星期日", -} - -get_time_function_desc = { - "type": "function", - "function": { - "name": "get_time", - "description": "获取今天日期或者当前时间信息", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, -} - - -@register_function("get_time", get_time_function_desc, ToolType.WAIT) -def get_time(): - """ - 获取当前的日期时间信息 - """ - now = datetime.now() - current_time = now.strftime("%H:%M:%S") - current_date = now.strftime("%Y-%m-%d") - current_weekday = WEEKDAY_MAP[now.strftime("%A")] - response_text = ( - f"当前日期: {current_date}，当前时间: {current_time}， {current_weekday}" - ) - - return ActionResponse(Action.REQLLM, response_text, None) - - get_lunar_function_desc = { "type": "function", "function": { "name": "get_lunar", "description": ( - "用于获取今天的阴历/农历和黄历信息。" + "用于具体日期的阴历/农历和黄历信息。" "用户可以指定查询内容，如：阴历日期、天干地支、节气、生肖、星座、八字、宜忌等。" "如果没有指定查询内容，则默认查询干支年和农历日期。" + "对于'今天农历是多少'、'今天农历日期'这样的基本查询，请直接使用context中的信息，不要调用此工具。" ), "parameters": { "type": "object", "properties": { + "date": { + "type": "string", + "description": "要查询的日期，格式为YYYY-MM-DD，例如2024-01-01。如果不提供，则使用当前日期", + }, "query": { "type": "string", "description": "要查询的内容，例如阴历日期、天干地支、节日、节气、生肖、星座、八字、宜忌等", - } + }, }, "required": [], }, @@ -63,23 +31,41 @@ get_lunar_function_desc = { @register_function("get_lunar", get_lunar_function_desc, ToolType.WAIT) -def get_lunar(query=None): +def get_lunar(date=None, query=None): """ 用于获取当前的阴历/农历，和天干地支、节气、生肖、星座、八字、宜忌等黄历信息 """ - now = datetime.now() - current_time = now.strftime("%H:%M:%S") + from core.utils.cache.manager import cache_manager, CacheType + + # 如果提供了日期参数，则使用指定日期；否则使用当前日期 + if date: + try: + now = datetime.strptime(date, "%Y-%m-%d") + except ValueError: + return ActionResponse( + Action.REQLLM, + f"日期格式错误，请使用YYYY-MM-DD格式，例如：2024-01-01", + None, + ) + else: + now = datetime.now() + current_date = now.strftime("%Y-%m-%d") - current_weekday = WEEKDAY_MAP[now.strftime("%A")] # 如果 query 为 None，则使用默认文本 if query is None: query = "默认查询干支年和农历日期" + + # 尝试从缓存获取农历信息 + lunar_cache_key = f"lunar_info_{current_date}" + cached_lunar_info = cache_manager.get(CacheType.LUNAR, lunar_cache_key) + if cached_lunar_info: + return ActionResponse(Action.REQLLM, cached_lunar_info, None) + response_text = f"根据以下信息回应用户的查询请求，并提供与{query}相关的信息：\n" lunar = cnlunar.Lunar(now, godType="8char") response_text += ( - f"当前公历日期: {current_date}，当前时间: {current_time}，{current_weekday}\n" "农历信息：\n" "%s年%s%s\n" % (lunar.lunarYearCn, lunar.lunarMonthCn[:-1], lunar.lunarDayCn) + "干支: %s年 %s月 %s日\n" % (lunar.year8Char, lunar.month8Char, lunar.day8Char) @@ -135,4 +121,7 @@ def get_lunar(query=None): + "(默认返回干支年和农历日期；仅在要求查询宜忌信息时才返回本日宜忌)" ) + # 缓存农历信息 + cache_manager.set(CacheType.LUNAR, lunar_cache_key, response_text) + return ActionResponse(Action.REQLLM, response_text, None) diff --git a/main/xiaozhi-server/plugins_func/functions/get_weather.py b/main/xiaozhi-server/plugins_func/functions/get_weather.py index 75c15c7b..a3af6c03 100644 --- a/main/xiaozhi-server/plugins_func/functions/get_weather.py +++ b/main/xiaozhi-server/plugins_func/functions/get_weather.py @@ -151,20 +151,44 @@ def parse_weather_info(soup): @register_function("get_weather", GET_WEATHER_FUNCTION_DESC, ToolType.SYSTEM_CTL) def get_weather(conn, location: str = None, lang: str = "zh_CN"): - api_host = conn.config["plugins"]["get_weather"].get("api_host", "mj7p3y7naa.re.qweatherapi.com") - api_key = conn.config["plugins"]["get_weather"].get("api_key", "a861d0d5e7bf4ee1a83d9a9e4f96d4da") + from core.utils.cache.manager import cache_manager, CacheType + + api_host = conn.config["plugins"]["get_weather"].get( + "api_host", "mj7p3y7naa.re.qweatherapi.com" + ) + api_key = conn.config["plugins"]["get_weather"].get( + "api_key", "a861d0d5e7bf4ee1a83d9a9e4f96d4da" + ) default_location = conn.config["plugins"]["get_weather"]["default_location"] client_ip = conn.client_ip + # 优先使用用户提供的location参数 if not location: # 通过客户端IP解析城市 if client_ip: - # 动态解析IP对应的城市信息 - ip_info = get_ip_info(client_ip, logger) - location = ip_info.get("city") if ip_info and "city" in ip_info else None + # 先从缓存获取IP对应的城市信息 + cached_ip_info = cache_manager.get(CacheType.IP_INFO, client_ip) + if cached_ip_info: + location = cached_ip_info.get("city") + else: + # 缓存未命中，调用API获取 + ip_info = get_ip_info(client_ip, logger) + if ip_info: + cache_manager.set(CacheType.IP_INFO, client_ip, ip_info) + location = ip_info.get("city") + + if not location: + location = default_location else: - # 若IP解析失败或无IP，使用默认位置 + # 若无IP，使用默认位置 location = default_location + # 尝试从缓存获取完整天气报告 + weather_cache_key = f"full_weather_{location}_{lang}" + cached_weather_report = cache_manager.get(CacheType.WEATHER, weather_cache_key) + if cached_weather_report: + return ActionResponse(Action.REQLLM, cached_weather_report, None) + + # 缓存未命中，获取实时天气数据 city_info = fetch_city_info(location, api_key, api_host) if not city_info: return ActionResponse( @@ -192,4 +216,7 @@ def get_weather(conn, location: str = None, lang: str = "zh_CN"): # 提示语 weather_report += "\n（如需某一天的具体天气，请告诉我日期）" + # 缓存完整的天气报告 + cache_manager.set(CacheType.WEATHER, weather_cache_key, weather_report) + return ActionResponse(Action.REQLLM, weather_report, None) From 0636b95b87e3bb0a701336ffd6390d1ddb7aa9a8 Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Tue, 8 Jul 2025 17:10:57 +0800 Subject: [PATCH 18/35] =?UTF-8?q?=E5=A3=B0=E7=BA=B9=E8=A1=A8=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E9=9F=B3=E9=A2=91=E5=AD=97=E6=AE=B5=20--202507081646.?= =?UTF-8?q?sql=20=E6=B7=BB=E5=8A=A0=E5=AD=97=E6=AE=B5sql=20--AgentVoicePri?= =?UTF-8?q?ntEntity.java=20=E5=AF=B9=E5=BA=94=E5=AE=9E=E4=BD=93=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=20--db.changelog-master.yaml=20=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AD=97=E6=AE=B5sql=E7=9A=84=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../modules/agent/entity/AgentVoicePrintEntity.java | 4 ++++ .../src/main/resources/db/changelog/202507081646.sql | 3 +++ .../main/resources/db/changelog/db.changelog-master.yaml | 9 ++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202507081646.sql diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentVoicePrintEntity.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentVoicePrintEntity.java index 9f9e6b53..99f2f122 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentVoicePrintEntity.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/entity/AgentVoicePrintEntity.java @@ -24,6 +24,10 @@ public class AgentVoicePrintEntity { * 关联的智能体id */ private String agentId; + /** + * 关联的音频id + */ + private String audioId; /** * 声纹来源的人姓名 */ diff --git a/main/manager-api/src/main/resources/db/changelog/202507081646.sql b/main/manager-api/src/main/resources/db/changelog/202507081646.sql new file mode 100644 index 00000000..6631ae18 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507081646.sql @@ -0,0 +1,3 @@ +-- 智能体声纹添加新字段 +ALTER TABLE ai_agent_voice_print + ADD COLUMN audio_id VARCHAR(32) NOT NULL COMMENT '音频ID'; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 1f7fdcea..2a520d16 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -253,4 +253,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202507041018.sql \ No newline at end of file + path: classpath:db/changelog/202507041018.sql + - changeSet: + id: 202507081646 + author: zjy + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202507081646.sql \ No newline at end of file From 3ed13b3ff50dbc800d1e8ef7b2c127b2d9fd56f9 Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Tue, 8 Jul 2025 17:32:44 +0800 Subject: [PATCH 19/35] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=BC=8F=E6=B4=9E=20--?= =?UTF-8?q?AgentChatHistoryServiceImpl.java=20=E7=94=A8=E6=88=B7=E8=81=8A?= =?UTF-8?q?=E5=A4=A9=E8=AE=B0=E5=BD=95=E4=B8=8D=E4=B8=80=E5=AE=9A=E9=83=BD?= =?UTF-8?q?=E5=B8=A6=E6=9C=89=E9=9F=B3=E9=A2=91=EF=BC=8C=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E6=97=B6=E8=A6=81=E6=8E=92=E9=99=A4=E4=B8=8D=E5=B8=A6=E9=9F=B3?= =?UTF-8?q?=E9=A2=91=E7=9A=84=E6=9C=80=E8=BF=9150=E6=9D=A1=E8=81=8A?= =?UTF-8?q?=E5=A4=A9=E8=AE=B0=E5=BD=95=20--AgentChatHistoryService.java=20?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=96=B9=E6=B3=95=E6=8F=8F=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../modules/agent/service/AgentChatHistoryService.java | 4 ++-- .../agent/service/impl/AgentChatHistoryServiceImpl.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatHistoryService.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatHistoryService.java index b69cd7b0..88e7c962 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatHistoryService.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatHistoryService.java @@ -47,9 +47,9 @@ public interface AgentChatHistoryService extends IService getRecentlyFiftyByAgentId(String agentId); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatHistoryServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatHistoryServiceImpl.java index 980115ad..47220703 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatHistoryServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatHistoryServiceImpl.java @@ -101,7 +101,8 @@ public class AgentChatHistoryServiceImpl extends ServiceImpl wrapper = new LambdaQueryWrapper<>(); wrapper.select(AgentChatHistoryEntity::getContent,AgentChatHistoryEntity::getAudioId) .eq(AgentChatHistoryEntity::getAgentId, agentId) - .eq(AgentChatHistoryEntity::getChatType, AgentChatHistoryType.USER.getValue()); + .eq(AgentChatHistoryEntity::getChatType, AgentChatHistoryType.USER.getValue()) + .isNotNull(AgentChatHistoryEntity::getAudioId); // 构建分页查询，查询前50页数据 Page pageParam = new Page<>(0, 50); From 5e18a46e3fa0c2a279587224a827be7ab4c3f90d Mon Sep 17 00:00:00 2001 From: JianYu Zheng <2375294554@qq.com> Date: Wed, 9 Jul 2025 11:48:02 +0800 Subject: [PATCH 20/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=96=B0=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=EF=BC=8C=E5=A3=B0=E7=BA=B9=E9=80=89=E6=8B=A9=E7=9A=84?= =?UTF-8?q?=E6=97=B6=E5=80=99=E5=8F=AF=E4=BB=A5=E5=90=AC=E5=8F=96=E9=9F=B3?= =?UTF-8?q?=E9=A2=91=E6=96=87=E4=BB=B6=20--VoicePrintDialog.vue=20?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=90=AC=E5=8F=96=E9=9F=B3=E9=A2=91=E7=9A=84?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/VoicePrintDialog.vue | 57 ++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/main/manager-web/src/components/VoicePrintDialog.vue b/main/manager-web/src/components/VoicePrintDialog.vue index a9f861db..b549f6e2 100644 --- a/main/manager-web/src/components/VoicePrintDialog.vue +++ b/main/manager-web/src/components/VoicePrintDialog.vue @@ -14,8 +14,13 @@ - + + {{ item.content }} + + + + @@ -71,6 +76,8 @@ export default { return { dialogKey: Date.now(), saving: false, + playingAudioId: null, + audioElement: null, valueTypeOptions: [ { audioId: '', content: '' } ], @@ -88,6 +95,45 @@ export default { }; }, methods: { + getAudioIconClass(audioId) { + if (this.playingAudioId === audioId) { + return 'el-icon-loading'; + } + return 'el-icon-video-play'; + }, + playAudio(audioId) { + if (this.playingAudioId === audioId) { + // 如果正在播放当前音频，则停止播放 + if (this.audioElement) { + this.audioElement.pause(); + this.audioElement = null; + } + this.playingAudioId = null; + return; + } + + // 停止当前正在播放的音频 + if (this.audioElement) { + this.audioElement.pause(); + this.audioElement = null; + } + + // 先获取音频下载ID + this.playingAudioId = audioId; + api.agent.getAudioId(audioId, (res) => { + if (res.data && res.data.data) { + // 使用获取到的下载ID播放音频 + this.audioElement = new Audio(api.getServiceUrl() + `/agent/play/${res.data.data}`); + + this.audioElement.onended = () => { + this.playingAudioId = null; + this.audioElement = null; + }; + + this.audioElement.play(); + } + }); + }, submit() { this.$refs.form.validate((valid) => { if (valid) { @@ -148,6 +194,13 @@ export default {