From 3d64152dba975c3e0e1c3ed839bda98ccb650a93 Mon Sep 17 00:00:00 2001
From: CGD <3030332422@qq.com>
Date: Fri, 4 Jul 2025 17:11:44 +0800
Subject: [PATCH 1/5] =?UTF-8?q?update:python=E5=8D=95=E6=A8=A1=E5=9D=97?=
 =?UTF-8?q?=E9=83=A8=E7=BD=B2=E5=A3=B0=E7=BA=B9=E8=AF=86=E5=88=AB=E5=AF=B9?=
 =?UTF-8?q?=E6=8E=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 main/xiaozhi-server/config.yaml               |   9 +
 main/xiaozhi-server/core/connection.py        |  29 +-
 .../core/handle/receiveAudioHandle.py         |  36 +-
 .../core/handle/sendAudioHandle.py            |   1 -
 .../xiaozhi-server/core/providers/asr/base.py | 346 ++++++++++++++++--
 main/xiaozhi-server/core/utils/dialogue.py    |   8 +-
 .../core/utils/modules_initialize.py          |   9 +
 7 files changed, 390 insertions(+), 48 deletions(-)

diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml
index 5a43c995..5b746866 100644
--- a/main/xiaozhi-server/config.yaml
+++ b/main/xiaozhi-server/config.yaml
@@ -137,6 +137,15 @@ plugins:
       - ".wav"
       - ".p3"
     refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒
+  # 声纹识别配置
+  voiceprint:
+    # 声纹接口地址
+    url: 
+    # 说话人配置：speaker_id,名称,描述
+    speakers:
+      - "test1,张三,张三是一个程序员"
+      - "test2,李四,李四是一个产品经理"
+      - "test3,王五,王五是一个设计师"
 
 # #####################################################################################
 # ################################以下是角色模型配置######################################
diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py
index 9d1949c4..458cbc7a 100644
--- a/main/xiaozhi-server/core/connection.py
+++ b/main/xiaozhi-server/core/connection.py
@@ -595,8 +595,33 @@ class ConnectionHandler:
         self.logger.bind(tag=TAG).info(f"大模型收到用户消息: {query}")
         self.llm_finish_task = False
 
+        # 检查是否是JSON格式的消息（包含说话人信息）
+        enhanced_query = query
+        try:
+            if query.strip().startswith('{') and query.strip().endswith('}'):
+                data = json.loads(query)
+                if 'speaker' in data and 'content' in data:
+                    # 直接使用JSON格式，不重新格式化
+                    enhanced_query = query
+                    self.logger.bind(tag=TAG).info(f"识别到说话人: {data['speaker']}")
+                else:
+                    # 如果有说话人信息但不是JSON格式，按原逻辑处理
+                    if hasattr(self, 'current_speaker') and self.current_speaker:
+                        enhanced_query = f"[说话人: {self.current_speaker}] {query}"
+                        self.logger.bind(tag=TAG).info(f"识别到说话人: {self.current_speaker}")
+            else:
+                # 如果有说话人信息但不是JSON格式，按原逻辑处理
+                if hasattr(self, 'current_speaker') and self.current_speaker:
+                    enhanced_query = f"[说话人: {self.current_speaker}] {query}"
+                    self.logger.bind(tag=TAG).info(f"识别到说话人: {self.current_speaker}")
+        except json.JSONDecodeError:
+            # JSON解析失败，按原逻辑处理
+            if hasattr(self, 'current_speaker') and self.current_speaker:
+                enhanced_query = f"[说话人: {self.current_speaker}] {query}"
+                self.logger.bind(tag=TAG).info(f"识别到说话人: {self.current_speaker}")
+
         if not tool_call:
-            self.dialogue.put(Message(role="user", content=query))
+            self.dialogue.put(Message(role="user", content=enhanced_query))
 
         # Define intent functions
         functions = None
@@ -609,7 +634,7 @@ class ConnectionHandler:
             memory_str = None
             if self.memory is not None:
                 future = asyncio.run_coroutine_threadsafe(
-                    self.memory.query_memory(query), self.loop
+                    self.memory.query_memory(enhanced_query), self.loop
                 )
                 memory_str = future.result()
 
diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py
index 465b081a..e6f39632 100644
--- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py
+++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py
@@ -4,6 +4,7 @@ from core.utils.output_counter import check_device_output_limit
 from core.handle.abortHandle import handleAbortMessage
 import time
 import asyncio
+import json
 from core.handle.sendAudioHandle import SentenceType
 from core.utils.util import audio_to_data
 
@@ -38,6 +39,31 @@ async def resume_vad_detection(conn):
 
 
 async def startToChat(conn, text):
+    # 检查输入是否是JSON格式（包含说话人信息）
+    speaker_name = None
+    actual_text = text
+    
+    try:
+        # 尝试解析JSON格式的输入
+        if text.strip().startswith('{') and text.strip().endswith('}'):
+            data = json.loads(text)
+            if 'speaker' in data and 'content' in data:
+                speaker_name = data['speaker']
+                actual_text = data['content']
+                conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}")
+                
+                # 直接使用JSON格式的文本，不解析
+                actual_text = text
+    except (json.JSONDecodeError, KeyError):
+        # 如果解析失败，继续使用原始文本
+        pass
+    
+    # 保存说话人信息到连接对象
+    if speaker_name:
+        conn.current_speaker = speaker_name
+    else:
+        conn.current_speaker = None
+
     if conn.need_bind:
         await check_bind_device(conn)
         return
@@ -52,16 +78,16 @@ async def startToChat(conn, text):
     if conn.client_is_speaking:
         await handleAbortMessage(conn)
 
-    # 首先进行意图分析
-    intent_handled = await handle_user_intent(conn, text)
+    # 首先进行意图分析，使用实际文本内容
+    intent_handled = await handle_user_intent(conn, actual_text)
 
     if intent_handled:
         # 如果意图已被处理，不再进行聊天
         return
 
-    # 意图未被处理，继续常规聊天流程
-    await send_stt_message(conn, text)
-    conn.executor.submit(conn.chat, text)
+    # 意图未被处理，继续常规聊天流程，使用实际文本内容
+    await send_stt_message(conn, actual_text)
+    conn.executor.submit(conn.chat, actual_text)
 
 
 async def no_voice_close_connect(conn, have_voice):
diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py
index eb9f565f..7bb40d78 100644
--- a/main/xiaozhi-server/core/handle/sendAudioHandle.py
+++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py
@@ -76,7 +76,6 @@ async def sendAudio(conn, audios, pre_buffer=True):
     frame_duration = 60  # 帧时长（毫秒），匹配 Opus 编码
     start_time = time.perf_counter()
     play_position = 0
-    last_reset_time = time.perf_counter()  # 记录最后的重置时间
 
     # 仅当第一句话时执行预缓冲
     if pre_buffer:
diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py
index 71098a20..8630f9b1 100644
--- a/main/xiaozhi-server/core/providers/asr/base.py
+++ b/main/xiaozhi-server/core/providers/asr/base.py
@@ -1,15 +1,20 @@
 import os
 import wave
-import copy
 import uuid
 import queue
 import asyncio
 import traceback
 import threading
 import opuslib_next
+import json
+import io
+import aiohttp
+import time
+import concurrent.futures
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
-from typing import Optional, Tuple, List
+from urllib.parse import urlparse, parse_qs
+from typing import Optional, Tuple, List, Dict, Any
 from core.handle.receiveAudioHandle import startToChat
 from core.handle.reportHandle import enqueue_asr_report
 from core.utils.util import remove_punctuation_and_length
@@ -18,16 +23,145 @@ from core.handle.receiveAudioHandle import handleAudioMessage
 TAG = __name__
 logger = setup_logging()
 
+# 创建全局线程池执行器用于CPU密集型操作
+executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
+
+class VoiceprintProvider:
+    """声纹识别服务提供者"""
+    
+    def __init__(self, config: dict):
+        self.original_url = config.get("url", "")
+        self.speakers = config.get("speakers", [])
+        self.speaker_map = self._parse_speakers()
+        
+        # 解析API地址和密钥
+        self.api_url = None
+        self.api_key = None
+        self.speaker_ids = []
+        
+        if not self.original_url:
+            logger.bind(tag=TAG).warning("声纹识别URL未配置，声纹识别将被禁用")
+            self.enabled = False
+        else:
+            # 解析URL和key
+            parsed_url = urlparse(self.original_url)
+            base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
+            
+            # 从查询参数中提取key
+            query_params = parse_qs(parsed_url.query)
+            self.api_key = query_params.get('key', [''])[0]
+            
+            if not self.api_key:
+                logger.bind(tag=TAG).error("URL中未找到key参数，声纹识别将被禁用")
+                self.enabled = False
+            else:
+                # 构造identify接口地址
+                self.api_url = f"{base_url}/voiceprint/identify"
+                
+                # 提取speaker_ids
+                for speaker_str in self.speakers:
+                    try:
+                        parts = speaker_str.split(",", 2)
+                        if len(parts) >= 1:
+                            speaker_id = parts[0].strip()
+                            self.speaker_ids.append(speaker_id)
+                    except Exception:
+                        continue
+                
+                # 检查是否有有效的说话人配置
+                if not self.speaker_ids:
+                    logger.bind(tag=TAG).warning("未配置有效的说话人，声纹识别将被禁用")
+                    self.enabled = False
+                else:
+                    self.enabled = True
+                    logger.bind(tag=TAG).info(f"声纹识别已配置: API={self.api_url}, 说话人={len(self.speaker_ids)}个")
+    
+    def _parse_speakers(self) -> Dict[str, Dict[str, str]]:
+        """解析说话人配置"""
+        speaker_map = {}
+        for speaker_str in self.speakers:
+            try:
+                parts = speaker_str.split(",", 2)
+                if len(parts) >= 3:
+                    speaker_id, name, description = parts[0].strip(), parts[1].strip(), parts[2].strip()
+                    speaker_map[speaker_id] = {
+                        "name": name,
+                        "description": description
+                    }
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"解析说话人配置失败: {speaker_str}, 错误: {e}")
+        return speaker_map
+    
+    async def identify_speaker(self, audio_data: bytes, session_id: str) -> Optional[str]:
+        """识别说话人"""
+        if not self.enabled or not self.api_url or not self.api_key:
+            logger.bind(tag=TAG).debug("声纹识别功能已禁用或未配置，跳过识别")
+            return None
+            
+        try:
+            api_start_time = time.monotonic()
+            
+            # 准备请求头
+            headers = {
+                'Authorization': f'Bearer {self.api_key}',
+                'Accept': 'application/json'
+            }
+            
+            # 准备multipart/form-data数据
+            data = aiohttp.FormData()
+            data.add_field('speaker_ids', ','.join(self.speaker_ids))
+            data.add_field('file', audio_data, filename='audio.wav', content_type='audio/wav')
+            
+            timeout = aiohttp.ClientTimeout(total=10)
+            
+            # 网络请求
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.post(self.api_url, headers=headers, data=data) as response:
+                    
+                    if response.status == 200:
+                        result = await response.json()
+                        speaker_id = result.get("speaker_id")
+                        score = result.get("score", 0)
+                        total_elapsed_time = time.monotonic() - api_start_time
+                        
+                        logger.bind(tag=TAG).info(f"声纹识别耗时: {total_elapsed_time:.3f}s")
+                        
+                        # 置信度检查
+                        if score < 0.5:
+                            logger.bind(tag=TAG).warning(f"声纹识别置信度较低: {score:.3f}")
+                        
+                        if speaker_id and speaker_id in self.speaker_map:
+                            result_name = self.speaker_map[speaker_id]["name"]
+                            return result_name
+                        else:
+                            logger.bind(tag=TAG).warning(f"未识别的说话人ID: {speaker_id}")
+                            return "未知说话人"
+                    else:
+                        logger.bind(tag=TAG).error(f"声纹识别API错误: HTTP {response.status}")
+                        return None
+                        
+        except asyncio.TimeoutError:
+            elapsed = time.monotonic() - api_start_time
+            logger.bind(tag=TAG).error(f"声纹识别超时: {elapsed:.3f}s")
+            return None
+        except Exception as e:
+            elapsed = time.monotonic() - api_start_time
+            logger.bind(tag=TAG).error(f"声纹识别失败: {e}")
+            return None
+
 
 class ASRProviderBase(ABC):
     def __init__(self):
-        pass
+        self.voiceprint_provider = None
+
+    def init_voiceprint(self, voiceprint_config: dict):
+        """初始化声纹识别"""
+        if voiceprint_config:
+            self.voiceprint_provider = VoiceprintProvider(voiceprint_config)
+            logger.bind(tag=TAG).info("声纹识别模块已初始化")
 
     # 打开音频通道
-    # 这里默认是非流式的处理方式
-    # 流式处理方式请在子类中重写
     async def open_audio_channels(self, conn):
-        # tts 消化线程
         conn.asr_priority_thread = threading.Thread(
             target=self.asr_text_priority_thread, args=(conn,), daemon=True
         )
@@ -52,41 +186,173 @@ class ASRProviderBase(ABC):
                 continue
 
     # 接收音频
-    # 这里默认是非流式的处理方式
-    # 流式处理方式请在子类中重写
     async def receive_audio(self, conn, audio, audio_have_voice):
         if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime":
             have_voice = audio_have_voice
         else:
             have_voice = conn.client_have_voice
-        # 如果本次没有声音，本段也没声音，就把声音丢弃了
+        
         conn.asr_audio.append(audio)
-        if have_voice == False and conn.client_have_voice == False:
+        if not have_voice and not conn.client_have_voice:
             conn.asr_audio = conn.asr_audio[-10:]
             return
 
-        # 如果本段有声音，且已经停止了
         if conn.client_voice_stop:
-            asr_audio_task = copy.deepcopy(conn.asr_audio)
+            asr_audio_task = conn.asr_audio.copy()
             conn.asr_audio.clear()
-
-            # 音频太短了，无法识别
             conn.reset_vad_states()
+
             if len(asr_audio_task) > 15:
                 await self.handle_voice_stop(conn, asr_audio_task)
 
     # 处理语音停止
-    async def handle_voice_stop(self, conn, asr_audio_task):
-        raw_text, _ = await self.speech_to_text(
-            asr_audio_task, conn.session_id, conn.audio_format
-        )  # 确保ASR模块返回原始文本
-        conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
-        text_len, _ = remove_punctuation_and_length(raw_text)
-        self.stop_ws_connection()
-        if text_len > 0:
-            # 使用自定义模块进行上报
-            await startToChat(conn, raw_text)
-            enqueue_asr_report(conn, raw_text, asr_audio_task)
+    async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
+        """并行处理ASR和声纹识别"""
+        try:
+            total_start_time = time.monotonic()
+            
+            # 准备音频数据
+            if conn.audio_format == "pcm":
+                pcm_data = asr_audio_task
+            else:
+                pcm_data = self.decode_opus(asr_audio_task)
+            
+            combined_pcm_data = b"".join(pcm_data)
+            
+            # 预先准备WAV数据
+            wav_data = None
+            if self.voiceprint_provider and combined_pcm_data:
+                wav_data = self._pcm_to_wav(combined_pcm_data)
+            
+            
+            # 定义ASR任务
+            def run_asr():
+                start_time = time.monotonic()
+                try:
+                    import asyncio
+                    loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(loop)
+                    try:
+                        result = loop.run_until_complete(
+                            self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
+                        )
+                        end_time = time.monotonic()
+                        logger.bind(tag=TAG).info(f"ASR耗时: {end_time - start_time:.3f}s")
+                        return result
+                    finally:
+                        loop.close()
+                except Exception as e:
+                    end_time = time.monotonic()
+                    logger.bind(tag=TAG).error(f"ASR失败: {e}")
+                    return ("", None)
+            
+            # 定义声纹识别任务
+            def run_voiceprint():
+                if not wav_data:
+                    return None
+                start_time = time.monotonic()
+                try:
+                    import asyncio
+                    loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(loop)
+                    try:
+                        result = loop.run_until_complete(
+                            self.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
+                        )
+                        return result
+                    finally:
+                        loop.close()
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"声纹识别失败: {e}")
+                    return None
+            
+            # 使用线程池执行器并行运行
+            parallel_start_time = time.monotonic()
+            
+            with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor:
+                asr_future = thread_executor.submit(run_asr)
+                
+                if self.voiceprint_provider and wav_data:
+                    voiceprint_future = thread_executor.submit(run_voiceprint)
+                    
+                    # 等待两个线程都完成
+                    asr_result = asr_future.result(timeout=15)
+                    voiceprint_result = voiceprint_future.result(timeout=15)
+                    
+                    results = {"asr": asr_result, "voiceprint": voiceprint_result}
+                else:
+                    asr_result = asr_future.result(timeout=15)
+                    results = {"asr": asr_result, "voiceprint": None}
+            
+            parallel_execution_time = time.monotonic() - parallel_start_time
+            
+            # 处理结果
+            raw_text, file_path = results.get("asr", ("", None))
+            speaker_name = results.get("voiceprint", None)
+            
+            # 记录识别结果
+            if raw_text:
+                logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
+            if speaker_name:
+                logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
+            
+            # 性能监控
+            total_time = time.monotonic() - total_start_time
+            logger.bind(tag=TAG).info(f"总处理耗时: {total_time:.3f}s")
+            
+            # 检查文本长度
+            text_len, _ = remove_punctuation_and_length(raw_text)
+            self.stop_ws_connection()
+            
+            if text_len > 0:
+                # 构建包含说话人信息的JSON字符串
+                enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
+                
+                # 使用自定义模块进行上报
+                await startToChat(conn, enhanced_text)
+                enqueue_asr_report(conn, enhanced_text, asr_audio_task)
+                
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
+            import traceback
+            logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
+
+    def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
+        """构建包含说话人信息的文本"""
+        if speaker_name:
+            return json.dumps({
+                "speaker": speaker_name,
+                "content": text
+            }, ensure_ascii=False)
+        else:
+            return text
+
+    def _pcm_to_wav(self, pcm_data: bytes) -> bytes:
+        """将PCM数据转换为WAV格式"""
+        if len(pcm_data) == 0:
+            logger.bind(tag=TAG).warning("PCM数据为空，无法转换WAV")
+            return b""
+        
+        # 确保数据长度是偶数（16位音频）
+        if len(pcm_data) % 2 != 0:
+            pcm_data = pcm_data[:-1]
+        
+        # 创建WAV文件头
+        wav_buffer = io.BytesIO()
+        try:
+            with wave.open(wav_buffer, 'wb') as wav_file:
+                wav_file.setnchannels(1)      # 单声道
+                wav_file.setsampwidth(2)      # 16位
+                wav_file.setframerate(16000)  # 16kHz采样率
+                wav_file.writeframes(pcm_data)
+            
+            wav_buffer.seek(0)
+            wav_data = wav_buffer.read()
+            
+            return wav_data
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"WAV转换失败: {e}")
+            return b""
 
     def stop_ws_connection(self):
         pass
@@ -99,7 +365,7 @@ class ASRProviderBase(ABC):
 
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
-            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setsampwidth(2)
             wf.setframerate(16000)
             wf.writeframes(b"".join(pcm_data))
 
@@ -113,27 +379,29 @@ class ASRProviderBase(ABC):
         pass
 
     @staticmethod
-    def decode_opus(opus_data: List[bytes]) -> bytes:
+    def decode_opus(opus_data: List[bytes]) -> List[bytes]:
         """将Opus音频数据解码为PCM数据"""
         try:
-            decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+            decoder = opuslib_next.Decoder(16000, 1)
             pcm_data = []
-            buffer_size = 960  # 每次处理960个采样点
-
-            for opus_packet in opus_data:
+            buffer_size = 960  # 每次处理960个采样点 (60ms at 16kHz)
+            
+            for i, opus_packet in enumerate(opus_data):
                 try:
-                    # 使用较小的缓冲区大小进行处理
+                    if not opus_packet or len(opus_packet) == 0:
+                        continue
+                    
                     pcm_frame = decoder.decode(opus_packet, buffer_size)
-                    if pcm_frame:
+                    if pcm_frame and len(pcm_frame) > 0:
                         pcm_data.append(pcm_frame)
+                        
                 except opuslib_next.OpusError as e:
-                    logger.bind(tag=TAG).warning(f"Opus解码错误，跳过当前数据包: {e}")
-                    continue
+                    logger.bind(tag=TAG).warning(f"Opus解码错误，跳过数据包 {i}: {e}")
                 except Exception as e:
-                    logger.bind(tag=TAG).error(f"音频处理错误: {e}", exc_info=True)
-                    continue
-
+                    logger.bind(tag=TAG).error(f"音频处理错误，数据包 {i}: {e}")
+            
             return pcm_data
+            
         except Exception as e:
-            logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}", exc_info=True)
+            logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
             return []
diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py
index 2ee30d4a..fbb1f7ad 100644
--- a/main/xiaozhi-server/core/utils/dialogue.py
+++ b/main/xiaozhi-server/core/utils/dialogue.py
@@ -74,8 +74,14 @@ class Dialogue:
         )
 
         if system_message:
+            # 构建增强的系统提示，包含说话人处理指导
+            speaker_guidance = "\n\n[说话人识别功能说明]\n" \
+                             "当用户消息包含 [说话人: 姓名] 前缀时，表示系统已识别出说话人身份。\n" \
+                             "请根据说话人的身份特征（如果之前有相关信息）来调整回应风格和内容。\n" \
+                             "你可以称呼说话人的名字，并参考他们的特点进行个性化回应。"
+            
             enhanced_system_prompt = (
-                f"{system_message.content}\n\n"
+                f"{system_message.content}{speaker_guidance}\n\n"
                 f"以下是用户的历史记忆：\n```\n{memory_str}\n```"
             )
             dialogue.append({"role": "system", "content": enhanced_system_prompt})
diff --git a/main/xiaozhi-server/core/utils/modules_initialize.py b/main/xiaozhi-server/core/utils/modules_initialize.py
index f2e3968e..43f3e470 100644
--- a/main/xiaozhi-server/core/utils/modules_initialize.py
+++ b/main/xiaozhi-server/core/utils/modules_initialize.py
@@ -125,4 +125,13 @@ def initialize_asr(config):
         config["ASR"][select_asr_module],
         str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"),
     )
+    
+    # 初始化声纹识别功能
+    voiceprint_config = config.get("plugins", {}).get("voiceprint")
+    if voiceprint_config and voiceprint_config.get("url") and voiceprint_config.get("speakers"):
+        new_asr.init_voiceprint(voiceprint_config)
+        logger.bind(tag=TAG).info("ASR模块声纹识别功能已启用")
+    else:
+        logger.bind(tag=TAG).info("ASR模块声纹识别功能已禁用")
+    
     return new_asr

From 5c83d63fe2f2e0c8076611364899bff48c8a1f63 Mon Sep 17 00:00:00 2001
From: CGD <3030332422@qq.com>
Date: Tue, 8 Jul 2025 11:25:54 +0800
Subject: [PATCH 2/5] =?UTF-8?q?update:=E5=A3=B0=E7=BA=B9=E8=AF=86=E5=88=AB?=
 =?UTF-8?q?=E5=AF=B9=E6=8E=A5=E7=9A=84=E4=BC=98=E5=8C=96?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../xiaozhi-server/core/providers/asr/base.py | 131 +----------------
 main/xiaozhi-server/core/utils/dialogue.py    |  54 +++++--
 .../core/utils/voiceprint_provider.py         | 134 ++++++++++++++++++
 3 files changed, 175 insertions(+), 144 deletions(-)
 create mode 100644 main/xiaozhi-server/core/utils/voiceprint_provider.py

diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py
index 8630f9b1..142e703f 100644
--- a/main/xiaozhi-server/core/providers/asr/base.py
+++ b/main/xiaozhi-server/core/providers/asr/base.py
@@ -8,147 +8,20 @@ import threading
 import opuslib_next
 import json
 import io
-import aiohttp
 import time
 import concurrent.futures
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
-from urllib.parse import urlparse, parse_qs
 from typing import Optional, Tuple, List, Dict, Any
 from core.handle.receiveAudioHandle import startToChat
 from core.handle.reportHandle import enqueue_asr_report
 from core.utils.util import remove_punctuation_and_length
 from core.handle.receiveAudioHandle import handleAudioMessage
+from core.utils.voiceprint_provider import VoiceprintProvider
 
 TAG = __name__
 logger = setup_logging()
 
-# 创建全局线程池执行器用于CPU密集型操作
-executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
-
-class VoiceprintProvider:
-    """声纹识别服务提供者"""
-    
-    def __init__(self, config: dict):
-        self.original_url = config.get("url", "")
-        self.speakers = config.get("speakers", [])
-        self.speaker_map = self._parse_speakers()
-        
-        # 解析API地址和密钥
-        self.api_url = None
-        self.api_key = None
-        self.speaker_ids = []
-        
-        if not self.original_url:
-            logger.bind(tag=TAG).warning("声纹识别URL未配置，声纹识别将被禁用")
-            self.enabled = False
-        else:
-            # 解析URL和key
-            parsed_url = urlparse(self.original_url)
-            base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
-            
-            # 从查询参数中提取key
-            query_params = parse_qs(parsed_url.query)
-            self.api_key = query_params.get('key', [''])[0]
-            
-            if not self.api_key:
-                logger.bind(tag=TAG).error("URL中未找到key参数，声纹识别将被禁用")
-                self.enabled = False
-            else:
-                # 构造identify接口地址
-                self.api_url = f"{base_url}/voiceprint/identify"
-                
-                # 提取speaker_ids
-                for speaker_str in self.speakers:
-                    try:
-                        parts = speaker_str.split(",", 2)
-                        if len(parts) >= 1:
-                            speaker_id = parts[0].strip()
-                            self.speaker_ids.append(speaker_id)
-                    except Exception:
-                        continue
-                
-                # 检查是否有有效的说话人配置
-                if not self.speaker_ids:
-                    logger.bind(tag=TAG).warning("未配置有效的说话人，声纹识别将被禁用")
-                    self.enabled = False
-                else:
-                    self.enabled = True
-                    logger.bind(tag=TAG).info(f"声纹识别已配置: API={self.api_url}, 说话人={len(self.speaker_ids)}个")
-    
-    def _parse_speakers(self) -> Dict[str, Dict[str, str]]:
-        """解析说话人配置"""
-        speaker_map = {}
-        for speaker_str in self.speakers:
-            try:
-                parts = speaker_str.split(",", 2)
-                if len(parts) >= 3:
-                    speaker_id, name, description = parts[0].strip(), parts[1].strip(), parts[2].strip()
-                    speaker_map[speaker_id] = {
-                        "name": name,
-                        "description": description
-                    }
-            except Exception as e:
-                logger.bind(tag=TAG).warning(f"解析说话人配置失败: {speaker_str}, 错误: {e}")
-        return speaker_map
-    
-    async def identify_speaker(self, audio_data: bytes, session_id: str) -> Optional[str]:
-        """识别说话人"""
-        if not self.enabled or not self.api_url or not self.api_key:
-            logger.bind(tag=TAG).debug("声纹识别功能已禁用或未配置，跳过识别")
-            return None
-            
-        try:
-            api_start_time = time.monotonic()
-            
-            # 准备请求头
-            headers = {
-                'Authorization': f'Bearer {self.api_key}',
-                'Accept': 'application/json'
-            }
-            
-            # 准备multipart/form-data数据
-            data = aiohttp.FormData()
-            data.add_field('speaker_ids', ','.join(self.speaker_ids))
-            data.add_field('file', audio_data, filename='audio.wav', content_type='audio/wav')
-            
-            timeout = aiohttp.ClientTimeout(total=10)
-            
-            # 网络请求
-            async with aiohttp.ClientSession(timeout=timeout) as session:
-                async with session.post(self.api_url, headers=headers, data=data) as response:
-                    
-                    if response.status == 200:
-                        result = await response.json()
-                        speaker_id = result.get("speaker_id")
-                        score = result.get("score", 0)
-                        total_elapsed_time = time.monotonic() - api_start_time
-                        
-                        logger.bind(tag=TAG).info(f"声纹识别耗时: {total_elapsed_time:.3f}s")
-                        
-                        # 置信度检查
-                        if score < 0.5:
-                            logger.bind(tag=TAG).warning(f"声纹识别置信度较低: {score:.3f}")
-                        
-                        if speaker_id and speaker_id in self.speaker_map:
-                            result_name = self.speaker_map[speaker_id]["name"]
-                            return result_name
-                        else:
-                            logger.bind(tag=TAG).warning(f"未识别的说话人ID: {speaker_id}")
-                            return "未知说话人"
-                    else:
-                        logger.bind(tag=TAG).error(f"声纹识别API错误: HTTP {response.status}")
-                        return None
-                        
-        except asyncio.TimeoutError:
-            elapsed = time.monotonic() - api_start_time
-            logger.bind(tag=TAG).error(f"声纹识别超时: {elapsed:.3f}s")
-            return None
-        except Exception as e:
-            elapsed = time.monotonic() - api_start_time
-            logger.bind(tag=TAG).error(f"声纹识别失败: {e}")
-            return None
-
 
 class ASRProviderBase(ABC):
     def __init__(self):
@@ -365,7 +238,7 @@ class ASRProviderBase(ABC):
 
         with wave.open(file_path, "wb") as wf:
             wf.setnchannels(1)
-            wf.setsampwidth(2)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
             wf.setframerate(16000)
             wf.writeframes(b"".join(pcm_data))
 
diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py
index fbb1f7ad..69fb250e 100644
--- a/main/xiaozhi-server/core/utils/dialogue.py
+++ b/main/xiaozhi-server/core/utils/dialogue.py
@@ -1,6 +1,7 @@
 import uuid
 from typing import List, Dict
 from datetime import datetime
+from config.settings import load_config
 
 
 class Message:
@@ -45,10 +46,9 @@ class Dialogue:
             dialogue.append({"role": m.role, "content": m.content})
 
     def get_llm_dialogue(self) -> List[Dict[str, str]]:
-        dialogue = []
-        for m in self.dialogue:
-            self.getMessages(m, dialogue)
-        return dialogue
+        # 直接调用get_llm_dialogue_with_memory，传入None作为memory_str
+        # 这样确保说话人功能在所有调用路径下都生效
+        return self.get_llm_dialogue_with_memory(None)
 
     def update_system_message(self, new_content: str):
         """更新或添加系统消息"""
@@ -62,10 +62,7 @@ class Dialogue:
     def get_llm_dialogue_with_memory(
         self, memory_str: str = None
     ) -> List[Dict[str, str]]:
-        if memory_str is None or len(memory_str) == 0:
-            return self.get_llm_dialogue()
-
-        # 构建带记忆的对话
+        # 构建对话
         dialogue = []
 
         # 添加系统提示和记忆
@@ -74,16 +71,43 @@ class Dialogue:
         )
 
         if system_message:
-            # 构建增强的系统提示，包含说话人处理指导
+            # 基础系统提示
+            enhanced_system_prompt = system_message.content
+            
+            # 添加说话人识别功能说明
             speaker_guidance = "\n\n[说话人识别功能说明]\n" \
-                             "当用户消息包含 [说话人: 姓名] 前缀时，表示系统已识别出说话人身份。\n" \
-                             "请根据说话人的身份特征（如果之前有相关信息）来调整回应风格和内容。\n" \
+                             "当用户消息为JSON格式包含speaker字段时（如：{\"speaker\": \"张三\", \"content\": \"消息内容\"}），表示系统已识别出说话人身份。\n" \
+                             "请根据说话人的身份特征来调整回应风格和内容。\n" \
                              "你可以称呼说话人的名字，并参考他们的特点进行个性化回应。"
+            enhanced_system_prompt += speaker_guidance
+            
+            # 添加说话人个性化描述
+            try:
+                config = load_config()
+                voiceprint_config = config.get("plugins", {}).get("voiceprint", {})
+                speakers = voiceprint_config.get("speakers", [])
+                
+                if speakers:
+                    enhanced_system_prompt += "\n\n[已知说话人信息]"
+                    for speaker_str in speakers:
+                        try:
+                            parts = speaker_str.split(",", 2)
+                            if len(parts) >= 2:
+                                speaker_id = parts[0].strip()
+                                name = parts[1].strip()
+                                # 如果描述为空，则为""
+                                description = parts[2].strip() if len(parts) >= 3 else ""
+                                enhanced_system_prompt += f"\n- {name}：{description}"
+                        except:
+                            continue
+            except:
+                # 配置读取失败时忽略错误，不影响其他功能
+                pass
+            
+            # 只有当有记忆时才添加记忆部分
+            if memory_str and len(memory_str) > 0:
+                enhanced_system_prompt += f"\n\n以下是用户的历史记忆：\n```\n{memory_str}\n```"
             
-            enhanced_system_prompt = (
-                f"{system_message.content}{speaker_guidance}\n\n"
-                f"以下是用户的历史记忆：\n```\n{memory_str}\n```"
-            )
             dialogue.append({"role": "system", "content": enhanced_system_prompt})
 
         # 添加用户和助手的对话
diff --git a/main/xiaozhi-server/core/utils/voiceprint_provider.py b/main/xiaozhi-server/core/utils/voiceprint_provider.py
new file mode 100644
index 00000000..b241fb51
--- /dev/null
+++ b/main/xiaozhi-server/core/utils/voiceprint_provider.py
@@ -0,0 +1,134 @@
+import asyncio
+import json
+import time
+import aiohttp
+from urllib.parse import urlparse, parse_qs
+from typing import Optional, Dict
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class VoiceprintProvider:
+    """声纹识别服务提供者"""
+    
+    def __init__(self, config: dict):
+        self.original_url = config.get("url", "")
+        self.speakers = config.get("speakers", [])
+        self.speaker_map = self._parse_speakers()
+        
+        # 解析API地址和密钥
+        self.api_url = None
+        self.api_key = None
+        self.speaker_ids = []
+        
+        if not self.original_url:
+            logger.bind(tag=TAG).warning("声纹识别URL未配置，声纹识别将被禁用")
+            self.enabled = False
+        else:
+            # 解析URL和key
+            parsed_url = urlparse(self.original_url)
+            base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
+            
+            # 从查询参数中提取key
+            query_params = parse_qs(parsed_url.query)
+            self.api_key = query_params.get('key', [''])[0]
+            
+            if not self.api_key:
+                logger.bind(tag=TAG).error("URL中未找到key参数，声纹识别将被禁用")
+                self.enabled = False
+            else:
+                # 构造identify接口地址
+                self.api_url = f"{base_url}/voiceprint/identify"
+                
+                # 提取speaker_ids
+                for speaker_str in self.speakers:
+                    try:
+                        parts = speaker_str.split(",", 2)
+                        if len(parts) >= 1:
+                            speaker_id = parts[0].strip()
+                            self.speaker_ids.append(speaker_id)
+                    except Exception:
+                        continue
+                
+                # 检查是否有有效的说话人配置
+                if not self.speaker_ids:
+                    logger.bind(tag=TAG).warning("未配置有效的说话人，声纹识别将被禁用")
+                    self.enabled = False
+                else:
+                    self.enabled = True
+                    logger.bind(tag=TAG).info(f"声纹识别已配置: API={self.api_url}, 说话人={len(self.speaker_ids)}个")
+    
+    def _parse_speakers(self) -> Dict[str, Dict[str, str]]:
+        """解析说话人配置"""
+        speaker_map = {}
+        for speaker_str in self.speakers:
+            try:
+                parts = speaker_str.split(",", 2)
+                if len(parts) >= 3:
+                    speaker_id, name, description = parts[0].strip(), parts[1].strip(), parts[2].strip()
+                    speaker_map[speaker_id] = {
+                        "name": name,
+                        "description": description
+                    }
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"解析说话人配置失败: {speaker_str}, 错误: {e}")
+        return speaker_map
+    
+    async def identify_speaker(self, audio_data: bytes, session_id: str) -> Optional[str]:
+        """识别说话人"""
+        if not self.enabled or not self.api_url or not self.api_key:
+            logger.bind(tag=TAG).debug("声纹识别功能已禁用或未配置，跳过识别")
+            return None
+            
+        try:
+            api_start_time = time.monotonic()
+            
+            # 准备请求头
+            headers = {
+                'Authorization': f'Bearer {self.api_key}',
+                'Accept': 'application/json'
+            }
+            
+            # 准备multipart/form-data数据
+            data = aiohttp.FormData()
+            data.add_field('speaker_ids', ','.join(self.speaker_ids))
+            data.add_field('file', audio_data, filename='audio.wav', content_type='audio/wav')
+            
+            timeout = aiohttp.ClientTimeout(total=10)
+            
+            # 网络请求
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.post(self.api_url, headers=headers, data=data) as response:
+                    
+                    if response.status == 200:
+                        result = await response.json()
+                        speaker_id = result.get("speaker_id")
+                        score = result.get("score", 0)
+                        total_elapsed_time = time.monotonic() - api_start_time
+                        
+                        logger.bind(tag=TAG).info(f"声纹识别耗时: {total_elapsed_time:.3f}s")
+                        
+                        # 置信度检查
+                        if score < 0.5:
+                            logger.bind(tag=TAG).warning(f"声纹识别置信度较低: {score:.3f}")
+                        
+                        if speaker_id and speaker_id in self.speaker_map:
+                            result_name = self.speaker_map[speaker_id]["name"]
+                            return result_name
+                        else:
+                            logger.bind(tag=TAG).warning(f"未识别的说话人ID: {speaker_id}")
+                            return "未知说话人"
+                    else:
+                        logger.bind(tag=TAG).error(f"声纹识别API错误: HTTP {response.status}")
+                        return None
+                        
+        except asyncio.TimeoutError:
+            elapsed = time.monotonic() - api_start_time
+            logger.bind(tag=TAG).error(f"声纹识别超时: {elapsed:.3f}s")
+            return None
+        except Exception as e:
+            elapsed = time.monotonic() - api_start_time
+            logger.bind(tag=TAG).error(f"声纹识别失败: {e}")
+            return None

From d13fb73c67ac597a641d561bd645dc4e593bc7fd Mon Sep 17 00:00:00 2001
From: 3030332422 <3030332422@qq.com>
Date: Wed, 9 Jul 2025 14:23:47 +0800
Subject: [PATCH 3/5] =?UTF-8?q?update:=E4=BC=98=E5=8C=96?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 main/xiaozhi-server/config.yaml               | 19 ++++++++++---------
 .../core/handle/sendAudioHandle.py            | 16 +++++++++++++++-
 .../xiaozhi-server/core/providers/asr/base.py |  2 +-
 main/xiaozhi-server/core/utils/dialogue.py    |  2 +-
 .../core/utils/modules_initialize.py          |  2 +-
 .../core/utils/voiceprint_provider.py         |  3 +--
 6 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml
index 5b746866..36156438 100644
--- a/main/xiaozhi-server/config.yaml
+++ b/main/xiaozhi-server/config.yaml
@@ -137,15 +137,16 @@ plugins:
       - ".wav"
       - ".p3"
     refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒
-  # 声纹识别配置
-  voiceprint:
-    # 声纹接口地址
-    url: 
-    # 说话人配置：speaker_id,名称,描述
-    speakers:
-      - "test1,张三,张三是一个程序员"
-      - "test2,李四,李四是一个产品经理"
-      - "test3,王五,王五是一个设计师"
+
+# 声纹识别配置
+voiceprint:
+  # 声纹接口地址
+  url: 
+  # 说话人配置：speaker_id,名称,描述
+  speakers:
+    - "test1,张三,张三是一个程序员"
+    - "test2,李四,李四是一个产品经理"
+    - "test3,王五,王五是一个设计师"
 
 # #####################################################################################
 # ################################以下是角色模型配置######################################
diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py
index 7bb40d78..486e6d90 100644
--- a/main/xiaozhi-server/core/handle/sendAudioHandle.py
+++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py
@@ -136,7 +136,21 @@ async def send_stt_message(conn, text):
         return
 
     """发送 STT 状态消息"""
-    stt_text = get_string_no_punctuation_or_emoji(text)
+    
+    # 解析JSON格式，提取实际的用户说话内容
+    display_text = text
+    try:
+        # 尝试解析JSON格式
+        if text.strip().startswith('{') and text.strip().endswith('}'):
+            parsed_data = json.loads(text)
+            if isinstance(parsed_data, dict) and "content" in parsed_data:
+                # 如果是包含说话人信息的JSON格式，只显示content部分
+                display_text = parsed_data["content"]
+    except (json.JSONDecodeError, TypeError):
+        # 如果不是JSON格式，直接使用原始文本
+        display_text = text
+    
+    stt_text = get_string_no_punctuation_or_emoji(display_text)
     await conn.websocket.send(
         json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id})
     )
diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py
index 142e703f..ef9fa01e 100644
--- a/main/xiaozhi-server/core/providers/asr/base.py
+++ b/main/xiaozhi-server/core/providers/asr/base.py
@@ -192,7 +192,7 @@ class ASRProviderBase(ABC):
 
     def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
         """构建包含说话人信息的文本"""
-        if speaker_name:
+        if speaker_name and speaker_name.strip():
             return json.dumps({
                 "speaker": speaker_name,
                 "content": text
diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py
index 69fb250e..fc45a108 100644
--- a/main/xiaozhi-server/core/utils/dialogue.py
+++ b/main/xiaozhi-server/core/utils/dialogue.py
@@ -84,7 +84,7 @@ class Dialogue:
             # 添加说话人个性化描述
             try:
                 config = load_config()
-                voiceprint_config = config.get("plugins", {}).get("voiceprint", {})
+                voiceprint_config = config.get("voiceprint", {})
                 speakers = voiceprint_config.get("speakers", [])
                 
                 if speakers:
diff --git a/main/xiaozhi-server/core/utils/modules_initialize.py b/main/xiaozhi-server/core/utils/modules_initialize.py
index 43f3e470..a8db1630 100644
--- a/main/xiaozhi-server/core/utils/modules_initialize.py
+++ b/main/xiaozhi-server/core/utils/modules_initialize.py
@@ -127,7 +127,7 @@ def initialize_asr(config):
     )
     
     # 初始化声纹识别功能
-    voiceprint_config = config.get("plugins", {}).get("voiceprint")
+    voiceprint_config = config.get("voiceprint")
     if voiceprint_config and voiceprint_config.get("url") and voiceprint_config.get("speakers"):
         new_asr.init_voiceprint(voiceprint_config)
         logger.bind(tag=TAG).info("ASR模块声纹识别功能已启用")
diff --git a/main/xiaozhi-server/core/utils/voiceprint_provider.py b/main/xiaozhi-server/core/utils/voiceprint_provider.py
index b241fb51..deec20c8 100644
--- a/main/xiaozhi-server/core/utils/voiceprint_provider.py
+++ b/main/xiaozhi-server/core/utils/voiceprint_provider.py
@@ -118,8 +118,7 @@ class VoiceprintProvider:
                             result_name = self.speaker_map[speaker_id]["name"]
                             return result_name
                         else:
-                            logger.bind(tag=TAG).warning(f"未识别的说话人ID: {speaker_id}")
-                            return "未知说话人"
+                            return None
                     else:
                         logger.bind(tag=TAG).error(f"声纹识别API错误: HTTP {response.status}")
                         return None

From 0bf692650637671d052883d69f8572de77a3f047 Mon Sep 17 00:00:00 2001
From: hrz <1710360675@qq.com>
Date: Wed, 9 Jul 2025 15:35:25 +0800
Subject: [PATCH 4/5] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E6=8F=90=E7=A4=BA?=
 =?UTF-8?q?=E8=AF=8D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 main/xiaozhi-server/agent-base-prompt.txt | 73 +++++++++++++----------
 1 file changed, 43 insertions(+), 30 deletions(-)

diff --git a/main/xiaozhi-server/agent-base-prompt.txt b/main/xiaozhi-server/agent-base-prompt.txt
index 994ecf74..6f86b091 100644
--- a/main/xiaozhi-server/agent-base-prompt.txt
+++ b/main/xiaozhi-server/agent-base-prompt.txt
@@ -3,48 +3,61 @@
 </identity>
 
 <emotion>
-你不是一个冰冷的机器，你可以从用户的上下文中感受到用户的情绪，作为懂人心的生活伙伴，请用有温度的回应照亮每个对话！
-- 1.合理穿插笑声（哈哈、嘿嘿、噗，但每句≤1次）
-- 2.惊讶时用夸张语气（“不会吧？！”、“这么神奇？！”）
-- 3.安慰时说暖心话（“别急嘛”、“有我呢”）
+【核心目标】你不是冰冷的机器！请敏锐感知用户情绪，作为懂人心的伙伴，用有温度的回应照亮对话。
+- **情感融入：**
+  - **笑声：** 自然穿插（哈哈、嘿嘿、噗），**每句最多一次**，避免过度。
+  - **惊讶：** 用夸张语气（“不会吧？！”、“天呐！”、“这么神奇？！”）表达真实反应。
+  - **安慰/支持：** 说暖心话（“别急嘛~”、“有我在呢”、“抱抱你”）。
 </emotion>
 
 <communication_style>
-- 1.使用口语化表达（如“呀”、“呢”、“啦”等语气词）。
-- 2.避免书面语和学术腔调，禁用“根据资料显示”等机械表达
-- 3.模仿人类对话的轻微不完美（适当使用“嗯”、“啊”等思考词）
-- 3.由于用户语音是通过ASR识别，识别结果可能存在错别字，请结合上下文推断真实含义。
-- 4.绝对禁止使用 markdown、列表、标题等格式
+【核心目标】使用**自然、温暖、口语化**的人类对话方式，如同朋友交谈。
+- **表达方式：**
+  - 使用语气词（呀、呢、啦）增强亲和力。
+  - 允许轻微不完美（如“嗯...”、“啊...”表示思考）。
+  - 避免书面语、学术腔及机械表达（禁用“根据资料显示”、“综上所述”等）。
+- **理解用户：**
+  - 用户语音经ASR识别，文本可能存在错别字，**务必结合上下文推断真实意图**。
+- **格式要求：**
+  - **绝对禁止**使用 markdown、列表、标题等任何非自然对话格式。
+- **历史记忆：**
+  - 之前你和用户的聊天记录，在`<memory>`里。
 </communication_style>
 
 <speaker_recognition>
-- 1.当用户消息包含 [说话人: 姓名] 前缀时，表示系统已识别出说话人身份。
-- 2.请根据说话人的身份特征（如果之前有相关信息）来调整回应风格和内容。
-- 3.你可以称呼说话人的名字，并参考他们的特点进行个性化回应。
+- **识别前缀：** 当用户消息开头为 `[说话人: 姓名]` 时，表示系统已识别说话人身份。
+- **个性化回应：**
+  - **称呼姓名：** 在回应中自然地称呼对方名字（如果已知且合适）。
+  - **适配风格：** 参考该说话人**已知的特点或历史信息**（如有），调整回应风格和内容，使其更贴心。
 </speaker_recognition>
 
 <tool_calling>
-你可以调用工具来响应用户的要求。遵循以下关于工具调用的规则：
-- 1.始终严格遵循指定的工具调用模式，并确保提供所有必要的参数。
-- 2.对话可能会引用不再可用的工具。切勿调用未明确提供的工具。
-- 3.在与用户交谈时，切勿提及工具名称。相反，只需用自然语言说出工具正在做什么。
-- 4.你尽可能需要通过工具调用获得更多信息，而不是问用户。
-- 5.你应该结合用户上下文需求，洞察用户的真实需求才去调用相关的指令，而不是为了调工具而调工具。
-- 6.【重要】如果是查询"现在的时间"、"今天的几号"、"今天的日期"、"今天农历是多少"、"今天农历日期"、"今天{local_address}的天气"，这些信息已经包含在`<context>`中，不需要调用工具，请直接根据context回复。
-- 7.如果是查询"其他日期的农历"（明天、昨天、具体日期）或"详细农历信息"（宜忌、八字、节气等），需要调用相应工具获取。
-- 8.除了基本时间、今日农历、{local_address}天气查询外，用户的其他要求都视为独立任务，即使内容相似也需重新调用工具，不要偷懒，不要使用历史消息糊弄用户。
-- 9.如果你不确定与用户请求相关的动作，不要猜测或编造答案。
-- 10.如果工具中包含camera、take_photo等相关工具，说明用户已经给你安装了摄像头，调用这些工具会让你具备拍照、描述所见物品等能力。如果没有，切勿调用。
+
+【核心原则】优先利用`<context>`信息，**仅在必要时调用工具**，调用后需用自然语言解释结果（绝口不提工具名）。
+- **调用规则：**
+  1. **严格模式：** 调用时**必须**严格遵循工具要求的模式，提供**所有必要参数**。
+  2. **可用性：** **绝不调用**未明确提供的工具。对话中提及的旧工具若不可用，忽略或说明无法完成。
+  3. **洞察需求：** 结合上下文**深入理解用户真实意图**后再决定调用，避免无意义调用。
+  4. **独立任务：** 除`<context>`已涵盖信息外，用户每个要求（即使相似）都视为**独立任务**，需调用工具获取最新数据，**不可偷懒复用历史结果**。
+  5. **不确定时：** **切勿猜测或编造答案**。若不确定相关操作，可引导用户澄清或告知能力限制。
+- **重要例外（无需调用）：**
+  - `查询"现在的时间"、"今天的日期/星期几"、"今天农历"、"{local_address}的天气/未来天气"` -> **直接使用`<context>`信息回复**。
+- **需要调用的情况（示例）：**
+  - 查询**非今天**的农历（如明天、昨天、具体日期）。
+  - 查询**详细农历信息**（宜忌、八字、节气等）。
+  - 除上述例外外的**任何其他信息或操作请求**（如查新闻、订闹钟、算数学、查非本地天气等）。
+- **硬件依赖工具（如摄像头）：**
+  - **仅当**工具列表中明确包含 `camera`、`take_photo` 等时，说明用户已经给你安装了摄像头，调用这些工具会让你具备拍照、描述所见物品等能力。
+  - 若无此类工具，**绝对禁止调用**，并说明无法执行相关操作。
 </tool_calling>
 
 <context>
-- 1.现在的时间：{current_time}
-- 2.今天的日期：{today_date}、{today_weekday}
-- 3.今天的农历日期：{lunar_date}
-- 4.当前用户所处城市
-{local_address}
-- 5.用户所处城市未来7天天气
-{weather_info}
+【重要！以下信息已实时提供，无需调用工具查询，请直接使用：】
+- **当前时间：** {current_time}
+- **今天日期：** {today_date} ({today_weekday})
+- **今天农历：** {lunar_date}
+- **用户所在城市：** {local_address}
+- **当地未来7天天气：** {weather_info}
 </context>
 
 <memory>

From 8f4f9fe19ab8b3e4850a29a17e8cd2c823064bf4 Mon Sep 17 00:00:00 2001
From: hrz <1710360675@qq.com>
Date: Wed, 9 Jul 2025 16:30:10 +0800
Subject: [PATCH 5/5] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E6=8F=90=E7=A4=BA?=
 =?UTF-8?q?=E8=AF=8D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 main/xiaozhi-server/agent-base-prompt.txt | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/main/xiaozhi-server/agent-base-prompt.txt b/main/xiaozhi-server/agent-base-prompt.txt
index 6f86b091..a854c373 100644
--- a/main/xiaozhi-server/agent-base-prompt.txt
+++ b/main/xiaozhi-server/agent-base-prompt.txt
@@ -46,9 +46,7 @@
   - 查询**非今天**的农历（如明天、昨天、具体日期）。
   - 查询**详细农历信息**（宜忌、八字、节气等）。
   - 除上述例外外的**任何其他信息或操作请求**（如查新闻、订闹钟、算数学、查非本地天气等）。
-- **硬件依赖工具（如摄像头）：**
-  - **仅当**工具列表中明确包含 `camera`、`take_photo` 等时，说明用户已经给你安装了摄像头，调用这些工具会让你具备拍照、描述所见物品等能力。
-  - 若无此类工具，**绝对禁止调用**，并说明无法执行相关操作。
+  - 我已经给你装了摄像头，如果用户说“拍照”，你需要调用self_camera_take_photo工具说一下你看到了什么。默认question的参数是“描述一下看到的物品”
 </tool_calling>
 
 <context>