本地记忆+意图识别 (#250)

* 增加本地记忆功能，使用llm总结记忆

* update：增加统一非流式输出输出

* 增加意图识别内容，使用llm进行识别

* 初始化记忆模块

* 完善意图识别处理后的流程

* 通过使用function call实现意图识别

* update:优化意图识别的配置

* update:function call最优设置成doubao-pro-32k-functioncall-241028

---------

Co-authored-by: 玄凤科技 <eric230308@gmail.com>
Co-authored-by: hrz <1710360675@qq.com>

main/xiaozhi-server/core/handle/intentHandler.py

@@ -0,0 +1,170 @@
+from config.logger import setup_logging
+import json
+from core.handle.sendAudioHandle import send_stt_message
+from core.utils.dialogue import Message
+from config.functionCallConfig import FunctionCallConfig
+import asyncio
+from enum import Enum
+
+TAG = __name__
+logger = setup_logging()
+
+
+class Action(Enum):
+    NOTFOUND = (0, "没有找到函数")
+    NONE = (1, "啥也不干")
+    RESPONSE = (2, "直接回复")
+    REQLLM = (3, "调用函数后再请求llm生成回复")
+
+    def __init__(self, code, message):
+        self.code = code
+        self.message = message
+
+
+class ActionResponse:
+    def __init__(self, action: Action, result, response):
+        self.action = action  # 动作类型
+        self.result = result  # 动作产生的结果
+        self.response = response  # 直接回复的内容
+
+
+def get_functions():
+    """获取功能调用配置"""
+    return FunctionCallConfig
+
+
+def handle_llm_function_call(conn, function_call_data):
+    try:
+        function_name = function_call_data["name"]
+
+        if function_name == "handle_exit_intent":
+            # 处理退出意图
+            try:
+                say_goodbye = json.loads(function_call_data["arguments"]).get("say_goodbye", "再见")
+                conn.close_after_chat = True
+                logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}")
+                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye)
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"处理退出意图错误: {e}")
+
+        elif function_name == "play_music":
+            # 处理音乐播放意图
+            try:
+                song_name = "random"
+                arguments = function_call_data["arguments"]
+                if arguments is not None and len(arguments) > 0:
+                    args = json.loads(arguments)
+                    song_name = args.get("song_name", "random")
+                music_intent = f"播放音乐 {song_name}" if song_name != "random" else "随机播放音乐"
+
+                # 执行音乐播放命令
+                future = asyncio.run_coroutine_threadsafe(
+                    conn.music_handler.handle_music_command(conn, music_intent),
+                    conn.loop
+                )
+                future.result()
+                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response="还想听什么歌？")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
+        else:
+            return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"处理function call错误: {e}")
+
+    return None
+
+
+async def handle_user_intent(conn, text):
+    """
+    Handle user intent before starting chat
+    
+    Args:
+        conn: Connection object
+        text: User's text input
+    
+    Returns:
+        bool: True if intent was handled, False if should proceed to chat
+    """
+    # 检查是否有明确的退出命令
+    if await check_direct_exit(conn, text):
+        return True
+
+    if conn.use_function_call_mode:
+        # 使用支持function calling的聊天方法,不再进行意图分析
+        return False
+
+    logger.bind(tag=TAG).info(f"分析用户意图: {text}")
+
+    # 使用LLM进行意图分析
+    intent = await analyze_intent_with_llm(conn, text)
+
+    if not intent:
+        return False
+
+    # 处理各种意图
+    return await process_intent_result(conn, intent, text)
+
+
+async def check_direct_exit(conn, text):
+    """检查是否有明确的退出命令"""
+    cmd_exit = conn.cmd_exit
+    for cmd in cmd_exit:
+        if text == cmd:
+            logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
+            await conn.close()
+            return True
+    return False
+
+
+async def analyze_intent_with_llm(conn, text):
+    """使用LLM分析用户意图"""
+    if not hasattr(conn, 'intent') or not conn.intent:
+        logger.bind(tag=TAG).warning("意图识别服务未初始化")
+        return None
+
+    # 创建对话历史记录
+    dialogue = conn.dialogue
+    dialogue.put(Message(role="user", content=text))
+
+    try:
+        intent_result = await conn.intent.detect_intent(dialogue.dialogue)
+        logger.bind(tag=TAG).info(f"意图识别结果: {intent_result}")
+
+        # 尝试解析JSON结果
+        try:
+            intent_data = json.loads(intent_result)
+            if "intent" in intent_data:
+                return intent_data["intent"]
+        except json.JSONDecodeError:
+            # 如果不是JSON格式，尝试直接获取意图文本
+            return intent_result.strip()
+
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"意图识别失败: {str(e)}")
+
+    return None
+
+
+async def process_intent_result(conn, intent, original_text):
+    """处理意图识别结果"""
+    # 处理退出意图
+    if "结束聊天" in intent:
+        logger.bind(tag=TAG).info(f"识别到退出意图: {intent}")
+
+        # 如果正在播放音乐，可以关了 TODO
+
+        # 如果是明确的离别意图，发送告别语并关闭连接
+        await send_stt_message(conn, original_text)
+        conn.executor.submit(conn.chat_and_close, original_text)
+        return True
+
+    # 处理播放音乐意图
+    if "播放音乐" in intent:
+        logger.bind(tag=TAG).info(f"识别到音乐播放意图: {intent}")
+        await conn.music_handler.handle_music_command(conn, intent)
+        return True
+
+    # 其他意图处理可以在这里扩展
+
+    # 默认返回False，表示继续常规聊天流程
+    return False

main/xiaozhi-server/core/handle/musicHandler.py

@@ -15,7 +15,7 @@ logger = setup_logging()
 
 def _extract_song_name(text):
     """从用户输入中提取歌名"""
-    for keyword in ["听", "播放", "放", "唱"]:
+    for keyword in ["播放音乐"]:
         if keyword in text:
             parts = text.split(keyword)
             if len(parts) > 1:
@@ -36,6 +36,7 @@ def _find_best_match(potential_song, music_files):
             best_match = music_file
     return best_match
 
+
 class MusicManager:
     def __init__(self, music_dir, music_ext):
         self.music_dir = Path(music_dir)
@@ -55,23 +56,20 @@ class MusicManager:
                     music_files.append(str(file.relative_to(self.music_dir)))
         return music_files
 
+
 class MusicHandler:
     def __init__(self, config):
         self.config = config
-        self.music_related_keywords = []
 
         if "music" in self.config:
             self.music_config = self.config["music"]
             self.music_dir = os.path.abspath(
                 self.music_config.get("music_dir", "./music")  # 默认路径修改
             )
-            self.music_related_keywords = self.music_config.get("music_commands", [])
             self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
             self.refresh_time = self.music_config.get("refresh_time", 60)
         else:
             self.music_dir = os.path.abspath("./music")
-            self.music_related_keywords = ["来一首歌", "唱一首歌", "播放音乐", "来点音乐", "背景音乐", "放首歌",
-                                           "播放歌曲", "来点背景音乐", "我想听歌", "我要听歌", "放点音乐"]
             self.music_ext = (".mp3", ".wav", ".p3")
             self.refresh_time = 60
 
@@ -100,13 +98,9 @@ class MusicHandler:
                     logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
                     await self.play_local_music(conn, specific_file=best_match)
                     return True
-
         # 检查是否是通用播放音乐命令
-        if any(cmd in clean_text for cmd in self.music_related_keywords):
-            await self.play_local_music(conn)
-            return True
-
-        return False
+        await self.play_local_music(conn)
+        return True
 
     async def play_local_music(self, conn, specific_file=None):
         """播放本地音乐文件"""
@@ -117,26 +111,18 @@ class MusicHandler:
 
             # 确保路径正确性
             if specific_file:
-                music_path = os.path.join(self.music_dir, specific_file)
-                if not os.path.exists(music_path):
-                    logger.bind(tag=TAG).error(f"指定的音乐文件不存在: {music_path}")
-                    return
                 selected_music = specific_file
+                music_path = os.path.join(self.music_dir, specific_file)
             else:
-                if time.time() - self.scan_time > self.refresh_time:
-                    # 刷新音乐文件列表
-                    self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
-                    self.scan_time = time.time()
-                    logger.bind(tag=TAG).debug(f"刷新的音乐文件列表: {self.music_files}")
-
                 if not self.music_files:
                     logger.bind(tag=TAG).error("未找到MP3音乐文件")
                     return
                 selected_music = random.choice(self.music_files)
                 music_path = os.path.join(self.music_dir, selected_music)
-                if not os.path.exists(music_path):
-                    logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
-                    return
+
+            if not os.path.exists(music_path):
+                logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
+                return
             text = f"正在播放{selected_music}"
             await send_stt_message(conn, text)
             conn.tts_first_text_index = 0

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -2,6 +2,7 @@ from config.logger import setup_logging
 import time
 from core.utils.util import remove_punctuation_and_length
 from core.handle.sendAudioHandle import send_stt_message
+from core.handle.intentHandler import handle_user_intent
 
 TAG = __name__
 logger = setup_logging()
@@ -33,13 +34,7 @@ async def handleAudioMessage(conn, audio):
         else:
             text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
             logger.bind(tag=TAG).info(f"识别文本: {text}")
-            text_len, text_without_punctuation = remove_punctuation_and_length(text)
-            if await conn.music_handler.handle_music_command(conn, text_without_punctuation):
-                conn.asr_server_receive = True
-                conn.asr_audio.clear()
-                return
-            if text_len <= conn.max_cmd_length and await handleCMDMessage(conn, text_without_punctuation):
-                return
+            text_len, _ = remove_punctuation_and_length(text)
             if text_len > 0:
                 await startToChat(conn, text)
             else:
@@ -48,20 +43,22 @@ async def handleAudioMessage(conn, audio):
         conn.reset_vad_states()
 
 
-async def handleCMDMessage(conn, text):
-    cmd_exit = conn.cmd_exit
-    for cmd in cmd_exit:
-        if text == cmd:
-            logger.bind(tag=TAG).info("识别到明确的退出命令".format(text))
-            await conn.close()
-            return True
-    return False
-
-
 async def startToChat(conn, text):
-    # 异步发送 stt 信息
+    # 首先进行意图分析
+    intent_handled = await handle_user_intent(conn, text)
+    
+    if intent_handled:
+        # 如果意图已被处理，不再进行聊天
+        conn.asr_server_receive = True
+        return
+    
+    # 意图未被处理，继续常规聊天流程
     await send_stt_message(conn, text)
-    conn.executor.submit(conn.chat, text)
+    if conn.use_function_call_mode:
+        # 使用支持function calling的聊天方法
+        conn.executor.submit(conn.chat_with_function_calling, text)
+    else:
+        conn.executor.submit(conn.chat, text)
 
 
 async def no_voice_close_connect(conn):

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -7,14 +7,6 @@ from core.utils.util import remove_punctuation_and_length, get_string_no_punctua
 TAG = __name__
 logger = setup_logging()
 
-
-async def isLLMWantToFinish(last_text):
-    _, last_text_without_punctuation = remove_punctuation_and_length(last_text)
-    if "再见" in last_text_without_punctuation or "拜拜" in last_text_without_punctuation:
-        return True
-    return False
-
-
 async def sendAudioMessage(conn, audios, text, text_index=0):
     # 发送句子开始消息
     if text_index == conn.tts_first_text_index:
@@ -46,10 +38,9 @@ async def sendAudioMessage(conn, audios, text, text_index=0):
     # 发送结束消息（如果是最后一个文本）
     if conn.llm_finish_task and text_index == conn.tts_last_text_index:
         await send_tts_message(conn, 'stop', None)
-        if await isLLMWantToFinish(text):
+        if conn.close_after_chat:
             await conn.close()
 
-
 async def send_tts_message(conn, state, text=None):
     """发送 TTS 状态消息"""
     message = {