diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 52596ba6..a87bccb6 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -20,7 +20,7 @@ from plugins_func.register import Action from config.private_config import PrivateConfig from core.auth import AuthMiddleware, AuthenticationError from core.utils.auth_code_gen import AuthCodeGenerator -import plugins_func.loadplugins +import plugins_func.loadplugins TAG = __name__ @@ -30,7 +30,7 @@ class TTSException(RuntimeError): class ConnectionHandler: - def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory, _intent): + def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _memory, _intent): self.config = config self.logger = setup_logging() self.auth = AuthMiddleware(config) @@ -91,7 +91,6 @@ class ConnectionHandler: self.private_config = None self.auth_code_gen = AuthCodeGenerator.get_instance() self.is_device_verified = False # 添加设备验证状态标志 - self.music_handler = _music self.close_after_chat = False # 是否在聊天结束后关闭连接 self.use_function_call_mode = False if self.config["selected_module"]["Intent"] == 'function_call': diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index 320c4793..1799516c 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -1,5 +1,6 @@ from config.logger import setup_logging import json +import uuid from core.handle.sendAudioHandle import send_stt_message from core.utils.util import remove_punctuation_and_length @@ -8,30 +9,16 @@ logger = setup_logging() async def handle_user_intent(conn, text): - """ - Handle user intent before starting chat - - Args: - conn: Connection object - text: User's text input - - Returns: - bool: True if intent was handled, False if should proceed to chat - """ # 检查是否有明确的退出命令 if await check_direct_exit(conn, text): return True - if conn.use_function_call_mode: # 使用支持function calling的聊天方法,不再进行意图分析 return False - # 使用LLM进行意图分析 intent = await analyze_intent_with_llm(conn, text) - if not intent: return False - # 处理各种意图 return await process_intent_result(conn, intent, text) @@ -58,7 +45,6 @@ async def analyze_intent_with_llm(conn, text): dialogue = conn.dialogue try: intent_result = await conn.intent.detect_intent(conn, dialogue.dialogue, text) - # 尝试解析JSON结果 try: intent_data = json.loads(intent_result) @@ -79,9 +65,6 @@ async def process_intent_result(conn, intent, original_text): # 处理退出意图 if "结束聊天" in intent: logger.bind(tag=TAG).info(f"识别到退出意图: {intent}") - - # 如果正在播放音乐,可以关了 TODO - # 如果是明确的离别意图,发送告别语并关闭连接 await send_stt_message(conn, original_text) conn.executor.submit(conn.chat_and_close, original_text) @@ -90,10 +73,37 @@ async def process_intent_result(conn, intent, original_text): # 处理播放音乐意图 if "播放音乐" in intent: logger.bind(tag=TAG).info(f"识别到音乐播放意图: {intent}") - await conn.music_handler.handle_music_command(conn, intent) + # 调用play_music函数来播放音乐 + song_name = extract_text_in_brackets(intent) + function_id = str(uuid.uuid4().hex) + function_name = "play_music" + function_arguments = '{ "song_name": "' + song_name + '" }' + + function_call_data = { + "name": function_name, + "id": function_id, + "arguments": function_arguments + } + conn.func_handler.handle_llm_function_call(conn, function_call_data) return True # 其他意图处理可以在这里扩展 # 默认返回False,表示继续常规聊天流程 return False + + +def extract_text_in_brackets(s): + """ + 从字符串中提取中括号内的文字 + + :param s: 输入字符串 + :return: 中括号内的文字,如果不存在则返回空字符串 + """ + left_bracket_index = s.find('[') + right_bracket_index = s.find(']') + + if left_bracket_index != -1 and right_bracket_index != -1 and left_bracket_index < right_bracket_index: + return s[left_bracket_index + 1:right_bracket_index] + else: + return "" \ No newline at end of file diff --git a/main/xiaozhi-server/core/handle/musicHandler.py b/main/xiaozhi-server/core/handle/musicHandler.py deleted file mode 100644 index b650b847..00000000 --- a/main/xiaozhi-server/core/handle/musicHandler.py +++ /dev/null @@ -1,139 +0,0 @@ -from config.logger import setup_logging -import os -import random -import difflib -import re -import traceback -from pathlib import Path -import time -from core.handle.sendAudioHandle import send_stt_message -from core.utils import p3 - -TAG = __name__ -logger = setup_logging() - - -def _extract_song_name(text): - """从用户输入中提取歌名""" - for keyword in ["播放音乐"]: - if keyword in text: - parts = text.split(keyword) - if len(parts) > 1: - return parts[1].strip() - return None - - -def _find_best_match(potential_song, music_files): - """查找最匹配的歌曲""" - best_match = None - highest_ratio = 0 - - for music_file in music_files: - song_name = os.path.splitext(music_file)[0] - ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio() - if ratio > highest_ratio and ratio > 0.4: - highest_ratio = ratio - best_match = music_file - return best_match - - -class MusicManager: - def __init__(self, music_dir, music_ext): - self.music_dir = Path(music_dir) - self.music_ext = music_ext - - def get_music_files(self): - music_files = [] - for file in self.music_dir.rglob("*"): - # 判断是否是文件 - if file.is_file(): - # 获取文件扩展名 - ext = file.suffix.lower() - # 判断扩展名是否在列表中 - if ext in self.music_ext: - # music_files.append(str(file.resolve())) # 添加绝对路径 - # 添加相对路径 - music_files.append(str(file.relative_to(self.music_dir))) - return music_files - - -class MusicHandler: - def __init__(self, config): - self.config = config - - if "music" in self.config: - self.music_config = self.config["music"] - self.music_dir = os.path.abspath( - self.music_config.get("music_dir", "./music") # 默认路径修改 - ) - self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3")) - self.refresh_time = self.music_config.get("refresh_time", 60) - else: - self.music_dir = os.path.abspath("./music") - self.music_ext = (".mp3", ".wav", ".p3") - self.refresh_time = 60 - - # 获取音乐文件列表 - self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files() - self.scan_time = time.time() - logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}") - - async def handle_music_command(self, conn, text): - """处理音乐播放指令""" - clean_text = re.sub(r'[^\w\s]', '', text).strip() - logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}") - - # 尝试匹配具体歌名 - if os.path.exists(self.music_dir): - if time.time() - self.scan_time > self.refresh_time: - # 刷新音乐文件列表 - self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files() - self.scan_time = time.time() - logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}") - - potential_song = _extract_song_name(clean_text) - if potential_song: - best_match = _find_best_match(potential_song, self.music_files) - if best_match: - logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}") - await self.play_local_music(conn, specific_file=best_match) - return True - # 检查是否是通用播放音乐命令 - await self.play_local_music(conn) - return True - - async def play_local_music(self, conn, specific_file=None): - """播放本地音乐文件""" - try: - if not os.path.exists(self.music_dir): - logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}") - return - - # 确保路径正确性 - if specific_file: - selected_music = specific_file - music_path = os.path.join(self.music_dir, specific_file) - else: - if not self.music_files: - logger.bind(tag=TAG).error("未找到MP3音乐文件") - return - selected_music = random.choice(self.music_files) - music_path = os.path.join(self.music_dir, selected_music) - - if not os.path.exists(music_path): - logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}") - return - text = f"正在播放{selected_music}" - await send_stt_message(conn, text) - conn.tts_first_text_index = 0 - conn.tts_last_text_index = 0 - conn.llm_finish_task = True - if music_path.endswith(".p3"): - opus_packets, duration = p3.decode_opus_from_file(music_path) - else: - opus_packets, duration = conn.tts.audio_to_opus_data(music_path) - conn.audio_play_queue.put((opus_packets, selected_music, 0)) - - except Exception as e: - logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}") - logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}") \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py index 04319438..9d5d2507 100644 --- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py +++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py @@ -1,12 +1,13 @@ from typing import List, Dict from ..base import IntentProviderBase +from plugins_func.functions.play_music import initialize_music_handler from config.logger import setup_logging -import re +import re + TAG = __name__ logger = setup_logging() - class IntentProvider(IntentProviderBase): def __init__(self, config): super().__init__(config) @@ -73,8 +74,8 @@ class IntentProvider(IntentProviderBase): "你现在可以使用的音乐的名称如下(使用标志):\n" ) return prompt - - async def detect_intent(self, conn, dialogue_history: List[Dict], text:str) -> str: + + async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str: if not self.llm: raise ValueError("LLM provider not set") @@ -89,7 +90,9 @@ class IntentProvider(IntentProviderBase): msgStr += f"User: {text}\n" user_prompt = f"当前的对话如下:\n{msgStr}" - prompt_music = f"{self.promot}\n{conn.music_handler.music_files}\n" + music_config = initialize_music_handler(conn) + music_file_names = music_config["music_file_names"] + prompt_music = f"{self.promot}\n{music_file_names}\n" logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}") # 使用LLM进行意图识别 intent = self.llm.response_no_stream( @@ -100,10 +103,9 @@ class IntentProvider(IntentProviderBase): # 使用正则表达式提取 {} 中的内容 match = re.search(r'\{.*?\}', intent) if match: - result = match.group(0) # 获取匹配到的内容(包含 {}) - print(result) # 输出:{intent: '播放音乐 [中秋月]'} + result = match.group(0) intent = result else: intent = "{intent: '继续聊天'}" logger.bind(tag=TAG).info(f"Detected intent: {intent}") - return intent.strip() \ No newline at end of file + return intent.strip() diff --git a/main/xiaozhi-server/core/websocket_server.py b/main/xiaozhi-server/core/websocket_server.py index 16748f34..41f14515 100644 --- a/main/xiaozhi-server/core/websocket_server.py +++ b/main/xiaozhi-server/core/websocket_server.py @@ -2,7 +2,6 @@ import asyncio import websockets from config.logger import setup_logging from core.connection import ConnectionHandler -from core.handle.musicHandler import MusicHandler from core.utils.util import get_local_ip from core.utils import asr, vad, llm, tts, memory, intent @@ -13,7 +12,7 @@ class WebSocketServer: def __init__(self, config: dict): self.config = config self.logger = setup_logging() - self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent = self._create_processing_instances() + self._vad, self._asr, self._llm, self._tts, self._memory, self.intent = self._create_processing_instances() self.active_connections = set() # 添加全局连接记录 def _create_processing_instances(self): @@ -50,7 +49,6 @@ class WebSocketServer: self.config["TTS"][self.config["selected_module"]["TTS"]], self.config["delete_audio"] ), - MusicHandler(self.config), memory.create_instance(memory_cls_name, memory_cfg), intent.create_instance( self.config["selected_module"]["Intent"] @@ -80,7 +78,7 @@ class WebSocketServer: async def _handle_connection(self, websocket): """处理新连接,每次创建独立的ConnectionHandler""" # 创建ConnectionHandler时传入当前server实例 - handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent) + handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._memory, self.intent) self.active_connections.add(handler) try: await handler.handle_connection(websocket) diff --git a/main/xiaozhi-server/plugins_func/functions/play_music.py b/main/xiaozhi-server/plugins_func/functions/play_music.py index 9e3372a3..b9220ea5 100644 --- a/main/xiaozhi-server/plugins_func/functions/play_music.py +++ b/main/xiaozhi-server/plugins_func/functions/play_music.py @@ -1,10 +1,22 @@ -from plugins_func.register import register_function,ToolType, ActionResponse, Action from config.logger import setup_logging +import os +import re +import time +import random import asyncio +import difflib +import traceback +from pathlib import Path +from core.utils import p3 +from core.handle.sendAudioHandle import send_stt_message +from plugins_func.register import register_function,ToolType, ActionResponse, Action + TAG = __name__ logger = setup_logging() +MUSIC_CACHE = {} + play_music_function_desc = { "type": "function", "function": { @@ -29,12 +41,157 @@ def play_music(conn, song_name: str): try: music_intent = f"播放音乐 {song_name}" if song_name != "random" else "随机播放音乐" - # 执行音乐播放命令 + # 检查事件循环状态 + if not conn.loop.is_running(): + logger.bind(tag=TAG).error("事件循环未运行,无法提交任务") + return ActionResponse(action=Action.RESPONSE, result="系统繁忙", response="请稍后再试") + + # 提交异步任务 future = asyncio.run_coroutine_threadsafe( - conn.music_handler.handle_music_command(conn, music_intent), + handle_music_command(conn, music_intent), conn.loop ) - future.result() - return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response="还想听什么歌?") + + # 非阻塞回调处理 + def handle_done(f): + try: + f.result() # 可在此处理成功逻辑 + logger.bind(tag=TAG).info("播放完成") + except Exception as e: + logger.bind(tag=TAG).error(f"播放失败: {e}") + + future.add_done_callback(handle_done) + + return ActionResponse(action=Action.RESPONSE, result="指令已接收", response="正在为您播放音乐") except Exception as e: - logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}") \ No newline at end of file + logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}") + return ActionResponse(action=Action.RESPONSE, result=str(e), response="播放音乐时出错了") + + +def _extract_song_name(text): + """从用户输入中提取歌名""" + for keyword in ["播放音乐"]: + if keyword in text: + parts = text.split(keyword) + if len(parts) > 1: + return parts[1].strip() + return None + + +def _find_best_match(potential_song, music_files): + """查找最匹配的歌曲""" + best_match = None + highest_ratio = 0 + + for music_file in music_files: + song_name = os.path.splitext(music_file)[0] + ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio() + if ratio > highest_ratio and ratio > 0.4: + highest_ratio = ratio + best_match = music_file + return best_match + + +def get_music_files(music_dir, music_ext): + music_dir = Path(music_dir) + music_files = [] + music_file_names = [] + for file in music_dir.rglob("*"): + # 判断是否是文件 + if file.is_file(): + # 获取文件扩展名 + ext = file.suffix.lower() + # 判断扩展名是否在列表中 + if ext in music_ext: + # 添加相对路径 + music_files.append(str(file.relative_to(music_dir))) + music_file_names.append(os.path.splitext(str(file.relative_to(music_dir)))[0]) + return music_files, music_file_names + + +def initialize_music_handler(conn): + global MUSIC_CACHE + if MUSIC_CACHE == {}: + logger.bind(tag=TAG).info(f"实例化音乐:") + if "music" in conn.config: + MUSIC_CACHE["music_config"] = conn.config["music"] + MUSIC_CACHE["music_dir"] = os.path.abspath( + MUSIC_CACHE["music_config"].get("music_dir", "./music") # 默认路径修改 + ) + MUSIC_CACHE["music_ext"] = MUSIC_CACHE["music_config"].get("music_ext", (".mp3", ".wav", ".p3")) + MUSIC_CACHE["refresh_time"] = MUSIC_CACHE["music_config"].get("refresh_time", 60) + else: + MUSIC_CACHE["music_dir"] = os.path.abspath("./music") + MUSIC_CACHE["music_ext"] = (".mp3", ".wav", ".p3") + MUSIC_CACHE["refresh_time"] = 60 + # 获取音乐文件列表 + MUSIC_CACHE["music_files"], MUSIC_CACHE["music_file_names"] = get_music_files(MUSIC_CACHE["music_dir"], + MUSIC_CACHE["music_ext"]) + MUSIC_CACHE["scan_time"] = time.time() + return MUSIC_CACHE + + +async def handle_music_command(conn, text): + initialize_music_handler(conn) + global MUSIC_CACHE + + """处理音乐播放指令""" + clean_text = re.sub(r'[^\w\s]', '', text).strip() + logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}") + + # 尝试匹配具体歌名 + if os.path.exists(MUSIC_CACHE["music_dir"]): + if time.time() - MUSIC_CACHE["scan_time"] > MUSIC_CACHE["refresh_time"]: + # 刷新音乐文件列表 + MUSIC_CACHE["music_files"], MUSIC_CACHE["music_file_names"] = get_music_files(MUSIC_CACHE["music_dir"], + MUSIC_CACHE["music_ext"]) + MUSIC_CACHE["scan_time"] = time.time() + + potential_song = _extract_song_name(clean_text) + if potential_song: + best_match = _find_best_match(potential_song, MUSIC_CACHE["music_files"]) + if best_match: + logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}") + await play_local_music(conn, specific_file=best_match) + return True + # 检查是否是通用播放音乐命令 + await play_local_music(conn) + return True + + +async def play_local_music(conn, specific_file=None): + global MUSIC_CACHE + """播放本地音乐文件""" + try: + if not os.path.exists(MUSIC_CACHE["music_dir"]): + logger.bind(tag=TAG).error(f"音乐目录不存在: " + MUSIC_CACHE["music_dir"]) + return + + # 确保路径正确性 + if specific_file: + selected_music = specific_file + music_path = os.path.join(MUSIC_CACHE["music_dir"], specific_file) + else: + if not MUSIC_CACHE["music_files"]: + logger.bind(tag=TAG).error("未找到MP3音乐文件") + return + selected_music = random.choice(MUSIC_CACHE["music_files"]) + music_path = os.path.join(MUSIC_CACHE["music_dir"], selected_music) + + if not os.path.exists(music_path): + logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}") + return + text = f"正在播放{selected_music}" + await send_stt_message(conn, text) + conn.tts_first_text_index = 0 + conn.tts_last_text_index = 0 + conn.llm_finish_task = True + if music_path.endswith(".p3"): + opus_packets, duration = p3.decode_opus_from_file(music_path) + else: + opus_packets, duration = conn.tts.audio_to_opus_data(music_path) + conn.audio_play_queue.put((opus_packets, selected_music, 0)) + + except Exception as e: + logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}") + logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")