diff --git a/config.yaml b/config.yaml index 5dd2793e..fd8c2601 100644 --- a/config.yaml +++ b/config.yaml @@ -312,3 +312,19 @@ module_test: - "你好,请介绍一下你自己" - "What's the weather like today?" - "请用100字概括量子计算的基本原理和应用前景" + +# 本地音乐播放配置 +music: + music_commands: + - "来一首歌" + - "唱一首歌" + - "播放音乐" + - "来点音乐" + - "背景音乐" + - "放首歌" + - "播放歌曲" + - "来点背景音乐" + - "我想听歌" + - "我要听歌" + - "放点音乐" + music_dir: "./music" # 音乐文件存放路径 \ No newline at end of file diff --git a/core/connection.py b/core/connection.py index ead53080..70dd9887 100644 --- a/core/connection.py +++ b/core/connection.py @@ -15,15 +15,18 @@ from core.utils.dialogue import Message, Dialogue from core.handle.textHandle import handleTextMessage from core.utils.util import get_string_no_punctuation_or_emoji from concurrent.futures import ThreadPoolExecutor, TimeoutError -from core.handle.audioHandle import handleAudioMessage, sendAudioMessage +from core.handle.sendAudioHandle import sendAudioMessage +from core.handle.receiveAudioHandle import handleAudioMessage from config.private_config import PrivateConfig from core.auth import AuthMiddleware, AuthenticationError -from core.utils.auth_code_gen import AuthCodeGenerator # 添加导入 +from core.utils.auth_code_gen import AuthCodeGenerator + TAG = __name__ + class ConnectionHandler: - def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts): + def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music): self.config = config self.logger = setup_logging() self.auth = AuthMiddleware(config) @@ -81,23 +84,25 @@ class ConnectionHandler: for cmd in self.cmd_exit: if len(cmd) > self.max_cmd_length: self.max_cmd_length = len(cmd) - + self.private_config = None self.auth_code_gen = AuthCodeGenerator.get_instance() self.is_device_verified = False # 添加设备验证状态标志 - + self.music_handler = _music async def handle_connection(self, ws): try: # 获取并验证headers self.headers = dict(ws.request.headers) - self.logger.bind(tag=TAG).info(f"New connection request - Headers: {self.headers}") + # 获取客户端ip地址 + client_ip = ws.remote_address[0] + self.logger.bind(tag=TAG).info(f"{client_ip} conn - Headers: {self.headers}") # 进行认证 await self.auth.authenticate(self.headers) device_id = self.headers.get("device-id", None) - + # Load private configuration if device_id is provided bUsePrivateConfig = self.config.get("use_private_config", False) self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}") @@ -108,10 +113,10 @@ class ConnectionHandler: # 判断是否已经绑定 owner = self.private_config.get_owner() self.is_device_verified = owner is not None - + if self.is_device_verified: - await self.private_config.update_last_chat_time() - + await self.private_config.update_last_chat_time() + llm, tts = self.private_config.create_private_instances() if all([llm, tts]): self.llm = llm @@ -171,7 +176,7 @@ class ConnectionHandler: date_time = time.strftime("%Y-%m-%d %H:%M", time.localtime()) self.prompt = self.prompt.replace("{date_time}", date_time) self.dialogue.put(Message(role="system", content=self.prompt)) - + async def _check_and_broadcast_auth_code(self): """检查设备绑定状态并广播认证码""" if not self.private_config.get_owner(): @@ -191,7 +196,7 @@ class ConnectionHandler: # 如果不使用私有配置,就不需要验证 return False return not self.is_device_verified - + def chat(self, query): # 如果设备未验证,就发送验证码 if self.isNeedAuth(): @@ -204,7 +209,7 @@ class ConnectionHandler: finally: loop.close() return True - + self.dialogue.put(Message(role="user", content=query)) response_message = [] start = 0 @@ -321,6 +326,8 @@ class ConnectionHandler: async def close(self): """资源清理方法""" + + # 清理其他资源 self.stop_event.set() self.executor.shutdown(wait=False) if self.websocket: diff --git a/core/handle/musicHandler.py b/core/handle/musicHandler.py new file mode 100644 index 00000000..a3d865d7 --- /dev/null +++ b/core/handle/musicHandler.py @@ -0,0 +1,109 @@ +from config.logger import setup_logging +import os +import random +import difflib +import re +import traceback +from core.handle.sendAudioHandle import sendAudioMessage, send_stt_message + +TAG = __name__ +logger = setup_logging() + + +def _extract_song_name(text): + """从用户输入中提取歌名""" + for keyword in ["听", "播放", "放", "唱"]: + if keyword in text: + parts = text.split(keyword) + if len(parts) > 1: + return parts[1].strip() + return None + + +def _find_best_match(potential_song, music_files): + """查找最匹配的歌曲""" + best_match = None + highest_ratio = 0 + + for music_file in music_files: + song_name = os.path.splitext(music_file)[0] + ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio() + if ratio > highest_ratio and ratio > 0.4: + highest_ratio = ratio + best_match = music_file + return best_match + + +class MusicHandler: + def __init__(self, config): + self.config = config + self.music_related_keywords = [] + + if "music" in self.config: + self.music_config = self.config["music"] + self.music_dir = os.path.abspath( + self.music_config.get("music_dir", "./music") # 默认路径修改 + ) + self.music_related_keywords = self.music_config.get("music_commands", []) + else: + self.music_dir = os.path.abspath("./music") + self.music_related_keywords = ["来一首歌", "唱一首歌", "播放音乐", "来点音乐", "背景音乐", "放首歌", + "播放歌曲", "来点背景音乐", "我想听歌", "我要听歌", "放点音乐"] + + async def handle_music_command(self, conn, text): + """处理音乐播放指令""" + clean_text = re.sub(r'[^\w\s]', '', text).strip() + logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}") + + # 尝试匹配具体歌名 + if os.path.exists(self.music_dir): + music_files = [f for f in os.listdir(self.music_dir) if f.endswith('.mp3')] + logger.bind(tag=TAG).debug(f"找到的音乐文件: {music_files}") + + potential_song = _extract_song_name(clean_text) + if potential_song: + best_match = _find_best_match(potential_song, music_files) + if best_match: + logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}") + await self.play_local_music(conn, specific_file=best_match) + return True + + # 检查是否是通用播放音乐命令 + if any(cmd in clean_text for cmd in self.music_related_keywords): + await self.play_local_music(conn) + return True + + return False + + async def play_local_music(self, conn, specific_file=None): + """播放本地音乐文件""" + try: + if not os.path.exists(self.music_dir): + logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}") + return + + # 确保路径正确性 + if specific_file: + music_path = os.path.join(self.music_dir, specific_file) + if not os.path.exists(music_path): + logger.bind(tag=TAG).error(f"指定的音乐文件不存在: {music_path}") + return + selected_music = specific_file + else: + music_files = [f for f in os.listdir(self.music_dir) if f.endswith('.mp3')] + if not music_files: + logger.bind(tag=TAG).error("未找到MP3音乐文件") + return + selected_music = random.choice(music_files) + music_path = os.path.join(self.music_dir, selected_music) + text = f"正在播放{selected_music}" + await send_stt_message(conn, text) + conn.tts_first_text = selected_music + conn.tts_last_text = selected_music + conn.llm_finish_task = True + opus_packets, duration = conn.tts.wav_to_opus_data(music_path) + await sendAudioMessage(conn, opus_packets, duration, selected_music) + + except Exception as e: + logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}") + logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}") diff --git a/core/handle/receiveAudioHandle.py b/core/handle/receiveAudioHandle.py new file mode 100644 index 00000000..59e0734d --- /dev/null +++ b/core/handle/receiveAudioHandle.py @@ -0,0 +1,81 @@ +from config.logger import setup_logging +import asyncio +import time +from core.utils.util import remove_punctuation_and_length +from core.handle.sendAudioHandle import schedule_with_interrupt, send_stt_message + +TAG = __name__ +logger = setup_logging() + + +async def handleAudioMessage(conn, audio): + if not conn.asr_server_receive: + logger.bind(tag=TAG).debug(f"前期数据处理中,暂停接收") + return + if conn.client_listen_mode == "auto": + have_voice = conn.vad.is_vad(conn, audio) + else: + have_voice = conn.client_have_voice + + # 如果本次没有声音,本段也没声音,就把声音丢弃了 + if have_voice == False and conn.client_have_voice == False: + await no_voice_close_connect(conn) + conn.asr_audio.clear() + return + conn.client_no_voice_last_time = 0.0 + conn.asr_audio.append(audio) + # 如果本段有声音,且已经停止了 + if conn.client_voice_stop: + conn.client_abort = False + conn.asr_server_receive = False + # 音频太短了,无法识别 + if len(conn.asr_audio) < 3: + conn.asr_server_receive = True + else: + text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id) + logger.bind(tag=TAG).info(f"识别文本: {text}") + text_len, text_without_punctuation = remove_punctuation_and_length(text) + if await conn.music_handler.handle_music_command(conn, text_without_punctuation): + conn.asr_server_receive = True + conn.asr_audio.clear() + return + if text_len <= conn.max_cmd_length and await handleCMDMessage(conn, text_without_punctuation): + return + if text_len > 0: + await startToChat(conn, text) + else: + conn.asr_server_receive = True + conn.asr_audio.clear() + conn.reset_vad_states() + + +async def handleCMDMessage(conn, text): + cmd_exit = conn.cmd_exit + for cmd in cmd_exit: + if text == cmd: + logger.bind(tag=TAG).info("识别到明确的退出命令".format(text)) + await conn.close() + return True + return False + + +async def startToChat(conn, text): + # 异步发送 stt 信息 + stt_task = asyncio.create_task( + schedule_with_interrupt(0, send_stt_message(conn, text)) + ) + conn.scheduled_tasks.append(stt_task) + conn.executor.submit(conn.chat, text) + + +async def no_voice_close_connect(conn): + if conn.client_no_voice_last_time == 0.0: + conn.client_no_voice_last_time = time.time() * 1000 + else: + no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time + close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120) + if no_voice_time > 1000 * close_connection_no_voice_time: + conn.client_abort = False + conn.asr_server_receive = False + prompt = "时间过得真快,我都好久没说话了。请你用十个字左右话跟我告别,以“再见”或“拜拜”为结尾" + await startToChat(conn, prompt) diff --git a/core/handle/audioHandle.py b/core/handle/sendAudioHandle.py similarity index 51% rename from core/handle/audioHandle.py rename to core/handle/sendAudioHandle.py index f25c4869..bcf639a7 100644 --- a/core/handle/audioHandle.py +++ b/core/handle/sendAudioHandle.py @@ -8,56 +8,6 @@ TAG = __name__ logger = setup_logging() -async def handleAudioMessage(conn, audio): - if not conn.asr_server_receive: - logger.bind(tag=TAG).debug(f"前期数据处理中,暂停接收") - return - if conn.client_listen_mode == "auto": - have_voice = conn.vad.is_vad(conn, audio) - else: - have_voice = conn.client_have_voice - - # 如果本次没有声音,本段也没声音,就把声音丢弃了 - if have_voice == False and conn.client_have_voice == False: - await no_voice_close_connect(conn) - conn.asr_audio.clear() - return - conn.client_no_voice_last_time = 0.0 - conn.asr_audio.append(audio) - # 如果本段有声音,且已经停止了 - if conn.client_voice_stop: - conn.client_abort = False - conn.asr_server_receive = False - # 音频太短了,无法识别 - if len(conn.asr_audio) < 3: - conn.asr_server_receive = True - else: - text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id) - logger.bind(tag=TAG).info(f"识别文本: {text}") - text_len, text_without_punctuation = remove_punctuation_and_length(text) - if text_len <= conn.max_cmd_length and await handleCMDMessage(conn, text_without_punctuation): - return - if text_len > 0: - await startToChat(conn, text) - else: - conn.asr_server_receive = True - conn.asr_audio.clear() - conn.reset_vad_states() - -async def handleCMDMessage(conn, text): - cmd_exit = conn.cmd_exit - for cmd in cmd_exit: - if text == cmd: - logger.bind(tag=TAG).info("识别到明确的退出命令".format(text)) - await finishToChat(conn) - return True - return False - - -async def finishToChat(conn): - await conn.close() - - async def isLLMWantToFinish(conn): first_text = conn.tts_first_text last_text = conn.tts_last_text @@ -70,15 +20,6 @@ async def isLLMWantToFinish(conn): return False -async def startToChat(conn, text): - # 异步发送 stt 信息 - stt_task = asyncio.create_task( - schedule_with_interrupt(0, send_stt_message(conn, text)) - ) - conn.scheduled_tasks.append(stt_task) - conn.executor.submit(conn.chat, text) - - async def sendAudioMessage(conn, audios, duration, text): base_delay = conn.tts_duration @@ -107,7 +48,7 @@ async def sendAudioMessage(conn, audios, duration, text): conn.scheduled_tasks.append(stop_task) if await isLLMWantToFinish(conn): finish_task = asyncio.create_task( - schedule_with_interrupt(stop_duration, finishToChat(conn)) + schedule_with_interrupt(stop_duration, await conn.close()) ) conn.scheduled_tasks.append(finish_task) @@ -152,16 +93,3 @@ async def schedule_with_interrupt(delay, coro): await coro except asyncio.CancelledError: pass - - -async def no_voice_close_connect(conn): - if conn.client_no_voice_last_time == 0.0: - conn.client_no_voice_last_time = time.time() * 1000 - else: - no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time - close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120) - if no_voice_time > 1000 * close_connection_no_voice_time: - conn.client_abort = False - conn.asr_server_receive = False - prompt = "时间过得真快,我都好久没说话了。请你用十个字左右话跟我告别,以“再见”或“拜拜”为结尾" - await startToChat(conn, prompt) \ No newline at end of file diff --git a/core/handle/textHandle.py b/core/handle/textHandle.py index 8e34aa72..2eb7304d 100644 --- a/core/handle/textHandle.py +++ b/core/handle/textHandle.py @@ -2,7 +2,7 @@ from config.logger import setup_logging import json from core.handle.abortHandle import handleAbortMessage from core.handle.helloHandle import handleHelloMessage -from core.handle.audioHandle import startToChat +from core.handle.receiveAudioHandle import startToChat from core.handle.iotHandle import handleIotDescriptors TAG = __name__ @@ -39,7 +39,6 @@ async def handleTextMessage(conn, message): await startToChat(conn, msg_json["text"]) elif msg_json["type"] == "iot": if "descriptors" in msg_json: - pass - # await handleIotDescriptors(conn, msg_json["descriptors"]) + await handleIotDescriptors(conn, msg_json["descriptors"]) except json.JSONDecodeError: await conn.websocket.send(message) diff --git a/core/websocket_server.py b/core/websocket_server.py index 8e09e0fe..907ef22f 100644 --- a/core/websocket_server.py +++ b/core/websocket_server.py @@ -2,6 +2,7 @@ import asyncio import websockets from config.logger import setup_logging from core.connection import ConnectionHandler +from core.handle.musicHandler import MusicHandler from core.utils.util import get_local_ip from core.utils import asr, vad, llm, tts @@ -12,7 +13,8 @@ class WebSocketServer: def __init__(self, config: dict): self.config = config self.logger = setup_logging() - self._vad, self._asr, self._llm, self._tts = self._create_processing_instances() + self._vad, self._asr, self._llm, self._tts, self._music = self._create_processing_instances() + self.active_connections = set() # 添加全局连接记录 def _create_processing_instances(self): """创建处理模块实例""" @@ -43,7 +45,8 @@ class WebSocketServer: self.config["TTS"][self.config["selected_module"]["TTS"]]["type"], self.config["TTS"][self.config["selected_module"]["TTS"]], self.config["delete_audio"] - ) + ), + MusicHandler(self.config) ) async def start(self): @@ -62,5 +65,10 @@ class WebSocketServer: async def _handle_connection(self, websocket): """处理新连接,每次创建独立的ConnectionHandler""" - handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts) - await handler.handle_connection(websocket) + # 创建ConnectionHandler时传入当前server实例 + handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._music) + self.active_connections.add(handler) + try: + await handler.handle_connection(websocket) + finally: + self.active_connections.discard(handler) diff --git a/music/一念千年_国风版.mp3 b/music/一念千年_国风版.mp3 new file mode 100644 index 00000000..31801be1 Binary files /dev/null and b/music/一念千年_国风版.mp3 differ diff --git a/music/中秋月.mp3 b/music/中秋月.mp3 new file mode 100644 index 00000000..abde2912 Binary files /dev/null and b/music/中秋月.mp3 differ diff --git a/music/廉波老矣,尚能饭否.mp3 b/music/廉波老矣,尚能饭否.mp3 new file mode 100644 index 00000000..ef9d056f Binary files /dev/null and b/music/廉波老矣,尚能饭否.mp3 differ diff --git a/requirements.txt b/requirements.txt index bfa9ed80..2ccc0584 100755 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,4 @@ aiohttp_cors==0.7.0 ormsgpack==1.7.0 ruamel.yaml==0.18.10 loguru==0.7.3 -requests>=2.0.0 \ No newline at end of file +requests==2.32.3 \ No newline at end of file