Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 15:13:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into dev

Browse Source
This commit is contained in:
pengzhisheng
2025-03-17 09:59:11 +08:00
parent d3571942d7 6eac25425c
commit 7d474e17fa
10 changed files with 294 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/ISSUE_TEMPLATE/bug_report.md
+34
View File
@@ -0,0 +1,34 @@
---
name: Bug 报告（Bug Report）
about: 反馈项目中的缺陷或问题
title: "[Bug] 简短描述问题"
labels: bug
assignees: ''
---
## 🐛 问题描述
<!-- 清晰简洁地描述问题是什么 -->
## 🔍 复现步骤
<!-- 详细描述复现问题的步骤 -->
1. 打开 '...'
2. 点击 '...'
3. 滚动到 '...'
4. 看到错误
## 🤔 预期行为
<!-- 简要描述预期的正确行为 -->
## 😯 截图
<!-- 如果适用，添加问题的截图 -->
## 🖥️ 环境信息
- 操作系统: [例如 Windows 10]
- 浏览器: [例如 Chrome 89]
- 项目版本: [例如 1.0.0]
- Python版本: [例如 3.9.13]
- Jdk版本：[例如 java 21.0.5 2024-10-15 LTS]
- Nodejs版本：[例如 v20.14.0]
## 📋 其他信息
<!-- 在此添加关于此问题的任何其他上下文信息 -->
.github/ISSUE_TEMPLATE/code_improvement.md
+19
View File
@@ -0,0 +1,19 @@
---
name: 代码优化建议（Code Improvement）
about: 提出对现有代码的优化或改进建议
title: "[Improvement] 简短描述改进内容"
labels: refactor
assignees: ''
---
## 💡 改进描述
<!-- 描述需要改进的代码部分以及存在的问题 -->
## 🌟 改进建议
<!-- 提出具体的改进方案或思路 -->
## 🛠️ 相关代码
<!-- 如果可能，提供相关代码片段或链接 -->
## 📋 其他信息
<!-- 在此添加任何其他相关信息 -->
.github/ISSUE_TEMPLATE/documentation_improvement.md
+16
View File
@@ -0,0 +1,16 @@
---
name: 文档改进建议（Documentation Improvement）
about: 提出对项目文档的改进或补充建议
title: "[Docs] 简短描述改进内容"
labels: documentation
assignees: ''
---
## 📚 改进描述
<!-- 描述需要改进的文档部分以及存在的问题 -->
## ✨ 改进建议
<!-- 提出具体的改进方案或内容 -->
## 📋 其他信息
<!-- 在此添加任何其他相关信息 -->
.github/ISSUE_TEMPLATE/feature_request.md
+19
View File
@@ -0,0 +1,19 @@
---
name: 功能请求（Feature Request）
about: 提出新的功能或改进建议
title: "[Feature] 简短描述功能"
labels: enhancement
assignees: ''
---
## 🚀 需求描述
<!-- 清晰简洁地描述你希望发生什么 -->
## 🎯 解决方案
<!-- 描述你认为可行的解决方案或实现方式 -->
## 📝 备选方案
<!-- 描述你考虑过的替代方案或功能 -->
## 📋 其他信息
<!-- 在此添加关于需求的任何其他上下文信息 -->
main/xiaozhi-server/core/connection.py
+2 -3
View File
@@ -20,7 +20,7 @@ from plugins_func.register import Action
from config.private_config import PrivateConfig
from core.auth import AuthMiddleware, AuthenticationError
from core.utils.auth_code_gen import AuthCodeGenerator
import plugins_func.loadplugins
import plugins_func.loadplugins
TAG = __name__
@@ -30,7 +30,7 @@ class TTSException(RuntimeError):
class ConnectionHandler:
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory, _intent):
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _memory, _intent):
self.config = config
self.logger = setup_logging()
self.auth = AuthMiddleware(config)
@@ -91,7 +91,6 @@ class ConnectionHandler:
self.private_config = None
self.auth_code_gen = AuthCodeGenerator.get_instance()
self.is_device_verified = False # 添加设备验证状态标志
self.music_handler = _music
self.close_after_chat = False # 是否在聊天结束后关闭连接
self.use_function_call_mode = False
if self.config["selected_module"]["Intent"] == 'function_call':
main/xiaozhi-server/core/handle/intentHandler.py
+29 -19
View File
@@ -1,5 +1,6 @@
from config.logger import setup_logging
import json
import uuid
from core.handle.sendAudioHandle import send_stt_message
from core.utils.util import remove_punctuation_and_length
@@ -8,30 +9,16 @@ logger = setup_logging()
async def handle_user_intent(conn, text):
"""
Handle user intent before starting chat
Args:
conn: Connection object
text: User's text input
Returns:
bool: True if intent was handled, False if should proceed to chat
"""
# 检查是否有明确的退出命令
if await check_direct_exit(conn, text):
return True
if conn.use_function_call_mode:
# 使用支持function calling的聊天方法,不再进行意图分析
return False
# 使用LLM进行意图分析
intent = await analyze_intent_with_llm(conn, text)
if not intent:
return False
# 处理各种意图
return await process_intent_result(conn, intent, text)
@@ -58,7 +45,6 @@ async def analyze_intent_with_llm(conn, text):
dialogue = conn.dialogue
try:
intent_result = await conn.intent.detect_intent(conn, dialogue.dialogue, text)
# 尝试解析JSON结果
try:
intent_data = json.loads(intent_result)
@@ -79,9 +65,6 @@ async def process_intent_result(conn, intent, original_text):
# 处理退出意图
if "结束聊天" in intent:
logger.bind(tag=TAG).info(f"识别到退出意图: {intent}")
# 如果正在播放音乐，可以关了 TODO
# 如果是明确的离别意图，发送告别语并关闭连接
await send_stt_message(conn, original_text)
conn.executor.submit(conn.chat_and_close, original_text)
@@ -90,10 +73,37 @@ async def process_intent_result(conn, intent, original_text):
# 处理播放音乐意图
if "播放音乐" in intent:
logger.bind(tag=TAG).info(f"识别到音乐播放意图: {intent}")
await conn.music_handler.handle_music_command(conn, intent)
# 调用play_music函数来播放音乐
song_name = extract_text_in_brackets(intent)
function_id = str(uuid.uuid4().hex)
function_name = "play_music"
function_arguments = '{ "song_name": "' + song_name + '" }'
function_call_data = {
"name": function_name,
"id": function_id,
"arguments": function_arguments
}
conn.func_handler.handle_llm_function_call(conn, function_call_data)
return True
# 其他意图处理可以在这里扩展
# 默认返回False，表示继续常规聊天流程
return False
def extract_text_in_brackets(s):
"""
从字符串中提取中括号内的文字
:param s: 输入字符串
:return: 中括号内的文字，如果不存在则返回空字符串
"""
left_bracket_index = s.find('[')
right_bracket_index = s.find(']')
if left_bracket_index != -1 and right_bracket_index != -1 and left_bracket_index < right_bracket_index:
return s[left_bracket_index + 1:right_bracket_index]
else:
return ""
main/xiaozhi-server/core/handle/musicHandler.py
-139
View File
@@ -1,139 +0,0 @@
from config.logger import setup_logging
import os
import random
import difflib
import re
import traceback
from pathlib import Path
import time
from core.handle.sendAudioHandle import send_stt_message
from core.utils import p3
TAG = __name__
logger = setup_logging()
def _extract_song_name(text):
"""从用户输入中提取歌名"""
for keyword in ["播放音乐"]:
if keyword in text:
parts = text.split(keyword)
if len(parts) > 1:
return parts[1].strip()
return None
def _find_best_match(potential_song, music_files):
"""查找最匹配的歌曲"""
best_match = None
highest_ratio = 0
for music_file in music_files:
song_name = os.path.splitext(music_file)[0]
ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
if ratio > highest_ratio and ratio > 0.4:
highest_ratio = ratio
best_match = music_file
return best_match
class MusicManager:
def __init__(self, music_dir, music_ext):
self.music_dir = Path(music_dir)
self.music_ext = music_ext
def get_music_files(self):
music_files = []
for file in self.music_dir.rglob("*"):
# 判断是否是文件
if file.is_file():
# 获取文件扩展名
ext = file.suffix.lower()
# 判断扩展名是否在列表中
if ext in self.music_ext:
# music_files.append(str(file.resolve())) # 添加绝对路径
# 添加相对路径
music_files.append(str(file.relative_to(self.music_dir)))
return music_files
class MusicHandler:
def __init__(self, config):
self.config = config
if "music" in self.config:
self.music_config = self.config["music"]
self.music_dir = os.path.abspath(
self.music_config.get("music_dir", "./music") # 默认路径修改
)
self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
self.refresh_time = self.music_config.get("refresh_time", 60)
else:
self.music_dir = os.path.abspath("./music")
self.music_ext = (".mp3", ".wav", ".p3")
self.refresh_time = 60
# 获取音乐文件列表
self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
self.scan_time = time.time()
logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}")
async def handle_music_command(self, conn, text):
"""处理音乐播放指令"""
clean_text = re.sub(r'[^\w\s]', '', text).strip()
logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
# 尝试匹配具体歌名
if os.path.exists(self.music_dir):
if time.time() - self.scan_time > self.refresh_time:
# 刷新音乐文件列表
self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
self.scan_time = time.time()
logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}")
potential_song = _extract_song_name(clean_text)
if potential_song:
best_match = _find_best_match(potential_song, self.music_files)
if best_match:
logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
await self.play_local_music(conn, specific_file=best_match)
return True
# 检查是否是通用播放音乐命令
await self.play_local_music(conn)
return True
async def play_local_music(self, conn, specific_file=None):
"""播放本地音乐文件"""
try:
if not os.path.exists(self.music_dir):
logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}")
return
# 确保路径正确性
if specific_file:
selected_music = specific_file
music_path = os.path.join(self.music_dir, specific_file)
else:
if not self.music_files:
logger.bind(tag=TAG).error("未找到MP3音乐文件")
return
selected_music = random.choice(self.music_files)
music_path = os.path.join(self.music_dir, selected_music)
if not os.path.exists(music_path):
logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
return
text = f"正在播放{selected_music}"
await send_stt_message(conn, text)
conn.tts_first_text_index = 0
conn.tts_last_text_index = 0
conn.llm_finish_task = True
if music_path.endswith(".p3"):
opus_packets, duration = p3.decode_opus_from_file(music_path)
else:
opus_packets, duration = conn.tts.audio_to_opus_data(music_path)
conn.audio_play_queue.put((opus_packets, selected_music, 0))
except Exception as e:
logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py
+10 -8
View File
@@ -1,12 +1,13 @@
from typing import List, Dict
from ..base import IntentProviderBase
from plugins_func.functions.play_music import initialize_music_handler
from config.logger import setup_logging
import re
import re
TAG = __name__
logger = setup_logging()
class IntentProvider(IntentProviderBase):
def __init__(self, config):
super().__init__(config)
@@ -73,8 +74,8 @@ class IntentProvider(IntentProviderBase):
"你现在可以使用的音乐的名称如下(使用<start>和<end>标志):\n"
)
return prompt
async def detect_intent(self, conn, dialogue_history: List[Dict], text:str) -> str:
async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
if not self.llm:
raise ValueError("LLM provider not set")
@@ -89,7 +90,9 @@ class IntentProvider(IntentProviderBase):
msgStr += f"User: {text}\n"
user_prompt = f"当前的对话如下：\n{msgStr}"
prompt_music = f"{self.promot}\n<start>{conn.music_handler.music_files}\n<end>"
music_config = initialize_music_handler(conn)
music_file_names = music_config["music_file_names"]
prompt_music = f"{self.promot}\n<start>{music_file_names}\n<end>"
logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
# 使用LLM进行意图识别
intent = self.llm.response_no_stream(
@@ -100,10 +103,9 @@ class IntentProvider(IntentProviderBase):
# 使用正则表达式提取 {} 中的内容
match = re.search(r'\{.*?\}', intent)
if match:
result = match.group(0) # 获取匹配到的内容（包含 {}）
print(result) # 输出：{intent: '播放音乐 [中秋月]'}
result = match.group(0)
intent = result
else:
intent = "{intent: '继续聊天'}"
logger.bind(tag=TAG).info(f"Detected intent: {intent}")
return intent.strip()
return intent.strip()
main/xiaozhi-server/core/websocket_server.py
+2 -4
View File
@@ -2,7 +2,6 @@ import asyncio
import websockets
from config.logger import setup_logging
from core.connection import ConnectionHandler
from core.handle.musicHandler import MusicHandler
from core.utils.util import get_local_ip
from core.utils import asr, vad, llm, tts, memory, intent
@@ -13,7 +12,7 @@ class WebSocketServer:
def __init__(self, config: dict):
self.config = config
self.logger = setup_logging()
self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent = self._create_processing_instances()
self._vad, self._asr, self._llm, self._tts, self._memory, self.intent = self._create_processing_instances()
self.active_connections = set() # 添加全局连接记录
def _create_processing_instances(self):
@@ -50,7 +49,6 @@ class WebSocketServer:
self.config["TTS"][self.config["selected_module"]["TTS"]],
self.config["delete_audio"]
),
MusicHandler(self.config),
memory.create_instance(memory_cls_name, memory_cfg),
intent.create_instance(
self.config["selected_module"]["Intent"]
@@ -80,7 +78,7 @@ class WebSocketServer:
async def _handle_connection(self, websocket):
"""处理新连接，每次创建独立的ConnectionHandler"""
# 创建ConnectionHandler时传入当前server实例
handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent)
handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._memory, self.intent)
self.active_connections.add(handler)
try:
await handler.handle_connection(websocket)
main/xiaozhi-server/plugins_func/functions/play_music.py
+163 -6
View File
@@ -1,10 +1,22 @@
from plugins_func.register import register_function,ToolType, ActionResponse, Action
from config.logger import setup_logging
import os
import re
import time
import random
import asyncio
import difflib
import traceback
from pathlib import Path
from core.utils import p3
from core.handle.sendAudioHandle import send_stt_message
from plugins_func.register import register_function,ToolType, ActionResponse, Action
TAG = __name__
logger = setup_logging()
MUSIC_CACHE = {}
play_music_function_desc = {
"type": "function",
"function": {
@@ -29,12 +41,157 @@ def play_music(conn, song_name: str):
try:
music_intent = f"播放音乐 {song_name}" if song_name != "random" else "随机播放音乐"
# 执行音乐播放命令
# 检查事件循环状态
if not conn.loop.is_running():
logger.bind(tag=TAG).error("事件循环未运行，无法提交任务")
return ActionResponse(action=Action.RESPONSE, result="系统繁忙", response="请稍后再试")
# 提交异步任务
future = asyncio.run_coroutine_threadsafe(
conn.music_handler.handle_music_command(conn, music_intent),
handle_music_command(conn, music_intent),
conn.loop
)
future.result()
return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response="还想听什么歌？")
# 非阻塞回调处理
def handle_done(f):
try:
f.result() # 可在此处理成功逻辑
logger.bind(tag=TAG).info("播放完成")
except Exception as e:
logger.bind(tag=TAG).error(f"播放失败: {e}")
future.add_done_callback(handle_done)
return ActionResponse(action=Action.RESPONSE, result="指令已接收", response="正在为您播放音乐")
except Exception as e:
logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
return ActionResponse(action=Action.RESPONSE, result=str(e), response="播放音乐时出错了")
def _extract_song_name(text):
"""从用户输入中提取歌名"""
for keyword in ["播放音乐"]:
if keyword in text:
parts = text.split(keyword)
if len(parts) > 1:
return parts[1].strip()
return None
def _find_best_match(potential_song, music_files):
"""查找最匹配的歌曲"""
best_match = None
highest_ratio = 0
for music_file in music_files:
song_name = os.path.splitext(music_file)[0]
ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
if ratio > highest_ratio and ratio > 0.4:
highest_ratio = ratio
best_match = music_file
return best_match
def get_music_files(music_dir, music_ext):
music_dir = Path(music_dir)
music_files = []
music_file_names = []
for file in music_dir.rglob("*"):
# 判断是否是文件
if file.is_file():
# 获取文件扩展名
ext = file.suffix.lower()
# 判断扩展名是否在列表中
if ext in music_ext:
# 添加相对路径
music_files.append(str(file.relative_to(music_dir)))
music_file_names.append(os.path.splitext(str(file.relative_to(music_dir)))[0])
return music_files, music_file_names
def initialize_music_handler(conn):
global MUSIC_CACHE
if MUSIC_CACHE == {}:
logger.bind(tag=TAG).info(f"实例化音乐:")
if "music" in conn.config:
MUSIC_CACHE["music_config"] = conn.config["music"]
MUSIC_CACHE["music_dir"] = os.path.abspath(
MUSIC_CACHE["music_config"].get("music_dir", "./music") # 默认路径修改
)
MUSIC_CACHE["music_ext"] = MUSIC_CACHE["music_config"].get("music_ext", (".mp3", ".wav", ".p3"))
MUSIC_CACHE["refresh_time"] = MUSIC_CACHE["music_config"].get("refresh_time", 60)
else:
MUSIC_CACHE["music_dir"] = os.path.abspath("./music")
MUSIC_CACHE["music_ext"] = (".mp3", ".wav", ".p3")
MUSIC_CACHE["refresh_time"] = 60
# 获取音乐文件列表
MUSIC_CACHE["music_files"], MUSIC_CACHE["music_file_names"] = get_music_files(MUSIC_CACHE["music_dir"],
MUSIC_CACHE["music_ext"])
MUSIC_CACHE["scan_time"] = time.time()
return MUSIC_CACHE
async def handle_music_command(conn, text):
initialize_music_handler(conn)
global MUSIC_CACHE
"""处理音乐播放指令"""
clean_text = re.sub(r'[^\w\s]', '', text).strip()
logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
# 尝试匹配具体歌名
if os.path.exists(MUSIC_CACHE["music_dir"]):
if time.time() - MUSIC_CACHE["scan_time"] > MUSIC_CACHE["refresh_time"]:
# 刷新音乐文件列表
MUSIC_CACHE["music_files"], MUSIC_CACHE["music_file_names"] = get_music_files(MUSIC_CACHE["music_dir"],
MUSIC_CACHE["music_ext"])
MUSIC_CACHE["scan_time"] = time.time()
potential_song = _extract_song_name(clean_text)
if potential_song:
best_match = _find_best_match(potential_song, MUSIC_CACHE["music_files"])
if best_match:
logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
await play_local_music(conn, specific_file=best_match)
return True
# 检查是否是通用播放音乐命令
await play_local_music(conn)
return True
async def play_local_music(conn, specific_file=None):
global MUSIC_CACHE
"""播放本地音乐文件"""
try:
if not os.path.exists(MUSIC_CACHE["music_dir"]):
logger.bind(tag=TAG).error(f"音乐目录不存在: " + MUSIC_CACHE["music_dir"])
return
# 确保路径正确性
if specific_file:
selected_music = specific_file
music_path = os.path.join(MUSIC_CACHE["music_dir"], specific_file)
else:
if not MUSIC_CACHE["music_files"]:
logger.bind(tag=TAG).error("未找到MP3音乐文件")
return
selected_music = random.choice(MUSIC_CACHE["music_files"])
music_path = os.path.join(MUSIC_CACHE["music_dir"], selected_music)
if not os.path.exists(music_path):
logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
return
text = f"正在播放{selected_music}"
await send_stt_message(conn, text)
conn.tts_first_text_index = 0
conn.tts_last_text_index = 0
conn.llm_finish_task = True
if music_path.endswith(".p3"):
opus_packets, duration = p3.decode_opus_from_file(music_path)
else:
opus_packets, duration = conn.tts.audio_to_opus_data(music_path)
conn.audio_play_queue.put((opus_packets, selected_music, 0))
except Exception as e:
logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")