mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-29 04:13:56 +08:00
重新划分目录 (#214)
* 🦄 refactor(web): 修改zhikongtaiweb到web * 🦄 refactor: 重写前端 路由守护尚未写完 * 🦄 refactor: 标准化路由 * update:前端重写，保留后端 * update:添加前端代码 * update:pip转成poetry启动 * update:增加mem0ai包依赖 * update:文档增加mem0ai的描述 * feat: play online mp3 (#181) Co-authored-by: 欣南科技 <huangrongzhuang@xin-nan.com> * 修改前端代码 * update:调整项目目录 * update:优化 * update:配置文件去除8002端口 * update:增加开发说明 * update:更新开发协议 --------- Co-authored-by: kalicyh <34980061+kaliCYH@users.noreply.github.com> Co-authored-by: hrz <1710360675@qq.com> Co-authored-by: freshlife001 <talent@mises.site> Co-authored-by: CGD <3030332422@qq.com>
This commit is contained in:
欣南科技
co-authored by
kalicyh
hrz
freshlife001
CGD
kalicyh
hrz
freshlife001
CGD
parent
8b151d07c2
commit
0e43748fdc
@@ -0,0 +1,16 @@
|
||||
import json
|
||||
import queue
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleAbortMessage(conn):
|
||||
logger.bind(tag=TAG).info("Abort message received")
|
||||
# 设置成打断状态，会自动打断llm、tts任务
|
||||
conn.client_abort = True
|
||||
# 打断客户端说话状态
|
||||
await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
|
||||
conn.clearSpeakStatus()
|
||||
logger.bind(tag=TAG).info("Abort message received-end")
|
||||
@@ -0,0 +1,8 @@
|
||||
import json
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleHelloMessage(conn):
|
||||
await conn.websocket.send(json.dumps(conn.welcome_msg))
|
||||
@@ -0,0 +1,152 @@
|
||||
import json
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class IotDescriptor:
|
||||
"""
|
||||
A class to represent an IoT descriptor.
|
||||
Attributes:
|
||||
----------
|
||||
name : str
|
||||
The name of the IoT descriptor.
|
||||
description : str
|
||||
A brief description of the IoT descriptor.
|
||||
properties : dict
|
||||
A dictionary containing properties of the IoT descriptor.
|
||||
methods : dict
|
||||
A dictionary containing methods of the IoT descriptor.
|
||||
-------
|
||||
"""
|
||||
|
||||
def __init__(self, name, description, properties, methods):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.properties = []
|
||||
self.methods = []
|
||||
|
||||
# 根据描述创建属性
|
||||
for key, value in properties.items():
|
||||
# "volume":{"description":"当前音量 值","type":"number"}
|
||||
"""
|
||||
等价于
|
||||
{
|
||||
'name': 名字,
|
||||
'description': 描述,
|
||||
'value': 0
|
||||
}
|
||||
"""
|
||||
# setattr(self, key, {}) # 创建一个空字典, 名字是属性名
|
||||
property_item = globals()[key] = {} # 创建一个空字典, 名字是属性名
|
||||
property_item['name'] = key
|
||||
property_item["description"] = value["description"]
|
||||
if value["type"] == "number":
|
||||
property_item["value"] = 0
|
||||
elif value["type"] == "boolean":
|
||||
property_item["value"] = False
|
||||
else:
|
||||
property_item["value"] = ""
|
||||
self.properties.append(property_item)
|
||||
|
||||
# 根据描述创建方法
|
||||
for key, value in methods.items():
|
||||
# "SetVolume": {"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}}
|
||||
"""
|
||||
等价于
|
||||
SetVolume = {
|
||||
`description`: 描述,
|
||||
`volume`: {
|
||||
`description`: 描述,
|
||||
`value`: 0
|
||||
}
|
||||
}
|
||||
"""
|
||||
# setattr(self, key, {}) # 创建一个空字典, 名字是方法名
|
||||
method = globals()[key] = {} # 创建一个空字典, 名字是方法名
|
||||
method["description"] = value["description"]
|
||||
method['name'] = key
|
||||
for k, v in value["parameters"].items():
|
||||
# 不同的参数解析
|
||||
method[k] = {}
|
||||
method[k]["description"] = v["description"]
|
||||
if v["type"] == "number":
|
||||
method[k]["value"] = 0
|
||||
elif v["type"] == "boolean":
|
||||
method[k]["value"] = False
|
||||
else:
|
||||
method[k]["value"] = ""
|
||||
|
||||
self.methods.append(method)
|
||||
|
||||
|
||||
async def handleIotDescriptors(conn, descriptors):
|
||||
"""
|
||||
处理物联网描述
|
||||
示例: [{
|
||||
"name":"Speaker",
|
||||
"description":"当前 AI 机器人的扬声器",
|
||||
"properties":{
|
||||
"volume":{"description":"当前音量 值","type":"number"} 可以有boolean, number, string三种类型
|
||||
},
|
||||
"methods":{
|
||||
"SetVolume":{
|
||||
"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}
|
||||
}
|
||||
}
|
||||
}]
|
||||
descriptors: 描述列表
|
||||
"""
|
||||
for descriptor in descriptors:
|
||||
iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
|
||||
descriptor["methods"])
|
||||
conn.iot_descriptors[descriptor["name"]] = iot_descriptor
|
||||
|
||||
# 暂时从配置文件中设置音量，后期通过意图识别控制音量
|
||||
default_iot_volume = 100
|
||||
if "iot" in conn.config:
|
||||
default_iot_volume = conn.config["iot"]["Speaker"]["volume"]
|
||||
logger.bind(tag=TAG).info(f"服务端设置音量为{default_iot_volume}")
|
||||
await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": default_iot_volume})
|
||||
|
||||
|
||||
async def send_iot_conn(conn, name, method_name, parameters):
|
||||
"""
|
||||
发送物联网指令
|
||||
name: 设备名称 "Speaker"
|
||||
method: 方法 "SetVolume"
|
||||
parameters: 参数, 是一个字典 {"volume": 100}
|
||||
发送示例:
|
||||
{
|
||||
"type": "iot",
|
||||
"commands": [
|
||||
{
|
||||
"name" : "Speaker",
|
||||
"method": "SetVolume",
|
||||
"parameters": {
|
||||
"volume": 100
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
for key, value in conn.iot_descriptors.items():
|
||||
if key == name:
|
||||
# 找到了设备
|
||||
for method in value.methods:
|
||||
# 找到了方法
|
||||
if method["name"] == method_name:
|
||||
await conn.websocket.send(json.dumps({
|
||||
"type": "iot",
|
||||
"commands": [
|
||||
{
|
||||
"name": name,
|
||||
"method": method_name,
|
||||
"parameters": parameters
|
||||
}
|
||||
]
|
||||
}))
|
||||
return
|
||||
logger.bind(tag=TAG).error(f"未找到方法{method_name}")
|
||||
@@ -0,0 +1,153 @@
|
||||
from config.logger import setup_logging
|
||||
import os
|
||||
import random
|
||||
import difflib
|
||||
import re
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
import time
|
||||
from core.handle.sendAudioHandle import send_stt_message
|
||||
from core.utils import p3
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
def _extract_song_name(text):
|
||||
"""从用户输入中提取歌名"""
|
||||
for keyword in ["听", "播放", "放", "唱"]:
|
||||
if keyword in text:
|
||||
parts = text.split(keyword)
|
||||
if len(parts) > 1:
|
||||
return parts[1].strip()
|
||||
return None
|
||||
|
||||
|
||||
def _find_best_match(potential_song, music_files):
|
||||
"""查找最匹配的歌曲"""
|
||||
best_match = None
|
||||
highest_ratio = 0
|
||||
|
||||
for music_file in music_files:
|
||||
song_name = os.path.splitext(music_file)[0]
|
||||
ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
|
||||
if ratio > highest_ratio and ratio > 0.4:
|
||||
highest_ratio = ratio
|
||||
best_match = music_file
|
||||
return best_match
|
||||
|
||||
class MusicManager:
|
||||
def __init__(self, music_dir, music_ext):
|
||||
self.music_dir = Path(music_dir)
|
||||
self.music_ext = music_ext
|
||||
|
||||
def get_music_files(self):
|
||||
music_files = []
|
||||
for file in self.music_dir.rglob("*"):
|
||||
# 判断是否是文件
|
||||
if file.is_file():
|
||||
# 获取文件扩展名
|
||||
ext = file.suffix.lower()
|
||||
# 判断扩展名是否在列表中
|
||||
if ext in self.music_ext:
|
||||
# music_files.append(str(file.resolve())) # 添加绝对路径
|
||||
# 添加相对路径
|
||||
music_files.append(str(file.relative_to(self.music_dir)))
|
||||
return music_files
|
||||
|
||||
class MusicHandler:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.music_related_keywords = []
|
||||
|
||||
if "music" in self.config:
|
||||
self.music_config = self.config["music"]
|
||||
self.music_dir = os.path.abspath(
|
||||
self.music_config.get("music_dir", "./music") # 默认路径修改
|
||||
)
|
||||
self.music_related_keywords = self.music_config.get("music_commands", [])
|
||||
self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
|
||||
self.refresh_time = self.music_config.get("refresh_time", 60)
|
||||
else:
|
||||
self.music_dir = os.path.abspath("./music")
|
||||
self.music_related_keywords = ["来一首歌", "唱一首歌", "播放音乐", "来点音乐", "背景音乐", "放首歌",
|
||||
"播放歌曲", "来点背景音乐", "我想听歌", "我要听歌", "放点音乐"]
|
||||
self.music_ext = (".mp3", ".wav", ".p3")
|
||||
self.refresh_time = 60
|
||||
|
||||
# 获取音乐文件列表
|
||||
self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
|
||||
self.scan_time = time.time()
|
||||
logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}")
|
||||
|
||||
async def handle_music_command(self, conn, text):
|
||||
"""处理音乐播放指令"""
|
||||
clean_text = re.sub(r'[^\w\s]', '', text).strip()
|
||||
logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
|
||||
|
||||
# 尝试匹配具体歌名
|
||||
if os.path.exists(self.music_dir):
|
||||
if time.time() - self.scan_time > self.refresh_time:
|
||||
# 刷新音乐文件列表
|
||||
self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
|
||||
self.scan_time = time.time()
|
||||
logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}")
|
||||
|
||||
potential_song = _extract_song_name(clean_text)
|
||||
if potential_song:
|
||||
best_match = _find_best_match(potential_song, self.music_files)
|
||||
if best_match:
|
||||
logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
|
||||
await self.play_local_music(conn, specific_file=best_match)
|
||||
return True
|
||||
|
||||
# 检查是否是通用播放音乐命令
|
||||
if any(cmd in clean_text for cmd in self.music_related_keywords):
|
||||
await self.play_local_music(conn)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
async def play_local_music(self, conn, specific_file=None):
|
||||
"""播放本地音乐文件"""
|
||||
try:
|
||||
if not os.path.exists(self.music_dir):
|
||||
logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}")
|
||||
return
|
||||
|
||||
# 确保路径正确性
|
||||
if specific_file:
|
||||
music_path = os.path.join(self.music_dir, specific_file)
|
||||
if not os.path.exists(music_path):
|
||||
logger.bind(tag=TAG).error(f"指定的音乐文件不存在: {music_path}")
|
||||
return
|
||||
selected_music = specific_file
|
||||
else:
|
||||
if time.time() - self.scan_time > self.refresh_time:
|
||||
# 刷新音乐文件列表
|
||||
self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
|
||||
self.scan_time = time.time()
|
||||
logger.bind(tag=TAG).debug(f"刷新的音乐文件列表: {self.music_files}")
|
||||
|
||||
if not self.music_files:
|
||||
logger.bind(tag=TAG).error("未找到MP3音乐文件")
|
||||
return
|
||||
selected_music = random.choice(self.music_files)
|
||||
music_path = os.path.join(self.music_dir, selected_music)
|
||||
if not os.path.exists(music_path):
|
||||
logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
|
||||
return
|
||||
text = f"正在播放{selected_music}"
|
||||
await send_stt_message(conn, text)
|
||||
conn.tts_first_text_index = 0
|
||||
conn.tts_last_text_index = 0
|
||||
conn.llm_finish_task = True
|
||||
if music_path.endswith(".p3"):
|
||||
opus_packets, duration = p3.decode_opus_from_file(music_path)
|
||||
else:
|
||||
opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
|
||||
conn.audio_play_queue.put((opus_packets, selected_music, 0))
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
|
||||
logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
|
||||
@@ -0,0 +1,77 @@
|
||||
from config.logger import setup_logging
|
||||
import time
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from core.handle.sendAudioHandle import send_stt_message
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleAudioMessage(conn, audio):
|
||||
if not conn.asr_server_receive:
|
||||
logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
|
||||
return
|
||||
if conn.client_listen_mode == "auto":
|
||||
have_voice = conn.vad.is_vad(conn, audio)
|
||||
else:
|
||||
have_voice = conn.client_have_voice
|
||||
|
||||
# 如果本次没有声音，本段也没声音，就把声音丢弃了
|
||||
if have_voice == False and conn.client_have_voice == False:
|
||||
await no_voice_close_connect(conn)
|
||||
conn.asr_audio.clear()
|
||||
return
|
||||
conn.client_no_voice_last_time = 0.0
|
||||
conn.asr_audio.append(audio)
|
||||
# 如果本段有声音，且已经停止了
|
||||
if conn.client_voice_stop:
|
||||
conn.client_abort = False
|
||||
conn.asr_server_receive = False
|
||||
# 音频太短了，无法识别
|
||||
if len(conn.asr_audio) < 3:
|
||||
conn.asr_server_receive = True
|
||||
else:
|
||||
text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
|
||||
logger.bind(tag=TAG).info(f"识别文本: {text}")
|
||||
text_len, text_without_punctuation = remove_punctuation_and_length(text)
|
||||
if await conn.music_handler.handle_music_command(conn, text_without_punctuation):
|
||||
conn.asr_server_receive = True
|
||||
conn.asr_audio.clear()
|
||||
return
|
||||
if text_len <= conn.max_cmd_length and await handleCMDMessage(conn, text_without_punctuation):
|
||||
return
|
||||
if text_len > 0:
|
||||
await startToChat(conn, text)
|
||||
else:
|
||||
conn.asr_server_receive = True
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
|
||||
|
||||
async def handleCMDMessage(conn, text):
|
||||
cmd_exit = conn.cmd_exit
|
||||
for cmd in cmd_exit:
|
||||
if text == cmd:
|
||||
logger.bind(tag=TAG).info("识别到明确的退出命令".format(text))
|
||||
await conn.close()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def startToChat(conn, text):
|
||||
# 异步发送 stt 信息
|
||||
await send_stt_message(conn, text)
|
||||
conn.executor.submit(conn.chat, text)
|
||||
|
||||
|
||||
async def no_voice_close_connect(conn):
|
||||
if conn.client_no_voice_last_time == 0.0:
|
||||
conn.client_no_voice_last_time = time.time() * 1000
|
||||
else:
|
||||
no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
|
||||
close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
|
||||
if no_voice_time > 1000 * close_connection_no_voice_time:
|
||||
conn.client_abort = False
|
||||
conn.asr_server_receive = False
|
||||
prompt = "时间过得真快，我都好久没说话了。请你用十个字左右话跟我告别，以“再见”或“拜拜”为结尾"
|
||||
await startToChat(conn, prompt)
|
||||
@@ -0,0 +1,83 @@
|
||||
from config.logger import setup_logging
|
||||
import json
|
||||
import asyncio
|
||||
import time
|
||||
from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def isLLMWantToFinish(last_text):
|
||||
_, last_text_without_punctuation = remove_punctuation_and_length(last_text)
|
||||
if "再见" in last_text_without_punctuation or "拜拜" in last_text_without_punctuation:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def sendAudioMessage(conn, audios, text, text_index=0):
|
||||
# 发送句子开始消息
|
||||
if text_index == conn.tts_first_text_index:
|
||||
logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
|
||||
await send_tts_message(conn, "sentence_start", text)
|
||||
|
||||
# 初始化流控参数
|
||||
frame_duration = 60 # 毫秒
|
||||
start_time = time.perf_counter() # 使用高精度计时器
|
||||
play_position = 0 # 已播放的时长（毫秒）
|
||||
|
||||
for opus_packet in audios:
|
||||
if conn.client_abort:
|
||||
return
|
||||
|
||||
# 计算当前包的预期发送时间
|
||||
expected_time = start_time + (play_position / 1000)
|
||||
current_time = time.perf_counter()
|
||||
|
||||
# 等待直到预期时间
|
||||
delay = expected_time - current_time
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
# 发送音频包
|
||||
await conn.websocket.send(opus_packet)
|
||||
play_position += frame_duration # 更新播放位置
|
||||
await send_tts_message(conn, "sentence_end", text)
|
||||
# 发送结束消息（如果是最后一个文本）
|
||||
if conn.llm_finish_task and text_index == conn.tts_last_text_index:
|
||||
await send_tts_message(conn, 'stop', None)
|
||||
if await isLLMWantToFinish(text):
|
||||
await conn.close()
|
||||
|
||||
|
||||
async def send_tts_message(conn, state, text=None):
|
||||
"""发送 TTS 状态消息"""
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": conn.session_id
|
||||
}
|
||||
if text is not None:
|
||||
message["text"] = text
|
||||
|
||||
await conn.websocket.send(json.dumps(message))
|
||||
if state == "stop":
|
||||
conn.clearSpeakStatus()
|
||||
|
||||
|
||||
async def send_stt_message(conn, text):
|
||||
"""发送 STT 状态消息"""
|
||||
stt_text = get_string_no_punctuation_or_emoji(text)
|
||||
await conn.websocket.send(json.dumps({
|
||||
"type": "stt",
|
||||
"text": stt_text,
|
||||
"session_id": conn.session_id}
|
||||
))
|
||||
await conn.websocket.send(
|
||||
json.dumps({
|
||||
"type": "llm",
|
||||
"text": "😊",
|
||||
"emotion": "happy",
|
||||
"session_id": conn.session_id}
|
||||
))
|
||||
await send_tts_message(conn, "start")
|
||||
@@ -0,0 +1,44 @@
|
||||
from config.logger import setup_logging
|
||||
import json
|
||||
from core.handle.abortHandle import handleAbortMessage
|
||||
from core.handle.helloHandle import handleHelloMessage
|
||||
from core.handle.receiveAudioHandle import startToChat
|
||||
from core.handle.iotHandle import handleIotDescriptors
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleTextMessage(conn, message):
|
||||
"""处理文本消息"""
|
||||
logger.bind(tag=TAG).info(f"收到文本消息：{message}")
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, int):
|
||||
await conn.websocket.send(message)
|
||||
return
|
||||
if msg_json["type"] == "hello":
|
||||
await handleHelloMessage(conn)
|
||||
elif msg_json["type"] == "abort":
|
||||
await handleAbortMessage(conn)
|
||||
elif msg_json["type"] == "listen":
|
||||
if "mode" in msg_json:
|
||||
conn.client_listen_mode = msg_json["mode"]
|
||||
logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}")
|
||||
if msg_json["state"] == "start":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = False
|
||||
elif msg_json["state"] == "stop":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = True
|
||||
elif msg_json["state"] == "detect":
|
||||
conn.asr_server_receive = False
|
||||
conn.client_have_voice = False
|
||||
conn.asr_audio.clear()
|
||||
if "text" in msg_json:
|
||||
await startToChat(conn, msg_json["text"])
|
||||
elif msg_json["type"] == "iot":
|
||||
if "descriptors" in msg_json:
|
||||
await handleIotDescriptors(conn, msg_json["descriptors"])
|
||||
except json.JSONDecodeError:
|
||||
await conn.websocket.send(message)
|
||||
Reference in New Issue
Block a user