mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-27 09:33:55 +08:00
merge main
This commit is contained in:
lizhongxiang
@@ -0,0 +1,16 @@
|
||||
import json
|
||||
import queue
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleAbortMessage(conn):
|
||||
logger.bind(tag=TAG).info("Abort message received")
|
||||
# 设置成打断状态，会自动打断llm、tts任务
|
||||
conn.client_abort = True
|
||||
# 打断客户端说话状态
|
||||
await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
|
||||
conn.clearSpeakStatus()
|
||||
logger.bind(tag=TAG).info("Abort message received-end")
|
||||
@@ -0,0 +1,8 @@
|
||||
import json
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleHelloMessage(conn):
|
||||
await conn.websocket.send(json.dumps(conn.welcome_msg))
|
||||
@@ -0,0 +1,170 @@
|
||||
from config.logger import setup_logging
|
||||
import json
|
||||
from core.handle.sendAudioHandle import send_stt_message
|
||||
from core.utils.dialogue import Message
|
||||
from config.functionCallConfig import FunctionCallConfig
|
||||
import asyncio
|
||||
from enum import Enum
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class Action(Enum):
|
||||
NOTFOUND = (0, "没有找到函数")
|
||||
NONE = (1, "啥也不干")
|
||||
RESPONSE = (2, "直接回复")
|
||||
REQLLM = (3, "调用函数后再请求llm生成回复")
|
||||
|
||||
def __init__(self, code, message):
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
class ActionResponse:
|
||||
def __init__(self, action: Action, result, response):
|
||||
self.action = action # 动作类型
|
||||
self.result = result # 动作产生的结果
|
||||
self.response = response # 直接回复的内容
|
||||
|
||||
|
||||
def get_functions():
|
||||
"""获取功能调用配置"""
|
||||
return FunctionCallConfig
|
||||
|
||||
|
||||
def handle_llm_function_call(conn, function_call_data):
|
||||
try:
|
||||
function_name = function_call_data["name"]
|
||||
|
||||
if function_name == "handle_exit_intent":
|
||||
# 处理退出意图
|
||||
try:
|
||||
say_goodbye = json.loads(function_call_data["arguments"]).get("say_goodbye", "再见")
|
||||
conn.close_after_chat = True
|
||||
logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}")
|
||||
return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理退出意图错误: {e}")
|
||||
|
||||
elif function_name == "play_music":
|
||||
# 处理音乐播放意图
|
||||
try:
|
||||
song_name = "random"
|
||||
arguments = function_call_data["arguments"]
|
||||
if arguments is not None and len(arguments) > 0:
|
||||
args = json.loads(arguments)
|
||||
song_name = args.get("song_name", "random")
|
||||
music_intent = f"播放音乐 {song_name}" if song_name != "random" else "随机播放音乐"
|
||||
|
||||
# 执行音乐播放命令
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
conn.music_handler.handle_music_command(conn, music_intent),
|
||||
conn.loop
|
||||
)
|
||||
future.result()
|
||||
return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response="还想听什么歌？")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
|
||||
else:
|
||||
return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理function call错误: {e}")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def handle_user_intent(conn, text):
|
||||
"""
|
||||
Handle user intent before starting chat
|
||||
|
||||
Args:
|
||||
conn: Connection object
|
||||
text: User's text input
|
||||
|
||||
Returns:
|
||||
bool: True if intent was handled, False if should proceed to chat
|
||||
"""
|
||||
# 检查是否有明确的退出命令
|
||||
if await check_direct_exit(conn, text):
|
||||
return True
|
||||
|
||||
if conn.use_function_call_mode:
|
||||
# 使用支持function calling的聊天方法,不再进行意图分析
|
||||
return False
|
||||
|
||||
logger.bind(tag=TAG).info(f"分析用户意图: {text}")
|
||||
|
||||
# 使用LLM进行意图分析
|
||||
intent = await analyze_intent_with_llm(conn, text)
|
||||
|
||||
if not intent:
|
||||
return False
|
||||
|
||||
# 处理各种意图
|
||||
return await process_intent_result(conn, intent, text)
|
||||
|
||||
|
||||
async def check_direct_exit(conn, text):
|
||||
"""检查是否有明确的退出命令"""
|
||||
cmd_exit = conn.cmd_exit
|
||||
for cmd in cmd_exit:
|
||||
if text == cmd:
|
||||
logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
|
||||
await conn.close()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def analyze_intent_with_llm(conn, text):
|
||||
"""使用LLM分析用户意图"""
|
||||
if not hasattr(conn, 'intent') or not conn.intent:
|
||||
logger.bind(tag=TAG).warning("意图识别服务未初始化")
|
||||
return None
|
||||
|
||||
# 创建对话历史记录
|
||||
dialogue = conn.dialogue
|
||||
dialogue.put(Message(role="user", content=text))
|
||||
|
||||
try:
|
||||
intent_result = await conn.intent.detect_intent(dialogue.dialogue)
|
||||
logger.bind(tag=TAG).info(f"意图识别结果: {intent_result}")
|
||||
|
||||
# 尝试解析JSON结果
|
||||
try:
|
||||
intent_data = json.loads(intent_result)
|
||||
if "intent" in intent_data:
|
||||
return intent_data["intent"]
|
||||
except json.JSONDecodeError:
|
||||
# 如果不是JSON格式，尝试直接获取意图文本
|
||||
return intent_result.strip()
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"意图识别失败: {str(e)}")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def process_intent_result(conn, intent, original_text):
|
||||
"""处理意图识别结果"""
|
||||
# 处理退出意图
|
||||
if "结束聊天" in intent:
|
||||
logger.bind(tag=TAG).info(f"识别到退出意图: {intent}")
|
||||
|
||||
# 如果正在播放音乐，可以关了 TODO
|
||||
|
||||
# 如果是明确的离别意图，发送告别语并关闭连接
|
||||
await send_stt_message(conn, original_text)
|
||||
conn.executor.submit(conn.chat_and_close, original_text)
|
||||
return True
|
||||
|
||||
# 处理播放音乐意图
|
||||
if "播放音乐" in intent:
|
||||
logger.bind(tag=TAG).info(f"识别到音乐播放意图: {intent}")
|
||||
await conn.music_handler.handle_music_command(conn, intent)
|
||||
return True
|
||||
|
||||
# 其他意图处理可以在这里扩展
|
||||
|
||||
# 默认返回False，表示继续常规聊天流程
|
||||
return False
|
||||
@@ -0,0 +1,193 @@
|
||||
import json
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class IotDescriptor:
|
||||
"""
|
||||
A class to represent an IoT descriptor.
|
||||
Attributes:
|
||||
----------
|
||||
name : str
|
||||
The name of the IoT descriptor.
|
||||
description : str
|
||||
A brief description of the IoT descriptor.
|
||||
properties : dict
|
||||
A dictionary containing properties of the IoT descriptor.
|
||||
methods : dict
|
||||
A dictionary containing methods of the IoT descriptor.
|
||||
-------
|
||||
"""
|
||||
|
||||
def __init__(self, name, description, properties, methods):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.properties = []
|
||||
self.methods = []
|
||||
|
||||
# 根据描述创建属性
|
||||
for key, value in properties.items():
|
||||
# "volume":{"description":"当前音量 值","type":"number"}
|
||||
"""
|
||||
等价于
|
||||
{
|
||||
'name': 名字,
|
||||
'description': 描述,
|
||||
'value': 0
|
||||
}
|
||||
"""
|
||||
# setattr(self, key, {}) # 创建一个空字典, 名字是属性名
|
||||
property_item = globals()[key] = {} # 创建一个空字典, 名字是属性名
|
||||
property_item['name'] = key
|
||||
property_item["description"] = value["description"]
|
||||
if value["type"] == "number":
|
||||
property_item["value"] = 0
|
||||
elif value["type"] == "boolean":
|
||||
property_item["value"] = False
|
||||
else:
|
||||
property_item["value"] = ""
|
||||
self.properties.append(property_item)
|
||||
|
||||
# 根据描述创建方法
|
||||
for key, value in methods.items():
|
||||
# "SetVolume": {"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}}
|
||||
"""
|
||||
等价于
|
||||
SetVolume = {
|
||||
`description`: 描述,
|
||||
`volume`: {
|
||||
`description`: 描述,
|
||||
`value`: 0
|
||||
}
|
||||
}
|
||||
"""
|
||||
# setattr(self, key, {}) # 创建一个空字典, 名字是方法名
|
||||
method = globals()[key] = {} # 创建一个空字典, 名字是方法名
|
||||
method["description"] = value["description"]
|
||||
method['name'] = key
|
||||
for k, v in value["parameters"].items():
|
||||
# 不同的参数解析
|
||||
method[k] = {}
|
||||
method[k]["description"] = v["description"]
|
||||
if v["type"] == "number":
|
||||
method[k]["value"] = 0
|
||||
elif v["type"] == "boolean":
|
||||
method[k]["value"] = False
|
||||
else:
|
||||
method[k]["value"] = ""
|
||||
|
||||
self.methods.append(method)
|
||||
|
||||
|
||||
async def handleIotDescriptors(conn, descriptors):
|
||||
"""
|
||||
处理物联网描述
|
||||
示例: [{
|
||||
"name":"Speaker",
|
||||
"description":"当前 AI 机器人的扬声器",
|
||||
"properties":{
|
||||
"volume":{"description":"当前音量 值","type":"number"} 可以有boolean, number, string三种类型
|
||||
},
|
||||
"methods":{
|
||||
"SetVolume":{
|
||||
"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}
|
||||
}
|
||||
}
|
||||
}]
|
||||
descriptors: 描述列表
|
||||
"""
|
||||
for descriptor in descriptors:
|
||||
iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
|
||||
descriptor["methods"])
|
||||
conn.iot_descriptors[descriptor["name"]] = iot_descriptor
|
||||
|
||||
# 暂时从配置文件中设置音量，后期通过意图识别控制音量
|
||||
default_iot_volume = 100
|
||||
if "iot" in conn.config:
|
||||
default_iot_volume = conn.config["iot"]["Speaker"]["volume"]
|
||||
logger.bind(tag=TAG).info(f"服务端设置音量为{default_iot_volume}")
|
||||
await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": default_iot_volume})
|
||||
async def handleIotStatus(conn, states):
|
||||
"""
|
||||
处理物联网状态
|
||||
示例: [{
|
||||
"name":"Speaker",
|
||||
"state":{
|
||||
"volume":100
|
||||
}
|
||||
}]
|
||||
states: 状态列表
|
||||
"""
|
||||
for state in states:
|
||||
for key, value in conn.iot_descriptors.items():
|
||||
if key == state["name"]:
|
||||
for property_item in value.properties:
|
||||
# properties为字典列表, 记录各种属性
|
||||
for k, v in state["state"].items():
|
||||
# state为字典, 记录各种属性的值, 是需要记录的信息
|
||||
if property_item["name"] == k:
|
||||
# 检查一下属性是不是相同的
|
||||
if type(v) != type(property_item["value"]):
|
||||
logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
|
||||
break
|
||||
else:
|
||||
property_item["value"] = v
|
||||
logger.bind(tag=TAG).info(f"物联网状态更新: {key} , {property_item['name']} = {v}")
|
||||
break
|
||||
break
|
||||
|
||||
async def get_iot_status(conn, name, property_name):
|
||||
"""
|
||||
获取物联网状态
|
||||
name: 设备名称 "Speaker"
|
||||
property_name: 属性名称 "volume"
|
||||
返回值: 属性值, 实际的属性有int, bool和str三种类型
|
||||
"""
|
||||
for key, value in conn.iot_descriptors.items():
|
||||
if key == name:
|
||||
for property_item in value.properties:
|
||||
if property_item["name"] == property_name:
|
||||
return property_item["value"]
|
||||
return None
|
||||
|
||||
async def send_iot_conn(conn, name, method_name, parameters):
|
||||
"""
|
||||
发送物联网指令
|
||||
name: 设备名称 "Speaker"
|
||||
method: 方法 "SetVolume"
|
||||
parameters: 参数, 是一个字典 {"volume": 100}
|
||||
发送示例:
|
||||
{
|
||||
"type": "iot",
|
||||
"commands": [
|
||||
{
|
||||
"name" : "Speaker",
|
||||
"method": "SetVolume",
|
||||
"parameters": {
|
||||
"volume": 100
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
for key, value in conn.iot_descriptors.items():
|
||||
if key == name:
|
||||
# 找到了设备
|
||||
for method in value.methods:
|
||||
# 找到了方法
|
||||
if method["name"] == method_name:
|
||||
await conn.websocket.send(json.dumps({
|
||||
"type": "iot",
|
||||
"commands": [
|
||||
{
|
||||
"name": name,
|
||||
"method": method_name,
|
||||
"parameters": parameters
|
||||
}
|
||||
]
|
||||
}))
|
||||
return
|
||||
logger.bind(tag=TAG).error(f"未找到方法{method_name}")
|
||||
@@ -0,0 +1,139 @@
|
||||
from config.logger import setup_logging
|
||||
import os
|
||||
import random
|
||||
import difflib
|
||||
import re
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
import time
|
||||
from core.handle.sendAudioHandle import send_stt_message
|
||||
from core.utils import p3
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
def _extract_song_name(text):
|
||||
"""从用户输入中提取歌名"""
|
||||
for keyword in ["播放音乐"]:
|
||||
if keyword in text:
|
||||
parts = text.split(keyword)
|
||||
if len(parts) > 1:
|
||||
return parts[1].strip()
|
||||
return None
|
||||
|
||||
|
||||
def _find_best_match(potential_song, music_files):
|
||||
"""查找最匹配的歌曲"""
|
||||
best_match = None
|
||||
highest_ratio = 0
|
||||
|
||||
for music_file in music_files:
|
||||
song_name = os.path.splitext(music_file)[0]
|
||||
ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
|
||||
if ratio > highest_ratio and ratio > 0.4:
|
||||
highest_ratio = ratio
|
||||
best_match = music_file
|
||||
return best_match
|
||||
|
||||
|
||||
class MusicManager:
|
||||
def __init__(self, music_dir, music_ext):
|
||||
self.music_dir = Path(music_dir)
|
||||
self.music_ext = music_ext
|
||||
|
||||
def get_music_files(self):
|
||||
music_files = []
|
||||
for file in self.music_dir.rglob("*"):
|
||||
# 判断是否是文件
|
||||
if file.is_file():
|
||||
# 获取文件扩展名
|
||||
ext = file.suffix.lower()
|
||||
# 判断扩展名是否在列表中
|
||||
if ext in self.music_ext:
|
||||
# music_files.append(str(file.resolve())) # 添加绝对路径
|
||||
# 添加相对路径
|
||||
music_files.append(str(file.relative_to(self.music_dir)))
|
||||
return music_files
|
||||
|
||||
|
||||
class MusicHandler:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
|
||||
if "music" in self.config:
|
||||
self.music_config = self.config["music"]
|
||||
self.music_dir = os.path.abspath(
|
||||
self.music_config.get("music_dir", "./music") # 默认路径修改
|
||||
)
|
||||
self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
|
||||
self.refresh_time = self.music_config.get("refresh_time", 60)
|
||||
else:
|
||||
self.music_dir = os.path.abspath("./music")
|
||||
self.music_ext = (".mp3", ".wav", ".p3")
|
||||
self.refresh_time = 60
|
||||
|
||||
# 获取音乐文件列表
|
||||
self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
|
||||
self.scan_time = time.time()
|
||||
logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}")
|
||||
|
||||
async def handle_music_command(self, conn, text):
|
||||
"""处理音乐播放指令"""
|
||||
clean_text = re.sub(r'[^\w\s]', '', text).strip()
|
||||
logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
|
||||
|
||||
# 尝试匹配具体歌名
|
||||
if os.path.exists(self.music_dir):
|
||||
if time.time() - self.scan_time > self.refresh_time:
|
||||
# 刷新音乐文件列表
|
||||
self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
|
||||
self.scan_time = time.time()
|
||||
logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}")
|
||||
|
||||
potential_song = _extract_song_name(clean_text)
|
||||
if potential_song:
|
||||
best_match = _find_best_match(potential_song, self.music_files)
|
||||
if best_match:
|
||||
logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
|
||||
await self.play_local_music(conn, specific_file=best_match)
|
||||
return True
|
||||
# 检查是否是通用播放音乐命令
|
||||
await self.play_local_music(conn)
|
||||
return True
|
||||
|
||||
async def play_local_music(self, conn, specific_file=None):
|
||||
"""播放本地音乐文件"""
|
||||
try:
|
||||
if not os.path.exists(self.music_dir):
|
||||
logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}")
|
||||
return
|
||||
|
||||
# 确保路径正确性
|
||||
if specific_file:
|
||||
selected_music = specific_file
|
||||
music_path = os.path.join(self.music_dir, specific_file)
|
||||
else:
|
||||
if not self.music_files:
|
||||
logger.bind(tag=TAG).error("未找到MP3音乐文件")
|
||||
return
|
||||
selected_music = random.choice(self.music_files)
|
||||
music_path = os.path.join(self.music_dir, selected_music)
|
||||
|
||||
if not os.path.exists(music_path):
|
||||
logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
|
||||
return
|
||||
text = f"正在播放{selected_music}"
|
||||
await send_stt_message(conn, text)
|
||||
conn.tts_first_text_index = 0
|
||||
conn.tts_last_text_index = 0
|
||||
conn.llm_finish_task = True
|
||||
if music_path.endswith(".p3"):
|
||||
opus_packets, duration = p3.decode_opus_from_file(music_path)
|
||||
else:
|
||||
opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
|
||||
conn.audio_play_queue.put((opus_packets, selected_music, 0))
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
|
||||
logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
|
||||
@@ -0,0 +1,74 @@
|
||||
from config.logger import setup_logging
|
||||
import time
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from core.handle.sendAudioHandle import send_stt_message
|
||||
from core.handle.intentHandler import handle_user_intent
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleAudioMessage(conn, audio):
|
||||
if not conn.asr_server_receive:
|
||||
logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
|
||||
return
|
||||
if conn.client_listen_mode == "auto":
|
||||
have_voice = conn.vad.is_vad(conn, audio)
|
||||
else:
|
||||
have_voice = conn.client_have_voice
|
||||
|
||||
# 如果本次没有声音，本段也没声音，就把声音丢弃了
|
||||
if have_voice == False and conn.client_have_voice == False:
|
||||
await no_voice_close_connect(conn)
|
||||
conn.asr_audio.clear()
|
||||
return
|
||||
conn.client_no_voice_last_time = 0.0
|
||||
conn.asr_audio.append(audio)
|
||||
# 如果本段有声音，且已经停止了
|
||||
if conn.client_voice_stop:
|
||||
conn.client_abort = False
|
||||
conn.asr_server_receive = False
|
||||
# 音频太短了，无法识别
|
||||
if len(conn.asr_audio) < 3:
|
||||
conn.asr_server_receive = True
|
||||
else:
|
||||
text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
|
||||
logger.bind(tag=TAG).info(f"识别文本: {text}")
|
||||
text_len, _ = remove_punctuation_and_length(text)
|
||||
if text_len > 0:
|
||||
await startToChat(conn, text)
|
||||
else:
|
||||
conn.asr_server_receive = True
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
|
||||
|
||||
async def startToChat(conn, text):
|
||||
# 首先进行意图分析
|
||||
intent_handled = await handle_user_intent(conn, text)
|
||||
|
||||
if intent_handled:
|
||||
# 如果意图已被处理，不再进行聊天
|
||||
conn.asr_server_receive = True
|
||||
return
|
||||
|
||||
# 意图未被处理，继续常规聊天流程
|
||||
await send_stt_message(conn, text)
|
||||
if conn.use_function_call_mode:
|
||||
# 使用支持function calling的聊天方法
|
||||
conn.executor.submit(conn.chat_with_function_calling, text)
|
||||
else:
|
||||
conn.executor.submit(conn.chat, text)
|
||||
|
||||
|
||||
async def no_voice_close_connect(conn):
|
||||
if conn.client_no_voice_last_time == 0.0:
|
||||
conn.client_no_voice_last_time = time.time() * 1000
|
||||
else:
|
||||
no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
|
||||
close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
|
||||
if no_voice_time > 1000 * close_connection_no_voice_time:
|
||||
conn.client_abort = False
|
||||
conn.asr_server_receive = False
|
||||
prompt = "时间过得真快，我都好久没说话了。请你用十个字左右话跟我告别，以“再见”或“拜拜”为结尾"
|
||||
await startToChat(conn, prompt)
|
||||
@@ -0,0 +1,136 @@
|
||||
import traceback
|
||||
|
||||
from config.logger import setup_logging
|
||||
import json
|
||||
import asyncio
|
||||
import time
|
||||
from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
async def sendAudioMessageStream(conn, audios_queue, text, text_index=0):
|
||||
# 发送句子开始消息
|
||||
if text_index == conn.tts_first_text_index:
|
||||
logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
|
||||
await send_tts_message(conn, "sentence_start", text)
|
||||
|
||||
# 初始化流控参数
|
||||
frame_duration = 60 # 毫秒
|
||||
start_time = time.time() # 使用高精度计时器
|
||||
play_position = 0 # 已播放的时长（毫秒）
|
||||
time_out_stop = False
|
||||
while True:
|
||||
try:
|
||||
start_get_queue = time.time()
|
||||
# 尝试获取数据，如果没有数据，则等待一小段时间再试
|
||||
audio_data_chunke = None
|
||||
try:
|
||||
audio_data_chunke = audios_queue.get(timeout=5) # 设置超时为1秒
|
||||
except Exception as e:
|
||||
# 如果超时，继续等待
|
||||
logger.bind(tag=TAG).error(f"获取队列超时～{e}")
|
||||
|
||||
audio_opus_datas = audio_data_chunke.get('data') if audio_data_chunke else None
|
||||
duration = audio_data_chunke.get('duration') if audio_data_chunke else 0
|
||||
|
||||
if audio_data_chunke:
|
||||
start_time = time.time()
|
||||
# 检查是否超过 5 秒没有数据
|
||||
if time.time() - start_time > 15:
|
||||
logger.bind(tag=TAG).error("超过15秒没有数据，退出。")
|
||||
break
|
||||
|
||||
if audio_data_chunke and audio_data_chunke.get("end", True):
|
||||
break
|
||||
|
||||
if audio_opus_datas:
|
||||
queue_duration = time.time() - start_get_queue
|
||||
last_duration = conn.tts_duration - queue_duration
|
||||
if last_duration <= 0:
|
||||
last_duration = 0
|
||||
conn.tts_duration = duration + last_duration
|
||||
for opus_packet in audio_opus_datas:
|
||||
await conn.websocket.send(opus_packet)
|
||||
start_time = time.time() # 更新获取数据的时间
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发生错误: {e}")
|
||||
traceback.print_exc() # 打印错误堆栈
|
||||
await send_tts_message(conn, "sentence_end", text)
|
||||
|
||||
print(f'{text_index}-{conn.tts_last_text_index}')
|
||||
# 发送结束消息（如果是最后一个文本）
|
||||
if conn.llm_finish_task and text_index == conn.tts_last_text_index:
|
||||
if conn.tts_duration and conn.tts_duration > 0:
|
||||
await asyncio.sleep(conn.tts_duration)
|
||||
await send_tts_message(conn, 'stop', None)
|
||||
if await isLLMWantToFinish(text):
|
||||
await conn.close()
|
||||
|
||||
|
||||
|
||||
async def sendAudioMessage(conn, audios, text, text_index=0):
|
||||
# 发送句子开始消息
|
||||
if text_index == conn.tts_first_text_index:
|
||||
logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
|
||||
await send_tts_message(conn, "sentence_start", text)
|
||||
|
||||
# 初始化流控参数
|
||||
frame_duration = 60 # 毫秒
|
||||
start_time = time.perf_counter() # 使用高精度计时器
|
||||
play_position = 0 # 已播放的时长（毫秒）
|
||||
|
||||
for opus_packet in audios:
|
||||
if conn.client_abort:
|
||||
return
|
||||
|
||||
# 计算当前包的预期发送时间
|
||||
expected_time = start_time + (play_position / 1000)
|
||||
current_time = time.perf_counter()
|
||||
|
||||
# 等待直到预期时间
|
||||
delay = expected_time - current_time
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
# 发送音频包
|
||||
await conn.websocket.send(opus_packet)
|
||||
play_position += frame_duration # 更新播放位置
|
||||
await send_tts_message(conn, "sentence_end", text)
|
||||
# 发送结束消息（如果是最后一个文本）
|
||||
if conn.llm_finish_task and text_index == conn.tts_last_text_index:
|
||||
await send_tts_message(conn, 'stop', None)
|
||||
if conn.close_after_chat:
|
||||
await conn.close()
|
||||
|
||||
async def send_tts_message(conn, state, text=None):
|
||||
"""发送 TTS 状态消息"""
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": conn.session_id
|
||||
}
|
||||
if text is not None:
|
||||
message["text"] = text
|
||||
|
||||
await conn.websocket.send(json.dumps(message))
|
||||
if state == "stop":
|
||||
conn.clearSpeakStatus()
|
||||
|
||||
|
||||
async def send_stt_message(conn, text):
|
||||
"""发送 STT 状态消息"""
|
||||
stt_text = get_string_no_punctuation_or_emoji(text)
|
||||
await conn.websocket.send(json.dumps({
|
||||
"type": "stt",
|
||||
"text": stt_text,
|
||||
"session_id": conn.session_id}
|
||||
))
|
||||
await conn.websocket.send(
|
||||
json.dumps({
|
||||
"type": "llm",
|
||||
"text": "😊",
|
||||
"emotion": "happy",
|
||||
"session_id": conn.session_id}
|
||||
))
|
||||
await send_tts_message(conn, "start")
|
||||
@@ -0,0 +1,46 @@
|
||||
from config.logger import setup_logging
|
||||
import json
|
||||
from core.handle.abortHandle import handleAbortMessage
|
||||
from core.handle.helloHandle import handleHelloMessage
|
||||
from core.handle.receiveAudioHandle import startToChat
|
||||
from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleTextMessage(conn, message):
|
||||
"""处理文本消息"""
|
||||
logger.bind(tag=TAG).info(f"收到文本消息：{message}")
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, int):
|
||||
await conn.websocket.send(message)
|
||||
return
|
||||
if msg_json["type"] == "hello":
|
||||
await handleHelloMessage(conn)
|
||||
elif msg_json["type"] == "abort":
|
||||
await handleAbortMessage(conn)
|
||||
elif msg_json["type"] == "listen":
|
||||
if "mode" in msg_json:
|
||||
conn.client_listen_mode = msg_json["mode"]
|
||||
logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}")
|
||||
if msg_json["state"] == "start":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = False
|
||||
elif msg_json["state"] == "stop":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = True
|
||||
elif msg_json["state"] == "detect":
|
||||
conn.asr_server_receive = False
|
||||
conn.client_have_voice = False
|
||||
conn.asr_audio.clear()
|
||||
if "text" in msg_json:
|
||||
await startToChat(conn, msg_json["text"])
|
||||
elif msg_json["type"] == "iot":
|
||||
if "descriptors" in msg_json:
|
||||
await handleIotDescriptors(conn, msg_json["descriptors"])
|
||||
if "states" in msg_json:
|
||||
await handleIotStatus(conn, msg_json["states"])
|
||||
except json.JSONDecodeError:
|
||||
await conn.websocket.send(message)
|
||||
Reference in New Issue
Block a user