update:合并非tts代码

This commit is contained in:
hrz
2025-05-21 13:18:12 +08:00
parent ede2bc6a4e
commit 851365fb58
65 changed files with 5423 additions and 1943 deletions
@@ -3,14 +3,16 @@ import queue
from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
async def handleAbortMessage(conn):
logger.bind(tag=TAG).info("Abort message received")
conn.logger.bind(tag=TAG).info("Abort message received")
# 设置成打断状态,会自动打断llm、tts任务
conn.client_abort = True
conn.clear_queues()
# 打断客户端说话状态
await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
await conn.websocket.send(
json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id})
)
conn.clearSpeakStatus()
logger.bind(tag=TAG).info("Abort message received-end")
conn.logger.bind(tag=TAG).info("Abort message received-end")
@@ -4,7 +4,6 @@ from plugins_func.register import FunctionRegistry, ActionResponse, Action, Tool
from plugins_func.functions.hass_init import append_devices_to_prompt
TAG = __name__
logger = setup_logging()
class FunctionHandler:
@@ -40,7 +39,9 @@ class FunctionHandler:
for func in self.functions_desc:
func_names.append(func["function"]["name"])
# 打印当前支持的函数列表
logger.bind(tag=TAG).info(f"当前支持的函数列表: {func_names}")
self.conn.logger.bind(tag=TAG, session_id=self.conn.session_id).info(
f"当前支持的函数列表: {func_names}"
)
return func_names
def get_functions(self):
@@ -57,7 +58,9 @@ class FunctionHandler:
def register_config_functions(self):
"""注册配置中的函数,可以不同客户端使用不同的配置"""
for func in self.config["Intent"]["function_call"].get("functions", []):
for func in self.config["Intent"][self.config["selected_module"]["Intent"]].get(
"functions", []
):
self.function_registry.register_function(func)
"""home assistant需要初始化提示词"""
@@ -77,7 +80,9 @@ class FunctionHandler:
func = funcItem.func
arguments = function_call_data["arguments"]
arguments = json.loads(arguments) if arguments else {}
logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
self.conn.logger.bind(tag=TAG).debug(
f"调用函数: {function_name}, 参数: {arguments}"
)
if (
funcItem.type == ToolType.SYSTEM_CTL
or funcItem.type == ToolType.IOT_CTL
@@ -92,6 +97,6 @@ class FunctionHandler:
action=Action.NOTFOUND, result="没有找到对应的函数", response=""
)
except Exception as e:
logger.bind(tag=TAG).error(f"处理function call错误: {e}")
self.conn.logger.bind(tag=TAG).error(f"处理function call错误: {e}")
return None
+24 -18
View File
@@ -1,6 +1,4 @@
import json
from config.logger import setup_logging
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType
from core.handle.sendAudioHandle import send_stt_message
from core.utils.util import remove_punctuation_and_length
import shutil
@@ -9,7 +7,7 @@ import os
import random
import time
logger = setup_logging()
TAG = __name__
WAKEUP_CONFIG = {
"dir": "config/assets/",
@@ -21,7 +19,16 @@ WAKEUP_CONFIG = {
}
async def handleHelloMessage(conn):
async def handleHelloMessage(conn, msg_json):
"""处理hello消息"""
audio_params = msg_json.get("audio_params")
if audio_params:
format = audio_params.get("format")
conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}")
conn.audio_format = format
conn.asr.set_audio_format(format)
conn.welcome_msg["audio_params"] = audio_params
await conn.websocket.send(json.dumps(conn.welcome_msg))
@@ -44,21 +51,11 @@ async def checkWakeupWords(conn, text):
if file is None:
asyncio.create_task(wakeupWordsResponse(conn))
return False
opus_packets, duration = conn.tts.audio_to_opus_data(file)
opus_packets, _ = conn.tts.audio_to_opus_data(file)
text_hello = WAKEUP_CONFIG["text"]
if not text_hello:
text_hello = text
conn.tts.tts_audio_queue.put(
TTSMessageDTO(
u_id=conn.u_id,
msg_type=MsgType.TTS_TEXT_RESPONSE,
content=opus_packets,
tts_finish_text="",
sentence_type=None,
duration=0,
)
)
conn.audio_play_queue.put((opus_packets, text_hello, 0))
if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
asyncio.create_task(wakeupWordsResponse(conn))
return True
@@ -80,11 +77,20 @@ def getWakeupWordFile(file_name):
async def wakeupWordsResponse(conn):
wait_max_time = 5
while conn.llm is None or not conn.llm.response_no_stream:
await asyncio.sleep(1)
wait_max_time -= 1
if wait_max_time <= 0:
conn.logger.bind(tag=TAG).error("连接对象没有llm")
return
"""唤醒词响应"""
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
# TODO 将result转换为tts_message_dto
tts_file = None
if result is None or result == "":
return
tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
if tts_file is not None and os.path.exists(tts_file):
file_type = os.path.splitext(tts_file)[1]
+91 -67
View File
@@ -10,7 +10,6 @@ from plugins_func.register import (
)
TAG = __name__
logger = setup_logging()
def wrap_async_function(async_func):
@@ -21,7 +20,7 @@ def wrap_async_function(async_func):
# 获取连接对象(第一个参数)
conn = args[0]
if not hasattr(conn, "loop"):
logger.bind(tag=TAG).error("Connection对象没有loop属性")
conn.logger.bind(tag=TAG).error("Connection对象没有loop属性")
return ActionResponse(
Action.ERROR,
"Connection对象没有loop属性",
@@ -35,7 +34,7 @@ def wrap_async_function(async_func):
# 等待结果返回
return future.result()
except Exception as e:
logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
conn.logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
return ActionResponse(Action.ERROR, str(e), f"执行操作时出错: {e}")
return wrapper
@@ -57,7 +56,7 @@ def create_iot_function(device_name, method_name, method_info):
response_failure = "操作失败"
# 打印响应参数
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).debug(
f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
)
@@ -86,7 +85,9 @@ def create_iot_function(device_name, method_name, method_info):
return ActionResponse(Action.RESPONSE, result, response)
except Exception as e:
logger.bind(tag=TAG).error(f"执行{device_name}{method_name}操作失败: {e}")
conn.logger.bind(tag=TAG).error(
f"执行{device_name}{method_name}操作失败: {e}"
)
# 操作失败时使用大模型提供的失败响应
response = response_failure
@@ -104,7 +105,7 @@ def create_iot_query_function(device_name, prop_name, prop_info):
async def iot_query_function(conn, response_success=None, response_failure=None):
try:
# 打印响应参数
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).info(
f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
)
@@ -122,7 +123,9 @@ def create_iot_query_function(device_name, prop_name, prop_info):
return ActionResponse(Action.ERROR, f"属性{prop_name}不存在", response)
except Exception as e:
logger.bind(tag=TAG).error(f"查询{device_name}{prop_name}时出错: {e}")
conn.logger.bind(tag=TAG).error(
f"查询{device_name}{prop_name}时出错: {e}"
)
# 查询出错时使用大模型提供的失败响应
response = response_failure
@@ -144,33 +147,34 @@ class IotDescriptor:
self.methods = []
# 根据描述创建属性
for key, value in properties.items():
property_item = globals()[key] = {}
property_item["name"] = key
property_item["description"] = value["description"]
if value["type"] == "number":
property_item["value"] = 0
elif value["type"] == "boolean":
property_item["value"] = False
else:
property_item["value"] = ""
self.properties.append(property_item)
if properties is not None:
for key, value in properties.items():
property_item = {}
property_item["name"] = key
property_item["description"] = value["description"]
if value["type"] == "number":
property_item["value"] = 0
elif value["type"] == "boolean":
property_item["value"] = False
else:
property_item["value"] = ""
self.properties.append(property_item)
# 根据描述创建方法
for key, value in methods.items():
method = globals()[key] = {}
method["description"] = value["description"]
method["name"] = key
for k, v in value["parameters"].items():
method[k] = {}
method[k]["description"] = v["description"]
if v["type"] == "number":
method[k]["value"] = 0
elif v["type"] == "boolean":
method[k]["value"] = False
else:
method[k]["value"] = ""
self.methods.append(method)
if methods is not None:
for key, value in methods.items():
method = {}
method["description"] = value["description"]
method["name"] = key
# 检查方法是否有参数
if "parameters" in value:
method["parameters"] = {}
for k, v in value["parameters"].items():
method["parameters"][k] = {
"description": v["description"],
"type": v["type"],
}
self.methods.append(method)
def register_device_type(descriptor):
@@ -219,13 +223,19 @@ def register_device_type(descriptor):
func_name = f"{device_name.lower()}_{method_name.lower()}"
# 创建参数字典,添加原有参数
parameters = {
param_name: {
"type": param_info["type"],
"description": param_info["description"],
parameters = {}
required_params = []
# 如果方法有参数,则添加参数信息
if "parameters" in method_info:
parameters = {
param_name: {
"type": param_info["type"],
"description": param_info["description"],
}
for param_name, param_info in method_info["parameters"].items()
}
for param_name, param_info in method_info["parameters"].items()
}
required_params = list(method_info["parameters"].keys())
# 添加响应参数
parameters.update(
@@ -242,7 +252,6 @@ def register_device_type(descriptor):
)
# 构建必须参数列表(原有参数 + 响应参数)
required_params = list(method_info["parameters"].keys())
required_params.extend(["response_success", "response_failure"])
func_desc = {
@@ -269,19 +278,36 @@ def register_device_type(descriptor):
# 用于接受前端设备推送的搜索iot描述
async def handleIotDescriptors(conn, descriptors):
if not conn.use_function_call_mode:
return
wait_max_time = 5
while conn.func_handler is None or not conn.func_handler.finish_init:
await asyncio.sleep(1)
wait_max_time -= 1
if wait_max_time <= 0:
logger.bind(tag=TAG).error("连接对象没有func_handler")
conn.logger.bind(tag=TAG).debug("连接对象没有func_handler")
return
"""处理物联网描述"""
functions_changed = False
for descriptor in descriptors:
# 如果descriptor没有properties和methods,则直接跳过
if "properties" not in descriptor and "methods" not in descriptor:
continue
# 处理缺失properties的情况
if "properties" not in descriptor:
descriptor["properties"] = {}
# 从methods中提取所有参数作为properties
if "methods" in descriptor:
for method_name, method_info in descriptor["methods"].items():
if "parameters" in method_info:
for param_name, param_info in method_info["parameters"].items():
# 将参数信息转换为属性信息
descriptor["properties"][param_name] = {
"description": param_info["description"],
"type": param_info["type"],
}
# 创建IOT设备描述符
iot_descriptor = IotDescriptor(
descriptor["name"],
@@ -291,7 +317,7 @@ async def handleIotDescriptors(conn, descriptors):
)
conn.iot_descriptors[descriptor["name"]] = iot_descriptor
if conn.use_function_call_mode:
if conn.load_function_plugin:
# 注册或获取设备类型
type_id = register_device_type(descriptor)
device_functions = device_type_registry.get_device_functions(type_id)
@@ -300,7 +326,7 @@ async def handleIotDescriptors(conn, descriptors):
if hasattr(conn, "func_handler"):
for func_name in device_functions:
conn.func_handler.function_registry.register_function(func_name)
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).info(
f"注册IOT函数到function handler: {func_name}"
)
functions_changed = True
@@ -309,8 +335,8 @@ async def handleIotDescriptors(conn, descriptors):
if functions_changed and hasattr(conn, "func_handler"):
conn.func_handler.upload_functions_desc()
func_names = conn.func_handler.current_support_functions()
logger.bind(tag=TAG).info(f"设备类型: {type_id}")
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).info(f"设备类型: {type_id}")
conn.logger.bind(tag=TAG).info(
f"更新function描述列表完成,当前支持的函数: {func_names}"
)
@@ -324,13 +350,13 @@ async def handleIotStatus(conn, states):
for k, v in state["state"].items():
if property_item["name"] == k:
if type(v) != type(property_item["value"]):
logger.bind(tag=TAG).error(
conn.logger.bind(tag=TAG).error(
f"属性{property_item['name']}的值类型不匹配"
)
break
else:
property_item["value"] = v
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).info(
f"物联网状态更新: {key} , {property_item['name']} = {v}"
)
break
@@ -344,7 +370,7 @@ async def get_iot_status(conn, name, property_name):
for property_item in value.properties:
if property_item["name"] == property_name:
return property_item["value"]
logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
return None
@@ -355,16 +381,16 @@ async def set_iot_status(conn, name, property_name, value):
for property_item in iot_descriptor.properties:
if property_item["name"] == property_name:
if type(value) != type(property_item["value"]):
logger.bind(tag=TAG).error(
conn.logger.bind(tag=TAG).error(
f"属性{property_item['name']}的值类型不匹配"
)
return
property_item["value"] = value
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).info(
f"物联网状态更新: {name} , {property_name} = {value}"
)
return
logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
async def send_iot_conn(conn, name, method_name, parameters):
@@ -375,19 +401,17 @@ async def send_iot_conn(conn, name, method_name, parameters):
for method in value.methods:
# 找到了方法
if method["name"] == method_name:
await conn.websocket.send(
json.dumps(
{
"type": "iot",
"commands": [
{
"name": name,
"method": method_name,
"parameters": parameters,
}
],
}
)
)
# 构建命令对象
command = {
"name": name,
"method": method_name,
}
# 只有当参数不为空时才添加parameters字段
if parameters:
command["parameters"] = parameters
send_message = json.dumps({"type": "iot", "commands": [command]})
await conn.websocket.send(send_message)
conn.logger.bind(tag=TAG).info(f"发送物联网指令: {send_message}")
return
logger.bind(tag=TAG).error(f"未找到方法{method_name}")
conn.logger.bind(tag=TAG).error(f"未找到方法{method_name}")
@@ -0,0 +1,145 @@
"""
TTS上报功能已集成到ConnectionHandler类中。
上报功能包括:
1. 每个连接对象拥有自己的上报队列和处理线程
2. 上报线程的生命周期与连接对象绑定
3. 使用ConnectionHandler.enqueue_tts_report方法进行上报
具体实现请参考core/connection.py中的相关代码。
"""
import opuslib_next
from config.manage_api_client import report as manage_report
TAG = __name__
def report(conn, type, text, opus_data):
"""执行聊天记录上报操作
Args:
conn: 连接对象
type: 上报类型,1为用户,2为智能体
text: 合成文本
opus_data: opus音频数据
"""
try:
if opus_data:
audio_data = opus_to_wav(conn, opus_data)
else:
audio_data = None
# 执行上报
manage_report(
mac_address=conn.device_id,
session_id=conn.session_id,
chat_type=type,
content=text,
audio=audio_data,
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"聊天记录上报失败: {e}")
def opus_to_wav(conn, opus_data):
"""将Opus数据转换为WAV格式的字节流
Args:
output_dir: 输出目录(保留参数以保持接口兼容)
opus_data: opus音频数据
Returns:
bytes: WAV格式的音频数据
"""
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
if not pcm_data:
raise ValueError("没有有效的PCM数据")
# 创建WAV文件头
pcm_data_bytes = b"".join(pcm_data)
num_samples = len(pcm_data_bytes) // 2 # 16-bit samples
# WAV文件头
wav_header = bytearray()
wav_header.extend(b"RIFF") # ChunkID
wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little")) # ChunkSize
wav_header.extend(b"WAVE") # Format
wav_header.extend(b"fmt ") # Subchunk1ID
wav_header.extend((16).to_bytes(4, "little")) # Subchunk1Size
wav_header.extend((1).to_bytes(2, "little")) # AudioFormat (PCM)
wav_header.extend((1).to_bytes(2, "little")) # NumChannels
wav_header.extend((16000).to_bytes(4, "little")) # SampleRate
wav_header.extend((32000).to_bytes(4, "little")) # ByteRate
wav_header.extend((2).to_bytes(2, "little")) # BlockAlign
wav_header.extend((16).to_bytes(2, "little")) # BitsPerSample
wav_header.extend(b"data") # Subchunk2ID
wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little")) # Subchunk2Size
# 返回完整的WAV数据
return bytes(wav_header) + pcm_data_bytes
def enqueue_tts_report(conn, text, opus_data):
if not conn.read_config_from_api or conn.need_bind or not conn.report_tts_enable:
return
if conn.chat_history_conf == 0:
return
"""将TTS数据加入上报队列
Args:
conn: 连接对象
text: 合成文本
opus_data: opus音频数据
"""
try:
# 使用连接对象的队列,传入文本和二进制数据而非文件路径
if conn.chat_history_conf == 2:
conn.report_queue.put((2, text, opus_data))
conn.logger.bind(tag=TAG).debug(
f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
)
else:
conn.report_queue.put((2, text, None))
conn.logger.bind(tag=TAG).debug(
f"TTS数据已加入上报队列: {conn.device_id}, 不上报音频"
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}")
def enqueue_asr_report(conn, text, opus_data):
if not conn.read_config_from_api or conn.need_bind or not conn.report_asr_enable:
return
if conn.chat_history_conf == 0:
return
"""将ASR数据加入上报队列
Args:
conn: 连接对象
text: 合成文本
opus_data: opus音频数据
"""
try:
# 使用连接对象的队列,传入文本和二进制数据而非文件路径
if conn.chat_history_conf == 2:
conn.report_queue.put((1, text, opus_data))
conn.logger.bind(tag=TAG).debug(
f"ASR数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
)
else:
conn.report_queue.put((1, text, None))
conn.logger.bind(tag=TAG).debug(
f"ASR数据已加入上报队列: {conn.device_id}, 不上报音频"
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"加入ASR上报队列失败: {text}, {e}")
+96 -6
View File
@@ -1,33 +1,37 @@
from config.logger import setup_logging
import json
from core.handle.abortHandle import handleAbortMessage
from core.handle.helloHandle import handleHelloMessage
from core.utils.util import remove_punctuation_and_length
from core.utils.util import remove_punctuation_and_length, filter_sensitive_info
from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
from core.handle.sendAudioHandle import send_stt_message, send_tts_message
from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
from core.handle.reportHandle import enqueue_asr_report
import asyncio
TAG = __name__
logger = setup_logging()
async def handleTextMessage(conn, message):
"""处理文本消息"""
logger.bind(tag=TAG).info(f"收到文本消息:{message}")
try:
msg_json = json.loads(message)
if isinstance(msg_json, int):
conn.logger.bind(tag=TAG).info(f"收到文本消息:{message}")
await conn.websocket.send(message)
return
if msg_json["type"] == "hello":
await handleHelloMessage(conn)
conn.logger.bind(tag=TAG).info(f"收到hello消息:{message}")
await handleHelloMessage(conn, msg_json)
elif msg_json["type"] == "abort":
conn.logger.bind(tag=TAG).info(f"收到abort消息:{message}")
await handleAbortMessage(conn)
elif msg_json["type"] == "listen":
conn.logger.bind(tag=TAG).info(f"收到listen消息:{message}")
if "mode" in msg_json:
conn.client_listen_mode = msg_json["mode"]
logger.bind(tag=TAG).debug(f"客户端拾音模式:{conn.client_listen_mode}")
conn.logger.bind(tag=TAG).debug(
f"客户端拾音模式:{conn.client_listen_mode}"
)
if msg_json["state"] == "start":
conn.client_have_voice = True
conn.client_voice_stop = False
@@ -53,13 +57,99 @@ async def handleTextMessage(conn, message):
# 如果是唤醒词,且关闭了唤醒词回复,就不用回答
await send_stt_message(conn, text)
await send_tts_message(conn, "stop", None)
elif is_wakeup_words:
# 上报纯文字数据(复用ASR上报功能,但不提供音频数据)
enqueue_asr_report(conn, "嘿,你好呀", [])
await startToChat(conn, "嘿,你好呀")
else:
# 上报纯文字数据(复用ASR上报功能,但不提供音频数据)
enqueue_asr_report(conn, text, [])
# 否则需要LLM对文字内容进行答复
await startToChat(conn, text)
elif msg_json["type"] == "iot":
conn.logger.bind(tag=TAG).info(f"收到iot消息:{message}")
if "descriptors" in msg_json:
asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
if "states" in msg_json:
asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
elif msg_json["type"] == "server":
# 记录日志时过滤敏感信息
conn.logger.bind(tag=TAG).info(
f"收到服务器消息:{filter_sensitive_info(msg_json)}"
)
# 如果配置是从API读取的,则需要验证secret
if not conn.read_config_from_api:
return
# 获取post请求的secret
post_secret = msg_json.get("content", {}).get("secret", "")
secret = conn.config["manager-api"].get("secret", "")
# 如果secret不匹配,则返回
if post_secret != secret:
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "服务器密钥验证失败",
}
)
)
return
# 动态更新配置
if msg_json["action"] == "update_config":
try:
# 更新WebSocketServer的配置
if not conn.server:
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "无法获取服务器实例",
"content": {"action": "update_config"},
}
)
)
return
if not await conn.server.update_config():
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "更新服务器配置失败",
"content": {"action": "update_config"},
}
)
)
return
# 发送成功响应
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "success",
"message": "配置更新成功",
"content": {"action": "update_config"},
}
)
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"更新配置失败: {str(e)}")
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": f"更新配置失败: {str(e)}",
"content": {"action": "update_config"},
}
)
)
# 重启服务器
elif msg_json["action"] == "restart":
await conn.handle_restart(msg_json)
except json.JSONDecodeError:
await conn.websocket.send(message)