mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-30 05:13:59 +08:00
update:合并非tts代码
This commit is contained in:
@@ -3,14 +3,16 @@ import queue
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleAbortMessage(conn):
|
||||
logger.bind(tag=TAG).info("Abort message received")
|
||||
conn.logger.bind(tag=TAG).info("Abort message received")
|
||||
# 设置成打断状态,会自动打断llm、tts任务
|
||||
conn.client_abort = True
|
||||
conn.clear_queues()
|
||||
# 打断客户端说话状态
|
||||
await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
|
||||
await conn.websocket.send(
|
||||
json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id})
|
||||
)
|
||||
conn.clearSpeakStatus()
|
||||
logger.bind(tag=TAG).info("Abort message received-end")
|
||||
conn.logger.bind(tag=TAG).info("Abort message received-end")
|
||||
|
||||
@@ -4,7 +4,6 @@ from plugins_func.register import FunctionRegistry, ActionResponse, Action, Tool
|
||||
from plugins_func.functions.hass_init import append_devices_to_prompt
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class FunctionHandler:
|
||||
@@ -40,7 +39,9 @@ class FunctionHandler:
|
||||
for func in self.functions_desc:
|
||||
func_names.append(func["function"]["name"])
|
||||
# 打印当前支持的函数列表
|
||||
logger.bind(tag=TAG).info(f"当前支持的函数列表: {func_names}")
|
||||
self.conn.logger.bind(tag=TAG, session_id=self.conn.session_id).info(
|
||||
f"当前支持的函数列表: {func_names}"
|
||||
)
|
||||
return func_names
|
||||
|
||||
def get_functions(self):
|
||||
@@ -57,7 +58,9 @@ class FunctionHandler:
|
||||
|
||||
def register_config_functions(self):
|
||||
"""注册配置中的函数,可以不同客户端使用不同的配置"""
|
||||
for func in self.config["Intent"]["function_call"].get("functions", []):
|
||||
for func in self.config["Intent"][self.config["selected_module"]["Intent"]].get(
|
||||
"functions", []
|
||||
):
|
||||
self.function_registry.register_function(func)
|
||||
|
||||
"""home assistant需要初始化提示词"""
|
||||
@@ -77,7 +80,9 @@ class FunctionHandler:
|
||||
func = funcItem.func
|
||||
arguments = function_call_data["arguments"]
|
||||
arguments = json.loads(arguments) if arguments else {}
|
||||
logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
|
||||
self.conn.logger.bind(tag=TAG).debug(
|
||||
f"调用函数: {function_name}, 参数: {arguments}"
|
||||
)
|
||||
if (
|
||||
funcItem.type == ToolType.SYSTEM_CTL
|
||||
or funcItem.type == ToolType.IOT_CTL
|
||||
@@ -92,6 +97,6 @@ class FunctionHandler:
|
||||
action=Action.NOTFOUND, result="没有找到对应的函数", response=""
|
||||
)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理function call错误: {e}")
|
||||
self.conn.logger.bind(tag=TAG).error(f"处理function call错误: {e}")
|
||||
|
||||
return None
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import json
|
||||
from config.logger import setup_logging
|
||||
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType
|
||||
from core.handle.sendAudioHandle import send_stt_message
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
import shutil
|
||||
@@ -9,7 +7,7 @@ import os
|
||||
import random
|
||||
import time
|
||||
|
||||
logger = setup_logging()
|
||||
TAG = __name__
|
||||
|
||||
WAKEUP_CONFIG = {
|
||||
"dir": "config/assets/",
|
||||
@@ -21,7 +19,16 @@ WAKEUP_CONFIG = {
|
||||
}
|
||||
|
||||
|
||||
async def handleHelloMessage(conn):
|
||||
async def handleHelloMessage(conn, msg_json):
|
||||
"""处理hello消息"""
|
||||
audio_params = msg_json.get("audio_params")
|
||||
if audio_params:
|
||||
format = audio_params.get("format")
|
||||
conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}")
|
||||
conn.audio_format = format
|
||||
conn.asr.set_audio_format(format)
|
||||
conn.welcome_msg["audio_params"] = audio_params
|
||||
|
||||
await conn.websocket.send(json.dumps(conn.welcome_msg))
|
||||
|
||||
|
||||
@@ -44,21 +51,11 @@ async def checkWakeupWords(conn, text):
|
||||
if file is None:
|
||||
asyncio.create_task(wakeupWordsResponse(conn))
|
||||
return False
|
||||
opus_packets, duration = conn.tts.audio_to_opus_data(file)
|
||||
opus_packets, _ = conn.tts.audio_to_opus_data(file)
|
||||
text_hello = WAKEUP_CONFIG["text"]
|
||||
if not text_hello:
|
||||
text_hello = text
|
||||
|
||||
conn.tts.tts_audio_queue.put(
|
||||
TTSMessageDTO(
|
||||
u_id=conn.u_id,
|
||||
msg_type=MsgType.TTS_TEXT_RESPONSE,
|
||||
content=opus_packets,
|
||||
tts_finish_text="",
|
||||
sentence_type=None,
|
||||
duration=0,
|
||||
)
|
||||
)
|
||||
conn.audio_play_queue.put((opus_packets, text_hello, 0))
|
||||
if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
|
||||
asyncio.create_task(wakeupWordsResponse(conn))
|
||||
return True
|
||||
@@ -80,11 +77,20 @@ def getWakeupWordFile(file_name):
|
||||
|
||||
|
||||
async def wakeupWordsResponse(conn):
|
||||
wait_max_time = 5
|
||||
while conn.llm is None or not conn.llm.response_no_stream:
|
||||
await asyncio.sleep(1)
|
||||
wait_max_time -= 1
|
||||
if wait_max_time <= 0:
|
||||
conn.logger.bind(tag=TAG).error("连接对象没有llm")
|
||||
return
|
||||
|
||||
"""唤醒词响应"""
|
||||
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
|
||||
result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
|
||||
# TODO 将result转换为tts_message_dto
|
||||
tts_file = None
|
||||
if result is None or result == "":
|
||||
return
|
||||
tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
|
||||
|
||||
if tts_file is not None and os.path.exists(tts_file):
|
||||
file_type = os.path.splitext(tts_file)[1]
|
||||
|
||||
@@ -10,7 +10,6 @@ from plugins_func.register import (
|
||||
)
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
def wrap_async_function(async_func):
|
||||
@@ -21,7 +20,7 @@ def wrap_async_function(async_func):
|
||||
# 获取连接对象(第一个参数)
|
||||
conn = args[0]
|
||||
if not hasattr(conn, "loop"):
|
||||
logger.bind(tag=TAG).error("Connection对象没有loop属性")
|
||||
conn.logger.bind(tag=TAG).error("Connection对象没有loop属性")
|
||||
return ActionResponse(
|
||||
Action.ERROR,
|
||||
"Connection对象没有loop属性",
|
||||
@@ -35,7 +34,7 @@ def wrap_async_function(async_func):
|
||||
# 等待结果返回
|
||||
return future.result()
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
|
||||
conn.logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
|
||||
return ActionResponse(Action.ERROR, str(e), f"执行操作时出错: {e}")
|
||||
|
||||
return wrapper
|
||||
@@ -57,7 +56,7 @@ def create_iot_function(device_name, method_name, method_info):
|
||||
response_failure = "操作失败"
|
||||
|
||||
# 打印响应参数
|
||||
logger.bind(tag=TAG).info(
|
||||
conn.logger.bind(tag=TAG).debug(
|
||||
f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
|
||||
)
|
||||
|
||||
@@ -86,7 +85,9 @@ def create_iot_function(device_name, method_name, method_info):
|
||||
|
||||
return ActionResponse(Action.RESPONSE, result, response)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"执行{device_name}的{method_name}操作失败: {e}")
|
||||
conn.logger.bind(tag=TAG).error(
|
||||
f"执行{device_name}的{method_name}操作失败: {e}"
|
||||
)
|
||||
|
||||
# 操作失败时使用大模型提供的失败响应
|
||||
response = response_failure
|
||||
@@ -104,7 +105,7 @@ def create_iot_query_function(device_name, prop_name, prop_info):
|
||||
async def iot_query_function(conn, response_success=None, response_failure=None):
|
||||
try:
|
||||
# 打印响应参数
|
||||
logger.bind(tag=TAG).info(
|
||||
conn.logger.bind(tag=TAG).info(
|
||||
f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
|
||||
)
|
||||
|
||||
@@ -122,7 +123,9 @@ def create_iot_query_function(device_name, prop_name, prop_info):
|
||||
|
||||
return ActionResponse(Action.ERROR, f"属性{prop_name}不存在", response)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"查询{device_name}的{prop_name}时出错: {e}")
|
||||
conn.logger.bind(tag=TAG).error(
|
||||
f"查询{device_name}的{prop_name}时出错: {e}"
|
||||
)
|
||||
|
||||
# 查询出错时使用大模型提供的失败响应
|
||||
response = response_failure
|
||||
@@ -144,33 +147,34 @@ class IotDescriptor:
|
||||
self.methods = []
|
||||
|
||||
# 根据描述创建属性
|
||||
for key, value in properties.items():
|
||||
property_item = globals()[key] = {}
|
||||
property_item["name"] = key
|
||||
property_item["description"] = value["description"]
|
||||
if value["type"] == "number":
|
||||
property_item["value"] = 0
|
||||
elif value["type"] == "boolean":
|
||||
property_item["value"] = False
|
||||
else:
|
||||
property_item["value"] = ""
|
||||
self.properties.append(property_item)
|
||||
if properties is not None:
|
||||
for key, value in properties.items():
|
||||
property_item = {}
|
||||
property_item["name"] = key
|
||||
property_item["description"] = value["description"]
|
||||
if value["type"] == "number":
|
||||
property_item["value"] = 0
|
||||
elif value["type"] == "boolean":
|
||||
property_item["value"] = False
|
||||
else:
|
||||
property_item["value"] = ""
|
||||
self.properties.append(property_item)
|
||||
|
||||
# 根据描述创建方法
|
||||
for key, value in methods.items():
|
||||
method = globals()[key] = {}
|
||||
method["description"] = value["description"]
|
||||
method["name"] = key
|
||||
for k, v in value["parameters"].items():
|
||||
method[k] = {}
|
||||
method[k]["description"] = v["description"]
|
||||
if v["type"] == "number":
|
||||
method[k]["value"] = 0
|
||||
elif v["type"] == "boolean":
|
||||
method[k]["value"] = False
|
||||
else:
|
||||
method[k]["value"] = ""
|
||||
self.methods.append(method)
|
||||
if methods is not None:
|
||||
for key, value in methods.items():
|
||||
method = {}
|
||||
method["description"] = value["description"]
|
||||
method["name"] = key
|
||||
# 检查方法是否有参数
|
||||
if "parameters" in value:
|
||||
method["parameters"] = {}
|
||||
for k, v in value["parameters"].items():
|
||||
method["parameters"][k] = {
|
||||
"description": v["description"],
|
||||
"type": v["type"],
|
||||
}
|
||||
self.methods.append(method)
|
||||
|
||||
|
||||
def register_device_type(descriptor):
|
||||
@@ -219,13 +223,19 @@ def register_device_type(descriptor):
|
||||
func_name = f"{device_name.lower()}_{method_name.lower()}"
|
||||
|
||||
# 创建参数字典,添加原有参数
|
||||
parameters = {
|
||||
param_name: {
|
||||
"type": param_info["type"],
|
||||
"description": param_info["description"],
|
||||
parameters = {}
|
||||
required_params = []
|
||||
|
||||
# 如果方法有参数,则添加参数信息
|
||||
if "parameters" in method_info:
|
||||
parameters = {
|
||||
param_name: {
|
||||
"type": param_info["type"],
|
||||
"description": param_info["description"],
|
||||
}
|
||||
for param_name, param_info in method_info["parameters"].items()
|
||||
}
|
||||
for param_name, param_info in method_info["parameters"].items()
|
||||
}
|
||||
required_params = list(method_info["parameters"].keys())
|
||||
|
||||
# 添加响应参数
|
||||
parameters.update(
|
||||
@@ -242,7 +252,6 @@ def register_device_type(descriptor):
|
||||
)
|
||||
|
||||
# 构建必须参数列表(原有参数 + 响应参数)
|
||||
required_params = list(method_info["parameters"].keys())
|
||||
required_params.extend(["response_success", "response_failure"])
|
||||
|
||||
func_desc = {
|
||||
@@ -269,19 +278,36 @@ def register_device_type(descriptor):
|
||||
|
||||
# 用于接受前端设备推送的搜索iot描述
|
||||
async def handleIotDescriptors(conn, descriptors):
|
||||
if not conn.use_function_call_mode:
|
||||
return
|
||||
wait_max_time = 5
|
||||
while conn.func_handler is None or not conn.func_handler.finish_init:
|
||||
await asyncio.sleep(1)
|
||||
wait_max_time -= 1
|
||||
if wait_max_time <= 0:
|
||||
logger.bind(tag=TAG).error("连接对象没有func_handler")
|
||||
conn.logger.bind(tag=TAG).debug("连接对象没有func_handler")
|
||||
return
|
||||
"""处理物联网描述"""
|
||||
functions_changed = False
|
||||
|
||||
for descriptor in descriptors:
|
||||
|
||||
# 如果descriptor没有properties和methods,则直接跳过
|
||||
if "properties" not in descriptor and "methods" not in descriptor:
|
||||
continue
|
||||
|
||||
# 处理缺失properties的情况
|
||||
if "properties" not in descriptor:
|
||||
descriptor["properties"] = {}
|
||||
# 从methods中提取所有参数作为properties
|
||||
if "methods" in descriptor:
|
||||
for method_name, method_info in descriptor["methods"].items():
|
||||
if "parameters" in method_info:
|
||||
for param_name, param_info in method_info["parameters"].items():
|
||||
# 将参数信息转换为属性信息
|
||||
descriptor["properties"][param_name] = {
|
||||
"description": param_info["description"],
|
||||
"type": param_info["type"],
|
||||
}
|
||||
|
||||
# 创建IOT设备描述符
|
||||
iot_descriptor = IotDescriptor(
|
||||
descriptor["name"],
|
||||
@@ -291,7 +317,7 @@ async def handleIotDescriptors(conn, descriptors):
|
||||
)
|
||||
conn.iot_descriptors[descriptor["name"]] = iot_descriptor
|
||||
|
||||
if conn.use_function_call_mode:
|
||||
if conn.load_function_plugin:
|
||||
# 注册或获取设备类型
|
||||
type_id = register_device_type(descriptor)
|
||||
device_functions = device_type_registry.get_device_functions(type_id)
|
||||
@@ -300,7 +326,7 @@ async def handleIotDescriptors(conn, descriptors):
|
||||
if hasattr(conn, "func_handler"):
|
||||
for func_name in device_functions:
|
||||
conn.func_handler.function_registry.register_function(func_name)
|
||||
logger.bind(tag=TAG).info(
|
||||
conn.logger.bind(tag=TAG).info(
|
||||
f"注册IOT函数到function handler: {func_name}"
|
||||
)
|
||||
functions_changed = True
|
||||
@@ -309,8 +335,8 @@ async def handleIotDescriptors(conn, descriptors):
|
||||
if functions_changed and hasattr(conn, "func_handler"):
|
||||
conn.func_handler.upload_functions_desc()
|
||||
func_names = conn.func_handler.current_support_functions()
|
||||
logger.bind(tag=TAG).info(f"设备类型: {type_id}")
|
||||
logger.bind(tag=TAG).info(
|
||||
conn.logger.bind(tag=TAG).info(f"设备类型: {type_id}")
|
||||
conn.logger.bind(tag=TAG).info(
|
||||
f"更新function描述列表完成,当前支持的函数: {func_names}"
|
||||
)
|
||||
|
||||
@@ -324,13 +350,13 @@ async def handleIotStatus(conn, states):
|
||||
for k, v in state["state"].items():
|
||||
if property_item["name"] == k:
|
||||
if type(v) != type(property_item["value"]):
|
||||
logger.bind(tag=TAG).error(
|
||||
conn.logger.bind(tag=TAG).error(
|
||||
f"属性{property_item['name']}的值类型不匹配"
|
||||
)
|
||||
break
|
||||
else:
|
||||
property_item["value"] = v
|
||||
logger.bind(tag=TAG).info(
|
||||
conn.logger.bind(tag=TAG).info(
|
||||
f"物联网状态更新: {key} , {property_item['name']} = {v}"
|
||||
)
|
||||
break
|
||||
@@ -344,7 +370,7 @@ async def get_iot_status(conn, name, property_name):
|
||||
for property_item in value.properties:
|
||||
if property_item["name"] == property_name:
|
||||
return property_item["value"]
|
||||
logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
|
||||
conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
|
||||
return None
|
||||
|
||||
|
||||
@@ -355,16 +381,16 @@ async def set_iot_status(conn, name, property_name, value):
|
||||
for property_item in iot_descriptor.properties:
|
||||
if property_item["name"] == property_name:
|
||||
if type(value) != type(property_item["value"]):
|
||||
logger.bind(tag=TAG).error(
|
||||
conn.logger.bind(tag=TAG).error(
|
||||
f"属性{property_item['name']}的值类型不匹配"
|
||||
)
|
||||
return
|
||||
property_item["value"] = value
|
||||
logger.bind(tag=TAG).info(
|
||||
conn.logger.bind(tag=TAG).info(
|
||||
f"物联网状态更新: {name} , {property_name} = {value}"
|
||||
)
|
||||
return
|
||||
logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
|
||||
conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
|
||||
|
||||
|
||||
async def send_iot_conn(conn, name, method_name, parameters):
|
||||
@@ -375,19 +401,17 @@ async def send_iot_conn(conn, name, method_name, parameters):
|
||||
for method in value.methods:
|
||||
# 找到了方法
|
||||
if method["name"] == method_name:
|
||||
await conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "iot",
|
||||
"commands": [
|
||||
{
|
||||
"name": name,
|
||||
"method": method_name,
|
||||
"parameters": parameters,
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
# 构建命令对象
|
||||
command = {
|
||||
"name": name,
|
||||
"method": method_name,
|
||||
}
|
||||
|
||||
# 只有当参数不为空时才添加parameters字段
|
||||
if parameters:
|
||||
command["parameters"] = parameters
|
||||
send_message = json.dumps({"type": "iot", "commands": [command]})
|
||||
await conn.websocket.send(send_message)
|
||||
conn.logger.bind(tag=TAG).info(f"发送物联网指令: {send_message}")
|
||||
return
|
||||
logger.bind(tag=TAG).error(f"未找到方法{method_name}")
|
||||
conn.logger.bind(tag=TAG).error(f"未找到方法{method_name}")
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
"""
|
||||
TTS上报功能已集成到ConnectionHandler类中。
|
||||
|
||||
上报功能包括:
|
||||
1. 每个连接对象拥有自己的上报队列和处理线程
|
||||
2. 上报线程的生命周期与连接对象绑定
|
||||
3. 使用ConnectionHandler.enqueue_tts_report方法进行上报
|
||||
|
||||
具体实现请参考core/connection.py中的相关代码。
|
||||
"""
|
||||
|
||||
import opuslib_next
|
||||
|
||||
from config.manage_api_client import report as manage_report
|
||||
|
||||
TAG = __name__
|
||||
|
||||
|
||||
def report(conn, type, text, opus_data):
|
||||
"""执行聊天记录上报操作
|
||||
|
||||
Args:
|
||||
conn: 连接对象
|
||||
type: 上报类型,1为用户,2为智能体
|
||||
text: 合成文本
|
||||
opus_data: opus音频数据
|
||||
"""
|
||||
try:
|
||||
if opus_data:
|
||||
audio_data = opus_to_wav(conn, opus_data)
|
||||
else:
|
||||
audio_data = None
|
||||
# 执行上报
|
||||
manage_report(
|
||||
mac_address=conn.device_id,
|
||||
session_id=conn.session_id,
|
||||
chat_type=type,
|
||||
content=text,
|
||||
audio=audio_data,
|
||||
)
|
||||
except Exception as e:
|
||||
conn.logger.bind(tag=TAG).error(f"聊天记录上报失败: {e}")
|
||||
|
||||
|
||||
def opus_to_wav(conn, opus_data):
|
||||
"""将Opus数据转换为WAV格式的字节流
|
||||
|
||||
Args:
|
||||
output_dir: 输出目录(保留参数以保持接口兼容)
|
||||
opus_data: opus音频数据
|
||||
|
||||
Returns:
|
||||
bytes: WAV格式的音频数据
|
||||
"""
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
if not pcm_data:
|
||||
raise ValueError("没有有效的PCM数据")
|
||||
|
||||
# 创建WAV文件头
|
||||
pcm_data_bytes = b"".join(pcm_data)
|
||||
num_samples = len(pcm_data_bytes) // 2 # 16-bit samples
|
||||
|
||||
# WAV文件头
|
||||
wav_header = bytearray()
|
||||
wav_header.extend(b"RIFF") # ChunkID
|
||||
wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little")) # ChunkSize
|
||||
wav_header.extend(b"WAVE") # Format
|
||||
wav_header.extend(b"fmt ") # Subchunk1ID
|
||||
wav_header.extend((16).to_bytes(4, "little")) # Subchunk1Size
|
||||
wav_header.extend((1).to_bytes(2, "little")) # AudioFormat (PCM)
|
||||
wav_header.extend((1).to_bytes(2, "little")) # NumChannels
|
||||
wav_header.extend((16000).to_bytes(4, "little")) # SampleRate
|
||||
wav_header.extend((32000).to_bytes(4, "little")) # ByteRate
|
||||
wav_header.extend((2).to_bytes(2, "little")) # BlockAlign
|
||||
wav_header.extend((16).to_bytes(2, "little")) # BitsPerSample
|
||||
wav_header.extend(b"data") # Subchunk2ID
|
||||
wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little")) # Subchunk2Size
|
||||
|
||||
# 返回完整的WAV数据
|
||||
return bytes(wav_header) + pcm_data_bytes
|
||||
|
||||
|
||||
def enqueue_tts_report(conn, text, opus_data):
|
||||
if not conn.read_config_from_api or conn.need_bind or not conn.report_tts_enable:
|
||||
return
|
||||
if conn.chat_history_conf == 0:
|
||||
return
|
||||
"""将TTS数据加入上报队列
|
||||
|
||||
Args:
|
||||
conn: 连接对象
|
||||
text: 合成文本
|
||||
opus_data: opus音频数据
|
||||
"""
|
||||
try:
|
||||
# 使用连接对象的队列,传入文本和二进制数据而非文件路径
|
||||
if conn.chat_history_conf == 2:
|
||||
conn.report_queue.put((2, text, opus_data))
|
||||
conn.logger.bind(tag=TAG).debug(
|
||||
f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
|
||||
)
|
||||
else:
|
||||
conn.report_queue.put((2, text, None))
|
||||
conn.logger.bind(tag=TAG).debug(
|
||||
f"TTS数据已加入上报队列: {conn.device_id}, 不上报音频"
|
||||
)
|
||||
except Exception as e:
|
||||
conn.logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}")
|
||||
|
||||
|
||||
def enqueue_asr_report(conn, text, opus_data):
|
||||
if not conn.read_config_from_api or conn.need_bind or not conn.report_asr_enable:
|
||||
return
|
||||
if conn.chat_history_conf == 0:
|
||||
return
|
||||
"""将ASR数据加入上报队列
|
||||
|
||||
Args:
|
||||
conn: 连接对象
|
||||
text: 合成文本
|
||||
opus_data: opus音频数据
|
||||
"""
|
||||
try:
|
||||
# 使用连接对象的队列,传入文本和二进制数据而非文件路径
|
||||
if conn.chat_history_conf == 2:
|
||||
conn.report_queue.put((1, text, opus_data))
|
||||
conn.logger.bind(tag=TAG).debug(
|
||||
f"ASR数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
|
||||
)
|
||||
else:
|
||||
conn.report_queue.put((1, text, None))
|
||||
conn.logger.bind(tag=TAG).debug(
|
||||
f"ASR数据已加入上报队列: {conn.device_id}, 不上报音频"
|
||||
)
|
||||
except Exception as e:
|
||||
conn.logger.bind(tag=TAG).error(f"加入ASR上报队列失败: {text}, {e}")
|
||||
@@ -1,33 +1,37 @@
|
||||
from config.logger import setup_logging
|
||||
import json
|
||||
from core.handle.abortHandle import handleAbortMessage
|
||||
from core.handle.helloHandle import handleHelloMessage
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from core.utils.util import remove_punctuation_and_length, filter_sensitive_info
|
||||
from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
|
||||
from core.handle.sendAudioHandle import send_stt_message, send_tts_message
|
||||
from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
|
||||
from core.handle.reportHandle import enqueue_asr_report
|
||||
import asyncio
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleTextMessage(conn, message):
|
||||
"""处理文本消息"""
|
||||
logger.bind(tag=TAG).info(f"收到文本消息:{message}")
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, int):
|
||||
conn.logger.bind(tag=TAG).info(f"收到文本消息:{message}")
|
||||
await conn.websocket.send(message)
|
||||
return
|
||||
if msg_json["type"] == "hello":
|
||||
await handleHelloMessage(conn)
|
||||
conn.logger.bind(tag=TAG).info(f"收到hello消息:{message}")
|
||||
await handleHelloMessage(conn, msg_json)
|
||||
elif msg_json["type"] == "abort":
|
||||
conn.logger.bind(tag=TAG).info(f"收到abort消息:{message}")
|
||||
await handleAbortMessage(conn)
|
||||
elif msg_json["type"] == "listen":
|
||||
conn.logger.bind(tag=TAG).info(f"收到listen消息:{message}")
|
||||
if "mode" in msg_json:
|
||||
conn.client_listen_mode = msg_json["mode"]
|
||||
logger.bind(tag=TAG).debug(f"客户端拾音模式:{conn.client_listen_mode}")
|
||||
conn.logger.bind(tag=TAG).debug(
|
||||
f"客户端拾音模式:{conn.client_listen_mode}"
|
||||
)
|
||||
if msg_json["state"] == "start":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = False
|
||||
@@ -53,13 +57,99 @@ async def handleTextMessage(conn, message):
|
||||
# 如果是唤醒词,且关闭了唤醒词回复,就不用回答
|
||||
await send_stt_message(conn, text)
|
||||
await send_tts_message(conn, "stop", None)
|
||||
elif is_wakeup_words:
|
||||
# 上报纯文字数据(复用ASR上报功能,但不提供音频数据)
|
||||
enqueue_asr_report(conn, "嘿,你好呀", [])
|
||||
await startToChat(conn, "嘿,你好呀")
|
||||
else:
|
||||
# 上报纯文字数据(复用ASR上报功能,但不提供音频数据)
|
||||
enqueue_asr_report(conn, text, [])
|
||||
# 否则需要LLM对文字内容进行答复
|
||||
await startToChat(conn, text)
|
||||
elif msg_json["type"] == "iot":
|
||||
conn.logger.bind(tag=TAG).info(f"收到iot消息:{message}")
|
||||
if "descriptors" in msg_json:
|
||||
asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
|
||||
if "states" in msg_json:
|
||||
asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
|
||||
elif msg_json["type"] == "server":
|
||||
# 记录日志时过滤敏感信息
|
||||
conn.logger.bind(tag=TAG).info(
|
||||
f"收到服务器消息:{filter_sensitive_info(msg_json)}"
|
||||
)
|
||||
# 如果配置是从API读取的,则需要验证secret
|
||||
if not conn.read_config_from_api:
|
||||
return
|
||||
# 获取post请求的secret
|
||||
post_secret = msg_json.get("content", {}).get("secret", "")
|
||||
secret = conn.config["manager-api"].get("secret", "")
|
||||
# 如果secret不匹配,则返回
|
||||
if post_secret != secret:
|
||||
await conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "server",
|
||||
"status": "error",
|
||||
"message": "服务器密钥验证失败",
|
||||
}
|
||||
)
|
||||
)
|
||||
return
|
||||
# 动态更新配置
|
||||
if msg_json["action"] == "update_config":
|
||||
try:
|
||||
# 更新WebSocketServer的配置
|
||||
if not conn.server:
|
||||
await conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "server",
|
||||
"status": "error",
|
||||
"message": "无法获取服务器实例",
|
||||
"content": {"action": "update_config"},
|
||||
}
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
if not await conn.server.update_config():
|
||||
await conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "server",
|
||||
"status": "error",
|
||||
"message": "更新服务器配置失败",
|
||||
"content": {"action": "update_config"},
|
||||
}
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
# 发送成功响应
|
||||
await conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "server",
|
||||
"status": "success",
|
||||
"message": "配置更新成功",
|
||||
"content": {"action": "update_config"},
|
||||
}
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
conn.logger.bind(tag=TAG).error(f"更新配置失败: {str(e)}")
|
||||
await conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "server",
|
||||
"status": "error",
|
||||
"message": f"更新配置失败: {str(e)}",
|
||||
"content": {"action": "update_config"},
|
||||
}
|
||||
)
|
||||
)
|
||||
# 重启服务器
|
||||
elif msg_json["action"] == "restart":
|
||||
await conn.handle_restart(msg_json)
|
||||
except json.JSONDecodeError:
|
||||
await conn.websocket.send(message)
|
||||
|
||||
Reference in New Issue
Block a user