Merge branch 'main' into future

This commit is contained in:
CGD
2025-09-05 10:14:52 +08:00
committed by GitHub
26 changed files with 500 additions and 199 deletions
@@ -237,7 +237,7 @@ public interface Constant {
/**
* 版本号
*/
public static final String VERSION = "0.7.5";
public static final String VERSION = "0.7.6";
/**
* 无效固件URL
@@ -222,7 +222,7 @@ function showAbout() {
title: `关于${import.meta.env.VITE_APP_TITLE}`,
content: `${import.meta.env.VITE_APP_TITLE}\n\n基于 Vue.js 3 + uni-app 构建的跨平台移动端管理应用,为小智ESP32智能硬件提供设备管理、智能体配置等功能。\n\n© 2025 xiaozhi-esp32-server`,
title: `关于小智智控台`,
content: `小智智控台\n\n基于 Vue.js 3 + uni-app 构建的跨平台移动端管理应用,为小智ESP32智能硬件提供设备管理、智能体配置等功能。\n\n© 2025 xiaozhi-esp32-server 0.7.5`,
content: `小智智控台\n\n基于 Vue.js 3 + uni-app 构建的跨平台移动端管理应用,为小智ESP32智能硬件提供设备管理、智能体配置等功能。\n\n© 2025 xiaozhi-esp32-server 0.7.6`,
showCancel: false,
confirmText: '确定',
})
@@ -698,15 +698,10 @@ export default {
font-size: 14px;
height: 36px;
box-sizing: border-box;
background-color: #f5f5f5;
}
::v-deep .el-input__inner {
background-color: #f5f5f5;
padding-right: 80px;
}
.url-input {
::v-deep .el-input__inner {
background-color: #f5f5f5 !important;
}
::v-deep .el-input__suffix {
right: 0;
+1 -1
View File
@@ -5,7 +5,7 @@ from config.config_loader import load_config
from config.settings import check_config_file
from datetime import datetime
SERVER_VERSION = "0.7.5"
SERVER_VERSION = "0.7.6"
_logger_initialized = False
+8 -157
View File
@@ -1,163 +1,14 @@
import json
import asyncio
from core.utils.util import filter_sensitive_info
from core.handle.abortHandle import handleAbortMessage
from core.handle.helloHandle import handleHelloMessage
from core.handle.reportHandle import enqueue_asr_report
from core.providers.tools.device_mcp import handle_mcp_message
from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
from core.handle.sendAudioHandle import send_stt_message, send_tts_message
from core.providers.tools.device_iot import handleIotDescriptors, handleIotStatus
from core.handle.textMessageHandlerRegistry import TextMessageHandlerRegistry
from core.handle.textMessageProcessor import TextMessageProcessor
TAG = __name__
# 全局处理器注册表
message_registry = TextMessageHandlerRegistry()
# 创建全局消息处理器实例
message_processor = TextMessageProcessor(message_registry)
async def handleTextMessage(conn, message):
"""处理文本消息"""
try:
msg_json = json.loads(message)
if isinstance(msg_json, int):
conn.logger.bind(tag=TAG).info(f"收到文本消息:{message}")
await conn.websocket.send(message)
return
if msg_json["type"] == "hello":
conn.logger.bind(tag=TAG).info(f"收到hello消息:{message}")
await handleHelloMessage(conn, msg_json)
elif msg_json["type"] == "abort":
conn.logger.bind(tag=TAG).info(f"收到abort消息:{message}")
await handleAbortMessage(conn)
elif msg_json["type"] == "listen":
conn.logger.bind(tag=TAG).info(f"收到listen消息:{message}")
if "mode" in msg_json:
conn.client_listen_mode = msg_json["mode"]
conn.logger.bind(tag=TAG).debug(
f"客户端拾音模式:{conn.client_listen_mode}"
)
if msg_json["state"] == "start":
conn.client_have_voice = True
conn.client_voice_stop = False
elif msg_json["state"] == "stop":
conn.client_have_voice = True
conn.client_voice_stop = True
if len(conn.asr_audio) > 0:
await handleAudioMessage(conn, b"")
elif msg_json["state"] == "detect":
conn.client_have_voice = False
conn.asr_audio.clear()
if "text" in msg_json:
original_text = msg_json["text"] # 保留设备上传的文本
# 识别是否是唤醒词
is_wakeup_words = original_text in conn.config.get("wakeup_words")
# 是否开启唤醒词回复
enable_greeting = conn.config.get("enable_greeting", True)
if is_wakeup_words and not enable_greeting:
# 如果是唤醒词,且关闭了唤醒词回复,就不用回答
await send_stt_message(conn, original_text)
await send_tts_message(conn, "stop", None)
conn.client_is_speaking = False
elif is_wakeup_words and enable_greeting:
conn.just_woken_up = True
# 上报纯文字数据(复用ASR上报功能,但不提供音频数据)
enqueue_asr_report(conn, "嘿,你好呀", [])
await startToChat(conn, "嘿,你好呀")
else:
# 上报纯文字数据(复用ASR上报功能,但不提供音频数据)
enqueue_asr_report(conn, original_text, [])
# 否则需要LLM对文字内容进行答复
await startToChat(conn, original_text)
elif msg_json["type"] == "iot":
conn.logger.bind(tag=TAG).info(f"收到iot消息:{message}")
if "descriptors" in msg_json:
asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
if "states" in msg_json:
asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
elif msg_json["type"] == "mcp":
conn.logger.bind(tag=TAG).info(f"收到mcp消息:{message[:100]}")
if "payload" in msg_json:
asyncio.create_task(
handle_mcp_message(conn, conn.mcp_client, msg_json["payload"])
)
elif msg_json["type"] == "server":
# 记录日志时过滤敏感信息
conn.logger.bind(tag=TAG).info(
f"收到服务器消息:{filter_sensitive_info(msg_json)}"
)
# 如果配置是从API读取的,则需要验证secret
if not conn.read_config_from_api:
return
# 获取post请求的secret
post_secret = msg_json.get("content", {}).get("secret", "")
secret = conn.config["manager-api"].get("secret", "")
# 如果secret不匹配,则返回
if post_secret != secret:
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "服务器密钥验证失败",
}
)
)
return
# 动态更新配置
if msg_json["action"] == "update_config":
try:
# 更新WebSocketServer的配置
if not conn.server:
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "无法获取服务器实例",
"content": {"action": "update_config"},
}
)
)
return
if not await conn.server.update_config():
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "更新服务器配置失败",
"content": {"action": "update_config"},
}
)
)
return
# 发送成功响应
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "success",
"message": "配置更新成功",
"content": {"action": "update_config"},
}
)
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"更新配置失败: {str(e)}")
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": f"更新配置失败: {str(e)}",
"content": {"action": "update_config"},
}
)
)
# 重启服务器
elif msg_json["action"] == "restart":
await conn.handle_restart(msg_json)
else:
conn.logger.bind(tag=TAG).error(f"收到未知类型消息:{message}")
except json.JSONDecodeError:
await conn.websocket.send(message)
await message_processor.process_message(conn, message)
@@ -0,0 +1,16 @@
from typing import Dict, Any
from core.handle.abortHandle import handleAbortMessage
from core.handle.textMessageHandler import TextMessageHandler
from core.handle.textMessageType import TextMessageType
class AbortTextMessageHandler(TextMessageHandler):
"""Abort消息处理器"""
@property
def message_type(self) -> TextMessageType:
return TextMessageType.ABORT
async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
await handleAbortMessage(conn)
@@ -0,0 +1,16 @@
from typing import Dict, Any
from core.handle.helloHandle import handleHelloMessage
from core.handle.textMessageHandler import TextMessageHandler
from core.handle.textMessageType import TextMessageType
class HelloTextMessageHandler(TextMessageHandler):
"""Hello消息处理器"""
@property
def message_type(self) -> TextMessageType:
return TextMessageType.HELLO
async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
await handleHelloMessage(conn, msg_json)
@@ -0,0 +1,20 @@
import asyncio
from typing import Dict, Any
from core.handle.textMessageHandler import TextMessageHandler
from core.handle.textMessageType import TextMessageType
from core.providers.tools.device_iot import handleIotStatus, handleIotDescriptors
class IotTextMessageHandler(TextMessageHandler):
"""IOT消息处理器"""
@property
def message_type(self) -> TextMessageType:
return TextMessageType.IOT
async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
if "descriptors" in msg_json:
asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
if "states" in msg_json:
asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
@@ -0,0 +1,63 @@
import time
from typing import Dict, Any
from core.handle.receiveAudioHandle import handleAudioMessage, startToChat
from core.handle.reportHandle import enqueue_asr_report
from core.handle.sendAudioHandle import send_stt_message, send_tts_message
from core.handle.textMessageHandler import TextMessageHandler
from core.handle.textMessageType import TextMessageType
from core.utils.util import remove_punctuation_and_length
TAG = __name__
class ListenTextMessageHandler(TextMessageHandler):
"""Listen消息处理器"""
@property
def message_type(self) -> TextMessageType:
return TextMessageType.LISTEN
async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
if "mode" in msg_json:
conn.client_listen_mode = msg_json["mode"]
conn.logger.bind(tag=TAG).debug(
f"客户端拾音模式:{conn.client_listen_mode}"
)
if msg_json["state"] == "start":
conn.client_have_voice = True
conn.client_voice_stop = False
elif msg_json["state"] == "stop":
conn.client_have_voice = True
conn.client_voice_stop = True
if len(conn.asr_audio) > 0:
await handleAudioMessage(conn, b"")
elif msg_json["state"] == "detect":
conn.client_have_voice = False
conn.asr_audio.clear()
if "text" in msg_json:
conn.last_activity_time = time.time() * 1000
original_text = msg_json["text"] # 保留原始文本
filtered_len, filtered_text = remove_punctuation_and_length(
original_text
)
# 识别是否是唤醒词
is_wakeup_words = filtered_text in conn.config.get("wakeup_words")
# 是否开启唤醒词回复
enable_greeting = conn.config.get("enable_greeting", True)
if is_wakeup_words and not enable_greeting:
# 如果是唤醒词,且关闭了唤醒词回复,就不用回答
await send_stt_message(conn, original_text)
await send_tts_message(conn, "stop", None)
conn.client_is_speaking = False
elif is_wakeup_words:
conn.just_woken_up = True
# 上报纯文字数据(复用ASR上报功能,但不提供音频数据)
enqueue_asr_report(conn, "嘿,你好呀", [])
await startToChat(conn, "嘿,你好呀")
else:
# 上报纯文字数据(复用ASR上报功能,但不提供音频数据)
enqueue_asr_report(conn, original_text, [])
# 否则需要LLM对文字内容进行答复
await startToChat(conn, original_text)
@@ -0,0 +1,20 @@
import asyncio
from typing import Dict, Any
from core.handle.textMessageHandler import TextMessageHandler
from core.handle.textMessageType import TextMessageType
from core.providers.tools.device_mcp import handle_mcp_message
class McpTextMessageHandler(TextMessageHandler):
"""MCP消息处理器"""
@property
def message_type(self) -> TextMessageType:
return TextMessageType.MCP
async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
if "payload" in msg_json:
asyncio.create_task(
handle_mcp_message(conn, conn.mcp_client, msg_json["payload"])
)
@@ -0,0 +1,92 @@
import asyncio
import json
from typing import Dict, Any
from core.handle.textMessageHandler import TextMessageHandler
from core.handle.textMessageType import TextMessageType
from core.providers.tools.device_mcp import handle_mcp_message
TAG = __name__
class ServerTextMessageHandler(TextMessageHandler):
"""MCP消息处理器"""
@property
def message_type(self) -> TextMessageType:
return TextMessageType.SERVER
async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
# 如果配置是从API读取的,则需要验证secret
if not conn.read_config_from_api:
return
# 获取post请求的secret
post_secret = msg_json.get("content", {}).get("secret", "")
secret = conn.config["manager-api"].get("secret", "")
# 如果secret不匹配,则返回
if post_secret != secret:
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "服务器密钥验证失败",
}
)
)
return
# 动态更新配置
if msg_json["action"] == "update_config":
try:
# 更新WebSocketServer的配置
if not conn.server:
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "无法获取服务器实例",
"content": {"action": "update_config"},
}
)
)
return
if not await conn.server.update_config():
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "更新服务器配置失败",
"content": {"action": "update_config"},
}
)
)
return
# 发送成功响应
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "success",
"message": "配置更新成功",
"content": {"action": "update_config"},
}
)
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"更新配置失败: {str(e)}")
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": f"更新配置失败: {str(e)}",
"content": {"action": "update_config"},
}
)
)
# 重启服务器
elif msg_json["action"] == "restart":
await conn.handle_restart(msg_json)
@@ -0,0 +1,21 @@
from abc import abstractmethod, ABC
from typing import Dict, Any
from core.handle.textMessageType import TextMessageType
TAG = __name__
class TextMessageHandler(ABC):
"""消息处理器抽象基类"""
@abstractmethod
async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
"""处理消息的抽象方法"""
pass
@property
@abstractmethod
def message_type(self) -> TextMessageType:
"""返回处理的消息类型"""
pass
@@ -0,0 +1,45 @@
from typing import Dict, Optional
from core.handle.textHandler.abortMessageHandler import AbortTextMessageHandler
from core.handle.textHandler.helloMessageHandler import HelloTextMessageHandler
from core.handle.textHandler.iotMessageHandler import IotTextMessageHandler
from core.handle.textHandler.listenMessageHandler import ListenTextMessageHandler
from core.handle.textHandler.mcpMessageHandler import McpTextMessageHandler
from core.handle.textMessageHandler import TextMessageHandler
from core.handle.textHandler.serverMessageHandler import ServerTextMessageHandler
TAG = __name__
class TextMessageHandlerRegistry:
"""消息处理器注册表"""
def __init__(self):
self._handlers: Dict[str, TextMessageHandler] = {}
self._register_default_handlers()
def _register_default_handlers(self) -> None:
"""注册默认的消息处理器"""
handlers = [
HelloTextMessageHandler(),
AbortTextMessageHandler(),
ListenTextMessageHandler(),
IotTextMessageHandler(),
McpTextMessageHandler(),
ServerTextMessageHandler(),
]
for handler in handlers:
self.register_handler(handler)
def register_handler(self, handler: TextMessageHandler) -> None:
"""注册消息处理器"""
self._handlers[handler.message_type.value] = handler
def get_handler(self, message_type: str) -> Optional[TextMessageHandler]:
"""获取消息处理器"""
return self._handlers.get(message_type)
def get_supported_types(self) -> list:
"""获取支持的消息类型"""
return list(self._handlers.keys())
@@ -0,0 +1,41 @@
import json
from core.handle.textMessageHandlerRegistry import TextMessageHandlerRegistry
TAG = __name__
class TextMessageProcessor:
"""消息处理器主类"""
def __init__(self, registry: TextMessageHandlerRegistry):
self.registry = registry
async def process_message(self, conn, message: str) -> None:
"""处理消息的主入口"""
try:
# 解析JSON消息
msg_json = json.loads(message)
# 处理JSON消息
if isinstance(msg_json, dict):
message_type = msg_json.get("type")
# 记录日志
conn.logger.bind(tag=TAG).info(f"收到{message_type}消息:{message}")
# 获取并执行处理器
handler = self.registry.get_handler(message_type)
if handler:
await handler.handle(conn, msg_json)
else:
conn.logger.bind(tag=TAG).error(f"收到未知类型消息:{message}")
# 处理纯数字消息
elif isinstance(msg_json, int):
conn.logger.bind(tag=TAG).info(f"收到数字消息:{message}")
await conn.websocket.send(message)
except json.JSONDecodeError:
# 非JSON消息直接转发
conn.logger.bind(tag=TAG).error(f"解析到错误的消息:{message}")
await conn.websocket.send(message)
@@ -0,0 +1,11 @@
from enum import Enum
class TextMessageType(Enum):
"""消息类型枚举"""
HELLO = "hello"
ABORT = "abort"
LISTEN = "listen"
IOT = "iot"
MCP = "mcp"
SERVER = "server"
@@ -143,8 +143,8 @@ class TTSProvider(TTSProviderBase):
data = {
"text": text,
"references": [
ServeReferenceAudio(audio=audio if audio else b"", text=text)
for text, audio in zip(ref_texts, byte_audios)
ServeReferenceAudio(audio=audio if audio else b"", text=ref_text)
for ref_text, audio in zip(ref_texts, byte_audios)
],
"reference_id": self.reference_id,
"normalize": self.normalize,
@@ -5,8 +5,9 @@ Opus编码工具类
import logging
import traceback
import numpy as np
from typing import Optional, Callable, Any
from typing import List, Optional
from opuslib_next import Encoder
from opuslib_next import constants
@@ -55,14 +56,13 @@ class OpusEncoderUtils:
self.encoder.reset_state()
self.buffer = np.array([], dtype=np.int16)
def encode_pcm_to_opus_stream(self, pcm_data: bytes, end_of_stream: bool, callback: Callable[[Any], Any]):
def encode_pcm_to_opus(self, pcm_data: bytes, end_of_stream: bool) -> List[bytes]:
"""
将PCM数据编码为Opus格式,以流式方式进行处理
将PCM数据编码为Opus格式
Args:
pcm_data: PCM字节数据
end_of_stream: 是否为流的结束,
callback: opus处理方法
end_of_stream: 是否为流的结束
Returns:
Opus数据包列表
@@ -76,6 +76,7 @@ class OpusEncoderUtils:
# 将新数据追加到缓冲区
self.buffer = np.append(self.buffer, new_samples)
opus_packets = []
offset = 0
# 处理所有完整帧
@@ -83,7 +84,7 @@ class OpusEncoderUtils:
frame = self.buffer[offset : offset + self.total_frame_size]
output = self._encode(frame)
if output:
callback(output)
opus_packets.append(output)
offset += self.total_frame_size
# 保留未处理的样本
@@ -97,9 +98,11 @@ class OpusEncoderUtils:
output = self._encode(last_frame)
if output:
callback(output)
opus_packets.append(output)
self.buffer = np.array([], dtype=np.int16)
return opus_packets
def _encode(self, frame: np.ndarray) -> Optional[bytes]:
"""编码一帧音频数据"""
try:
+27 -10
View File
@@ -1,12 +1,15 @@
import io
import struct
from typing import Callable, Any
def decode_opus_from_file(input_file):
"""
从p3文件中解码 Opus 数据,并返回一个 Opus 数据包的列表以及总时长。
"""
opus_datas = []
total_frames = 0
sample_rate = 16000 # 文件采样率
frame_duration_ms = 60 # 帧时长
frame_size = int(sample_rate * frame_duration_ms / 1000)
def decode_opus_from_file_stream(input_file, callback: Callable[[Any], Any]):
"""
从p3文件中解码 Opus 数据,由 callback 处理 Opus 数据包。
"""
with open(input_file, 'rb') as f:
while True:
# 读取头部(4字节):[1字节类型,1字节保留,2字节长度]
@@ -22,13 +25,23 @@ def decode_opus_from_file_stream(input_file, callback: Callable[[Any], Any]):
if len(opus_data) != data_len:
raise ValueError(f"Data length({len(opus_data)}) mismatch({data_len}) in the file.")
callback(opus_data)
opus_datas.append(opus_data)
total_frames += 1
# 计算总时长
total_duration = (total_frames * frame_duration_ms) / 1000.0
return opus_datas, total_duration
def decode_opus_from_bytes_stream(input_bytes, callback: Callable[[Any], Any]):
def decode_opus_from_bytes(input_bytes):
"""
从p3二进制数据中解码 Opus 数据,由 callback 处理 Opus 数据包。
从p3二进制数据中解码 Opus 数据,并返回一个 Opus 数据包的列表以及总时长
"""
import io
opus_datas = []
total_frames = 0
sample_rate = 16000 # 文件采样率
frame_duration_ms = 60 # 帧时长
frame_size = int(sample_rate * frame_duration_ms / 1000)
f = io.BytesIO(input_bytes)
while True:
@@ -39,4 +52,8 @@ def decode_opus_from_bytes_stream(input_bytes, callback: Callable[[Any], Any]):
opus_data = f.read(data_len)
if len(opus_data) != data_len:
raise ValueError(f"Data length({len(opus_data)}) mismatch({data_len}) in the bytes.")
callback(opus_data)
opus_datas.append(opus_data)
total_frames += 1
total_duration = (total_frames * frame_duration_ms) / 1000.0
return opus_datas, total_duration
@@ -125,7 +125,8 @@ def fetch_news_from_api(conn, source="thepaper"):
]["get_news_from_newsnow"].get("url"):
api_url = conn.config["plugins"]["get_news_from_newsnow"]["url"] + source
response = requests.get(api_url, timeout=10)
headers = {"User-Agent": "Mozilla/5.0"}
response = requests.get(api_url, headers=headers, timeout=10)
response.raise_for_status()
data = response.json()
@@ -144,7 +145,8 @@ def fetch_news_from_api(conn, source="thepaper"):
def fetch_news_detail(url):
"""获取新闻详情页内容并使用MarkItDown清理HTML"""
try:
response = requests.get(url, timeout=10)
headers = {"User-Agent": "Mozilla/5.0"}
response = requests.get(url, headers=headers, timeout=10)
response.raise_for_status()
# 使用MarkItDown清理HTML内容
@@ -212,6 +212,7 @@ async def play_local_music(conn, specific_file=None):
conn.logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
return
text = _get_random_play_prompt(selected_music)
await send_stt_message(conn, text)
conn.dialogue.put(Message(role="assistant", content=text))
if conn.intent_type == "intent_llm":