mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-28 01:53:53 +08:00
resolve merge conflict
This commit is contained in:
caixypromise
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(tree:*)",
|
||||
"Bash(find:*)"
|
||||
],
|
||||
"deny": [],
|
||||
"ask": []
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,7 @@
|
||||
3. **洞察需求：** 结合上下文**深入理解用户真实意图**后再决定调用，避免无意义调用。
|
||||
4. **独立任务：** 除`<context>`已涵盖信息外，用户每个要求（即使相似）都视为**独立任务**，需调用工具获取最新数据，**不可偷懒复用历史结果**。
|
||||
5. **不确定时：** **切勿猜测或编造答案**。若不确定相关操作，可引导用户澄清或告知能力限制。
|
||||
6. **多工具调用：** 当用户要求执行多个任务时，你会调用多个工具（数量不定）。**重要：在获取到所有工具结果后，你必须依次总结每个工具的查询结果**，不要遗漏任何一个。例如用户问"设备当前状态，某某地方的天气和社会新闻"，你要先说设备状态，再说天气情况，最后说新闻内容。
|
||||
- **重要例外（无需调用）：**
|
||||
- `查询"现在的时间"、"今天的日期/星期几"、"今天农历"、"{{local_address}}的天气/未来天气"` -> **直接使用`<context>`信息回复**。
|
||||
- **需要调用的情况（示例）：**
|
||||
@@ -63,6 +64,7 @@
|
||||
- 查询**详细农历信息**（宜忌、八字、节气等）。
|
||||
- 除上述例外外的**任何其他信息或操作请求**（如查新闻、订闹钟、算数学、查非本地天气等）。
|
||||
- 我已经给你装了摄像头，如果用户说“拍照”，你需要调用self_camera_take_photo工具说一下你看到了什么。默认question的参数是“描述一下看到的物品”
|
||||
- 当工具列表包含search_from_ragflow，说明可以使用知识库，你应该结合用户上下文意图并结合知识库的使用描述，推断是否要调用调用知识库。
|
||||
</tool_calling>
|
||||
|
||||
<context>
|
||||
@@ -72,6 +74,7 @@
|
||||
- **今天农历：** {{lunar_date}}
|
||||
- **用户所在城市：** {{local_address}}
|
||||
- **当地未来7天天气：** {{weather_info}}
|
||||
{{ dynamic_context }}
|
||||
</context>
|
||||
|
||||
<memory>
|
||||
|
||||
@@ -9,6 +9,7 @@ from core.utils.util import get_local_ip, validate_mcp_endpoint
|
||||
from core.http_server import SimpleHttpServer
|
||||
from core.xiaozhi_server_facade import XiaozhiServerFacade
|
||||
from core.utils.util import check_ffmpeg_installed
|
||||
from core.utils.gc_manager import get_gc_manager
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
@@ -46,21 +47,30 @@ async def main():
|
||||
check_ffmpeg_installed()
|
||||
config = load_config()
|
||||
|
||||
# 默认使用manager-api的secret作为auth_key
|
||||
# 如果secret为空，则生成随机密钥
|
||||
# auth_key用于jwt认证，比如视觉分析接口的jwt认证
|
||||
auth_key = config.get("manager-api", {}).get("secret", "")
|
||||
# auth_key优先级：配置文件server.auth_key > manager-api.secret > 自动生成
|
||||
# auth_key用于jwt认证，比如视觉分析接口的jwt认证、ota接口的token生成与websocket认证
|
||||
# 获取配置文件中的auth_key
|
||||
auth_key = config["server"].get("auth_key", "")
|
||||
|
||||
# 验证auth_key，无效则尝试使用manager-api.secret
|
||||
if not auth_key or len(auth_key) == 0 or "你" in auth_key:
|
||||
auth_key = str(uuid.uuid4().hex)
|
||||
auth_key = config.get("manager-api", {}).get("secret", "")
|
||||
# 验证secret，无效则生成随机密钥
|
||||
if not auth_key or len(auth_key) == 0 or "你" in auth_key:
|
||||
auth_key = str(uuid.uuid4().hex)
|
||||
|
||||
config["server"]["auth_key"] = auth_key
|
||||
|
||||
# 添加 stdin 监控任务
|
||||
stdin_task = asyncio.create_task(monitor_stdin())
|
||||
|
||||
# 启动全局GC管理器（5分钟清理一次）
|
||||
gc_manager = get_gc_manager(interval_seconds=300)
|
||||
await gc_manager.start()
|
||||
|
||||
# 启动小智服务器门面（支持WebSocket和MQTT）
|
||||
xiaozhi_server = XiaozhiServerFacade(config)
|
||||
xiaozhi_task = asyncio.create_task(xiaozhi_server.start())
|
||||
|
||||
# 启动 Simple http 服务器
|
||||
ota_server = SimpleHttpServer(config)
|
||||
ota_task = asyncio.create_task(ota_server.start())
|
||||
@@ -149,8 +159,11 @@ async def main():
|
||||
try:
|
||||
await xiaozhi_server.stop()
|
||||
except Exception as e:
|
||||
logger.error(f"停止小智服务器失败: {e}")
|
||||
logger.bind(tag=TAG).error(f"停止小智服务器失败: {e}")
|
||||
|
||||
# 停止全局GC管理器
|
||||
await gc_manager.stop()
|
||||
|
||||
# 取消所有任务（关键修复点）
|
||||
stdin_task.cancel()
|
||||
xiaozhi_task.cancel()
|
||||
|
||||
+155
-40
@@ -38,9 +38,16 @@ server:
|
||||
name: "your-device-name1" # 设备1标识
|
||||
- token: "your-token2" # 设备2的token
|
||||
name: "your-device-name2" # 设备2标识
|
||||
# 可选:设备白名单，如果设置了白名单，那么白名单的机器无论是什么token都可以连接。
|
||||
#allowed_devices:
|
||||
# - "24:0A:C4:1D:3B:F0" # MAC地址列表
|
||||
# 白名单设备ID列表
|
||||
# 如果属于白名单内的设备，不校验token，直接放行
|
||||
allowed_devices:
|
||||
- "11:22:33:44:55:66"
|
||||
# MQTT网关配置，用于通过OTA下发到设备，根据mqtt_gateway的.env文件配置，格式为host:port
|
||||
mqtt_gateway: null
|
||||
# MQTT签名密钥，用于生成MQTT连接密码，根据mqtt_gateway的.env文件配置
|
||||
mqtt_signature_key: null
|
||||
# UDP网关配置
|
||||
udp_gateway: null
|
||||
|
||||
# #####################################################################################
|
||||
# #############################协议配置（Protocol Configuration）########################
|
||||
@@ -88,7 +95,6 @@ mqtt_server:
|
||||
# enabled: true
|
||||
# port: 1883
|
||||
# public_ip: "your.server.ip" # 替换为实际IP
|
||||
|
||||
log:
|
||||
# 设置控制台输出的日志格式，时间、日志级别、标签、消息
|
||||
log_format: "<green>{time:YYMMDD HH:mm:ss}</green>[{version}_{selected_module}][<light-blue>{extra[tag]}</light-blue>]-<level>{level}</level>-<light-green>{message}</light-green>"
|
||||
@@ -117,6 +123,15 @@ enable_greeting: true
|
||||
enable_stop_tts_notify: false
|
||||
# 说完话是否开启提示音，音效地址
|
||||
stop_tts_notify_voice: "config/assets/tts_notify.mp3"
|
||||
# 是否启用WebSocket心跳保活机制
|
||||
enable_websocket_ping: false
|
||||
|
||||
|
||||
# TTS音频发送延迟配置
|
||||
# tts_audio_send_delay: 控制音频包发送间隔
|
||||
# 0: 使用精确时间控制，严格匹配音频帧率（默认，运行时按音频帧率计算）
|
||||
# > 0: 使用固定延迟（毫秒）发送，例如: 60
|
||||
tts_audio_send_delay: 0
|
||||
|
||||
exit_commands:
|
||||
- "退出"
|
||||
@@ -155,6 +170,15 @@ wakeup_words:
|
||||
# MCP接入点地址，地址格式为：ws://你的mcp接入点ip或者域名:端口号/mcp/?token=你的token
|
||||
# 详细教程 https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/mcp-endpoint-integration.md
|
||||
mcp_endpoint: 你的接入点 websocket地址
|
||||
|
||||
# 上下文源配置
|
||||
# 用于在系统提示词中注入动态数据，如健康数据、股票信息等
|
||||
# 可以添加多个上下文源
|
||||
context_providers:
|
||||
- url: ""
|
||||
headers:
|
||||
Authorization: ""
|
||||
|
||||
# 插件的基础配置
|
||||
plugins:
|
||||
# 获取天气插件的配置，这里填写你的api_key
|
||||
@@ -162,7 +186,10 @@ plugins:
|
||||
# 想稳定一点就自行申请替换，每天有1000次免费调用
|
||||
# 申请地址：https://console.qweather.com/#/apps/create-key/over
|
||||
# 申请后通过这个链接可以找到自己的apihost：https://console.qweather.com/setting?lang=zh
|
||||
get_weather: {"api_host":"mj7p3y7naa.re.qweatherapi.com", "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" }
|
||||
get_weather:
|
||||
api_host: "mj7p3y7naa.re.qweatherapi.com"
|
||||
api_key: "a861d0d5e7bf4ee1a83d9a9e4f96d4da"
|
||||
default_location: "广州"
|
||||
# 获取新闻插件的配置，这里根据需要的新闻类型传入对应的url链接，默认支持社会、科技、财经新闻
|
||||
# 更多类型的新闻列表查看 https://www.chinanews.com.cn/rss/
|
||||
get_news_from_chinanews:
|
||||
@@ -186,7 +213,15 @@ plugins:
|
||||
- ".wav"
|
||||
- ".p3"
|
||||
refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒
|
||||
|
||||
search_from_ragflow:
|
||||
# 知识库的描述信息，方便大语言模型知道什么时候调用
|
||||
description: "当用户问xxx时，调用本方法，使用知识库中的信息回答问题"
|
||||
# ragflow接口配置
|
||||
base_url: "http://192.168.0.8"
|
||||
# ragflow api访问令牌
|
||||
api_key: "ragflow-xxx"
|
||||
# ragflow知识库id
|
||||
dataset_ids: ["123456789"]
|
||||
# 声纹识别配置
|
||||
voiceprint:
|
||||
# 声纹接口地址
|
||||
@@ -196,6 +231,9 @@ voiceprint:
|
||||
- "test1,张三,张三是一个程序员"
|
||||
- "test2,李四,李四是一个产品经理"
|
||||
- "test3,王五,王五是一个设计师"
|
||||
# 声纹识别相似度阈值，范围0.0-1.0，默认0.4
|
||||
# 数值越高越严格，减少误识别但可能增加拒识率
|
||||
similarity_threshold: 0.4
|
||||
|
||||
# #####################################################################################
|
||||
# ################################以下是角色模型配置######################################
|
||||
@@ -215,6 +253,9 @@ prompt: |
|
||||
- 长篇大论，叽叽歪歪
|
||||
- 长时间严肃对话
|
||||
|
||||
# 默认系统提示词模板文件
|
||||
prompt_template: agent-base-prompt.txt
|
||||
|
||||
# 结束语prompt
|
||||
end_prompt:
|
||||
enable: true # 是否开启结束语
|
||||
@@ -272,6 +313,7 @@ Intent:
|
||||
functions:
|
||||
- change_role
|
||||
- get_weather
|
||||
# - search_from_ragflow
|
||||
# - get_news_from_chinanews
|
||||
- get_news_from_newsnow
|
||||
# play_music是服务器自带的音乐播放，hass_play_music是通过home assistant控制的独立外部程序音乐播放
|
||||
@@ -361,6 +403,8 @@ ASR:
|
||||
# 热词、替换词使用流程：https://www.volcengine.com/docs/6561/155738
|
||||
boosting_table_name: （选填）你的热词文件名称
|
||||
correct_table_name: （选填）你的替换词文件名称
|
||||
# 静音判定时长(ms)，默认200ms
|
||||
end_window_size: 200
|
||||
output_dir: tmp/
|
||||
TencentASR:
|
||||
# token申请地址：https://console.cloud.tencent.com/cam/capi
|
||||
@@ -440,8 +484,57 @@ ASR:
|
||||
base_url: https://api.groq.com/openai/v1/audio/transcriptions
|
||||
model_name: whisper-large-v3-turbo
|
||||
output_dir: tmp/
|
||||
|
||||
|
||||
VoskASR:
|
||||
# 官方网站：https://alphacephei.com/vosk/
|
||||
# 配置说明：
|
||||
# 1. VOSK是一个离线语音识别库，支持多种语言
|
||||
# 2. 需要先下载模型文件：https://alphacephei.com/vosk/models
|
||||
# 3. 中文模型推荐使用vosk-model-small-cn-0.22或vosk-model-cn-0.22
|
||||
# 4. 完全离线运行，无需网络连接
|
||||
# 5. 输出文件保存在tmp/目录
|
||||
# 使用步骤：
|
||||
# 1. 访问 https://alphacephei.com/vosk/models 下载对应的模型
|
||||
# 2. 解压模型文件到项目目录下的models/vosk/文件夹
|
||||
# 3. 在配置中指定正确的模型路径
|
||||
# 4. 注意：VOSK中文模型输出不带标点符号，词与词之间会有空格
|
||||
type: vosk
|
||||
model_path: 你的模型路径，如：models/vosk/vosk-model-small-cn-0.22
|
||||
output_dir: tmp/
|
||||
Qwen3ASRFlash:
|
||||
# 通义千问Qwen3-ASR-Flash语音识别服务，需要先在阿里云百炼平台创建API密钥
|
||||
# 申请步骤：
|
||||
# 1.登录阿里云百炼平台。https://bailian.console.aliyun.com/
|
||||
# 2.创建API-KEY https://bailian.console.aliyun.com/#/api-key
|
||||
# 3.Qwen3-ASR-Flash基于通义千问多模态基座，支持多语言识别、歌唱识别、噪声拒识等功能
|
||||
type: qwen3_asr_flash
|
||||
api_key: 你的阿里云百炼API密钥
|
||||
base_url: https://dashscope.aliyuncs.com/compatible-mode/v1
|
||||
model_name: qwen3-asr-flash
|
||||
output_dir: tmp/
|
||||
# ASR选项配置
|
||||
enable_lid: true # 自动语种检测
|
||||
enable_itn: true # 逆文本归一化
|
||||
#language: "zh" # 语种，支持zh、en、ja、ko等
|
||||
context: "" # 上下文信息，用于提高识别准确率，不超过10000 Token
|
||||
XunfeiStreamASR:
|
||||
# 讯飞流式语音识别服务
|
||||
# 需要先在讯飞开放平台创建应用，获取以下认证信息
|
||||
# 讯飞开放平台地址：https://www.xfyun.cn/
|
||||
# 创建应用后，在"我的应用"中获取：
|
||||
# - APPID
|
||||
# - APISecret
|
||||
# - APIKey
|
||||
type: xunfei_stream
|
||||
# 必填参数 - 讯飞开放平台应用信息
|
||||
app_id: 你的APPID
|
||||
api_key: 你的APIKey
|
||||
api_secret: 你的APISecret
|
||||
# 识别参数配置
|
||||
domain: slm # 识别领域，iat:日常用语，medical:医疗，finance:金融等
|
||||
language: zh_cn # 语言，zh_cn:中文，en_us:英文
|
||||
accent: mandarin # 方言，mandarin:普通话
|
||||
# 调整音频处理参数以提高长语音识别质量
|
||||
output_dir: tmp/
|
||||
|
||||
VAD:
|
||||
SileroVAD:
|
||||
@@ -464,7 +557,6 @@ LLM:
|
||||
temperature: 0.7 # 温度值
|
||||
max_tokens: 500 # 最大生成token数
|
||||
top_p: 1
|
||||
top_k: 50
|
||||
frequency_penalty: 0 # 频率惩罚
|
||||
AliAppLLM:
|
||||
# 定义LLM API类型
|
||||
@@ -599,6 +691,16 @@ VLLM:
|
||||
url: https://dashscope.aliyuncs.com/compatible-mode/v1
|
||||
# 可在这里找到你的api key https://bailian.console.aliyun.com/?apiKey=1#/api-key
|
||||
api_key: 你的api_key
|
||||
XunfeiSparkLLM:
|
||||
# 定义LLM API类型
|
||||
type: openai
|
||||
# 先新建应用，在下面的地址
|
||||
# 开通应用地址：https://console.xfyun.cn/app/myapp
|
||||
# 有免费额度，但也要开通服务，才能获取api_key
|
||||
# 每一个模型都需要单独开通，每一个模型的api_password都不同，例如Lite模型在https://console.xfyun.cn/services/cbm 开通
|
||||
base_url: https://ark.cn-beijing.volces.com/api/v3
|
||||
model_name: lite
|
||||
api_key: 你的api_password
|
||||
TTS:
|
||||
# 当前支持的type为edge、doubao，可自行适配
|
||||
EdgeTTS:
|
||||
@@ -637,6 +739,8 @@ TTS:
|
||||
access_token: 你的火山引擎语音合成服务access_token
|
||||
resource_id: volc.service_type.10029
|
||||
speaker: zh_female_wanwanxiaohe_moon_bigtts
|
||||
# 开启WebSocket连接复用，默认复用（注意：复用后设备处于聆听状态时空闲链接会占并发数）
|
||||
enable_ws_reuse: True
|
||||
speech_rate: 0
|
||||
loudness_rate: 0
|
||||
pitch: 0
|
||||
@@ -736,19 +840,13 @@ TTS:
|
||||
inp_refs: []
|
||||
sample_steps: 32
|
||||
if_sr: false
|
||||
MinimaxTTS:
|
||||
# Minimax语音合成服务，需要先在minimax平台创建账户充值，并获取登录信息
|
||||
# 平台地址：https://platform.minimaxi.com/
|
||||
# 充值地址：https://platform.minimaxi.com/user-center/payment/balance
|
||||
# group_id地址：https://platform.minimaxi.com/user-center/basic-information
|
||||
# api_key地址：https://platform.minimaxi.com/user-center/basic-information/interface-key
|
||||
# 定义TTS API类型
|
||||
type: minimax
|
||||
MinimaxTTSHTTPStream:
|
||||
# Minimax流式语音合成服务
|
||||
type: minimax_httpstream
|
||||
output_dir: tmp/
|
||||
group_id: 你的minimax平台groupID
|
||||
api_key: 你的minimax平台接口密钥
|
||||
model: "speech-01-turbo"
|
||||
# 此处设置将优先于voice_setting中voice_id的设置；如都不设置，默认为 female-shaonv
|
||||
voice_id: "female-shaonv"
|
||||
# 以下可不用设置，使用默认设置
|
||||
# voice_setting:
|
||||
@@ -762,7 +860,7 @@ TTS:
|
||||
# - "处理/(chu3)(li3)"
|
||||
# - "危险/dangerous"
|
||||
# audio_setting:
|
||||
# sample_rate: 32000
|
||||
# sample_rate: 24000
|
||||
# bitrate: 128000
|
||||
# format: "mp3"
|
||||
# channel: 1
|
||||
@@ -774,26 +872,6 @@ TTS:
|
||||
# voice_id: female-shaonv
|
||||
# weight: 1
|
||||
# language_boost: auto
|
||||
|
||||
# MinimaxTTSHTTPStream和MinimaxTTSWebSocketStream还在测试，测试完再开放
|
||||
#
|
||||
# MinimaxTTSHTTPStream:
|
||||
# # Minimax流式语音合成服务
|
||||
# type: minimax_httpstream
|
||||
# output_dir: tmp/
|
||||
# group_id: 你的minimax平台groupID
|
||||
# api_key: 你的minimax平台接口密钥
|
||||
# model: "speech-01-turbo"
|
||||
# voice_id: "female-shaonv"
|
||||
#
|
||||
# MinimaxTTSWebSocketStream:
|
||||
# type: minimax_webSocket
|
||||
# output_dir: tmp/
|
||||
# group_id: 你的minimax平台groupID
|
||||
# api_key: 你的minimax平台接口密钥
|
||||
# model: "speech-01-turbo"
|
||||
# voice_id: "female-shaonv"
|
||||
|
||||
AliyunTTS:
|
||||
# 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息
|
||||
# 平台地址：https://nls-portal.console.aliyun.com/
|
||||
@@ -961,4 +1039,41 @@ TTS:
|
||||
audio_format: "pcm"
|
||||
# 默认音色，如需其他音色可到项目assets文件夹下注册
|
||||
voice: "jay_klee"
|
||||
output_dir: tmp/
|
||||
output_dir: tmp/
|
||||
AliBLTTS:
|
||||
# 阿里百炼CosyVoice大模型流式文本语音合成
|
||||
# 可在这里找到你的 api_key https://bailian.console.aliyun.com/?apiKey=1#/api-key
|
||||
# cosyvoice-v3和部分音色需要申请开通
|
||||
type: alibl_stream
|
||||
api_key: 你的api_key
|
||||
model: "cosyvoice-v2"
|
||||
voice: "longcheng_v2"
|
||||
output_dir: tmp/
|
||||
# 以下可不用设置，使用默认设置
|
||||
# format: pcm # 音频格式：pcm、wav、mp3、opus
|
||||
# sample_rate: 24000 # 采样率：16000, 24000, 48000
|
||||
# volume: 50 # 音量：0-100
|
||||
# rate: 1 # 语速：0.5~2
|
||||
# pitch: 1 # 语调：0.5~2
|
||||
XunFeiTTS:
|
||||
# 讯飞TTS服务 官方网站：https://www.xfyun.cn/
|
||||
# 登录讯飞语音技术平台 https://console.xfyun.cn/app/myapp 创建相关应用
|
||||
# 选择需要的服务获取api相关配置 https://console.xfyun.cn/services/uts
|
||||
# 为需要使用的应用(APPID)购买相关服务 例如：超拟人合成 https://console.xfyun.cn/services/uts
|
||||
type: xunfei_stream
|
||||
api_url: wss://cbm01.cn-huabei-1.xf-yun.com/v1/private/mcd9m97e6
|
||||
app_id: 你的app_id
|
||||
api_secret: 你的api_secret
|
||||
api_key: 你的api_key
|
||||
voice: x5_lingxiaoxuan_flow
|
||||
output_dir: tmp/
|
||||
# 以下可不用设置，使用默认设置，注意V5音色不支持口语化配置
|
||||
# oral_level: mid # 口语化等级：high, mid, low
|
||||
# spark_assist: 1 # 是否通过大模型进行口语化 开启:1, 关闭:0
|
||||
# stop_split: 0 # 关闭服务端拆句 不关闭：0，关闭：1
|
||||
# remain: 0 # 是否保留原书面语的样子 保留:1, 不保留:0
|
||||
# format: raw # 音频格式：raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb
|
||||
# sample_rate: 24000 # 采样率：16000, 8000, 24000
|
||||
# volume: 50 # 音量：0-100
|
||||
# speed: 50 # 语速：0-100
|
||||
# pitch: 50 # 语调：0-100
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -256,7 +256,16 @@ def load_config():
|
||||
custom_config = read_config(custom_config_path)
|
||||
|
||||
if custom_config.get("manager-api", {}).get("url"):
|
||||
config = get_config_from_api(custom_config)
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
# 如果已经在事件循环中，使用异步版本
|
||||
config = asyncio.run_coroutine_threadsafe(
|
||||
get_config_from_api_async(custom_config), loop
|
||||
).result()
|
||||
except RuntimeError:
|
||||
# 如果不在事件循环中（启动时），创建新的事件循环
|
||||
config = asyncio.run(get_config_from_api_async(custom_config))
|
||||
else:
|
||||
# 合并配置
|
||||
config = merge_configs(default_config, custom_config)
|
||||
@@ -272,13 +281,13 @@ def load_config():
|
||||
return config
|
||||
|
||||
|
||||
def get_config_from_api(config):
|
||||
"""从Java API获取配置"""
|
||||
async def get_config_from_api_async(config):
|
||||
"""从Java API获取配置（异步版本）"""
|
||||
# 初始化API客户端
|
||||
init_service(config)
|
||||
|
||||
# 获取服务器配置
|
||||
config_data = get_server_config()
|
||||
config_data = await get_server_config()
|
||||
if config_data is None:
|
||||
raise Exception("Failed to fetch server config from API")
|
||||
|
||||
@@ -287,6 +296,7 @@ def get_config_from_api(config):
|
||||
"url": config["manager-api"].get("url", ""),
|
||||
"secret": config["manager-api"].get("secret", ""),
|
||||
}
|
||||
auth_enabled = config_data.get("server", {}).get("auth", {}).get("enabled", False)
|
||||
# server的配置以本地为准
|
||||
if config.get("server"):
|
||||
config_data["server"] = {
|
||||
@@ -296,12 +306,16 @@ def get_config_from_api(config):
|
||||
"vision_explain": config["server"].get("vision_explain", ""),
|
||||
"auth_key": config["server"].get("auth_key", ""),
|
||||
}
|
||||
config_data["server"]["auth"] = {"enabled": auth_enabled}
|
||||
# 如果服务器没有prompt_template，则从本地配置读取
|
||||
if not config_data.get("prompt_template"):
|
||||
config_data["prompt_template"] = config.get("prompt_template")
|
||||
return ConfigDict(config_data)
|
||||
|
||||
|
||||
def get_private_config_from_api(config, device_id, client_id):
|
||||
async def get_private_config_from_api(config, device_id, client_id):
|
||||
"""从Java API获取私有配置"""
|
||||
return get_agent_models(device_id, client_id, config["selected_module"])
|
||||
return await get_agent_models(device_id, client_id, config["selected_module"])
|
||||
|
||||
|
||||
def ensure_directories(config):
|
||||
|
||||
@@ -5,7 +5,7 @@ from config.config_loader import load_config
|
||||
from config.settings import check_config_file
|
||||
from datetime import datetime
|
||||
|
||||
SERVER_VERSION = "0.7.7"
|
||||
SERVER_VERSION = "0.8.10"
|
||||
_logger_initialized = False
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import os
|
||||
import time
|
||||
import base64
|
||||
from typing import Optional, Dict
|
||||
|
||||
@@ -20,7 +19,7 @@ class DeviceBindException(Exception):
|
||||
|
||||
class ManageApiClient:
|
||||
_instance = None
|
||||
_client = None
|
||||
_async_clients = {} # 为每个事件循环存储独立的客户端
|
||||
_secret = None
|
||||
|
||||
def __new__(cls, config):
|
||||
@@ -32,7 +31,7 @@ class ManageApiClient:
|
||||
|
||||
@classmethod
|
||||
def _init_client(cls, config):
|
||||
"""初始化持久化连接池"""
|
||||
"""初始化配置（延迟创建客户端）"""
|
||||
cls.config = config.get("manager-api")
|
||||
|
||||
if not cls.config:
|
||||
@@ -47,23 +46,41 @@ class ManageApiClient:
|
||||
cls._secret = cls.config.get("secret")
|
||||
cls.max_retries = cls.config.get("max_retries", 6) # 最大重试次数
|
||||
cls.retry_delay = cls.config.get("retry_delay", 10) # 初始重试延迟(秒)
|
||||
# NOTE(goody): 2025/4/16 http相关资源统一管理，后续可以增加线程池或者超时
|
||||
# 后续也可以统一配置apiToken之类的走通用的Auth
|
||||
cls._client = httpx.Client(
|
||||
base_url=cls.config.get("url"),
|
||||
headers={
|
||||
"User-Agent": f"PythonClient/2.0 (PID:{os.getpid()})",
|
||||
"Accept": "application/json",
|
||||
"Authorization": "Bearer " + cls._secret,
|
||||
},
|
||||
timeout=cls.config.get("timeout", 30), # 默认超时时间30秒
|
||||
)
|
||||
# 不在这里创建 AsyncClient，延迟到实际使用时创建
|
||||
cls._async_clients = {}
|
||||
|
||||
@classmethod
|
||||
def _request(cls, method: str, endpoint: str, **kwargs) -> Dict:
|
||||
"""发送单次HTTP请求并处理响应"""
|
||||
async def _ensure_async_client(cls):
|
||||
"""确保异步客户端已创建（为每个事件循环创建独立的客户端）"""
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
loop_id = id(loop)
|
||||
|
||||
# 为每个事件循环创建独立的客户端
|
||||
if loop_id not in cls._async_clients:
|
||||
cls._async_clients[loop_id] = httpx.AsyncClient(
|
||||
base_url=cls.config.get("url"),
|
||||
headers={
|
||||
"User-Agent": f"PythonClient/2.0 (PID:{os.getpid()})",
|
||||
"Accept": "application/json",
|
||||
"Authorization": "Bearer " + cls._secret,
|
||||
},
|
||||
timeout=cls.config.get("timeout", 30),
|
||||
)
|
||||
return cls._async_clients[loop_id]
|
||||
except RuntimeError:
|
||||
# 如果没有运行中的事件循环，创建一个临时的
|
||||
raise Exception("必须在异步上下文中调用")
|
||||
|
||||
@classmethod
|
||||
async def _async_request(cls, method: str, endpoint: str, **kwargs) -> Dict:
|
||||
"""发送单次异步HTTP请求并处理响应"""
|
||||
# 确保客户端已创建
|
||||
client = await cls._ensure_async_client()
|
||||
endpoint = endpoint.lstrip("/")
|
||||
response = cls._client.request(method, endpoint, **kwargs)
|
||||
response = await client.request(method, endpoint, **kwargs)
|
||||
response.raise_for_status()
|
||||
|
||||
result = response.json()
|
||||
@@ -96,22 +113,24 @@ class ManageApiClient:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _execute_request(cls, method: str, endpoint: str, **kwargs) -> Dict:
|
||||
"""带重试机制的请求执行器"""
|
||||
async def _execute_async_request(cls, method: str, endpoint: str, **kwargs) -> Dict:
|
||||
"""带重试机制的异步请求执行器"""
|
||||
import asyncio
|
||||
|
||||
retry_count = 0
|
||||
|
||||
while retry_count <= cls.max_retries:
|
||||
try:
|
||||
# 执行请求
|
||||
return cls._request(method, endpoint, **kwargs)
|
||||
# 执行异步请求
|
||||
return await cls._async_request(method, endpoint, **kwargs)
|
||||
except Exception as e:
|
||||
# 判断是否应该重试
|
||||
if retry_count < cls.max_retries and cls._should_retry(e):
|
||||
retry_count += 1
|
||||
print(
|
||||
f"{method} {endpoint} 请求失败，将在 {cls.retry_delay:.1f} 秒后进行第 {retry_count} 次重试"
|
||||
f"{method} {endpoint} 异步请求失败，将在 {cls.retry_delay:.1f} 秒后进行第 {retry_count} 次重试"
|
||||
)
|
||||
time.sleep(cls.retry_delay)
|
||||
await asyncio.sleep(cls.retry_delay)
|
||||
continue
|
||||
else:
|
||||
# 不重试，直接抛出异常
|
||||
@@ -119,22 +138,30 @@ class ManageApiClient:
|
||||
|
||||
@classmethod
|
||||
def safe_close(cls):
|
||||
"""安全关闭连接池"""
|
||||
if cls._client:
|
||||
cls._client.close()
|
||||
cls._instance = None
|
||||
"""安全关闭所有异步连接池"""
|
||||
import asyncio
|
||||
|
||||
for client in list(cls._async_clients.values()):
|
||||
try:
|
||||
asyncio.run(client.aclose())
|
||||
except Exception:
|
||||
pass
|
||||
cls._async_clients.clear()
|
||||
cls._instance = None
|
||||
|
||||
|
||||
def get_server_config() -> Optional[Dict]:
|
||||
async def get_server_config() -> Optional[Dict]:
|
||||
"""获取服务器基础配置"""
|
||||
return ManageApiClient._instance._execute_request("POST", "/config/server-base")
|
||||
return await ManageApiClient._instance._execute_async_request(
|
||||
"POST", "/config/server-base"
|
||||
)
|
||||
|
||||
|
||||
def get_agent_models(
|
||||
async def get_agent_models(
|
||||
mac_address: str, client_id: str, selected_module: Dict
|
||||
) -> Optional[Dict]:
|
||||
"""获取代理模型配置"""
|
||||
return ManageApiClient._instance._execute_request(
|
||||
return await ManageApiClient._instance._execute_async_request(
|
||||
"POST",
|
||||
"/config/agent-models",
|
||||
json={
|
||||
@@ -145,28 +172,26 @@ def get_agent_models(
|
||||
)
|
||||
|
||||
|
||||
def save_mem_local_short(mac_address: str, short_momery: str) -> Optional[Dict]:
|
||||
async def generate_and_save_chat_summary(session_id: str) -> Optional[Dict]:
|
||||
"""生成并保存聊天记录总结"""
|
||||
try:
|
||||
return ManageApiClient._instance._execute_request(
|
||||
"PUT",
|
||||
f"/agent/saveMemory/" + mac_address,
|
||||
json={
|
||||
"summaryMemory": short_momery,
|
||||
},
|
||||
return await ManageApiClient._instance._execute_async_request(
|
||||
"POST",
|
||||
f"/agent/chat-summary/{session_id}/save",
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"存储短期记忆到服务器失败: {e}")
|
||||
print(f"生成并保存聊天记录总结失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def report(
|
||||
async def report(
|
||||
mac_address: str, session_id: str, chat_type: int, content: str, audio, report_time
|
||||
) -> Optional[Dict]:
|
||||
"""带熔断的业务方法示例"""
|
||||
"""异步聊天记录上报"""
|
||||
if not content or not ManageApiClient._instance:
|
||||
return None
|
||||
try:
|
||||
return ManageApiClient._instance._execute_request(
|
||||
return await ManageApiClient._instance._execute_async_request(
|
||||
"POST",
|
||||
f"/agent/chat-history/report",
|
||||
json={
|
||||
|
||||
@@ -22,4 +22,6 @@ manager-api:
|
||||
# 如果使用docker部署，请使用填写成 http://xiaozhi-esp32-server-web:8002/xiaozhi
|
||||
url: http://127.0.0.1:8002/xiaozhi
|
||||
# 你的manager-api的token，就是刚才复制出来的server.secret
|
||||
secret: 你的server.secret值
|
||||
secret: 你的server.secret值
|
||||
# 默认系统提示词模板文件
|
||||
prompt_template: agent-base-prompt.txt
|
||||
@@ -10,7 +10,15 @@ class BaseHandler:
|
||||
def _add_cors_headers(self, response):
|
||||
"""添加CORS头信息"""
|
||||
response.headers["Access-Control-Allow-Headers"] = (
|
||||
"client-id, content-type, device-id"
|
||||
"client-id, content-type, device-id, authorization"
|
||||
)
|
||||
response.headers["Access-Control-Allow-Credentials"] = "true"
|
||||
response.headers["Access-Control-Allow-Origin"] = "*"
|
||||
|
||||
async def handle_options(self, request):
|
||||
"""处理OPTIONS请求，添加CORS头信息"""
|
||||
response = web.Response(body=b"", content_type="text/plain")
|
||||
self._add_cors_headers(response)
|
||||
# 添加允许的方法
|
||||
response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
|
||||
return response
|
||||
|
||||
@@ -1,15 +1,126 @@
|
||||
import json
|
||||
import time
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import re
|
||||
import glob
|
||||
from typing import Dict, List, Tuple
|
||||
from aiohttp import web
|
||||
from core.utils.util import get_local_ip
|
||||
|
||||
from core.auth import AuthManager
|
||||
from core.utils.util import get_local_ip, get_vision_url
|
||||
from core.api.base_handler import BaseHandler
|
||||
|
||||
TAG = __name__
|
||||
|
||||
|
||||
def _safe_basename(filename: str) -> str:
|
||||
# Prevent directory traversal
|
||||
return os.path.basename(filename)
|
||||
|
||||
|
||||
def _parse_version(ver: str) -> Tuple[int, ...]:
|
||||
# conservative parser: split by non-digit, keep numeric parts
|
||||
parts = re.findall(r"\d+", ver)
|
||||
return tuple(int(p) for p in parts) if parts else (0,)
|
||||
|
||||
|
||||
def _is_higher_version(a: str, b: str) -> bool:
|
||||
"""Return True if version string a > b (semver-like numeric compare)."""
|
||||
ta = _parse_version(a)
|
||||
tb = _parse_version(b)
|
||||
# compare tuple lexicographically, but allow different lengths
|
||||
maxlen = max(len(ta), len(tb))
|
||||
for i in range(maxlen):
|
||||
ai = ta[i] if i < len(ta) else 0
|
||||
bi = tb[i] if i < len(tb) else 0
|
||||
if ai > bi:
|
||||
return True
|
||||
if ai < bi:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
class OTAHandler(BaseHandler):
|
||||
def __init__(self, config: dict):
|
||||
super().__init__(config)
|
||||
auth_config = config["server"].get("auth", {})
|
||||
self.auth_enable = auth_config.get("enabled", False)
|
||||
# 设备白名单
|
||||
self.allowed_devices = set(auth_config.get("allowed_devices", []))
|
||||
secret_key = config["server"]["auth_key"]
|
||||
expire_seconds = auth_config.get("expire_seconds")
|
||||
self.auth = AuthManager(secret_key=secret_key, expire_seconds=expire_seconds)
|
||||
|
||||
# firmware storage
|
||||
self.bin_dir = os.path.join(os.getcwd(), "data", "bin")
|
||||
# cache structure: { 'updated_at': timestamp, 'ttl': seconds, 'files_by_model': { model: [(version, filename), ...] } }
|
||||
self._bin_cache: Dict = {
|
||||
"updated_at": 0,
|
||||
"ttl": config.get("firmware_cache_ttl", 30),
|
||||
"files_by_model": {},
|
||||
}
|
||||
|
||||
def _refresh_bin_cache_if_needed(self):
|
||||
now = int(time.time())
|
||||
ttl = int(self._bin_cache.get("ttl", 30))
|
||||
if now - int(
|
||||
self._bin_cache.get("updated_at", 0)
|
||||
) < ttl and self._bin_cache.get("files_by_model"):
|
||||
return
|
||||
|
||||
files_by_model: Dict[str, List[Tuple[str, str]]] = {}
|
||||
try:
|
||||
if not os.path.isdir(self.bin_dir):
|
||||
os.makedirs(self.bin_dir, exist_ok=True)
|
||||
|
||||
# match files like model_1.2.3.bin (allow dots, dashes, underscores in model and version)
|
||||
pattern = os.path.join(self.bin_dir, "*.bin")
|
||||
for path in glob.glob(pattern):
|
||||
fname = os.path.basename(path)
|
||||
# filename format: {model}_{version}.bin
|
||||
m = re.match(r"^(.+?)_([0-9][A-Za-z0-9\.\-_]*)\.bin$", fname)
|
||||
if not m:
|
||||
# skip files not conforming to naming rule
|
||||
continue
|
||||
model = m.group(1)
|
||||
version = m.group(2)
|
||||
files_by_model.setdefault(model, []).append((version, fname))
|
||||
|
||||
# sort versions for each model descending
|
||||
for model, items in files_by_model.items():
|
||||
items.sort(key=lambda it: _parse_version(it[0]), reverse=True)
|
||||
|
||||
self._bin_cache["files_by_model"] = files_by_model
|
||||
self._bin_cache["updated_at"] = now
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"Firmware cache refreshed: {len(files_by_model)} models"
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"刷新固件缓存失败: {e}")
|
||||
# keep previous cache if any
|
||||
|
||||
def generate_password_signature(self, content: str, secret_key: str) -> str:
|
||||
"""生成MQTT密码签名
|
||||
|
||||
Args:
|
||||
content: 签名内容 (clientId + '|' + username)
|
||||
secret_key: 密钥
|
||||
|
||||
Returns:
|
||||
str: Base64编码的HMAC-SHA256签名
|
||||
"""
|
||||
try:
|
||||
hmac_obj = hmac.new(
|
||||
secret_key.encode("utf-8"), content.encode("utf-8"), hashlib.sha256
|
||||
)
|
||||
signature = hmac_obj.digest()
|
||||
return base64.b64encode(signature).decode("utf-8")
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"生成MQTT密码签名失败: {e}")
|
||||
return ""
|
||||
|
||||
def _get_websocket_url(self, local_ip: str, port: int) -> str:
|
||||
"""获取websocket地址
|
||||
@@ -30,7 +141,14 @@ class OTAHandler(BaseHandler):
|
||||
return f"ws://{local_ip}:{port}/xiaozhi/v1/"
|
||||
|
||||
async def handle_post(self, request):
|
||||
"""处理 OTA POST 请求"""
|
||||
"""处理 OTA POST 请求
|
||||
|
||||
This handler will:
|
||||
- read device id/client id (as before)
|
||||
- attempt to determine device model and current firmware version (prefer headers, fallback to body)
|
||||
- check data/bin for newer firmware for that model
|
||||
- if found a newer firmware, set firmware.url to the download endpoint
|
||||
"""
|
||||
try:
|
||||
data = await request.text()
|
||||
self.logger.bind(tag=TAG).debug(f"OTA请求方法: {request.method}")
|
||||
@@ -43,30 +161,188 @@ class OTAHandler(BaseHandler):
|
||||
else:
|
||||
raise Exception("OTA请求设备ID为空")
|
||||
|
||||
data_json = json.loads(data)
|
||||
client_id = request.headers.get("client-id", "")
|
||||
if client_id:
|
||||
self.logger.bind(tag=TAG).info(f"OTA请求ClientID: {client_id}")
|
||||
else:
|
||||
raise Exception("OTA请求ClientID为空")
|
||||
|
||||
data_json = {}
|
||||
try:
|
||||
data_json = json.loads(data) if data else {}
|
||||
except Exception:
|
||||
data_json = {}
|
||||
|
||||
server_config = self.config["server"]
|
||||
port = int(server_config.get("port", 8000))
|
||||
# Distinguish ports:
|
||||
# - websocket_port is used to construct websocket URL (server["port"])
|
||||
# - http_port is used to construct OTA download URLs (server["http_port"])
|
||||
websocket_port = int(server_config.get("port", 8000))
|
||||
http_port = int(server_config.get("http_port", 8003))
|
||||
local_ip = get_local_ip()
|
||||
|
||||
# Determine device model (prefer headers)
|
||||
device_model = ""
|
||||
# header candidates
|
||||
for h in ("device-model", "device_model", "model"):
|
||||
if h in request.headers:
|
||||
device_model = request.headers.get(h, "").strip()
|
||||
break
|
||||
# body fallback
|
||||
if not device_model:
|
||||
try:
|
||||
if "board" in data_json and isinstance(data_json["board"], dict):
|
||||
device_model = data_json["board"].get("type", "")
|
||||
elif "model" in data_json:
|
||||
device_model = data_json.get("model", "")
|
||||
except Exception:
|
||||
device_model = ""
|
||||
if not device_model:
|
||||
device_model = "default"
|
||||
|
||||
# Determine device current version (prefer headers)
|
||||
device_version = ""
|
||||
for h in (
|
||||
"device-version",
|
||||
"device_version",
|
||||
"firmware-version",
|
||||
"app-version",
|
||||
"application-version",
|
||||
):
|
||||
if h in request.headers:
|
||||
device_version = request.headers.get(h, "").strip()
|
||||
break
|
||||
if not device_version:
|
||||
try:
|
||||
device_version = data_json.get("application", {}).get("version", "")
|
||||
except Exception:
|
||||
device_version = ""
|
||||
if not device_version:
|
||||
device_version = "0.0.0"
|
||||
|
||||
return_json = {
|
||||
"server_time": {
|
||||
"timestamp": int(round(time.time() * 1000)),
|
||||
"timezone_offset": server_config.get("timezone_offset", 8) * 60,
|
||||
},
|
||||
"firmware": {
|
||||
"version": data_json["application"].get("version", "1.0.0"),
|
||||
"version": device_version,
|
||||
"url": "",
|
||||
},
|
||||
"websocket": {
|
||||
"url": self._get_websocket_url(local_ip, port),
|
||||
},
|
||||
}
|
||||
|
||||
# existing mqtt/websocket logic (unchanged)
|
||||
mqtt_gateway_endpoint = server_config.get("mqtt_gateway")
|
||||
|
||||
if mqtt_gateway_endpoint: # 如果配置了非空字符串
|
||||
# 尝试从请求数据中获取设备型号（已解析 above）
|
||||
try:
|
||||
group_id = f"GID_{device_model}".replace(":", "_").replace(" ", "_")
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"获取设备型号失败: {e}")
|
||||
group_id = "GID_default"
|
||||
|
||||
mac_address_safe = device_id.replace(":", "_")
|
||||
mqtt_client_id = f"{group_id}@@@{mac_address_safe}@@@{mac_address_safe}"
|
||||
|
||||
# 构建用户数据
|
||||
user_data = {"ip": "unknown"}
|
||||
try:
|
||||
user_data_json = json.dumps(user_data)
|
||||
username = base64.b64encode(user_data_json.encode("utf-8")).decode(
|
||||
"utf-8"
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"生成用户名失败: {e}")
|
||||
username = ""
|
||||
|
||||
# 生成密码
|
||||
password = ""
|
||||
signature_key = server_config.get("mqtt_signature_key", "")
|
||||
if signature_key:
|
||||
password = self.generate_password_signature(
|
||||
mqtt_client_id + "|" + username, signature_key
|
||||
)
|
||||
if not password:
|
||||
password = "" # 签名失败则留空，由设备决定是否允许无密码
|
||||
else:
|
||||
self.logger.bind(tag=TAG).warning("缺少MQTT签名密钥，密码留空")
|
||||
|
||||
# 构建MQTT配置（直接使用 mqtt_gateway 字符串）
|
||||
return_json["mqtt"] = {
|
||||
"endpoint": mqtt_gateway_endpoint,
|
||||
"client_id": mqtt_client_id,
|
||||
"username": username,
|
||||
"password": password,
|
||||
"publish_topic": "device-server",
|
||||
"subscribe_topic": f"devices/p2p/{mac_address_safe}",
|
||||
}
|
||||
self.logger.bind(tag=TAG).info(f"为设备 {device_id} 下发MQTT网关配置")
|
||||
|
||||
else: # 未配置 mqtt_gateway，下发 WebSocket
|
||||
# 如果开启了认证，则进行认证校验
|
||||
token = ""
|
||||
if self.auth_enable:
|
||||
if self.allowed_devices:
|
||||
if device_id not in self.allowed_devices:
|
||||
token = self.auth.generate_token(client_id, device_id)
|
||||
else:
|
||||
token = self.auth.generate_token(client_id, device_id)
|
||||
# NOTE: use websocket_port here
|
||||
return_json["websocket"] = {
|
||||
"url": self._get_websocket_url(local_ip, websocket_port),
|
||||
"token": token,
|
||||
}
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"未配置MQTT网关，为设备 {device_id} 下发WebSocket配置"
|
||||
)
|
||||
|
||||
# Now check firmware files for updates
|
||||
try:
|
||||
self._refresh_bin_cache_if_needed()
|
||||
files_by_model = self._bin_cache.get("files_by_model", {})
|
||||
candidates = files_by_model.get(device_model, [])
|
||||
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"查找型号 {device_model} 的固件，找到 {len(candidates)} 个候选"
|
||||
)
|
||||
|
||||
chosen_url = ""
|
||||
chosen_version = device_version
|
||||
|
||||
# candidates are sorted descending by version
|
||||
for ver, fname in candidates:
|
||||
if _is_higher_version(ver, device_version):
|
||||
# build download url (only allow download via our download endpoint)
|
||||
chosen_version = ver
|
||||
# Use get_vision_url to get the base URL and replace the path
|
||||
vision_url = get_vision_url(self.config)
|
||||
# Replace the path from "/mcp/vision/explain" to "/xiaozhi/ota/download/{fname}"
|
||||
chosen_url = vision_url.replace(
|
||||
"/mcp/vision/explain", f"/xiaozhi/ota/download/{fname}"
|
||||
)
|
||||
break
|
||||
|
||||
if chosen_url:
|
||||
return_json["firmware"]["version"] = chosen_version
|
||||
return_json["firmware"]["url"] = chosen_url
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"为设备 {device_id} 下发固件 {chosen_version} [如果地址前缀有误，请检查配置文件中的server.vision_explain]-> {chosen_url} "
|
||||
)
|
||||
else:
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"设备 {device_id} 固件已是最新: {device_version}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"检查固件版本时出错: {e}")
|
||||
|
||||
response = web.Response(
|
||||
text=json.dumps(return_json, separators=(",", ":")),
|
||||
content_type="application/json",
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"OTA POST处理异常: {e}")
|
||||
return_json = {"success": False, "message": "request error."}
|
||||
response = web.Response(
|
||||
text=json.dumps(return_json, separators=(",", ":")),
|
||||
@@ -81,8 +357,9 @@ class OTAHandler(BaseHandler):
|
||||
try:
|
||||
server_config = self.config["server"]
|
||||
local_ip = get_local_ip()
|
||||
port = int(server_config.get("port", 8000))
|
||||
websocket_url = self._get_websocket_url(local_ip, port)
|
||||
# use websocket port for websocket URL
|
||||
websocket_port = int(server_config.get("port", 8000))
|
||||
websocket_url = self._get_websocket_url(local_ip, websocket_port)
|
||||
message = f"OTA接口运行正常，向设备发送的websocket地址是：{websocket_url}"
|
||||
response = web.Response(text=message, content_type="text/plain")
|
||||
except Exception as e:
|
||||
@@ -91,3 +368,48 @@ class OTAHandler(BaseHandler):
|
||||
finally:
|
||||
self._add_cors_headers(response)
|
||||
return response
|
||||
|
||||
async def handle_download(self, request):
|
||||
"""
|
||||
下载固件接口
|
||||
URL: /xiaozhi/ota/download/{filename}
|
||||
- 只允许下载 data/bin 目录下的 .bin 文件
|
||||
- filename 必须是 basename 且匹配安全的模式
|
||||
"""
|
||||
try:
|
||||
fname = request.match_info.get("filename", "")
|
||||
if not fname:
|
||||
raise web.HTTPBadRequest(text="filename required")
|
||||
|
||||
# sanitize
|
||||
fname = _safe_basename(fname)
|
||||
# pattern: allow letters, numbers, dot, underscore, dash
|
||||
if not re.match(r"^[A-Za-z0-9\.\-_]+\.bin$", fname):
|
||||
raise web.HTTPBadRequest(text="invalid filename")
|
||||
|
||||
file_path = os.path.join(self.bin_dir, fname)
|
||||
# ensure realpath is under bin_dir
|
||||
file_real = os.path.realpath(file_path)
|
||||
bin_dir_real = os.path.realpath(self.bin_dir)
|
||||
if (
|
||||
not file_real.startswith(bin_dir_real + os.sep)
|
||||
and file_real != bin_dir_real
|
||||
):
|
||||
raise web.HTTPForbidden(text="forbidden")
|
||||
|
||||
if not os.path.isfile(file_real):
|
||||
raise web.HTTPNotFound(text="file not found")
|
||||
|
||||
# use FileResponse to stream file
|
||||
resp = web.FileResponse(path=file_real)
|
||||
except web.HTTPError as e:
|
||||
resp = e
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"固件下载异常: {e}")
|
||||
resp = web.Response(text="download error", status=500)
|
||||
finally:
|
||||
try:
|
||||
self._add_cors_headers(resp)
|
||||
except Exception:
|
||||
pass
|
||||
return resp
|
||||
|
||||
@@ -2,6 +2,7 @@ import json
|
||||
import copy
|
||||
from aiohttp import web
|
||||
from config.logger import setup_logging
|
||||
from core.api.base_handler import BaseHandler
|
||||
from core.utils.util import get_vision_url, is_valid_image_file
|
||||
from core.utils.vllm import create_instance
|
||||
from config.config_loader import get_private_config_from_api
|
||||
@@ -16,10 +17,9 @@ TAG = __name__
|
||||
MAX_FILE_SIZE = 5 * 1024 * 1024
|
||||
|
||||
|
||||
class VisionHandler:
|
||||
class VisionHandler(BaseHandler):
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
self.logger = setup_logging()
|
||||
super().__init__(config)
|
||||
# 初始化认证工具
|
||||
self.auth = AuthToken(config["server"]["auth_key"])
|
||||
|
||||
@@ -96,7 +96,7 @@ class VisionHandler:
|
||||
current_config = copy.deepcopy(self.config)
|
||||
read_config_from_api = current_config.get("read_config_from_api", False)
|
||||
if read_config_from_api:
|
||||
current_config = get_private_config_from_api(
|
||||
current_config = await get_private_config_from_api(
|
||||
current_config,
|
||||
device_id,
|
||||
client_id,
|
||||
@@ -172,11 +172,3 @@ class VisionHandler:
|
||||
finally:
|
||||
self._add_cors_headers(response)
|
||||
return response
|
||||
|
||||
def _add_cors_headers(self, response):
|
||||
"""添加CORS头信息"""
|
||||
response.headers["Access-Control-Allow-Headers"] = (
|
||||
"client-id, content-type, device-id"
|
||||
)
|
||||
response.headers["Access-Control-Allow-Credentials"] = "true"
|
||||
response.headers["Access-Control-Allow-Origin"] = "*"
|
||||
|
||||
@@ -1,54 +1,118 @@
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
import hmac
|
||||
import base64
|
||||
import hashlib
|
||||
import time
|
||||
|
||||
|
||||
class AuthenticationError(Exception):
|
||||
"""认证异常"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class AuthMiddleware:
|
||||
def __init__(self, config):
|
||||
"""
|
||||
认证中间件（兼容旧版命名）
|
||||
用于 WebSocket/MQTT 连接认证
|
||||
"""
|
||||
|
||||
def __init__(self, config: dict):
|
||||
"""
|
||||
初始化认证中间件
|
||||
|
||||
Args:
|
||||
config: 配置字典，包含认证相关配置
|
||||
"""
|
||||
self.config = config
|
||||
self.auth_config = config["server"].get("auth", {})
|
||||
# 构建token查找表
|
||||
self.tokens = {
|
||||
item["token"]: item["name"]
|
||||
for item in self.auth_config.get("tokens", [])
|
||||
}
|
||||
# 设备白名单
|
||||
self.allowed_devices = set(
|
||||
self.auth_config.get("allowed_devices", [])
|
||||
)
|
||||
auth_config = config.get("server", {}).get("auth", {})
|
||||
self.enabled = auth_config.get("enabled", False)
|
||||
self.tokens = auth_config.get("tokens", [])
|
||||
self.allowed_devices = auth_config.get("allowed_devices", [])
|
||||
|
||||
async def authenticate(self, headers):
|
||||
"""验证连接请求"""
|
||||
# 检查是否启用认证
|
||||
if not self.auth_config.get("enabled", False):
|
||||
def authenticate(self, device_id: str, token: str = None) -> bool:
|
||||
"""
|
||||
验证设备认证
|
||||
|
||||
Args:
|
||||
device_id: 设备 ID
|
||||
token: 认证令牌
|
||||
|
||||
Returns:
|
||||
bool: 认证是否通过
|
||||
"""
|
||||
if not self.enabled:
|
||||
return True
|
||||
|
||||
# 检查设备是否在白名单中
|
||||
device_id = headers.get("device-id", "")
|
||||
|
||||
if self.allowed_devices and device_id in self.allowed_devices:
|
||||
|
||||
# 检查白名单
|
||||
if device_id in self.allowed_devices:
|
||||
return True
|
||||
|
||||
# 检查 token
|
||||
if token:
|
||||
for token_config in self.tokens:
|
||||
if token_config.get("token") == token:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
# 验证Authorization header
|
||||
auth_header = headers.get("authorization", "")
|
||||
if not auth_header.startswith("Bearer "):
|
||||
logger.bind(tag=TAG).error("Missing or invalid Authorization header")
|
||||
raise AuthenticationError("Missing or invalid Authorization header")
|
||||
|
||||
token = auth_header.split(" ")[1]
|
||||
if token not in self.tokens:
|
||||
logger.bind(tag=TAG).error(f"Invalid token: {token}")
|
||||
raise AuthenticationError("Invalid token")
|
||||
class AuthManager:
|
||||
"""
|
||||
统一授权认证管理器
|
||||
生成与验证 client_id device_id token（HMAC-SHA256）认证三元组
|
||||
token 中不含明文 client_id/device_id，只携带签名 + 时间戳; client_id/device_id在连接时传递
|
||||
在 MQTT 中 client_id: client_id, username: device_id, password: token
|
||||
在 Websocket 中，header:{Device-ID: device_id, Client-ID: client_id, Authorization: Bearer token, ......}
|
||||
"""
|
||||
|
||||
logger.bind(tag=TAG).info(f"Authentication successful - Device: {device_id}, Token: {self.tokens[token]}")
|
||||
return True
|
||||
def __init__(self, secret_key: str, expire_seconds: int = 60 * 60 * 24 * 30):
|
||||
if not expire_seconds or expire_seconds < 0:
|
||||
self.expire_seconds = 60 * 60 * 24 * 30
|
||||
else:
|
||||
self.expire_seconds = expire_seconds
|
||||
self.secret_key = secret_key
|
||||
|
||||
def get_token_name(self, token):
|
||||
"""获取token对应的设备名称"""
|
||||
return self.tokens.get(token)
|
||||
def _sign(self, content: str) -> str:
|
||||
"""HMAC-SHA256签名并Base64编码"""
|
||||
sig = hmac.new(
|
||||
self.secret_key.encode("utf-8"), content.encode("utf-8"), hashlib.sha256
|
||||
).digest()
|
||||
return base64.urlsafe_b64encode(sig).decode("utf-8").rstrip("=")
|
||||
|
||||
def generate_token(self, client_id: str, username: str) -> str:
|
||||
"""
|
||||
生成 token
|
||||
Args:
|
||||
client_id: 设备连接ID
|
||||
username: 设备用户名（通常为deviceId）
|
||||
Returns:
|
||||
str: token字符串
|
||||
"""
|
||||
ts = int(time.time())
|
||||
content = f"{client_id}|{username}|{ts}"
|
||||
signature = self._sign(content)
|
||||
# token仅包含签名与时间戳，不包含明文信息
|
||||
token = f"{signature}.{ts}"
|
||||
return token
|
||||
|
||||
def verify_token(self, token: str, client_id: str, username: str) -> bool:
|
||||
"""
|
||||
验证token有效性
|
||||
Args:
|
||||
token: 客户端传入的token
|
||||
client_id: 连接使用的client_id
|
||||
username: 连接使用的username
|
||||
"""
|
||||
try:
|
||||
sig_part, ts_str = token.split(".")
|
||||
ts = int(ts_str)
|
||||
if int(time.time()) - ts > self.expire_seconds:
|
||||
return False # 过期
|
||||
|
||||
expected_sig = self._sign(f"{client_id}|{username}|{ts}")
|
||||
if not hmac.compare_digest(sig_part, expected_sig):
|
||||
return False
|
||||
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@@ -10,6 +10,7 @@ import threading
|
||||
import traceback
|
||||
import subprocess
|
||||
import websockets
|
||||
|
||||
from core.utils.util import (
|
||||
extract_json_from_string,
|
||||
check_vad_update,
|
||||
@@ -69,9 +70,12 @@ class ConnectionHandler:
|
||||
self.logger = setup_logging()
|
||||
self.server = server # 保存server实例的引用
|
||||
|
||||
self.auth = AuthMiddleware(config)
|
||||
self.need_bind = False
|
||||
self.bind_code = None
|
||||
self.need_bind = False # 是否需要绑定设备
|
||||
self.bind_completed_event = asyncio.Event()
|
||||
self.bind_code = None # 绑定设备的验证码
|
||||
self.last_bind_prompt_time = 0 # 上次播放绑定提示的时间戳(秒)
|
||||
self.bind_prompt_interval = 60 # 绑定提示播放间隔(秒)
|
||||
|
||||
self.read_config_from_api = self.config.get("read_config_from_api", False)
|
||||
|
||||
self.websocket = None
|
||||
@@ -90,7 +94,7 @@ class ConnectionHandler:
|
||||
self.client_listen_mode = "auto"
|
||||
|
||||
# 线程任务相关
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.loop = None # 在 handle_connection 中获取运行中的事件循环
|
||||
self.stop_event = threading.Event()
|
||||
self.executor = ThreadPoolExecutor(max_workers=5)
|
||||
|
||||
@@ -117,9 +121,10 @@ class ConnectionHandler:
|
||||
# vad相关变量
|
||||
self.client_audio_buffer = bytearray()
|
||||
self.client_have_voice = False
|
||||
self.client_voice_window = deque(maxlen=5)
|
||||
self.first_activity_time = 0.0 # 记录首次活动的时间（毫秒）
|
||||
self.last_activity_time = 0.0 # 统一的活动时间戳（毫秒）
|
||||
self.client_voice_stop = False
|
||||
self.client_voice_window = deque(maxlen=5)
|
||||
self.last_is_voice = False
|
||||
|
||||
# asr相关变量
|
||||
@@ -156,8 +161,11 @@ class ConnectionHandler:
|
||||
# {"mcp":true} 表示启用MCP功能
|
||||
self.features = None
|
||||
|
||||
# 标记连接是否来自MQTT
|
||||
self.conn_from_mqtt_gateway = False
|
||||
|
||||
# 初始化提示词管理器
|
||||
self.prompt_manager = PromptManager(config, self.logger)
|
||||
self.prompt_manager = PromptManager(self.config, self.logger)
|
||||
|
||||
# 新增：会话上下文与组件管理器（会话级清理）
|
||||
self.session_context: SessionContext = SessionContext()
|
||||
@@ -166,27 +174,11 @@ class ConnectionHandler:
|
||||
|
||||
async def handle_connection(self, ws):
|
||||
try:
|
||||
# 获取运行中的事件循环（必须在异步上下文中）
|
||||
self.loop = asyncio.get_running_loop()
|
||||
|
||||
# 获取并验证headers
|
||||
self.headers = dict(ws.request.headers)
|
||||
|
||||
if self.headers.get("device-id", None) is None:
|
||||
# 尝试从 URL 的查询参数中获取 device-id
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
# 从 WebSocket 请求中获取路径
|
||||
request_path = ws.request.path
|
||||
if not request_path:
|
||||
self.logger.bind(tag=TAG).error("无法获取请求路径")
|
||||
return
|
||||
parsed_url = urlparse(request_path)
|
||||
query_params = parse_qs(parsed_url.query)
|
||||
if "device-id" in query_params:
|
||||
self.headers["device-id"] = query_params["device-id"][0]
|
||||
self.headers["client-id"] = query_params["client-id"][0]
|
||||
else:
|
||||
await ws.send("端口正常，如需测试连接，请使用test_page.html")
|
||||
await self.close(ws)
|
||||
return
|
||||
real_ip = self.headers.get("x-real-ip") or self.headers.get(
|
||||
"x-forwarded-for"
|
||||
)
|
||||
@@ -198,18 +190,24 @@ class ConnectionHandler:
|
||||
f"{self.client_ip} conn - Headers: {self.headers}"
|
||||
)
|
||||
|
||||
# 进行认证
|
||||
await self.auth.authenticate(self.headers)
|
||||
self.device_id = self.headers.get("device-id", None)
|
||||
|
||||
# 认证通过,继续处理
|
||||
self.websocket = ws
|
||||
self.device_id = self.headers.get("device-id", None)
|
||||
|
||||
# 更新会话上下文关键信息
|
||||
self.session_context.headers = self.headers
|
||||
self.session_context.device_id = self.device_id
|
||||
self.session_context.client_ip = self.client_ip
|
||||
|
||||
# 检查是否来自MQTT连接
|
||||
request_path = ws.request.path
|
||||
self.conn_from_mqtt_gateway = request_path.endswith("?from=mqtt_gateway")
|
||||
if self.conn_from_mqtt_gateway:
|
||||
self.logger.bind(tag=TAG).info("连接来自:MQTT网关")
|
||||
|
||||
# 初始化活动时间戳
|
||||
self.first_activity_time = time.time() * 1000
|
||||
self.last_activity_time = time.time() * 1000
|
||||
|
||||
# 启动超时检查任务
|
||||
@@ -219,10 +217,8 @@ class ConnectionHandler:
|
||||
self.welcome_msg = self.config.xiaozhi
|
||||
self.welcome_msg["session_id"] = self.session_id
|
||||
|
||||
# 获取差异化配置
|
||||
self._initialize_private_config()
|
||||
# 异步初始化
|
||||
self.executor.submit(self._initialize_components)
|
||||
# 在后台初始化配置和组件（完全不阻塞主循环）
|
||||
asyncio.create_task(self._background_initialize())
|
||||
|
||||
try:
|
||||
async for message in self.websocket:
|
||||
@@ -273,7 +269,9 @@ class ConnectionHandler:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
loop.run_until_complete(
|
||||
self.memory.save_memory(self.dialogue.dialogue)
|
||||
self.memory.save_memory(
|
||||
self.dialogue.dialogue, self.session_id
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"保存记忆失败: {e}")
|
||||
@@ -296,17 +294,116 @@ class ConnectionHandler:
|
||||
f"保存记忆后关闭连接失败: {close_error}"
|
||||
)
|
||||
|
||||
async def _discard_message_with_bind_prompt(self):
|
||||
"""丢弃消息并检查是否需要播放绑定提示"""
|
||||
current_time = time.time()
|
||||
# 检查是否需要播放绑定提示
|
||||
if current_time - self.last_bind_prompt_time >= self.bind_prompt_interval:
|
||||
self.last_bind_prompt_time = current_time
|
||||
# 复用现有的绑定提示逻辑
|
||||
from core.handle.receiveAudioHandle import check_bind_device
|
||||
|
||||
asyncio.create_task(check_bind_device(self))
|
||||
|
||||
async def _route_message(self, message):
|
||||
"""消息路由"""
|
||||
# 检查是否已经获取到真实的绑定状态
|
||||
if not self.bind_completed_event.is_set():
|
||||
# 还没有获取到真实状态，等待直到获取到真实状态或超时
|
||||
try:
|
||||
await asyncio.wait_for(self.bind_completed_event.wait(), timeout=1)
|
||||
except asyncio.TimeoutError:
|
||||
# 超时仍未获取到真实状态，丢弃消息
|
||||
await self._discard_message_with_bind_prompt()
|
||||
return
|
||||
|
||||
# 已经获取到真实状态，检查是否需要绑定
|
||||
if self.need_bind:
|
||||
# 需要绑定，丢弃消息
|
||||
await self._discard_message_with_bind_prompt()
|
||||
return
|
||||
|
||||
# 不需要绑定，继续处理消息
|
||||
|
||||
if isinstance(message, str):
|
||||
await handleTextMessage(self, message)
|
||||
elif isinstance(message, bytes):
|
||||
if self.vad is None:
|
||||
return
|
||||
if self.asr is None:
|
||||
if self.vad is None or self.asr is None:
|
||||
return
|
||||
|
||||
# 处理来自MQTT网关的音频包
|
||||
if self.conn_from_mqtt_gateway and len(message) >= 16:
|
||||
handled = await self._process_mqtt_audio_message(message)
|
||||
if handled:
|
||||
return
|
||||
|
||||
# 不需要头部处理或没有头部时，直接处理原始消息
|
||||
self.asr_audio_queue.put(message)
|
||||
|
||||
async def _process_mqtt_audio_message(self, message):
|
||||
"""
|
||||
处理来自MQTT网关的音频消息，解析16字节头部并提取音频数据
|
||||
|
||||
Args:
|
||||
message: 包含头部的音频消息
|
||||
|
||||
Returns:
|
||||
bool: 是否成功处理了消息
|
||||
"""
|
||||
try:
|
||||
# 提取头部信息
|
||||
timestamp = int.from_bytes(message[8:12], "big")
|
||||
audio_length = int.from_bytes(message[12:16], "big")
|
||||
|
||||
# 提取音频数据
|
||||
if audio_length > 0 and len(message) >= 16 + audio_length:
|
||||
# 有指定长度，提取精确的音频数据
|
||||
audio_data = message[16 : 16 + audio_length]
|
||||
# 基于时间戳进行排序处理
|
||||
self._process_websocket_audio(audio_data, timestamp)
|
||||
return True
|
||||
elif len(message) > 16:
|
||||
# 没有指定长度或长度无效，去掉头部后处理剩余数据
|
||||
audio_data = message[16:]
|
||||
self.asr_audio_queue.put(audio_data)
|
||||
return True
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"解析WebSocket音频包失败: {e}")
|
||||
|
||||
# 处理失败，返回False表示需要继续处理
|
||||
return False
|
||||
|
||||
def _process_websocket_audio(self, audio_data, timestamp):
|
||||
"""处理WebSocket格式的音频包"""
|
||||
# 初始化时间戳序列管理
|
||||
if not hasattr(self, "audio_timestamp_buffer"):
|
||||
self.audio_timestamp_buffer = {}
|
||||
self.last_processed_timestamp = 0
|
||||
self.max_timestamp_buffer_size = 20
|
||||
|
||||
# 如果时间戳是递增的，直接处理
|
||||
if timestamp >= self.last_processed_timestamp:
|
||||
self.asr_audio_queue.put(audio_data)
|
||||
self.last_processed_timestamp = timestamp
|
||||
|
||||
# 处理缓冲区中的后续包
|
||||
processed_any = True
|
||||
while processed_any:
|
||||
processed_any = False
|
||||
for ts in sorted(self.audio_timestamp_buffer.keys()):
|
||||
if ts > self.last_processed_timestamp:
|
||||
buffered_audio = self.audio_timestamp_buffer.pop(ts)
|
||||
self.asr_audio_queue.put(buffered_audio)
|
||||
self.last_processed_timestamp = ts
|
||||
processed_any = True
|
||||
break
|
||||
else:
|
||||
# 乱序包，暂存
|
||||
if len(self.audio_timestamp_buffer) < self.max_timestamp_buffer_size:
|
||||
self.audio_timestamp_buffer[timestamp] = audio_data
|
||||
else:
|
||||
self.asr_audio_queue.put(audio_data)
|
||||
|
||||
async def handle_restart(self, message):
|
||||
"""处理服务器重启请求"""
|
||||
try:
|
||||
@@ -357,6 +454,15 @@ class ConnectionHandler:
|
||||
|
||||
def _initialize_components(self):
|
||||
try:
|
||||
if self.tts is None:
|
||||
self.tts = self._initialize_tts()
|
||||
# 打开语音合成通道
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.tts.open_audio_channels(self), self.loop
|
||||
)
|
||||
if self.need_bind:
|
||||
self.bind_completed_event.set()
|
||||
return
|
||||
self.selected_module_str = build_module_string(
|
||||
self.config.get("selected_module", {})
|
||||
)
|
||||
@@ -380,17 +486,10 @@ class ConnectionHandler:
|
||||
|
||||
# 初始化声纹识别
|
||||
self._initialize_voiceprint()
|
||||
|
||||
# 打开语音识别通道
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.asr.open_audio_channels(self), self.loop
|
||||
)
|
||||
if self.tts is None:
|
||||
self.tts = self._initialize_tts()
|
||||
# 打开语音合成通道
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.tts.open_audio_channels(self), self.loop
|
||||
)
|
||||
|
||||
"""加载记忆"""
|
||||
self._initialize_memory()
|
||||
@@ -405,6 +504,7 @@ class ConnectionHandler:
|
||||
self.logger.bind(tag=TAG).error(f"实例化组件失败: {e}")
|
||||
|
||||
def _init_prompt_enhancement(self):
|
||||
|
||||
# 更新上下文信息
|
||||
self.prompt_manager.update_context_info(self, self.client_ip)
|
||||
enhanced_prompt = self.prompt_manager.build_enhanced_prompt(
|
||||
@@ -412,7 +512,7 @@ class ConnectionHandler:
|
||||
)
|
||||
if enhanced_prompt:
|
||||
self.change_system_prompt(enhanced_prompt)
|
||||
self.logger.bind(tag=TAG).info("系统提示词已增强更新")
|
||||
self.logger.bind(tag=TAG).debug("系统提示词已增强更新")
|
||||
|
||||
def _init_report_threads(self):
|
||||
"""初始化ASR和TTS上报线程"""
|
||||
@@ -440,7 +540,11 @@ class ConnectionHandler:
|
||||
|
||||
def _initialize_asr(self):
|
||||
"""初始化ASR"""
|
||||
if self._asr.interface_type == InterfaceType.LOCAL:
|
||||
if (
|
||||
self._asr is not None
|
||||
and hasattr(self._asr, "interface_type")
|
||||
and self._asr.interface_type == InterfaceType.LOCAL
|
||||
):
|
||||
# 如果公共ASR是本地服务，则直接返回
|
||||
# 因为本地一个实例ASR，可以被多个连接共享
|
||||
asr = self._asr
|
||||
@@ -456,29 +560,46 @@ class ConnectionHandler:
|
||||
try:
|
||||
voiceprint_config = self.config.get("voiceprint", {})
|
||||
if voiceprint_config:
|
||||
self.voiceprint_provider = VoiceprintProvider(voiceprint_config)
|
||||
self.logger.bind(tag=TAG).info("声纹识别功能已在连接时动态启用")
|
||||
voiceprint_provider = VoiceprintProvider(voiceprint_config)
|
||||
if voiceprint_provider is not None and voiceprint_provider.enabled:
|
||||
self.voiceprint_provider = voiceprint_provider
|
||||
self.logger.bind(tag=TAG).info("声纹识别功能已在连接时动态启用")
|
||||
else:
|
||||
self.logger.bind(tag=TAG).warning("声纹识别功能启用但配置不完整")
|
||||
else:
|
||||
self.logger.bind(tag=TAG).info("声纹识别功能未启用或配置不完整")
|
||||
self.logger.bind(tag=TAG).info("声纹识别功能未启用")
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).warning(f"声纹识别初始化失败: {str(e)}")
|
||||
|
||||
def _initialize_private_config(self):
|
||||
"""如果是从配置文件获取，则进行二次实例化"""
|
||||
async def _background_initialize(self):
|
||||
"""在后台初始化配置和组件（完全不阻塞主循环）"""
|
||||
try:
|
||||
# 异步获取差异化配置
|
||||
await self._initialize_private_config_async()
|
||||
# 在线程池中初始化组件
|
||||
self.executor.submit(self._initialize_components)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"后台初始化失败: {e}")
|
||||
|
||||
async def _initialize_private_config_async(self):
|
||||
"""从接口异步获取差异化配置（异步版本，不阻塞主循环）"""
|
||||
if not self.read_config_from_api:
|
||||
self.need_bind = False
|
||||
self.bind_completed_event.set()
|
||||
return
|
||||
"""从接口获取差异化的配置进行二次实例化，非全量重新实例化"""
|
||||
try:
|
||||
begin_time = time.time()
|
||||
private_config = get_private_config_from_api(
|
||||
private_config = await get_private_config_from_api(
|
||||
self.config,
|
||||
self.headers.get("device-id"),
|
||||
self.headers.get("client-id", self.headers.get("device-id")),
|
||||
)
|
||||
private_config["delete_audio"] = bool(self.config.get("delete_audio", True))
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"{time.time() - begin_time} 秒，获取差异化配置成功: {json.dumps(filter_sensitive_info(private_config), ensure_ascii=False)}"
|
||||
f"{time.time() - begin_time} 秒，异步获取差异化配置成功: {json.dumps(filter_sensitive_info(private_config), ensure_ascii=False)}"
|
||||
)
|
||||
self.need_bind = False
|
||||
self.bind_completed_event.set()
|
||||
except DeviceNotFoundException as e:
|
||||
self.need_bind = True
|
||||
private_config = {}
|
||||
@@ -488,7 +609,7 @@ class ConnectionHandler:
|
||||
private_config = {}
|
||||
except Exception as e:
|
||||
self.need_bind = True
|
||||
self.logger.bind(tag=TAG).error(f"获取差异化配置失败: {e}")
|
||||
self.logger.bind(tag=TAG).error(f"异步获取差异化配置失败: {e}")
|
||||
private_config = {}
|
||||
|
||||
init_llm, init_tts, init_memory, init_intent = (
|
||||
@@ -562,8 +683,14 @@ class ConnectionHandler:
|
||||
self.chat_history_conf = int(private_config["chat_history_conf"])
|
||||
if private_config.get("mcp_endpoint", None) is not None:
|
||||
self.config["mcp_endpoint"] = private_config["mcp_endpoint"]
|
||||
if private_config.get("context_providers", None) is not None:
|
||||
self.config["context_providers"] = private_config["context_providers"]
|
||||
|
||||
# 使用 run_in_executor 在线程池中执行 initialize_modules，避免阻塞主循环
|
||||
try:
|
||||
modules = initialize_modules(
|
||||
modules = await self.loop.run_in_executor(
|
||||
None, # 使用默认线程池
|
||||
initialize_modules,
|
||||
self.logger,
|
||||
private_config,
|
||||
init_vad,
|
||||
@@ -685,11 +812,12 @@ class ConnectionHandler:
|
||||
self.dialogue.update_system_message(self.prompt)
|
||||
|
||||
def chat(self, query, depth=0):
|
||||
self.logger.bind(tag=TAG).info(f"大模型收到用户消息: {query}")
|
||||
self.llm_finish_task = False
|
||||
if query is not None:
|
||||
self.logger.bind(tag=TAG).info(f"大模型收到用户消息: {query}")
|
||||
|
||||
# 为最顶层时新建会话ID和发送FIRST请求
|
||||
if depth == 0:
|
||||
self.llm_finish_task = False
|
||||
self.sentence_id = str(uuid.uuid4().hex)
|
||||
self.dialogue.put(Message(role="user", content=query))
|
||||
self.tts.tts_text_queue.put(
|
||||
@@ -700,9 +828,31 @@ class ConnectionHandler:
|
||||
)
|
||||
)
|
||||
|
||||
# 设置最大递归深度，避免无限循环，可根据实际需求调整
|
||||
MAX_DEPTH = 5
|
||||
force_final_answer = False # 标记是否强制最终回答
|
||||
|
||||
if depth >= MAX_DEPTH:
|
||||
self.logger.bind(tag=TAG).debug(
|
||||
f"已达到最大工具调用深度 {MAX_DEPTH}，将强制基于现有信息回答"
|
||||
)
|
||||
force_final_answer = True
|
||||
# 添加系统指令，要求 LLM 基于现有信息回答
|
||||
self.dialogue.put(
|
||||
Message(
|
||||
role="user",
|
||||
content="[系统提示] 已达到最大工具调用次数限制，请你基于目前已经获取的所有信息，直接给出最终答案。不要再尝试调用任何工具。",
|
||||
)
|
||||
)
|
||||
|
||||
# Define intent functions
|
||||
functions = None
|
||||
if self.intent_type == "function_call" and hasattr(self, "func_handler"):
|
||||
# 达到最大深度时，禁用工具调用，强制 LLM 直接回答
|
||||
if (
|
||||
self.intent_type == "function_call"
|
||||
and hasattr(self, "func_handler")
|
||||
and not force_final_answer
|
||||
):
|
||||
functions = self.func_handler.get_functions()
|
||||
response_message = []
|
||||
|
||||
@@ -737,9 +887,8 @@ class ConnectionHandler:
|
||||
|
||||
# 处理流式响应
|
||||
tool_call_flag = False
|
||||
function_name = None
|
||||
function_id = None
|
||||
function_arguments = ""
|
||||
# 支持多个并行工具调用 - 使用列表存储
|
||||
tool_calls_list = [] # 格式: [{"id": "", "name": "", "arguments": ""}]
|
||||
content_arguments = ""
|
||||
self.client_abort = False
|
||||
emotion_flag = True
|
||||
@@ -760,12 +909,7 @@ class ConnectionHandler:
|
||||
|
||||
if tools_call is not None and len(tools_call) > 0:
|
||||
tool_call_flag = True
|
||||
if tools_call[0].id is not None:
|
||||
function_id = tools_call[0].id
|
||||
if tools_call[0].function.name is not None:
|
||||
function_name = tools_call[0].function.name
|
||||
if tools_call[0].function.arguments is not None:
|
||||
function_arguments += tools_call[0].function.arguments
|
||||
self._merge_tool_calls(tool_calls_list, tools_call)
|
||||
else:
|
||||
content = response
|
||||
|
||||
@@ -791,16 +935,22 @@ class ConnectionHandler:
|
||||
# 处理function call
|
||||
if tool_call_flag:
|
||||
bHasError = False
|
||||
if function_id is None:
|
||||
# 处理基于文本的工具调用格式
|
||||
if len(tool_calls_list) == 0 and content_arguments:
|
||||
a = extract_json_from_string(content_arguments)
|
||||
if a is not None:
|
||||
try:
|
||||
content_arguments_json = json.loads(a)
|
||||
function_name = content_arguments_json["name"]
|
||||
function_arguments = json.dumps(
|
||||
content_arguments_json["arguments"], ensure_ascii=False
|
||||
tool_calls_list.append(
|
||||
{
|
||||
"id": str(uuid.uuid4().hex),
|
||||
"name": content_arguments_json["name"],
|
||||
"arguments": json.dumps(
|
||||
content_arguments_json["arguments"],
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
)
|
||||
function_id = str(uuid.uuid4().hex)
|
||||
except Exception as e:
|
||||
bHasError = True
|
||||
response_message.append(a)
|
||||
@@ -811,30 +961,43 @@ class ConnectionHandler:
|
||||
self.logger.bind(tag=TAG).error(
|
||||
f"function call error: {content_arguments}"
|
||||
)
|
||||
if not bHasError:
|
||||
|
||||
if not bHasError and len(tool_calls_list) > 0:
|
||||
# 如需要大模型先处理一轮，添加相关处理后的日志情况
|
||||
if len(response_message) > 0:
|
||||
text_buff = "".join(response_message)
|
||||
self.tts_MessageText = text_buff
|
||||
self.dialogue.put(Message(role="assistant", content=text_buff))
|
||||
response_message.clear()
|
||||
self.logger.bind(tag=TAG).debug(
|
||||
f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}"
|
||||
)
|
||||
function_call_data = {
|
||||
"name": function_name,
|
||||
"id": function_id,
|
||||
"arguments": function_arguments,
|
||||
}
|
||||
|
||||
# 使用统一工具处理器处理所有工具调用
|
||||
result = asyncio.run_coroutine_threadsafe(
|
||||
self.func_handler.handle_llm_function_call(
|
||||
self, function_call_data
|
||||
),
|
||||
self.loop,
|
||||
).result()
|
||||
self._handle_function_result(result, function_call_data, depth=depth)
|
||||
self.logger.bind(tag=TAG).debug(
|
||||
f"检测到 {len(tool_calls_list)} 个工具调用"
|
||||
)
|
||||
|
||||
# 收集所有工具调用的 Future
|
||||
futures_with_data = []
|
||||
for tool_call_data in tool_calls_list:
|
||||
self.logger.bind(tag=TAG).debug(
|
||||
f"function_name={tool_call_data['name']}, function_id={tool_call_data['id']}, function_arguments={tool_call_data['arguments']}"
|
||||
)
|
||||
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.func_handler.handle_llm_function_call(
|
||||
self, tool_call_data
|
||||
),
|
||||
self.loop,
|
||||
)
|
||||
futures_with_data.append((future, tool_call_data))
|
||||
|
||||
# 等待协程结束（实际等待时长为最慢的那个）
|
||||
tool_results = []
|
||||
for future, tool_call_data in futures_with_data:
|
||||
result = future.result()
|
||||
tool_results.append((result, tool_call_data))
|
||||
|
||||
# 统一处理所有工具调用结果
|
||||
if tool_results:
|
||||
self._handle_function_result(tool_results, depth=depth)
|
||||
|
||||
# 存储对话内容
|
||||
if len(response_message) > 0:
|
||||
@@ -849,60 +1012,69 @@ class ConnectionHandler:
|
||||
content_type=ContentType.ACTION,
|
||||
)
|
||||
)
|
||||
self.llm_finish_task = True
|
||||
# 使用lambda延迟计算，只有在DEBUG级别时才执行get_llm_dialogue()
|
||||
self.logger.bind(tag=TAG).debug(
|
||||
lambda: json.dumps(
|
||||
self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False
|
||||
self.llm_finish_task = True
|
||||
# 使用lambda延迟计算，只有在DEBUG级别时才执行get_llm_dialogue()
|
||||
self.logger.bind(tag=TAG).debug(
|
||||
lambda: json.dumps(
|
||||
self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
def _handle_function_result(self, result, function_call_data, depth):
|
||||
if result.action == Action.RESPONSE: # 直接回复前端
|
||||
text = result.response
|
||||
self.tts.tts_one_sentence(self, ContentType.TEXT, content_detail=text)
|
||||
self.dialogue.put(Message(role="assistant", content=text))
|
||||
elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
|
||||
text = result.result
|
||||
if text is not None and len(text) > 0:
|
||||
function_id = function_call_data["id"]
|
||||
function_name = function_call_data["name"]
|
||||
function_arguments = function_call_data["arguments"]
|
||||
self.dialogue.put(
|
||||
Message(
|
||||
role="assistant",
|
||||
tool_calls=[
|
||||
{
|
||||
"id": function_id,
|
||||
"function": {
|
||||
"arguments": "{}" if function_arguments == "" else function_arguments,
|
||||
"name": function_name,
|
||||
},
|
||||
"type": "function",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
)
|
||||
)
|
||||
def _handle_function_result(self, tool_results, depth):
|
||||
need_llm_tools = []
|
||||
|
||||
self.dialogue.put(
|
||||
Message(
|
||||
role="tool",
|
||||
tool_call_id=(
|
||||
str(uuid.uuid4()) if function_id is None else function_id
|
||||
for result, tool_call_data in tool_results:
|
||||
if result.action in [
|
||||
Action.RESPONSE,
|
||||
Action.NOTFOUND,
|
||||
Action.ERROR,
|
||||
]: # 直接回复前端
|
||||
text = result.response if result.response else result.result
|
||||
self.tts.tts_one_sentence(self, ContentType.TEXT, content_detail=text)
|
||||
self.dialogue.put(Message(role="assistant", content=text))
|
||||
elif result.action == Action.REQLLM:
|
||||
# 收集需要 LLM 处理的工具
|
||||
need_llm_tools.append((result, tool_call_data))
|
||||
else:
|
||||
pass
|
||||
|
||||
if need_llm_tools:
|
||||
all_tool_calls = [
|
||||
{
|
||||
"id": tool_call_data["id"],
|
||||
"function": {
|
||||
"arguments": (
|
||||
"{}"
|
||||
if tool_call_data["arguments"] == ""
|
||||
else tool_call_data["arguments"]
|
||||
),
|
||||
content=text,
|
||||
"name": tool_call_data["name"],
|
||||
},
|
||||
"type": "function",
|
||||
"index": idx,
|
||||
}
|
||||
for idx, (_, tool_call_data) in enumerate(need_llm_tools)
|
||||
]
|
||||
self.dialogue.put(Message(role="assistant", tool_calls=all_tool_calls))
|
||||
|
||||
for result, tool_call_data in need_llm_tools:
|
||||
text = result.result
|
||||
if text is not None and len(text) > 0:
|
||||
self.dialogue.put(
|
||||
Message(
|
||||
role="tool",
|
||||
tool_call_id=(
|
||||
str(uuid.uuid4())
|
||||
if tool_call_data["id"] is None
|
||||
else tool_call_data["id"]
|
||||
),
|
||||
content=text,
|
||||
)
|
||||
)
|
||||
)
|
||||
self.chat(text, depth=depth + 1)
|
||||
elif result.action == Action.NOTFOUND or result.action == Action.ERROR:
|
||||
text = result.response if result.response else result.result
|
||||
self.tts.tts_one_sentence(self, ContentType.TEXT, content_detail=text)
|
||||
self.dialogue.put(Message(role="assistant", content=text))
|
||||
else:
|
||||
pass
|
||||
|
||||
self.chat(None, depth=depth + 1)
|
||||
|
||||
def _report_worker(self):
|
||||
"""聊天记录上报工作线程"""
|
||||
@@ -930,9 +1102,9 @@ class ConnectionHandler:
|
||||
def _process_report(self, type, text, audio_data, report_time):
|
||||
"""处理上报任务"""
|
||||
try:
|
||||
# 执行上报（传入二进制数据）
|
||||
# report(self, type, text, audio_data, report_time) # 旧架构，已被新架构替代
|
||||
pass # 新架构中由ReportProcessor处理
|
||||
# 执行异步上报（在事件循环中运行）
|
||||
from core.handle.reportHandle import report
|
||||
asyncio.run(report(self, type, text, audio_data, report_time))
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"上报处理异常: {e}")
|
||||
finally:
|
||||
@@ -946,6 +1118,10 @@ class ConnectionHandler:
|
||||
async def close(self, ws=None):
|
||||
"""资源清理方法"""
|
||||
try:
|
||||
# 清理音频缓冲区
|
||||
if hasattr(self, "audio_buffer"):
|
||||
self.audio_buffer.clear()
|
||||
|
||||
# 取消超时任务
|
||||
if self.timeout_task and not self.timeout_task.done():
|
||||
self.timeout_task.cancel()
|
||||
@@ -1019,7 +1195,6 @@ class ConnectionHandler:
|
||||
f"关闭线程池时出错: {executor_error}"
|
||||
)
|
||||
self.executor = None
|
||||
|
||||
self.logger.bind(tag=TAG).info("连接资源已释放")
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"关闭连接时出错: {e}")
|
||||
@@ -1049,6 +1224,11 @@ class ConnectionHandler:
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
# 重置音频流控器（取消后台任务并清空队列）
|
||||
if hasattr(self, "audio_rate_controller") and self.audio_rate_controller:
|
||||
self.audio_rate_controller.reset()
|
||||
self.logger.bind(tag=TAG).debug("已重置音频流控器")
|
||||
|
||||
self.logger.bind(tag=TAG).debug(
|
||||
f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
|
||||
)
|
||||
@@ -1074,13 +1254,14 @@ class ConnectionHandler:
|
||||
"""检查连接超时"""
|
||||
try:
|
||||
while not self.stop_event.is_set():
|
||||
last_activity_time = self.last_activity_time
|
||||
if self.need_bind:
|
||||
last_activity_time = self.first_activity_time
|
||||
|
||||
# 检查是否超时（只有在时间戳已初始化的情况下）
|
||||
if self.last_activity_time > 0.0:
|
||||
if last_activity_time > 0.0:
|
||||
current_time = time.time() * 1000
|
||||
if (
|
||||
current_time - self.last_activity_time
|
||||
> self.timeout_seconds * 1000
|
||||
):
|
||||
if current_time - last_activity_time > self.timeout_seconds * 1000:
|
||||
if not self.stop_event.is_set():
|
||||
self.logger.bind(tag=TAG).info("连接超时，准备关闭")
|
||||
# 设置停止事件，防止重复处理
|
||||
@@ -1099,3 +1280,31 @@ class ConnectionHandler:
|
||||
self.logger.bind(tag=TAG).error(f"超时检查任务出错: {e}")
|
||||
finally:
|
||||
self.logger.bind(tag=TAG).info("超时检查任务已退出")
|
||||
|
||||
def _merge_tool_calls(self, tool_calls_list, tools_call):
|
||||
"""合并工具调用列表
|
||||
|
||||
Args:
|
||||
tool_calls_list: 已收集的工具调用列表
|
||||
tools_call: 新的工具调用
|
||||
"""
|
||||
for tool_call in tools_call:
|
||||
tool_index = getattr(tool_call, "index", None)
|
||||
if tool_index is None:
|
||||
if tool_call.function.name:
|
||||
# 有 function_name，说明是新的工具调用
|
||||
tool_index = len(tool_calls_list)
|
||||
else:
|
||||
tool_index = len(tool_calls_list) - 1 if tool_calls_list else 0
|
||||
|
||||
# 确保列表有足够的位置
|
||||
if tool_index >= len(tool_calls_list):
|
||||
tool_calls_list.append({"id": "", "name": "", "arguments": ""})
|
||||
|
||||
# 更新工具调用信息
|
||||
if tool_call.id:
|
||||
tool_calls_list[tool_index]["id"] = tool_call.id
|
||||
if tool_call.function.name:
|
||||
tool_calls_list[tool_index]["name"] = tool_call.function.name
|
||||
if tool_call.function.arguments:
|
||||
tool_calls_list[tool_index]["arguments"] += tool_call.function.arguments
|
||||
|
||||
@@ -6,7 +6,7 @@ from core.utils.dialogue import Message
|
||||
from core.utils.util import audio_to_data
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
from core.utils.wakeup_word import WakeupWordsConfig
|
||||
from core.handle.sendAudioHandle import sendAudioMessage, send_stt_message
|
||||
from core.handle.sendAudioHandle import sendAudioMessage, send_tts_message
|
||||
from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes
|
||||
from core.providers.tools.device_mcp import (
|
||||
MCPClient,
|
||||
@@ -17,8 +17,18 @@ from core.providers.tools.device_mcp import (
|
||||
TAG = __name__
|
||||
|
||||
WAKEUP_CONFIG = {
|
||||
"refresh_time": 5,
|
||||
"words": ["你好", "你好啊", "嘿，你好", "嗨"],
|
||||
"refresh_time": 10,
|
||||
"responses": [
|
||||
"我一直都在呢，您请说。",
|
||||
"在的呢，请随时吩咐我。",
|
||||
"来啦来啦，请告诉我吧。",
|
||||
"您请说，我正听着。",
|
||||
"请您讲话，我准备好了。",
|
||||
"请您说出指令吧。",
|
||||
"我认真听着呢，请讲。",
|
||||
"请问您需要什么帮助？",
|
||||
"我在这里，等候您的指令。",
|
||||
],
|
||||
}
|
||||
|
||||
# 创建全局的唤醒词配置管理器
|
||||
@@ -33,15 +43,15 @@ async def handleHelloMessage(conn, msg_json):
|
||||
audio_params = msg_json.get("audio_params")
|
||||
if audio_params:
|
||||
format = audio_params.get("format")
|
||||
conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}")
|
||||
conn.logger.bind(tag=TAG).debug(f"客户端音频格式: {format}")
|
||||
conn.audio_format = format
|
||||
conn.welcome_msg["audio_params"] = audio_params
|
||||
features = msg_json.get("features")
|
||||
if features:
|
||||
conn.logger.bind(tag=TAG).info(f"客户端特性: {features}")
|
||||
conn.logger.bind(tag=TAG).debug(f"客户端特性: {features}")
|
||||
conn.features = features
|
||||
if features.get("mcp"):
|
||||
conn.logger.bind(tag=TAG).info("客户端支持MCP")
|
||||
conn.logger.bind(tag=TAG).debug("客户端支持MCP")
|
||||
conn.mcp_client = MCPClient()
|
||||
# 发送初始化
|
||||
asyncio.create_task(send_mcp_initialize_message(conn))
|
||||
@@ -73,7 +83,7 @@ async def checkWakeupWords(conn, text):
|
||||
return False
|
||||
|
||||
conn.just_woken_up = True
|
||||
await send_stt_message(conn, text)
|
||||
await send_tts_message(conn, "start")
|
||||
|
||||
# 获取当前音色
|
||||
voice = getattr(conn.tts, "voice", "default")
|
||||
@@ -85,13 +95,13 @@ async def checkWakeupWords(conn, text):
|
||||
if not response or not response.get("file_path"):
|
||||
response = {
|
||||
"voice": "default",
|
||||
"file_path": "config/assets/wakeup_words.wav",
|
||||
"file_path": "config/assets/wakeup_words_short.wav",
|
||||
"time": 0,
|
||||
"text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",
|
||||
"text": "我在这里哦！",
|
||||
}
|
||||
|
||||
# 获取音频数据
|
||||
opus_packets = audio_to_data(response.get("file_path"))
|
||||
opus_packets = await audio_to_data(response.get("file_path"), use_cache=False)
|
||||
# 播放唤醒词回复
|
||||
conn.client_abort = False
|
||||
|
||||
@@ -110,7 +120,7 @@ async def checkWakeupWords(conn, text):
|
||||
|
||||
|
||||
async def wakeupWordsResponse(conn):
|
||||
if not conn.tts or not conn.llm or not conn.llm.response_no_stream:
|
||||
if not conn.tts:
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -118,16 +128,8 @@ async def wakeupWordsResponse(conn):
|
||||
if not await _wakeup_response_lock.acquire():
|
||||
return
|
||||
|
||||
# 生成唤醒词回复
|
||||
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
|
||||
question = (
|
||||
"此刻用户正在和你说```"
|
||||
+ wakeup_word
|
||||
+ "```。\n请你根据以上用户的内容进行20-30字回复。要符合系统设置的角色情感和态度，不要像机器人一样说话。\n"
|
||||
+ "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。"
|
||||
)
|
||||
|
||||
result = conn.llm.response_no_stream(conn.config["prompt"], question)
|
||||
# 从预定义回复列表中随机选择一个回复
|
||||
result = random.choice(WAKEUP_CONFIG["responses"])
|
||||
if not result or len(result) == 0:
|
||||
return
|
||||
|
||||
@@ -148,4 +150,4 @@ async def wakeupWordsResponse(conn):
|
||||
finally:
|
||||
# 确保在任何情况下都释放锁
|
||||
if _wakeup_response_lock.locked():
|
||||
_wakeup_response_lock.release()
|
||||
_wakeup_response_lock.release()
|
||||
|
||||
@@ -91,6 +91,30 @@ async def process_intent_result(conn, intent_result, original_text):
|
||||
if function_name == "continue_chat":
|
||||
return False
|
||||
|
||||
if function_name == "result_for_context":
|
||||
await send_stt_message(conn, original_text)
|
||||
conn.client_abort = False
|
||||
|
||||
def process_context_result():
|
||||
conn.dialogue.put(Message(role="user", content=original_text))
|
||||
|
||||
from core.utils.current_time import get_current_time_info
|
||||
|
||||
current_time, today_date, today_weekday, lunar_date = get_current_time_info()
|
||||
|
||||
# 构建带上下文的基础提示
|
||||
context_prompt = f"""当前时间：{current_time}
|
||||
今天日期：{today_date} ({today_weekday})
|
||||
今天农历：{lunar_date}
|
||||
|
||||
请根据以上信息回答用户的问题：{original_text}"""
|
||||
|
||||
response = conn.intent.replyResult(context_prompt, original_text)
|
||||
speak_txt(conn, response)
|
||||
|
||||
conn.executor.submit(process_context_result)
|
||||
return True
|
||||
|
||||
function_args = {}
|
||||
if "arguments" in intent_data["function_call"]:
|
||||
function_args = intent_data["function_call"]["arguments"]
|
||||
|
||||
@@ -14,26 +14,29 @@ async def handleAudioMessage(conn, audio):
|
||||
# 当前片段是否有人说话
|
||||
have_voice = conn.vad.is_vad(conn, audio)
|
||||
# 如果设备刚刚被唤醒，短暂忽略VAD检测
|
||||
if have_voice and hasattr(conn, "just_woken_up") and conn.just_woken_up:
|
||||
if hasattr(conn, "just_woken_up") and conn.just_woken_up:
|
||||
have_voice = False
|
||||
# 设置一个短暂延迟后恢复VAD检测
|
||||
conn.asr_audio.clear()
|
||||
if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
|
||||
conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
|
||||
return
|
||||
# manual 模式下不打断正在播放的内容
|
||||
if have_voice:
|
||||
if conn.client_is_speaking:
|
||||
if conn.client_is_speaking and conn.client_listen_mode != "manual":
|
||||
await handleAbortMessage(conn)
|
||||
# 设备长时间空闲检测，用于say goodbye
|
||||
await no_voice_close_connect(conn, have_voice)
|
||||
# 接收音频
|
||||
await conn.asr.receive_audio(conn, audio, have_voice)
|
||||
|
||||
|
||||
async def resume_vad_detection(conn):
|
||||
# 等待2秒后恢复VAD检测
|
||||
await asyncio.sleep(1)
|
||||
await asyncio.sleep(2)
|
||||
conn.just_woken_up = False
|
||||
|
||||
|
||||
async def startToChat(conn, text):
|
||||
# 检查输入是否是JSON格式（包含说话人信息）
|
||||
speaker_name = None
|
||||
@@ -41,11 +44,11 @@ async def startToChat(conn, text):
|
||||
|
||||
try:
|
||||
# 尝试解析JSON格式的输入
|
||||
if text.strip().startswith('{') and text.strip().endswith('}'):
|
||||
if text.strip().startswith("{") and text.strip().endswith("}"):
|
||||
data = json.loads(text)
|
||||
if 'speaker' in data and 'content' in data:
|
||||
speaker_name = data['speaker']
|
||||
actual_text = data['content']
|
||||
if "speaker" in data and "content" in data:
|
||||
speaker_name = data["speaker"]
|
||||
actual_text = data["content"]
|
||||
conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}")
|
||||
|
||||
# 直接使用JSON格式的文本，不解析
|
||||
@@ -71,7 +74,8 @@ async def startToChat(conn, text):
|
||||
):
|
||||
await max_out_size(conn)
|
||||
return
|
||||
if conn.client_is_speaking:
|
||||
# manual 模式下不打断正在播放的内容
|
||||
if conn.client_is_speaking and conn.client_listen_mode != "manual":
|
||||
await handleAbortMessage(conn)
|
||||
|
||||
# 首先进行意图分析，使用实际文本内容
|
||||
@@ -119,7 +123,7 @@ async def max_out_size(conn):
|
||||
text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！"
|
||||
await send_stt_message(conn, text)
|
||||
file_path = "config/assets/max_output_size.wav"
|
||||
opus_packets = audio_to_data(file_path)
|
||||
opus_packets = await audio_to_data(file_path)
|
||||
conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))
|
||||
conn.close_after_chat = True
|
||||
|
||||
@@ -138,7 +142,7 @@ async def check_bind_device(conn):
|
||||
|
||||
# 播放提示音
|
||||
music_path = "config/assets/bind_code.wav"
|
||||
opus_packets = audio_to_data(music_path)
|
||||
opus_packets = await audio_to_data(music_path)
|
||||
conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, text))
|
||||
|
||||
# 逐个播放数字
|
||||
@@ -146,7 +150,7 @@ async def check_bind_device(conn):
|
||||
try:
|
||||
digit = conn.bind_code[i]
|
||||
num_path = f"config/assets/bind_code/{digit}.wav"
|
||||
num_packets = audio_to_data(num_path)
|
||||
num_packets = await audio_to_data(num_path)
|
||||
conn.tts.tts_audio_queue.put((SentenceType.MIDDLE, num_packets, None))
|
||||
except Exception as e:
|
||||
conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
|
||||
@@ -158,5 +162,5 @@ async def check_bind_device(conn):
|
||||
text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
|
||||
await send_stt_message(conn, text)
|
||||
music_path = "config/assets/bind_not_found.wav"
|
||||
opus_packets = audio_to_data(music_path)
|
||||
opus_packets = await audio_to_data(music_path)
|
||||
conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))
|
||||
|
||||
@@ -10,7 +10,6 @@ TTS上报功能已集成到ConnectionHandler类中。
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import opuslib_next
|
||||
|
||||
from config.manage_api_client import report as manage_report
|
||||
@@ -18,7 +17,7 @@ from config.manage_api_client import report as manage_report
|
||||
TAG = __name__
|
||||
|
||||
|
||||
def report(conn, type, text, opus_data, report_time):
|
||||
async def report(conn, type, text, opus_data, report_time):
|
||||
"""执行聊天记录上报操作
|
||||
|
||||
Args:
|
||||
@@ -33,8 +32,8 @@ def report(conn, type, text, opus_data, report_time):
|
||||
audio_data = opus_to_wav(conn, opus_data)
|
||||
else:
|
||||
audio_data = None
|
||||
# 执行上报
|
||||
manage_report(
|
||||
# 执行异步上报
|
||||
await manage_report(
|
||||
mac_address=conn.device_id,
|
||||
session_id=conn.session_id,
|
||||
chat_type=type,
|
||||
@@ -56,41 +55,49 @@ def opus_to_wav(conn, opus_data):
|
||||
Returns:
|
||||
bytes: WAV格式的音频数据
|
||||
"""
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
decoder = None
|
||||
try:
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
if not pcm_data:
|
||||
raise ValueError("没有有效的PCM数据")
|
||||
if not pcm_data:
|
||||
raise ValueError("没有有效的PCM数据")
|
||||
|
||||
# 创建WAV文件头
|
||||
pcm_data_bytes = b"".join(pcm_data)
|
||||
num_samples = len(pcm_data_bytes) // 2 # 16-bit samples
|
||||
# 创建WAV文件头
|
||||
pcm_data_bytes = b"".join(pcm_data)
|
||||
num_samples = len(pcm_data_bytes) // 2 # 16-bit samples
|
||||
|
||||
# WAV文件头
|
||||
wav_header = bytearray()
|
||||
wav_header.extend(b"RIFF") # ChunkID
|
||||
wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little")) # ChunkSize
|
||||
wav_header.extend(b"WAVE") # Format
|
||||
wav_header.extend(b"fmt ") # Subchunk1ID
|
||||
wav_header.extend((16).to_bytes(4, "little")) # Subchunk1Size
|
||||
wav_header.extend((1).to_bytes(2, "little")) # AudioFormat (PCM)
|
||||
wav_header.extend((1).to_bytes(2, "little")) # NumChannels
|
||||
wav_header.extend((16000).to_bytes(4, "little")) # SampleRate
|
||||
wav_header.extend((32000).to_bytes(4, "little")) # ByteRate
|
||||
wav_header.extend((2).to_bytes(2, "little")) # BlockAlign
|
||||
wav_header.extend((16).to_bytes(2, "little")) # BitsPerSample
|
||||
wav_header.extend(b"data") # Subchunk2ID
|
||||
wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little")) # Subchunk2Size
|
||||
# WAV文件头
|
||||
wav_header = bytearray()
|
||||
wav_header.extend(b"RIFF") # ChunkID
|
||||
wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little")) # ChunkSize
|
||||
wav_header.extend(b"WAVE") # Format
|
||||
wav_header.extend(b"fmt ") # Subchunk1ID
|
||||
wav_header.extend((16).to_bytes(4, "little")) # Subchunk1Size
|
||||
wav_header.extend((1).to_bytes(2, "little")) # AudioFormat (PCM)
|
||||
wav_header.extend((1).to_bytes(2, "little")) # NumChannels
|
||||
wav_header.extend((16000).to_bytes(4, "little")) # SampleRate
|
||||
wav_header.extend((32000).to_bytes(4, "little")) # ByteRate
|
||||
wav_header.extend((2).to_bytes(2, "little")) # BlockAlign
|
||||
wav_header.extend((16).to_bytes(2, "little")) # BitsPerSample
|
||||
wav_header.extend(b"data") # Subchunk2ID
|
||||
wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little")) # Subchunk2Size
|
||||
|
||||
# 返回完整的WAV数据
|
||||
return bytes(wav_header) + pcm_data_bytes
|
||||
# 返回完整的WAV数据
|
||||
return bytes(wav_header) + pcm_data_bytes
|
||||
finally:
|
||||
if decoder is not None:
|
||||
try:
|
||||
del decoder
|
||||
except Exception as e:
|
||||
conn.logger.bind(tag=TAG).debug(f"释放decoder资源时出错: {e}")
|
||||
|
||||
|
||||
def enqueue_tts_report(conn, text, opus_data):
|
||||
|
||||
@@ -4,8 +4,13 @@ import asyncio
|
||||
from core.utils import textUtils
|
||||
from core.utils.util import audio_to_data
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
from core.utils.audioRateController import AudioRateController
|
||||
|
||||
TAG = __name__
|
||||
# 音频帧时长（毫秒）
|
||||
AUDIO_FRAME_DURATION = 60
|
||||
# 预缓冲包数量，直接发送以减少延迟
|
||||
PRE_BUFFER_COUNT = 5
|
||||
|
||||
|
||||
async def sendAudioMessage(conn, sentenceType, audios, text):
|
||||
@@ -15,7 +20,19 @@ async def sendAudioMessage(conn, sentenceType, audios, text):
|
||||
await send_tts_message(conn, "start", None)
|
||||
|
||||
if sentenceType == SentenceType.FIRST:
|
||||
await send_tts_message(conn, "sentence_start", text)
|
||||
# 同一句子的后续消息加入流控队列，其他情况立即发送
|
||||
if (
|
||||
hasattr(conn, "audio_rate_controller")
|
||||
and conn.audio_rate_controller
|
||||
and getattr(conn, "audio_flow_control", {}).get("sentence_id")
|
||||
== conn.sentence_id
|
||||
):
|
||||
conn.audio_rate_controller.add_message(
|
||||
lambda: send_tts_message(conn, "sentence_start", text)
|
||||
)
|
||||
else:
|
||||
# 新句子或流控器未初始化，立即发送
|
||||
await send_tts_message(conn, "sentence_start", text)
|
||||
|
||||
await sendAudio(conn, audios)
|
||||
# 发送句子开始消息
|
||||
@@ -23,85 +40,203 @@ async def sendAudioMessage(conn, sentenceType, audios, text):
|
||||
conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}")
|
||||
|
||||
# 发送结束消息（如果是最后一个文本）
|
||||
if conn.llm_finish_task and sentenceType == SentenceType.LAST:
|
||||
if sentenceType == SentenceType.LAST:
|
||||
await send_tts_message(conn, "stop", None)
|
||||
conn.client_is_speaking = False
|
||||
if conn.close_after_chat:
|
||||
await conn.close()
|
||||
|
||||
|
||||
# 播放音频
|
||||
async def sendAudio(conn, audios, frame_duration=60):
|
||||
async def _wait_for_audio_completion(conn):
|
||||
"""
|
||||
发送单个opus包，支持流控
|
||||
等待音频队列清空并等待预缓冲包播放完成
|
||||
|
||||
Args:
|
||||
conn: 连接对象
|
||||
opus_packet: 单个opus数据包
|
||||
pre_buffer: 快速发送音频
|
||||
frame_duration: 帧时长（毫秒），匹配 Opus 编码
|
||||
"""
|
||||
if hasattr(conn, "audio_rate_controller") and conn.audio_rate_controller:
|
||||
rate_controller = conn.audio_rate_controller
|
||||
conn.logger.bind(tag=TAG).debug(
|
||||
f"等待音频发送完成，队列中还有 {len(rate_controller.queue)} 个包"
|
||||
)
|
||||
await rate_controller.queue_empty_event.wait()
|
||||
|
||||
# 等待预缓冲包播放完成
|
||||
# 前N个包直接发送，增加2个网络抖动包，需要额外等待它们在客户端播放完成
|
||||
frame_duration_ms = rate_controller.frame_duration
|
||||
pre_buffer_playback_time = (PRE_BUFFER_COUNT + 2) * frame_duration_ms / 1000.0
|
||||
await asyncio.sleep(pre_buffer_playback_time)
|
||||
|
||||
conn.logger.bind(tag=TAG).debug("音频发送完成")
|
||||
|
||||
|
||||
async def _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence):
|
||||
"""
|
||||
发送带16字节头部的opus数据包给mqtt_gateway
|
||||
Args:
|
||||
conn: 连接对象
|
||||
opus_packet: opus数据包
|
||||
timestamp: 时间戳
|
||||
sequence: 序列号
|
||||
"""
|
||||
# 为opus数据包添加16字节头部
|
||||
header = bytearray(16)
|
||||
header[0] = 1 # type
|
||||
header[2:4] = len(opus_packet).to_bytes(2, "big") # payload length
|
||||
header[4:8] = sequence.to_bytes(4, "big") # sequence
|
||||
header[8:12] = timestamp.to_bytes(4, "big") # 时间戳
|
||||
header[12:16] = len(opus_packet).to_bytes(4, "big") # opus长度
|
||||
|
||||
# 发送包含头部的完整数据包
|
||||
complete_packet = bytes(header) + opus_packet
|
||||
await conn.websocket.send(complete_packet)
|
||||
|
||||
|
||||
async def sendAudio(conn, audios, frame_duration=AUDIO_FRAME_DURATION):
|
||||
"""
|
||||
发送音频包，使用 AudioRateController 进行精确的流量控制
|
||||
|
||||
Args:
|
||||
conn: 连接对象
|
||||
audios: 单个opus包(bytes) 或 opus包列表
|
||||
frame_duration: 帧时长（毫秒），默认使用全局常量AUDIO_FRAME_DURATION
|
||||
"""
|
||||
if audios is None or len(audios) == 0:
|
||||
return
|
||||
|
||||
if isinstance(audios, bytes):
|
||||
send_delay = conn.config.get("tts_audio_send_delay", -1) / 1000.0
|
||||
is_single_packet = isinstance(audios, bytes)
|
||||
|
||||
# 初始化或获取 RateController
|
||||
rate_controller, flow_control = _get_or_create_rate_controller(
|
||||
conn, frame_duration, is_single_packet
|
||||
)
|
||||
|
||||
# 统一转换为列表处理
|
||||
audio_list = [audios] if is_single_packet else audios
|
||||
|
||||
# 发送音频包
|
||||
await _send_audio_with_rate_control(
|
||||
conn, audio_list, rate_controller, flow_control, send_delay
|
||||
)
|
||||
|
||||
|
||||
def _get_or_create_rate_controller(conn, frame_duration, is_single_packet):
|
||||
"""
|
||||
获取或创建 RateController 和 flow_control
|
||||
|
||||
Args:
|
||||
conn: 连接对象
|
||||
frame_duration: 帧时长
|
||||
is_single_packet: 是否单包模式（True: TTS流式单包, False: 批量包）
|
||||
|
||||
Returns:
|
||||
(rate_controller, flow_control)
|
||||
"""
|
||||
# 判断是否需要重置：单包模式且 sentence_id 变化，或者控制器不存在
|
||||
need_reset = (
|
||||
is_single_packet
|
||||
and getattr(conn, "audio_flow_control", {}).get("sentence_id")
|
||||
!= conn.sentence_id
|
||||
) or not hasattr(conn, "audio_rate_controller")
|
||||
|
||||
if need_reset:
|
||||
# 创建或获取 rate_controller
|
||||
if not hasattr(conn, "audio_rate_controller"):
|
||||
conn.audio_rate_controller = AudioRateController(frame_duration)
|
||||
else:
|
||||
conn.audio_rate_controller.reset()
|
||||
|
||||
# 初始化 flow_control
|
||||
conn.audio_flow_control = {
|
||||
"packet_count": 0,
|
||||
"sequence": 0,
|
||||
"sentence_id": conn.sentence_id,
|
||||
}
|
||||
|
||||
# 启动后台发送循环
|
||||
_start_background_sender(
|
||||
conn, conn.audio_rate_controller, conn.audio_flow_control
|
||||
)
|
||||
|
||||
return conn.audio_rate_controller, conn.audio_flow_control
|
||||
|
||||
|
||||
def _start_background_sender(conn, rate_controller, flow_control):
|
||||
"""
|
||||
启动后台发送循环任务
|
||||
|
||||
Args:
|
||||
conn: 连接对象
|
||||
rate_controller: 速率控制器
|
||||
flow_control: 流控状态
|
||||
"""
|
||||
|
||||
async def send_callback(packet):
|
||||
# 检查是否应该中止
|
||||
if conn.client_abort:
|
||||
raise asyncio.CancelledError("客户端已中止")
|
||||
|
||||
conn.last_activity_time = time.time() * 1000
|
||||
await _do_send_audio(conn, packet, flow_control)
|
||||
conn.client_is_speaking = True
|
||||
|
||||
# 使用 start_sending 启动后台循环
|
||||
rate_controller.start_sending(send_callback)
|
||||
|
||||
|
||||
async def _send_audio_with_rate_control(
|
||||
conn, audio_list, rate_controller, flow_control, send_delay
|
||||
):
|
||||
"""
|
||||
使用 rate_controller 发送音频包
|
||||
|
||||
Args:
|
||||
conn: 连接对象
|
||||
audio_list: 音频包列表
|
||||
rate_controller: 速率控制器
|
||||
flow_control: 流控状态
|
||||
send_delay: 固定延迟（秒），-1表示使用动态流控
|
||||
"""
|
||||
for packet in audio_list:
|
||||
if conn.client_abort:
|
||||
return
|
||||
|
||||
conn.last_activity_time = time.time() * 1000
|
||||
|
||||
# 获取或初始化流控状态
|
||||
if not hasattr(conn, "audio_flow_control"):
|
||||
conn.audio_flow_control = {
|
||||
"last_send_time": 0,
|
||||
"packet_count": 0,
|
||||
"start_time": time.perf_counter(),
|
||||
}
|
||||
# 预缓冲：前N个包直接发送
|
||||
if flow_control["packet_count"] < PRE_BUFFER_COUNT:
|
||||
await _do_send_audio(conn, packet, flow_control)
|
||||
conn.client_is_speaking = True
|
||||
elif send_delay > 0:
|
||||
# 固定延迟模式
|
||||
await asyncio.sleep(send_delay)
|
||||
await _do_send_audio(conn, packet, flow_control)
|
||||
conn.client_is_speaking = True
|
||||
else:
|
||||
# 动态流控模式：仅添加到队列，由后台循环负责发送
|
||||
rate_controller.add_audio(packet)
|
||||
|
||||
flow_control = conn.audio_flow_control
|
||||
current_time = time.perf_counter()
|
||||
# 计算预期发送时间
|
||||
expected_time = flow_control["start_time"] + (
|
||||
flow_control["packet_count"] * frame_duration / 1000
|
||||
)
|
||||
delay = expected_time - current_time
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
# 发送数据包
|
||||
await conn.websocket.send(audios)
|
||||
async def _do_send_audio(conn, opus_packet, flow_control):
|
||||
"""
|
||||
执行实际的音频发送
|
||||
"""
|
||||
packet_index = flow_control.get("packet_count", 0)
|
||||
sequence = flow_control.get("sequence", 0)
|
||||
|
||||
# 更新流控状态
|
||||
flow_control["packet_count"] += 1
|
||||
flow_control["last_send_time"] = time.perf_counter()
|
||||
if conn.conn_from_mqtt_gateway:
|
||||
# 计算时间戳（基于播放位置）
|
||||
start_time = time.time()
|
||||
timestamp = int(start_time * 1000) % (2**32)
|
||||
await _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence)
|
||||
else:
|
||||
# 文件型音频走普通播放
|
||||
start_time = time.perf_counter()
|
||||
play_position = 0
|
||||
# 直接发送opus数据包
|
||||
await conn.websocket.send(opus_packet)
|
||||
|
||||
# 执行预缓冲
|
||||
pre_buffer_frames = min(3, len(audios))
|
||||
for i in range(pre_buffer_frames):
|
||||
await conn.websocket.send(audios[i])
|
||||
remaining_audios = audios[pre_buffer_frames:]
|
||||
|
||||
# 播放剩余音频帧
|
||||
for opus_packet in remaining_audios:
|
||||
if conn.client_abort:
|
||||
break
|
||||
|
||||
# 重置没有声音的状态
|
||||
conn.last_activity_time = time.time() * 1000
|
||||
|
||||
# 计算预期发送时间
|
||||
expected_time = start_time + (play_position / 1000)
|
||||
current_time = time.perf_counter()
|
||||
delay = expected_time - current_time
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
await conn.websocket.send(opus_packet)
|
||||
|
||||
play_position += frame_duration
|
||||
# 更新流控状态
|
||||
flow_control["packet_count"] = packet_index + 1
|
||||
flow_control["sequence"] = sequence + 1
|
||||
|
||||
|
||||
async def send_tts_message(conn, state, text=None):
|
||||
@@ -120,8 +255,10 @@ async def send_tts_message(conn, state, text=None):
|
||||
stop_tts_notify_voice = conn.config.get(
|
||||
"stop_tts_notify_voice", "config/assets/tts_notify.mp3"
|
||||
)
|
||||
audios = audio_to_data(stop_tts_notify_voice, is_opus=True)
|
||||
audios = await audio_to_data(stop_tts_notify_voice, is_opus=True)
|
||||
await sendAudio(conn, audios)
|
||||
# 等待所有音频包发送完成
|
||||
await _wait_for_audio_completion(conn)
|
||||
# 清除服务端讲话状态
|
||||
conn.clearSpeakStatus()
|
||||
|
||||
@@ -155,5 +292,4 @@ async def send_stt_message(conn, text):
|
||||
await conn.websocket.send(
|
||||
json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id})
|
||||
)
|
||||
conn.client_is_speaking = True
|
||||
await send_tts_message(conn, "start")
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
import time
|
||||
import asyncio
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.handle.receiveAudioHandle import handleAudioMessage, startToChat
|
||||
from core.handle.receiveAudioHandle import startToChat
|
||||
from core.handle.reportHandle import enqueue_asr_report
|
||||
from core.handle.sendAudioHandle import send_stt_message, send_tts_message
|
||||
from core.handle.textMessageHandler import TextMessageHandler
|
||||
from core.handle.textMessageType import TextMessageType
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
|
||||
TAG = __name__
|
||||
|
||||
@@ -29,8 +31,18 @@ class ListenTextMessageHandler(TextMessageHandler):
|
||||
elif msg_json["state"] == "stop":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = True
|
||||
if len(conn.asr_audio) > 0:
|
||||
await handleAudioMessage(conn, b"")
|
||||
if conn.asr.interface_type == InterfaceType.STREAM:
|
||||
# 流式模式下，发送结束请求
|
||||
asyncio.create_task(conn.asr._send_stop_request())
|
||||
else:
|
||||
# 非流式模式：直接触发ASR识别
|
||||
if len(conn.asr_audio) > 0:
|
||||
asr_audio_task = conn.asr_audio.copy()
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
|
||||
if len(asr_audio_task) > 0:
|
||||
await conn.asr.handle_voice_stop(conn, asr_audio_task)
|
||||
elif msg_json["state"] == "detect":
|
||||
conn.client_have_voice = False
|
||||
conn.asr_audio.clear()
|
||||
@@ -57,6 +69,7 @@ class ListenTextMessageHandler(TextMessageHandler):
|
||||
enqueue_asr_report(conn, "嘿，你好呀", [])
|
||||
await startToChat(conn, "嘿，你好呀")
|
||||
else:
|
||||
conn.just_woken_up = True
|
||||
# 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
|
||||
enqueue_asr_report(conn, original_text, [])
|
||||
# 否则需要LLM对文字内容进行答复
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import json
|
||||
import time
|
||||
from typing import Dict, Any
|
||||
|
||||
from core.handle.textMessageHandler import TextMessageHandler
|
||||
from core.handle.textMessageType import TextMessageType
|
||||
|
||||
TAG = __name__
|
||||
|
||||
|
||||
class PingMessageHandler(TextMessageHandler):
|
||||
"""Ping消息处理器，用于保持WebSocket连接"""
|
||||
|
||||
@property
|
||||
def message_type(self) -> TextMessageType:
|
||||
return TextMessageType.PING
|
||||
|
||||
async def handle(self, conn, msg_json: Dict[str, Any]) -> None:
|
||||
"""
|
||||
处理PING消息，发送PONG响应
|
||||
消息格式：{"type": "ping"}
|
||||
Args:
|
||||
conn: WebSocket连接对象
|
||||
msg_json: PING消息的JSON数据
|
||||
"""
|
||||
# 检查是否启用了WebSocket心跳功能
|
||||
enable_websocket_ping = conn.config.get("enable_websocket_ping", False)
|
||||
if not enable_websocket_ping:
|
||||
conn.logger.debug(f"WebSocket心跳功能未启用，忽略PING消息")
|
||||
return
|
||||
|
||||
try:
|
||||
conn.logger.debug(f"收到PING消息，发送PONG响应")
|
||||
conn.last_activity_time = time.time() * 1000
|
||||
# 构造PONG响应消息
|
||||
pong_message = {
|
||||
"type": "pong",
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
|
||||
}
|
||||
|
||||
# 发送PONG响应
|
||||
await conn.websocket.send(json.dumps(pong_message))
|
||||
|
||||
except Exception as e:
|
||||
conn.logger.error(f"处理PING消息时发生错误: {e}")
|
||||
@@ -7,6 +7,7 @@ from core.handle.textHandler.listenMessageHandler import ListenTextMessageHandle
|
||||
from core.handle.textHandler.mcpMessageHandler import McpTextMessageHandler
|
||||
from core.handle.textMessageHandler import TextMessageHandler
|
||||
from core.handle.textHandler.serverMessageHandler import ServerTextMessageHandler
|
||||
from core.handle.textHandler.pingMessageHandler import PingMessageHandler
|
||||
|
||||
TAG = __name__
|
||||
|
||||
@@ -27,6 +28,7 @@ class TextMessageHandlerRegistry:
|
||||
IotTextMessageHandler(),
|
||||
McpTextMessageHandler(),
|
||||
ServerTextMessageHandler(),
|
||||
PingMessageHandler(),
|
||||
]
|
||||
|
||||
for handler in handlers:
|
||||
|
||||
@@ -9,3 +9,4 @@ class TextMessageType(Enum):
|
||||
IOT = "iot"
|
||||
MCP = "mcp"
|
||||
SERVER = "server"
|
||||
PING = "ping"
|
||||
|
||||
@@ -33,38 +33,60 @@ class SimpleHttpServer:
|
||||
return f"ws://{local_ip}:{port}/xiaozhi/v1/"
|
||||
|
||||
async def start(self):
|
||||
server_config = self.config["server"]
|
||||
read_config_from_api = self.config.get("read_config_from_api", False)
|
||||
host = server_config.get("ip", "0.0.0.0")
|
||||
port = int(server_config.get("http_port", 8003))
|
||||
try:
|
||||
server_config = self.config["server"]
|
||||
read_config_from_api = self.config.get("read_config_from_api", False)
|
||||
host = server_config.get("ip", "0.0.0.0")
|
||||
port = int(server_config.get("http_port", 8003))
|
||||
|
||||
if port:
|
||||
app = web.Application()
|
||||
if port:
|
||||
app = web.Application()
|
||||
|
||||
if not read_config_from_api:
|
||||
# 如果没有开启智控台，只是单模块运行，就需要再添加简单OTA接口，用于下发websocket接口
|
||||
if not read_config_from_api:
|
||||
# 如果没有开启智控台，只是单模块运行，就需要再添加简单OTA接口，用于下发websocket接口
|
||||
app.add_routes(
|
||||
[
|
||||
web.get("/xiaozhi/ota/", self.ota_handler.handle_get),
|
||||
web.post("/xiaozhi/ota/", self.ota_handler.handle_post),
|
||||
web.options(
|
||||
"/xiaozhi/ota/", self.ota_handler.handle_options
|
||||
),
|
||||
# 下载接口，仅提供 data/bin/*.bin 下载
|
||||
web.get(
|
||||
"/xiaozhi/ota/download/{filename}",
|
||||
self.ota_handler.handle_download,
|
||||
),
|
||||
web.options(
|
||||
"/xiaozhi/ota/download/{filename}",
|
||||
self.ota_handler.handle_options,
|
||||
),
|
||||
]
|
||||
)
|
||||
# 添加路由
|
||||
app.add_routes(
|
||||
[
|
||||
web.get("/xiaozhi/ota/", self.ota_handler.handle_get),
|
||||
web.post("/xiaozhi/ota/", self.ota_handler.handle_post),
|
||||
web.options("/xiaozhi/ota/", self.ota_handler.handle_post),
|
||||
web.get("/mcp/vision/explain", self.vision_handler.handle_get),
|
||||
web.post(
|
||||
"/mcp/vision/explain", self.vision_handler.handle_post
|
||||
),
|
||||
web.options(
|
||||
"/mcp/vision/explain", self.vision_handler.handle_options
|
||||
),
|
||||
]
|
||||
)
|
||||
# 添加路由
|
||||
app.add_routes(
|
||||
[
|
||||
web.get("/mcp/vision/explain", self.vision_handler.handle_get),
|
||||
web.post("/mcp/vision/explain", self.vision_handler.handle_post),
|
||||
web.options("/mcp/vision/explain", self.vision_handler.handle_post),
|
||||
]
|
||||
)
|
||||
|
||||
# 运行服务
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, host, port)
|
||||
await site.start()
|
||||
# 运行服务
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, host, port)
|
||||
await site.start()
|
||||
|
||||
# 保持服务运行
|
||||
while True:
|
||||
await asyncio.sleep(3600) # 每隔 1 小时检查一次
|
||||
# 保持服务运行
|
||||
while True:
|
||||
await asyncio.sleep(3600) # 每隔 1 小时检查一次
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"HTTP服务器启动失败: {e}")
|
||||
import traceback
|
||||
|
||||
self.logger.bind(tag=TAG).error(f"错误堆栈: {traceback.format_exc()}")
|
||||
raise
|
||||
|
||||
@@ -8,8 +8,6 @@ import asyncio
|
||||
import requests
|
||||
import websockets
|
||||
import opuslib_next
|
||||
import random
|
||||
from typing import Optional, Tuple, List
|
||||
from urllib import parse
|
||||
from datetime import datetime
|
||||
from config.logger import setup_logging
|
||||
@@ -96,6 +94,8 @@ class ASRProvider(ASRProviderBase):
|
||||
self.delete_audio_file = delete_audio_file
|
||||
self.expire_time = None
|
||||
|
||||
self.task_id = uuid.uuid4().hex
|
||||
|
||||
# Token管理
|
||||
if self.access_key_id and self.access_key_secret:
|
||||
self._refresh_token()
|
||||
@@ -137,13 +137,13 @@ class ASRProvider(ASRProviderBase):
|
||||
conn.asr_audio.append(audio)
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
|
||||
# 只在有声音且没有连接时建立连接
|
||||
if audio_have_voice and not self.is_processing:
|
||||
# 只在有声音且没有连接时建立连接（排除正在停止的情况）
|
||||
if audio_have_voice and not self.is_processing and not self.asr_ws:
|
||||
try:
|
||||
await self._start_recognition(conn)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}")
|
||||
await self._cleanup(conn)
|
||||
await self._cleanup()
|
||||
return
|
||||
|
||||
if self.asr_ws and self.is_processing and self.server_ready:
|
||||
@@ -169,20 +169,22 @@ class ASRProvider(ASRProviderBase):
|
||||
ping_timeout=None,
|
||||
close_timeout=5,
|
||||
)
|
||||
|
||||
|
||||
self.task_id = uuid.uuid4().hex
|
||||
|
||||
logger.bind(tag=TAG).debug(f"WebSocket连接建立成功, task_id: {self.task_id}")
|
||||
|
||||
self.is_processing = True
|
||||
self.server_ready = False # 重置服务器准备状态
|
||||
self.forward_task = asyncio.create_task(self._forward_results(conn))
|
||||
|
||||
|
||||
# 发送开始请求
|
||||
start_request = {
|
||||
"header": {
|
||||
"namespace": "SpeechTranscriber",
|
||||
"name": "StartTranscription",
|
||||
"status": 20000000,
|
||||
"message_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"task_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"status_text": "Gateway:SUCCESS:Success.",
|
||||
"message_id": uuid.uuid4().hex,
|
||||
"task_id": self.task_id,
|
||||
"appkey": self.appkey
|
||||
},
|
||||
"payload": {
|
||||
@@ -196,23 +198,26 @@ class ASRProvider(ASRProviderBase):
|
||||
}
|
||||
}
|
||||
await self.asr_ws.send(json.dumps(start_request, ensure_ascii=False))
|
||||
logger.bind(tag=TAG).info("已发送开始请求，等待服务器准备...")
|
||||
logger.bind(tag=TAG).debug("已发送开始请求，等待服务器准备...")
|
||||
|
||||
async def _forward_results(self, conn):
|
||||
"""转发识别结果"""
|
||||
try:
|
||||
while self.asr_ws and not conn.stop_event.is_set():
|
||||
while not conn.stop_event.is_set():
|
||||
try:
|
||||
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
|
||||
result = json.loads(response)
|
||||
|
||||
|
||||
header = result.get("header", {})
|
||||
payload = result.get("payload", {})
|
||||
message_name = header.get("name", "")
|
||||
status = header.get("status", 0)
|
||||
|
||||
|
||||
if status != 20000000:
|
||||
if status in [40000004, 40010004]: # 连接超时或客户端断开
|
||||
if status == 40010004:
|
||||
logger.bind(tag=TAG).warning(f"请在服务端响应完成后再关闭链接，状态码: {status}")
|
||||
break
|
||||
if status in [40000004, 40010003]: # 连接超时或客户端断开
|
||||
logger.bind(tag=TAG).warning(f"连接问题，状态码: {status}")
|
||||
break
|
||||
elif status in [40270002, 40270003]: # 音频问题
|
||||
@@ -221,12 +226,12 @@ class ASRProvider(ASRProviderBase):
|
||||
else:
|
||||
logger.bind(tag=TAG).error(f"识别错误，状态码: {status}, 消息: {header.get('status_text', '')}")
|
||||
continue
|
||||
|
||||
|
||||
# 收到TranscriptionStarted表示服务器准备好接收音频数据
|
||||
if message_name == "TranscriptionStarted":
|
||||
self.server_ready = True
|
||||
logger.bind(tag=TAG).info("服务器已准备，开始发送缓存音频...")
|
||||
|
||||
logger.bind(tag=TAG).debug("服务器已准备，开始发送缓存音频...")
|
||||
|
||||
# 发送缓存音频
|
||||
if conn.asr_audio:
|
||||
for cached_audio in conn.asr_audio[-10:]:
|
||||
@@ -237,88 +242,89 @@ class ASRProvider(ASRProviderBase):
|
||||
logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}")
|
||||
break
|
||||
continue
|
||||
|
||||
if message_name == "TranscriptionResultChanged":
|
||||
# 中间结果
|
||||
text = payload.get("result", "")
|
||||
if text:
|
||||
self.text = text
|
||||
elif message_name == "SentenceEnd":
|
||||
# 最终结果
|
||||
# 句子结束（每个句子都会触发）
|
||||
text = payload.get("result", "")
|
||||
if text:
|
||||
self.text = text
|
||||
conn.reset_vad_states()
|
||||
# 传递缓存的音频数据
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清空缓存
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
break
|
||||
elif message_name == "TranscriptionCompleted":
|
||||
# 识别完成
|
||||
self.is_processing = False
|
||||
break
|
||||
|
||||
logger.bind(tag=TAG).info(f"识别到文本: {text}")
|
||||
|
||||
# 手动模式下累积识别结果
|
||||
if conn.client_listen_mode == "manual":
|
||||
if self.text:
|
||||
self.text += text
|
||||
else:
|
||||
self.text = text
|
||||
|
||||
# 手动模式下，只有在收到stop信号后才触发处理（仅处理一次）
|
||||
if conn.client_voice_stop:
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
if len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
break
|
||||
else:
|
||||
# 自动模式下直接覆盖
|
||||
self.text = text
|
||||
conn.reset_vad_states()
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
except websockets.exceptions.ConnectionClosed:
|
||||
logger.bind(tag=TAG).error("接收结果超时")
|
||||
break
|
||||
except websockets.ConnectionClosed:
|
||||
logger.bind(tag=TAG).info("ASR服务连接已关闭")
|
||||
self.is_processing = False
|
||||
break
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}")
|
||||
break
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}")
|
||||
finally:
|
||||
await self._cleanup(conn)
|
||||
# 清理连接的音频缓存
|
||||
await self._cleanup()
|
||||
if conn:
|
||||
if hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, 'asr_audio'):
|
||||
conn.asr_audio = []
|
||||
|
||||
async def _cleanup(self, conn):
|
||||
"""清理资源"""
|
||||
logger.bind(tag=TAG).info(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
|
||||
|
||||
# 清理连接的音频缓存
|
||||
if conn and hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
|
||||
# 判断是否需要发送终止请求
|
||||
should_stop = self.is_processing or self.server_ready
|
||||
|
||||
# 发送停止识别请求
|
||||
if self.asr_ws and should_stop:
|
||||
async def _send_stop_request(self):
|
||||
"""发送停止识别请求（不关闭连接）"""
|
||||
if self.asr_ws:
|
||||
try:
|
||||
# 先停止音频发送
|
||||
self.is_processing = False
|
||||
|
||||
stop_msg = {
|
||||
"header": {
|
||||
"namespace": "SpeechTranscriber",
|
||||
"name": "StopTranscription",
|
||||
"status": 20000000,
|
||||
"message_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"status_text": "Client:Stop",
|
||||
"message_id": uuid.uuid4().hex,
|
||||
"task_id": self.task_id,
|
||||
"appkey": self.appkey
|
||||
}
|
||||
}
|
||||
logger.bind(tag=TAG).info("正在发送ASR终止请求")
|
||||
logger.bind(tag=TAG).debug("停止识别请求已发送")
|
||||
await self.asr_ws.send(json.dumps(stop_msg, ensure_ascii=False))
|
||||
await asyncio.sleep(0.1)
|
||||
logger.bind(tag=TAG).info("ASR终止请求已发送")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR终止请求发送失败: {e}")
|
||||
|
||||
# 状态重置（在终止请求发送后）
|
||||
logger.bind(tag=TAG).error(f"发送停止识别请求失败: {e}")
|
||||
|
||||
async def _cleanup(self):
|
||||
"""清理资源（关闭连接）"""
|
||||
logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
|
||||
|
||||
# 状态重置
|
||||
self.is_processing = False
|
||||
self.server_ready = False
|
||||
logger.bind(tag=TAG).info("ASR状态已重置")
|
||||
logger.bind(tag=TAG).debug("ASR状态已重置")
|
||||
|
||||
# 清理任务
|
||||
if self.forward_task and not self.forward_task.done():
|
||||
self.forward_task.cancel()
|
||||
try:
|
||||
await asyncio.wait_for(self.forward_task, timeout=1.0)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}")
|
||||
finally:
|
||||
self.forward_task = None
|
||||
|
||||
# 关闭连接
|
||||
if self.asr_ws:
|
||||
try:
|
||||
@@ -329,8 +335,11 @@ class ASRProvider(ASRProviderBase):
|
||||
logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
|
||||
finally:
|
||||
self.asr_ws = None
|
||||
|
||||
logger.bind(tag=TAG).info("ASR会话清理完成")
|
||||
|
||||
# 清理任务引用
|
||||
self.forward_task = None
|
||||
|
||||
logger.bind(tag=TAG).debug("ASR会话清理完成")
|
||||
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format):
|
||||
"""获取识别结果"""
|
||||
@@ -340,4 +349,11 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
async def close(self):
|
||||
"""关闭资源"""
|
||||
await self._cleanup()
|
||||
await self._cleanup(None)
|
||||
if hasattr(self, 'decoder') and self.decoder is not None:
|
||||
try:
|
||||
del self.decoder
|
||||
self.decoder = None
|
||||
logger.bind(tag=TAG).debug("Aliyun decoder resources released")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"释放Aliyun decoder资源时出错: {e}")
|
||||
|
||||
@@ -9,7 +9,6 @@ import asyncio
|
||||
import traceback
|
||||
import threading
|
||||
import opuslib_next
|
||||
import concurrent.futures
|
||||
from abc import ABC, abstractmethod
|
||||
from config.logger import setup_logging
|
||||
from typing import Optional, Tuple, List
|
||||
@@ -118,121 +117,89 @@ class ASRProviderBase(ABC):
|
||||
|
||||
# 接收音频
|
||||
async def receive_audio(self, conn, audio, audio_have_voice):
|
||||
if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime":
|
||||
have_voice = audio_have_voice
|
||||
if conn.client_listen_mode == "manual":
|
||||
# 手动模式：缓存音频用于ASR识别
|
||||
conn.asr_audio.append(audio)
|
||||
else:
|
||||
have_voice = conn.client_have_voice
|
||||
|
||||
conn.asr_audio.append(audio)
|
||||
if not have_voice and not conn.client_have_voice:
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
return
|
||||
# 自动/实时模式：使用VAD检测
|
||||
have_voice = audio_have_voice
|
||||
|
||||
if conn.client_voice_stop:
|
||||
asr_audio_task = conn.asr_audio.copy()
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
conn.asr_audio.append(audio)
|
||||
if not have_voice and not conn.client_have_voice:
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
return
|
||||
|
||||
if len(asr_audio_task) > 15:
|
||||
await self.handle_voice_stop(conn, asr_audio_task)
|
||||
# 自动模式下通过VAD检测到语音停止时触发识别
|
||||
if conn.client_voice_stop:
|
||||
asr_audio_task = conn.asr_audio.copy()
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
|
||||
if len(asr_audio_task) > 15:
|
||||
await self.handle_voice_stop(conn, asr_audio_task)
|
||||
|
||||
# 处理语音停止
|
||||
async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
|
||||
"""并行处理ASR和声纹识别"""
|
||||
try:
|
||||
total_start_time = time.monotonic()
|
||||
|
||||
|
||||
# 准备音频数据
|
||||
if conn.audio_format == "pcm":
|
||||
pcm_data = asr_audio_task
|
||||
else:
|
||||
pcm_data = self.decode_opus(asr_audio_task)
|
||||
|
||||
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
|
||||
|
||||
# 预先准备WAV数据
|
||||
wav_data = None
|
||||
if conn.voiceprint_provider and combined_pcm_data:
|
||||
wav_data = self._pcm_to_wav(combined_pcm_data)
|
||||
|
||||
|
||||
# 定义ASR任务
|
||||
def run_asr():
|
||||
start_time = time.monotonic()
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
result = loop.run_until_complete(
|
||||
self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
|
||||
)
|
||||
end_time = time.monotonic()
|
||||
logger.bind(tag=TAG).info(f"ASR耗时: {end_time - start_time:.3f}s")
|
||||
return result
|
||||
finally:
|
||||
loop.close()
|
||||
except Exception as e:
|
||||
end_time = time.monotonic()
|
||||
logger.bind(tag=TAG).error(f"ASR失败: {e}")
|
||||
return ("", None)
|
||||
|
||||
# 定义声纹识别任务
|
||||
def run_voiceprint():
|
||||
if not wav_data:
|
||||
return None
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
# 使用连接的声纹识别提供者
|
||||
result = loop.run_until_complete(
|
||||
conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
|
||||
)
|
||||
return result
|
||||
finally:
|
||||
loop.close()
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"声纹识别失败: {e}")
|
||||
return None
|
||||
|
||||
# 使用线程池执行器并行运行
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor:
|
||||
asr_future = thread_executor.submit(run_asr)
|
||||
|
||||
if conn.voiceprint_provider and wav_data:
|
||||
voiceprint_future = thread_executor.submit(run_voiceprint)
|
||||
|
||||
# 等待两个线程都完成
|
||||
asr_result = asr_future.result(timeout=15)
|
||||
voiceprint_result = voiceprint_future.result(timeout=15)
|
||||
|
||||
results = {"asr": asr_result, "voiceprint": voiceprint_result}
|
||||
else:
|
||||
asr_result = asr_future.result(timeout=15)
|
||||
results = {"asr": asr_result, "voiceprint": None}
|
||||
|
||||
|
||||
# 处理结果
|
||||
raw_text, _ = results.get("asr", ("", None))
|
||||
speaker_name = results.get("voiceprint", None)
|
||||
|
||||
# 记录识别结果
|
||||
asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
|
||||
|
||||
if conn.voiceprint_provider and wav_data:
|
||||
voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
|
||||
# 并发等待两个结果
|
||||
asr_result, voiceprint_result = await asyncio.gather(
|
||||
asr_task, voiceprint_task, return_exceptions=True
|
||||
)
|
||||
else:
|
||||
asr_result = await asr_task
|
||||
voiceprint_result = None
|
||||
|
||||
# 记录识别结果 - 检查是否为异常
|
||||
if isinstance(asr_result, Exception):
|
||||
logger.bind(tag=TAG).error(f"ASR识别失败: {asr_result}")
|
||||
raw_text = ""
|
||||
else:
|
||||
raw_text, _ = asr_result
|
||||
|
||||
if isinstance(voiceprint_result, Exception):
|
||||
logger.bind(tag=TAG).error(f"声纹识别失败: {voiceprint_result}")
|
||||
speaker_name = ""
|
||||
else:
|
||||
speaker_name = voiceprint_result
|
||||
|
||||
if raw_text:
|
||||
logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
|
||||
if speaker_name:
|
||||
logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
|
||||
|
||||
|
||||
# 性能监控
|
||||
total_time = time.monotonic() - total_start_time
|
||||
logger.bind(tag=TAG).info(f"总处理耗时: {total_time:.3f}s")
|
||||
|
||||
logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s")
|
||||
|
||||
# 检查文本长度
|
||||
text_len, _ = remove_punctuation_and_length(raw_text)
|
||||
self.stop_ws_connection()
|
||||
|
||||
|
||||
if text_len > 0:
|
||||
# 构建包含说话人信息的JSON字符串
|
||||
enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
|
||||
|
||||
|
||||
# 使用自定义模块进行上报
|
||||
await startToChat(conn, enhanced_text)
|
||||
enqueue_asr_report(conn, enhanced_text, asr_audio_task)
|
||||
@@ -306,6 +273,7 @@ class ASRProviderBase(ABC):
|
||||
@staticmethod
|
||||
def decode_opus(opus_data: List[bytes]) -> List[bytes]:
|
||||
"""将Opus音频数据解码为PCM数据"""
|
||||
decoder = None
|
||||
try:
|
||||
decoder = opuslib_next.Decoder(16000, 1)
|
||||
pcm_data = []
|
||||
@@ -330,3 +298,9 @@ class ASRProviderBase(ABC):
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
|
||||
return []
|
||||
finally:
|
||||
if decoder is not None:
|
||||
try:
|
||||
del decoder
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"释放decoder资源时出错: {e}")
|
||||
|
||||
@@ -18,8 +18,6 @@ class ASRProvider(ASRProviderBase):
|
||||
self.interface_type = InterfaceType.STREAM
|
||||
self.config = config
|
||||
self.text = ""
|
||||
self.max_retries = 3
|
||||
self.retry_delay = 2
|
||||
self.decoder = opuslib_next.Decoder(16000, 1)
|
||||
self.asr_ws = None
|
||||
self.forward_task = None
|
||||
@@ -49,6 +47,8 @@ class ASRProvider(ASRProviderBase):
|
||||
self.channel = config.get("channel", 1)
|
||||
self.auth_method = config.get("auth_method", "token")
|
||||
self.secret = config.get("secret", "access_secret")
|
||||
end_window_size = config.get("end_window_size")
|
||||
self.end_window_size = int(end_window_size) if end_window_size else 200
|
||||
|
||||
async def open_audio_channels(self, conn):
|
||||
await super().open_audio_channels(conn)
|
||||
@@ -56,14 +56,13 @@ class ASRProvider(ASRProviderBase):
|
||||
async def receive_audio(self, conn, audio, audio_have_voice):
|
||||
conn.asr_audio.append(audio)
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
|
||||
# 存储音频数据
|
||||
if not hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
conn.asr_audio_for_voiceprint.append(audio)
|
||||
|
||||
|
||||
# 当没有音频数据时处理完整语音片段
|
||||
if not audio and len(conn.asr_audio_for_voiceprint) > 0:
|
||||
if conn.client_listen_mode != "manual" and not audio and len(conn.asr_audio_for_voiceprint) > 0:
|
||||
await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint)
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
|
||||
@@ -179,6 +178,7 @@ class ASRProvider(ASRProviderBase):
|
||||
payload.get("audio_info", {}).get("duration", 0) > 2000
|
||||
and not utterances
|
||||
and not payload["result"].get("text")
|
||||
and conn.client_listen_mode != "manual"
|
||||
):
|
||||
logger.bind(tag=TAG).error(f"识别文本：空")
|
||||
self.text = ""
|
||||
@@ -187,15 +187,44 @@ class ASRProvider(ASRProviderBase):
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
|
||||
# 专门处理没有文本的识别结果（手动模式下可能已经识别完成但是没松按键）
|
||||
elif not payload["result"].get("text") and not utterances:
|
||||
if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
break
|
||||
|
||||
for utterance in utterances:
|
||||
if utterance.get("definite", False):
|
||||
self.text = utterance["text"]
|
||||
current_text = utterance["text"]
|
||||
logger.bind(tag=TAG).info(
|
||||
f"识别到文本: {self.text}"
|
||||
f"识别到文本: {current_text}"
|
||||
)
|
||||
conn.reset_vad_states()
|
||||
if len(audio_data) > 15: # 确保有足够音频数据
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
|
||||
# 手动模式下累积识别结果
|
||||
if conn.client_listen_mode == "manual":
|
||||
if self.text:
|
||||
self.text += current_text
|
||||
else:
|
||||
self.text = current_text
|
||||
|
||||
# 在接收消息中途时收到停止信号
|
||||
if conn.client_voice_stop and len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("消息中途收到停止信号，触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
break
|
||||
else:
|
||||
# 自动模式下直接覆盖
|
||||
self.text = current_text
|
||||
conn.reset_vad_states()
|
||||
if len(audio_data) > 15: # 确保有足够音频数据
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
elif "error" in payload:
|
||||
error_msg = payload.get("error", "未知错误")
|
||||
@@ -227,8 +256,6 @@ class ASRProvider(ASRProviderBase):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, 'asr_audio'):
|
||||
conn.asr_audio = []
|
||||
if hasattr(conn, 'has_valid_voice'):
|
||||
conn.has_valid_voice = False
|
||||
|
||||
def stop_ws_connection(self):
|
||||
if self.asr_ws:
|
||||
@@ -236,6 +263,20 @@ class ASRProvider(ASRProviderBase):
|
||||
self.asr_ws = None
|
||||
self.is_processing = False
|
||||
|
||||
async def _send_stop_request(self):
|
||||
"""发送最后一个音频帧以通知服务器结束"""
|
||||
if self.asr_ws:
|
||||
try:
|
||||
# 发送结束标记的音频帧（gzip压缩的空数据）
|
||||
empty_payload = gzip.compress(b"")
|
||||
last_audio_request = bytearray(self.generate_last_audio_default_header())
|
||||
last_audio_request.extend(len(empty_payload).to_bytes(4, "big"))
|
||||
last_audio_request.extend(empty_payload)
|
||||
await self.asr_ws.send(last_audio_request)
|
||||
logger.bind(tag=TAG).debug("已发送结束音频帧")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"发送结束音频帧时出错: {e}")
|
||||
|
||||
def construct_request(self, reqid):
|
||||
req = {
|
||||
"app": {
|
||||
@@ -252,7 +293,7 @@ class ASRProvider(ASRProviderBase):
|
||||
"sequence": 1,
|
||||
"boosting_table_name": self.boosting_table_name,
|
||||
"correct_table_name": self.correct_table_name,
|
||||
"end_window_size": 200,
|
||||
"end_window_size": self.end_window_size,
|
||||
},
|
||||
"audio": {
|
||||
"format": self.format,
|
||||
@@ -370,6 +411,16 @@ class ASRProvider(ASRProviderBase):
|
||||
pass
|
||||
self.forward_task = None
|
||||
self.is_processing = False
|
||||
|
||||
# 显式释放decoder资源
|
||||
if hasattr(self, 'decoder') and self.decoder is not None:
|
||||
try:
|
||||
del self.decoder
|
||||
self.decoder = None
|
||||
logger.bind(tag=TAG).debug("Doubao decoder resources released")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")
|
||||
|
||||
# 清理所有连接的音频缓冲区
|
||||
if hasattr(self, '_connections'):
|
||||
for conn in self._connections.values():
|
||||
@@ -377,5 +428,3 @@ class ASRProvider(ASRProviderBase):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, 'asr_audio'):
|
||||
conn.asr_audio = []
|
||||
if hasattr(conn, 'has_valid_voice'):
|
||||
conn.has_valid_voice = False
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
import time
|
||||
import os
|
||||
import sys
|
||||
import io
|
||||
import sys
|
||||
import time
|
||||
import shutil
|
||||
import psutil
|
||||
import asyncio
|
||||
|
||||
from config.logger import setup_logging
|
||||
from typing import Optional, Tuple, List
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from funasr import AutoModel
|
||||
from funasr.utils.postprocess_utils import rich_transcription_postprocess
|
||||
import shutil
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
|
||||
TAG = __name__
|
||||
@@ -90,16 +92,17 @@ class ASRProvider(ASRProviderBase):
|
||||
else:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
# 语音识别
|
||||
# 语音识别 - 使用线程池避免阻塞事件循环
|
||||
start_time = time.time()
|
||||
result = self.model.generate(
|
||||
result = await asyncio.to_thread(
|
||||
self.model.generate,
|
||||
input=combined_pcm_data,
|
||||
cache={},
|
||||
language="auto",
|
||||
use_itn=True,
|
||||
batch_size_s=60,
|
||||
)
|
||||
text = rich_transcription_postprocess(result[0]["text"])
|
||||
text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"])
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Optional, Tuple, List
|
||||
import dashscope
|
||||
from config.logger import setup_logging
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
|
||||
tag = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool):
|
||||
super().__init__()
|
||||
# 音频文件上传类型，流式文本识别输出
|
||||
self.interface_type = InterfaceType.NON_STREAM
|
||||
"""Qwen3-ASR-Flash ASR初始化"""
|
||||
|
||||
# 配置参数
|
||||
self.api_key = config.get("api_key")
|
||||
if not self.api_key:
|
||||
raise ValueError("Qwen3-ASR-Flash 需要配置 api_key")
|
||||
|
||||
self.model_name = config.get("model_name", "qwen3-asr-flash")
|
||||
self.output_dir = config.get("output_dir", "./audio_output")
|
||||
self.delete_audio_file = delete_audio_file
|
||||
|
||||
# ASR选项配置
|
||||
self.enable_lid = config.get("enable_lid", True) # 自动语种检测
|
||||
self.enable_itn = config.get("enable_itn", True) # 逆文本归一化
|
||||
self.language = config.get("language", None) # 指定语种，默认自动检测
|
||||
self.context = config.get("context", "") # 上下文信息，用于提高识别准确率
|
||||
|
||||
# 确保输出目录存在
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
def _prepare_audio_file(self, pcm_data: bytes) -> str:
|
||||
"""将PCM数据转换为WAV文件并返回文件路径"""
|
||||
try:
|
||||
import wave
|
||||
|
||||
# 创建临时WAV文件
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file:
|
||||
temp_path = temp_file.name
|
||||
|
||||
# 写入WAV格式
|
||||
with wave.open(temp_path, 'wb') as wav_file:
|
||||
wav_file.setnchannels(1) # 单声道
|
||||
wav_file.setsampwidth(2) # 16位
|
||||
wav_file.setframerate(16000) # 16kHz采样率
|
||||
wav_file.writeframes(pcm_data)
|
||||
|
||||
return temp_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=tag).error(f"音频文件准备失败: {e}")
|
||||
return None
|
||||
|
||||
async def speech_to_text(
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus"
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
temp_file_path = None
|
||||
file_path = None
|
||||
|
||||
try:
|
||||
# 解码音频数据
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
if len(combined_pcm_data) == 0:
|
||||
logger.bind(tag=tag).warning("音频数据为空")
|
||||
return "", None
|
||||
|
||||
# 准备音频文件
|
||||
temp_file_path = self._prepare_audio_file(combined_pcm_data)
|
||||
if not temp_file_path:
|
||||
return "", None
|
||||
|
||||
# 保存音频文件（如果需要）
|
||||
if not self.delete_audio_file:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
# 构造请求消息
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"audio": temp_file_path}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
# 如果有上下文信息，添加system消息
|
||||
if self.context:
|
||||
messages.insert(0, {
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"text": self.context}
|
||||
]
|
||||
})
|
||||
|
||||
# 准备ASR选项
|
||||
asr_options = {
|
||||
"enable_lid": self.enable_lid,
|
||||
"enable_itn": self.enable_itn
|
||||
}
|
||||
|
||||
# 如果指定了语种，添加到选项中
|
||||
if self.language:
|
||||
asr_options["language"] = self.language
|
||||
|
||||
# 设置API密钥
|
||||
dashscope.api_key = self.api_key
|
||||
|
||||
# 发送流式请求
|
||||
response = dashscope.MultiModalConversation.call(
|
||||
model=self.model_name,
|
||||
messages=messages,
|
||||
result_format="message",
|
||||
asr_options=asr_options,
|
||||
stream=True
|
||||
)
|
||||
|
||||
# 处理流式响应
|
||||
full_text = ""
|
||||
for chunk in response:
|
||||
try:
|
||||
text = chunk["output"]["choices"][0]["message"].content[0]["text"]
|
||||
# 更新为最新的完整文本
|
||||
full_text = text.strip()
|
||||
except:
|
||||
pass
|
||||
|
||||
return full_text, file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=tag).error(f"语音识别失败: {e}")
|
||||
return "", file_path
|
||||
|
||||
finally:
|
||||
# 清理临时文件
|
||||
if temp_file_path and os.path.exists(temp_file_path):
|
||||
try:
|
||||
os.unlink(temp_file_path)
|
||||
except Exception as e:
|
||||
logger.bind(tag=tag).warning(f"清理临时文件失败: {e}")
|
||||
@@ -0,0 +1,114 @@
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
from typing import Optional, Tuple, List
|
||||
from .base import ASRProviderBase
|
||||
from config.logger import setup_logging
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
import vosk
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool = True):
|
||||
super().__init__()
|
||||
self.interface_type = InterfaceType.LOCAL
|
||||
self.model_path = config.get("model_path")
|
||||
self.output_dir = config.get("output_dir", "tmp/")
|
||||
self.delete_audio_file = delete_audio_file
|
||||
|
||||
# 初始化VOSK模型
|
||||
self.model = None
|
||||
self.recognizer = None
|
||||
self._load_model()
|
||||
|
||||
# 确保输出目录存在
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
def _load_model(self):
|
||||
"""加载VOSK模型"""
|
||||
try:
|
||||
if not os.path.exists(self.model_path):
|
||||
raise FileNotFoundError(f"VOSK模型路径不存在: {self.model_path}")
|
||||
|
||||
logger.bind(tag=TAG).info(f"正在加载VOSK模型: {self.model_path}")
|
||||
self.model = vosk.Model(self.model_path)
|
||||
|
||||
# 初始化VOSK识别器（采样率必须为16kHz）
|
||||
self.recognizer = vosk.KaldiRecognizer(self.model, 16000)
|
||||
|
||||
logger.bind(tag=TAG).info("VOSK模型加载成功")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"加载VOSK模型失败: {e}")
|
||||
raise
|
||||
|
||||
async def speech_to_text(
|
||||
self, audio_data: List[bytes], session_id: str, audio_format: str = "opus"
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
file_path = None
|
||||
try:
|
||||
# 检查模型是否加载成功
|
||||
if not self.model:
|
||||
logger.bind(tag=TAG).error("VOSK模型未加载，无法进行识别")
|
||||
return "", None
|
||||
|
||||
# 解码音频（如果原始格式是Opus）
|
||||
if audio_format == "pcm":
|
||||
pcm_data = audio_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(audio_data)
|
||||
|
||||
if not pcm_data:
|
||||
logger.bind(tag=TAG).warning("解码后的PCM数据为空，无法进行识别")
|
||||
return "", None
|
||||
|
||||
# 合并PCM数据
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
if len(combined_pcm_data) == 0:
|
||||
logger.bind(tag=TAG).warning("合并后的PCM数据为空")
|
||||
return "", None
|
||||
|
||||
# 判断是否保存为WAV文件
|
||||
if not self.delete_audio_file:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
|
||||
# 进行识别（VOSK推荐每次送入2000字节的数据）
|
||||
chunk_size = 2000
|
||||
text_result = ""
|
||||
|
||||
for i in range(0, len(combined_pcm_data), chunk_size):
|
||||
chunk = combined_pcm_data[i:i+chunk_size]
|
||||
if self.recognizer.AcceptWaveform(chunk):
|
||||
result = json.loads(self.recognizer.Result())
|
||||
text = result.get('text', '')
|
||||
if text:
|
||||
text_result += text + " "
|
||||
|
||||
# 获取最终结果
|
||||
final_result = json.loads(self.recognizer.FinalResult())
|
||||
final_text = final_result.get('text', '')
|
||||
if final_text:
|
||||
text_result += final_text
|
||||
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"VOSK语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text_result.strip()}"
|
||||
)
|
||||
|
||||
return text_result.strip(), file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"VOSK语音识别失败: {e}")
|
||||
return "", None
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
|
||||
@@ -0,0 +1,372 @@
|
||||
import json
|
||||
import hmac
|
||||
import base64
|
||||
import hashlib
|
||||
import asyncio
|
||||
import websockets
|
||||
import opuslib_next
|
||||
import gc
|
||||
from time import mktime
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlencode
|
||||
from typing import List
|
||||
from config.logger import setup_logging
|
||||
from wsgiref.handlers import format_date_time
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
# 帧状态常量
|
||||
STATUS_FIRST_FRAME = 0 # 第一帧的标识
|
||||
STATUS_CONTINUE_FRAME = 1 # 中间帧标识
|
||||
STATUS_LAST_FRAME = 2 # 最后一帧的标识
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__()
|
||||
self.interface_type = InterfaceType.STREAM
|
||||
self.config = config
|
||||
self.text = ""
|
||||
self.decoder = opuslib_next.Decoder(16000, 1)
|
||||
self.asr_ws = None
|
||||
self.forward_task = None
|
||||
self.is_processing = False
|
||||
self.server_ready = False
|
||||
|
||||
# 讯飞配置
|
||||
self.app_id = config.get("app_id")
|
||||
self.api_key = config.get("api_key")
|
||||
self.api_secret = config.get("api_secret")
|
||||
|
||||
if not all([self.app_id, self.api_key, self.api_secret]):
|
||||
raise ValueError("必须提供app_id、api_key和api_secret")
|
||||
|
||||
# 识别参数
|
||||
self.iat_params = {
|
||||
"domain": config.get("domain", "slm"),
|
||||
"language": config.get("language", "zh_cn"),
|
||||
"accent": config.get("accent", "mandarin"),
|
||||
"result": {"encoding": "utf8", "compress": "raw", "format": "plain"},
|
||||
}
|
||||
|
||||
self.output_dir = config.get("output_dir", "tmp/")
|
||||
self.delete_audio_file = delete_audio_file
|
||||
|
||||
def create_url(self) -> str:
|
||||
"""生成认证URL"""
|
||||
url = "ws://iat.cn-huabei-1.xf-yun.com/v1"
|
||||
# 生成RFC1123格式的时间戳
|
||||
now = datetime.now()
|
||||
date = format_date_time(mktime(now.timetuple()))
|
||||
|
||||
# 拼接字符串
|
||||
signature_origin = "host: " + "iat.cn-huabei-1.xf-yun.com" + "\n"
|
||||
signature_origin += "date: " + date + "\n"
|
||||
signature_origin += "GET " + "/v1 " + "HTTP/1.1"
|
||||
|
||||
# 进行hmac-sha256进行加密
|
||||
signature_sha = hmac.new(
|
||||
self.api_secret.encode("utf-8"),
|
||||
signature_origin.encode("utf-8"),
|
||||
digestmod=hashlib.sha256,
|
||||
).digest()
|
||||
signature_sha = base64.b64encode(signature_sha).decode(encoding="utf-8")
|
||||
|
||||
authorization_origin = (
|
||||
'api_key="%s", algorithm="%s", headers="%s", signature="%s"'
|
||||
% (self.api_key, "hmac-sha256", "host date request-line", signature_sha)
|
||||
)
|
||||
authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
# 将请求的鉴权参数组合为字典
|
||||
v = {
|
||||
"authorization": authorization,
|
||||
"date": date,
|
||||
"host": "iat.cn-huabei-1.xf-yun.com",
|
||||
}
|
||||
|
||||
# 拼接鉴权参数，生成url
|
||||
url = url + "?" + urlencode(v)
|
||||
return url
|
||||
|
||||
async def open_audio_channels(self, conn):
|
||||
await super().open_audio_channels(conn)
|
||||
|
||||
async def receive_audio(self, conn, audio, audio_have_voice):
|
||||
# 先调用父类方法处理基础逻辑
|
||||
await super().receive_audio(conn, audio, audio_have_voice)
|
||||
|
||||
# 存储音频数据用于声纹识别
|
||||
if not hasattr(conn, "asr_audio_for_voiceprint"):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
conn.asr_audio_for_voiceprint.append(audio)
|
||||
|
||||
# 如果本次有声音，且之前没有建立连接
|
||||
if audio_have_voice and self.asr_ws is None and not self.is_processing:
|
||||
try:
|
||||
await self._start_recognition(conn)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
|
||||
await self._cleanup()
|
||||
return
|
||||
|
||||
# 发送当前音频数据
|
||||
if self.asr_ws and self.is_processing and self.server_ready:
|
||||
try:
|
||||
pcm_frame = self.decoder.decode(audio, 960)
|
||||
await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"发送音频数据时发生错误: {e}")
|
||||
await self._cleanup()
|
||||
|
||||
async def _start_recognition(self, conn):
|
||||
"""开始识别会话"""
|
||||
try:
|
||||
self.is_processing = True
|
||||
# 建立WebSocket连接
|
||||
ws_url = self.create_url()
|
||||
logger.bind(tag=TAG).info(f"正在连接ASR服务: {ws_url[:50]}...")
|
||||
|
||||
# 如果为手动模式,设置超时时长为一分钟
|
||||
if conn.client_listen_mode == "manual":
|
||||
self.iat_params["eos"] = 60000
|
||||
|
||||
self.asr_ws = await websockets.connect(
|
||||
ws_url,
|
||||
max_size=1000000000,
|
||||
ping_interval=None,
|
||||
ping_timeout=None,
|
||||
close_timeout=10,
|
||||
)
|
||||
|
||||
logger.bind(tag=TAG).info("ASR WebSocket连接已建立")
|
||||
self.server_ready = False
|
||||
self.forward_task = asyncio.create_task(self._forward_results(conn))
|
||||
|
||||
# 发送首帧音频
|
||||
if conn.asr_audio and len(conn.asr_audio) > 0:
|
||||
first_audio = conn.asr_audio[-1] if conn.asr_audio else b""
|
||||
pcm_frame = (
|
||||
self.decoder.decode(first_audio, 960) if first_audio else b""
|
||||
)
|
||||
await self._send_audio_frame(pcm_frame, STATUS_FIRST_FRAME)
|
||||
self.server_ready = True
|
||||
logger.bind(tag=TAG).info("已发送首帧，开始识别")
|
||||
|
||||
# 发送缓存的音频数据
|
||||
for cached_audio in conn.asr_audio[-10:]:
|
||||
try:
|
||||
pcm_frame = self.decoder.decode(cached_audio, 960)
|
||||
await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).info(f"发送缓存音频数据时发生错误: {e}")
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
|
||||
if hasattr(e, "__cause__") and e.__cause__:
|
||||
logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
|
||||
if self.asr_ws:
|
||||
await self.asr_ws.close()
|
||||
self.asr_ws = None
|
||||
self.is_processing = False
|
||||
raise
|
||||
|
||||
async def _send_audio_frame(self, audio_data: bytes, status: int):
|
||||
"""发送音频帧"""
|
||||
if not self.asr_ws:
|
||||
return
|
||||
|
||||
audio_b64 = base64.b64encode(audio_data).decode("utf-8")
|
||||
|
||||
frame_data = {
|
||||
"header": {"status": status, "app_id": self.app_id},
|
||||
"parameter": {"iat": self.iat_params},
|
||||
"payload": {
|
||||
"audio": {"audio": audio_b64, "sample_rate": 16000, "encoding": "raw"}
|
||||
},
|
||||
}
|
||||
|
||||
await self.asr_ws.send(json.dumps(frame_data, ensure_ascii=False))
|
||||
|
||||
async def _forward_results(self, conn):
|
||||
"""转发识别结果"""
|
||||
try:
|
||||
while not conn.stop_event.is_set():
|
||||
try:
|
||||
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=60)
|
||||
result = json.loads(response)
|
||||
logger.bind(tag=TAG).debug(f"收到ASR结果: {result}")
|
||||
|
||||
header = result.get("header", {})
|
||||
payload = result.get("payload", {})
|
||||
code = header.get("code", 0)
|
||||
status = header.get("status", 0)
|
||||
|
||||
if code != 0:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"识别错误，错误码: {code}, 消息: {header.get('message', '')}"
|
||||
)
|
||||
if code in [10114, 10160]: # 连接问题
|
||||
break
|
||||
continue
|
||||
|
||||
# 处理识别结果
|
||||
if payload and "result" in payload:
|
||||
text_data = payload["result"]["text"]
|
||||
if text_data:
|
||||
# 解码base64文本
|
||||
decoded_text = base64.b64decode(text_data).decode("utf-8")
|
||||
text_json = json.loads(decoded_text)
|
||||
# 提取文本内容
|
||||
text_ws = text_json.get("ws", [])
|
||||
for i in text_ws:
|
||||
for j in i.get("cw", []):
|
||||
w = j.get("w", "")
|
||||
self.text += w
|
||||
|
||||
if status == 2:
|
||||
if conn.client_listen_mode == "manual":
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
if len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
break
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
logger.bind(tag=TAG).error("接收结果超时")
|
||||
break
|
||||
except websockets.ConnectionClosed:
|
||||
logger.bind(tag=TAG).info("ASR服务连接已关闭")
|
||||
self.is_processing = False
|
||||
break
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理ASR结果时发生错误: {str(e)}")
|
||||
if hasattr(e, "__cause__") and e.__cause__:
|
||||
logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
|
||||
self.is_processing = False
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR结果转发任务发生错误: {str(e)}")
|
||||
if hasattr(e, "__cause__") and e.__cause__:
|
||||
logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
|
||||
finally:
|
||||
# 清理连接资源
|
||||
await self._cleanup()
|
||||
|
||||
# 清理连接的音频缓存
|
||||
if conn:
|
||||
if hasattr(conn, "asr_audio_for_voiceprint"):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, "asr_audio"):
|
||||
conn.asr_audio = []
|
||||
|
||||
async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
|
||||
"""处理语音停止，发送最后一帧并处理识别结果"""
|
||||
try:
|
||||
# 先发送最后一帧表示音频结束
|
||||
if self.asr_ws and self.is_processing:
|
||||
try:
|
||||
await self._send_audio_frame(b"", STATUS_LAST_FRAME)
|
||||
logger.bind(tag=TAG).debug(f"已发送停止请求")
|
||||
|
||||
await asyncio.sleep(0.25)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
|
||||
|
||||
await super().handle_voice_stop(conn, asr_audio_task)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
|
||||
import traceback
|
||||
|
||||
logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
|
||||
|
||||
def stop_ws_connection(self):
|
||||
if self.asr_ws:
|
||||
asyncio.create_task(self.asr_ws.close())
|
||||
self.asr_ws = None
|
||||
self.is_processing = False
|
||||
|
||||
async def _send_stop_request(self):
|
||||
"""发送停止识别请求（不关闭连接）"""
|
||||
if self.asr_ws:
|
||||
try:
|
||||
# 先停止音频发送
|
||||
self.is_processing = False
|
||||
await self._send_audio_frame(b"", STATUS_LAST_FRAME)
|
||||
logger.bind(tag=TAG).debug("已发送停止请求")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
|
||||
|
||||
async def _cleanup(self):
|
||||
"""清理资源（关闭连接）"""
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}"
|
||||
)
|
||||
|
||||
# 状态重置
|
||||
self.is_processing = False
|
||||
self.server_ready = False
|
||||
logger.bind(tag=TAG).debug("ASR状态已重置")
|
||||
|
||||
# 关闭连接
|
||||
if self.asr_ws:
|
||||
try:
|
||||
logger.bind(tag=TAG).debug("正在关闭WebSocket连接")
|
||||
await asyncio.wait_for(self.asr_ws.close(), timeout=2.0)
|
||||
logger.bind(tag=TAG).debug("WebSocket连接已关闭")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
|
||||
finally:
|
||||
self.asr_ws = None
|
||||
|
||||
# 清理任务引用
|
||||
self.forward_task = None
|
||||
|
||||
logger.bind(tag=TAG).debug("ASR会话清理完成")
|
||||
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format):
|
||||
"""获取识别结果"""
|
||||
result = self.text
|
||||
self.text = ""
|
||||
return result, None
|
||||
|
||||
async def close(self):
|
||||
"""资源清理方法"""
|
||||
if self.asr_ws:
|
||||
await self.asr_ws.close()
|
||||
self.asr_ws = None
|
||||
if self.forward_task:
|
||||
self.forward_task.cancel()
|
||||
try:
|
||||
await self.forward_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self.forward_task = None
|
||||
self.is_processing = False
|
||||
|
||||
# 显式释放decoder资源
|
||||
if hasattr(self, 'decoder') and self.decoder is not None:
|
||||
try:
|
||||
del self.decoder
|
||||
self.decoder = None
|
||||
logger.bind(tag=TAG).debug("Xunfei decoder resources released")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
|
||||
|
||||
# 清理所有连接的音频缓冲区
|
||||
if hasattr(self, "_connections"):
|
||||
for conn in self._connections.values():
|
||||
if hasattr(conn, "asr_audio_for_voiceprint"):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, "asr_audio"):
|
||||
conn.asr_audio = []
|
||||
@@ -53,24 +53,32 @@ class IntentProvider(IntentProviderBase):
|
||||
functions_desc += "---\n"
|
||||
|
||||
prompt = (
|
||||
"【严格格式要求】你必须只能返回JSON格式，绝对不能返回任何自然语言！\n\n"
|
||||
"你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
|
||||
"【重要规则】以下类型的查询请直接返回result_for_context，无需调用函数：\n"
|
||||
"- 询问当前时间（如：现在几点、当前时间、查询时间等）\n"
|
||||
"- 询问今天日期（如：今天几号、今天星期几、今天是什么日期等）\n"
|
||||
"- 询问今天农历（如：今天农历几号、今天什么节气等）\n"
|
||||
"- 询问所在城市（如：我现在在哪里、你知道我在哪个城市吗等）"
|
||||
"系统会根据上下文信息直接构建回答。\n\n"
|
||||
"- 如果用户使用疑问词（如'怎么'、'为什么'、'如何'）询问退出相关的问题（例如'怎么退出了？'），注意这不是让你退出，请返回 {'function_call': {'name': 'continue_chat'}\n"
|
||||
"- 仅当用户明确使用'退出系统'、'结束对话'、'我不想和你说话了'等指令时，才触发 handle_exit_intent\n\n"
|
||||
f"{functions_desc}\n"
|
||||
"处理步骤:\n"
|
||||
"1. 分析用户输入，确定用户意图\n"
|
||||
"2. 从可用函数列表中选择最匹配的函数\n"
|
||||
"3. 如果找到匹配的函数，生成对应的function_call 格式\n"
|
||||
'4. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
|
||||
"2. 检查是否为上述基础信息查询（时间、日期等），如是则返回result_for_context\n"
|
||||
"3. 从可用函数列表中选择最匹配的函数\n"
|
||||
"4. 如果找到匹配的函数，生成对应的function_call 格式\n"
|
||||
'5. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
|
||||
"返回格式要求：\n"
|
||||
"1. 必须返回纯JSON格式\n"
|
||||
"1. 必须返回纯JSON格式，不要包含任何其他文字\n"
|
||||
"2. 必须包含function_call字段\n"
|
||||
"3. function_call必须包含name字段\n"
|
||||
"4. 如果函数需要参数，必须包含arguments字段\n\n"
|
||||
"示例：\n"
|
||||
"```\n"
|
||||
"用户: 现在几点了？\n"
|
||||
'返回: {"function_call": {"name": "get_time"}}\n'
|
||||
'返回: {"function_call": {"name": "result_for_context"}}\n'
|
||||
"```\n"
|
||||
"```\n"
|
||||
"用户: 当前电池电量是多少？\n"
|
||||
@@ -94,12 +102,15 @@ class IntentProvider(IntentProviderBase):
|
||||
"```\n\n"
|
||||
"注意：\n"
|
||||
"1. 只返回JSON格式，不要包含任何其他文字\n"
|
||||
'2. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
|
||||
"3. 确保返回的JSON格式正确，包含所有必要的字段\n"
|
||||
'2. 优先检查用户查询是否为基础信息（时间、日期等），如是则返回{"function_call": {"name": "result_for_context"}}，不需要arguments参数\n'
|
||||
'3. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
|
||||
"4. 确保返回的JSON格式正确，包含所有必要的字段\n"
|
||||
"5. result_for_context不需要任何参数，系统会自动从上下文获取信息\n"
|
||||
"特殊说明：\n"
|
||||
"- 当用户单次输入包含多个指令时（如'打开灯并且调高音量'）\n"
|
||||
"- 请返回多个function_call组成的JSON数组\n"
|
||||
"- 示例：{'function_calls': [{name:'light_on'}, {name:'volume_up'}]}"
|
||||
"- 示例：{'function_calls': [{name:'light_on'}, {name:'volume_up'}]}\n\n"
|
||||
"【最终警告】绝对禁止输出任何自然语言、表情符号或解释文字！只能输出有效JSON格式！违反此规则将导致系统错误！"
|
||||
)
|
||||
return prompt
|
||||
|
||||
@@ -190,7 +201,7 @@ class IntentProvider(IntentProviderBase):
|
||||
# 记录LLM调用完成时间
|
||||
llm_time = time.time() - llm_start_time
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"LLM意图识别完成, 模型: {model_info}, 调用耗时: {llm_time:.4f}秒"
|
||||
f"外挂的大模型意图识别完成, 模型: {model_info}, 调用耗时: {llm_time:.4f}秒"
|
||||
)
|
||||
|
||||
# 记录后处理开始时间
|
||||
@@ -223,8 +234,15 @@ class IntentProvider(IntentProviderBase):
|
||||
f"llm 识别到意图: {function_name}, 参数: {function_args}"
|
||||
)
|
||||
|
||||
# 如果是继续聊天，清理工具调用相关的历史消息
|
||||
if function_name == "continue_chat":
|
||||
# 处理不同类型的意图
|
||||
if function_name == "result_for_context":
|
||||
# 处理基础信息查询，直接从context构建结果
|
||||
logger.bind(tag=TAG).info(
|
||||
"检测到result_for_context意图，将使用上下文信息直接回答"
|
||||
)
|
||||
|
||||
elif function_name == "continue_chat":
|
||||
# 处理普通对话
|
||||
# 保留非工具相关的消息
|
||||
clean_history = [
|
||||
msg
|
||||
@@ -233,25 +251,15 @@ class IntentProvider(IntentProviderBase):
|
||||
]
|
||||
conn.dialogue.dialogue = clean_history
|
||||
|
||||
# 添加到缓存
|
||||
self.cache_manager.set(self.CacheType.INTENT, cache_key, intent)
|
||||
else:
|
||||
# 处理函数调用
|
||||
logger.bind(tag=TAG).info(f"检测到函数调用意图: {function_name}")
|
||||
|
||||
# 后处理时间
|
||||
postprocess_time = time.time() - postprocess_start_time
|
||||
logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
|
||||
|
||||
# 确保返回完全序列化的JSON字符串
|
||||
return intent
|
||||
else:
|
||||
# 添加到缓存
|
||||
self.cache_manager.set(self.CacheType.INTENT, cache_key, intent)
|
||||
|
||||
# 后处理时间
|
||||
postprocess_time = time.time() - postprocess_start_time
|
||||
logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
|
||||
|
||||
# 返回普通意图
|
||||
return intent
|
||||
# 统一缓存处理和返回
|
||||
self.cache_manager.set(self.CacheType.INTENT, cache_key, intent)
|
||||
postprocess_time = time.time() - postprocess_start_time
|
||||
logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
|
||||
return intent
|
||||
except json.JSONDecodeError:
|
||||
# 后处理时间
|
||||
postprocess_time = time.time() - postprocess_start_time
|
||||
@@ -259,4 +267,4 @@ class IntentProvider(IntentProviderBase):
|
||||
f"无法解析意图JSON: {intent}, 后处理耗时: {postprocess_time:.4f}秒"
|
||||
)
|
||||
# 如果解析失败，默认返回继续聊天意图
|
||||
return '{"intent": "继续聊天"}'
|
||||
return '{"function_call": {"name": "continue_chat"}}'
|
||||
|
||||
@@ -17,27 +17,26 @@ class LLMProvider(LLMProviderBase):
|
||||
self.base_url = config.get("base_url")
|
||||
else:
|
||||
self.base_url = config.get("url")
|
||||
# 增加timeout的配置项，单位为秒
|
||||
timeout = config.get("timeout", 300)
|
||||
self.timeout = int(timeout) if timeout else 300
|
||||
|
||||
param_defaults = {
|
||||
"max_tokens": (500, int),
|
||||
"temperature": (0.7, lambda x: round(float(x), 1)),
|
||||
"top_p": (1.0, lambda x: round(float(x), 1)),
|
||||
"frequency_penalty": (0, lambda x: round(float(x), 1)),
|
||||
"max_tokens": int,
|
||||
"temperature": lambda x: round(float(x), 1),
|
||||
"top_p": lambda x: round(float(x), 1),
|
||||
"frequency_penalty": lambda x: round(float(x), 1),
|
||||
}
|
||||
|
||||
for param, (default, converter) in param_defaults.items():
|
||||
for param, converter in param_defaults.items():
|
||||
value = config.get(param)
|
||||
try:
|
||||
setattr(
|
||||
self,
|
||||
param,
|
||||
converter(value) if value not in (None, "") else default,
|
||||
converter(value) if value not in (None, "") else None,
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
setattr(self, param, default)
|
||||
setattr(self, param, None)
|
||||
|
||||
logger.debug(
|
||||
f"意图识别参数初始化: {self.temperature}, {self.max_tokens}, {self.top_p}, {self.frequency_penalty}"
|
||||
@@ -48,34 +47,46 @@ class LLMProvider(LLMProviderBase):
|
||||
logger.bind(tag=TAG).error(model_key_msg)
|
||||
self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url, timeout=httpx.Timeout(self.timeout))
|
||||
|
||||
@staticmethod
|
||||
def normalize_dialogue(dialogue):
|
||||
"""自动修复 dialogue 中缺失 content 的消息"""
|
||||
for msg in dialogue:
|
||||
if "role" in msg and "content" not in msg:
|
||||
msg["content"] = ""
|
||||
return dialogue
|
||||
|
||||
def response(self, session_id, dialogue, **kwargs):
|
||||
try:
|
||||
responses = self.client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=dialogue,
|
||||
stream=True,
|
||||
max_tokens=kwargs.get("max_tokens", self.max_tokens),
|
||||
temperature=kwargs.get("temperature", self.temperature),
|
||||
top_p=kwargs.get("top_p", self.top_p),
|
||||
frequency_penalty=kwargs.get(
|
||||
"frequency_penalty", self.frequency_penalty
|
||||
),
|
||||
)
|
||||
dialogue = self.normalize_dialogue(dialogue)
|
||||
|
||||
request_params = {
|
||||
"model": self.model_name,
|
||||
"messages": dialogue,
|
||||
"stream": True,
|
||||
}
|
||||
|
||||
# 添加可选参数,只有当参数不为None时才添加
|
||||
optional_params = {
|
||||
"max_tokens": kwargs.get("max_tokens", self.max_tokens),
|
||||
"temperature": kwargs.get("temperature", self.temperature),
|
||||
"top_p": kwargs.get("top_p", self.top_p),
|
||||
"frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
|
||||
}
|
||||
|
||||
for key, value in optional_params.items():
|
||||
if value is not None:
|
||||
request_params[key] = value
|
||||
|
||||
responses = self.client.chat.completions.create(**request_params)
|
||||
|
||||
is_active = True
|
||||
for chunk in responses:
|
||||
try:
|
||||
# 检查是否存在有效的choice且content不为空
|
||||
delta = (
|
||||
chunk.choices[0].delta
|
||||
if getattr(chunk, "choices", None)
|
||||
else None
|
||||
)
|
||||
content = delta.content if hasattr(delta, "content") else ""
|
||||
delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
|
||||
content = getattr(delta, "content", "") if delta else ""
|
||||
except IndexError:
|
||||
content = ""
|
||||
if content:
|
||||
# 处理标签跨多个chunk的情况
|
||||
if "<think>" in content:
|
||||
is_active = False
|
||||
content = content.split("<think>")[0]
|
||||
@@ -88,19 +99,36 @@ class LLMProvider(LLMProviderBase):
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
|
||||
|
||||
def response_with_functions(self, session_id, dialogue, functions=None):
|
||||
def response_with_functions(self, session_id, dialogue, functions=None, **kwargs):
|
||||
try:
|
||||
stream = self.client.chat.completions.create(
|
||||
model=self.model_name, messages=dialogue, stream=True, tools=functions
|
||||
)
|
||||
dialogue = self.normalize_dialogue(dialogue)
|
||||
|
||||
request_params = {
|
||||
"model": self.model_name,
|
||||
"messages": dialogue,
|
||||
"stream": True,
|
||||
"tools": functions,
|
||||
}
|
||||
|
||||
optional_params = {
|
||||
"max_tokens": kwargs.get("max_tokens", self.max_tokens),
|
||||
"temperature": kwargs.get("temperature", self.temperature),
|
||||
"top_p": kwargs.get("top_p", self.top_p),
|
||||
"frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
|
||||
}
|
||||
|
||||
for key, value in optional_params.items():
|
||||
if value is not None:
|
||||
request_params[key] = value
|
||||
|
||||
stream = self.client.chat.completions.create(**request_params)
|
||||
|
||||
for chunk in stream:
|
||||
# 检查是否存在有效的choice且content不为空
|
||||
if getattr(chunk, "choices", None):
|
||||
yield chunk.choices[0].delta.content, chunk.choices[
|
||||
0
|
||||
].delta.tool_calls
|
||||
# 存在 CompletionUsage 消息时，生成 Token 消耗 log
|
||||
delta = chunk.choices[0].delta
|
||||
content = getattr(delta, "content", "")
|
||||
tool_calls = getattr(delta, "tool_calls", None)
|
||||
yield content, tool_calls
|
||||
elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
|
||||
usage_info = getattr(chunk, "usage", None)
|
||||
logger.bind(tag=TAG).info(
|
||||
|
||||
@@ -14,7 +14,7 @@ class MemoryProviderBase(ABC):
|
||||
self.llm = llm
|
||||
|
||||
@abstractmethod
|
||||
async def save_memory(self, msgs):
|
||||
async def save_memory(self, msgs, session_id=None):
|
||||
"""Save a new memory for specific role and return memory ID"""
|
||||
print("this is base func", msgs)
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ class MemoryProvider(MemoryProviderBase):
|
||||
logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
|
||||
self.use_mem0 = False
|
||||
|
||||
async def save_memory(self, msgs):
|
||||
async def save_memory(self, msgs, session_id=None):
|
||||
if not self.use_mem0:
|
||||
return None
|
||||
if len(msgs) < 2:
|
||||
@@ -41,9 +41,7 @@ class MemoryProvider(MemoryProviderBase):
|
||||
for message in msgs
|
||||
if message.role != "system"
|
||||
]
|
||||
result = self.client.add(
|
||||
messages, user_id=self.role_id, output_format=self.api_version
|
||||
)
|
||||
result = self.client.add(messages, user_id=self.role_id)
|
||||
logger.bind(tag=TAG).debug(f"Save memory result: {result}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
|
||||
@@ -53,9 +51,12 @@ class MemoryProvider(MemoryProviderBase):
|
||||
if not self.use_mem0:
|
||||
return ""
|
||||
try:
|
||||
results = self.client.search(
|
||||
query, user_id=self.role_id, output_format=self.api_version
|
||||
)
|
||||
if not getattr(self, "role_id", None):
|
||||
return ""
|
||||
|
||||
filters = {"user_id": self.role_id}
|
||||
|
||||
results = self.client.search(query, filters=filters)
|
||||
if not results or "results" not in results:
|
||||
return ""
|
||||
|
||||
|
||||
@@ -4,7 +4,8 @@ import json
|
||||
import os
|
||||
import yaml
|
||||
from config.config_loader import get_project_dir
|
||||
from config.manage_api_client import save_mem_local_short
|
||||
from config.manage_api_client import generate_and_save_chat_summary
|
||||
import asyncio
|
||||
from core.utils.util import check_model_key
|
||||
|
||||
|
||||
@@ -74,18 +75,6 @@ short_term_memory_prompt = """
|
||||
```
|
||||
"""
|
||||
|
||||
short_term_memory_prompt_only_content = """
|
||||
你是一个经验丰富的记忆总结者，擅长将对话内容进行总结摘要，遵循以下规则：
|
||||
1、总结user的重要信息，以便在未来的对话中提供更个性化的服务
|
||||
2、不要重复总结，不要遗忘之前记忆，除非原来的记忆超过了1800字内，否则不要遗忘、不要压缩用户的历史记忆
|
||||
3、用户操控的设备音量、播放音乐、天气、退出、不想对话等和用户本身无关的内容，这些信息不需要加入到总结中
|
||||
4、聊天内容中的今天的日期时间、今天的天气情况与用户事件无关的数据，这些信息如果当成记忆存储会影响后序对话，这些信息不需要加入到总结中
|
||||
5、不要把设备操控的成果结果和失败结果加入到总结中，也不要把用户的一些废话加入到总结中
|
||||
6、不要为了总结而总结，如果用户的聊天没有意义，请返回原来的历史记录也是可以的
|
||||
7、只需要返回总结摘要，严格控制在1800字内
|
||||
8、不要包含代码、xml，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
|
||||
"""
|
||||
|
||||
|
||||
def extract_json_data(json_code):
|
||||
start = json_code.find("```json")
|
||||
@@ -143,7 +132,7 @@ class MemoryProvider(MemoryProviderBase):
|
||||
with open(self.memory_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(all_memory, f, allow_unicode=True)
|
||||
|
||||
async def save_memory(self, msgs):
|
||||
async def save_memory(self, msgs, session_id=None):
|
||||
# 打印使用的模型信息
|
||||
model_info = getattr(self.llm, "model_name", str(self.llm.__class__.__name__))
|
||||
logger.bind(tag=TAG).debug(f"使用记忆保存模型: {model_info}")
|
||||
@@ -187,14 +176,12 @@ class MemoryProvider(MemoryProviderBase):
|
||||
except Exception as e:
|
||||
print("Error:", e)
|
||||
else:
|
||||
result = self.llm.response_no_stream(
|
||||
short_term_memory_prompt_only_content,
|
||||
msgStr,
|
||||
max_tokens=2000,
|
||||
temperature=0.2,
|
||||
)
|
||||
save_mem_local_short(self.role_id, result)
|
||||
logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
|
||||
# 当save_to_file为False时，调用Java端的聊天记录总结接口
|
||||
summary_id = session_id if session_id else self.role_id
|
||||
await generate_and_save_chat_summary(summary_id)
|
||||
logger.bind(tag=TAG).info(
|
||||
f"Save memory successful - Role: {self.role_id}, Session: {session_id}"
|
||||
)
|
||||
|
||||
return self.short_memory
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ class MemoryProvider(MemoryProviderBase):
|
||||
def __init__(self, config, summary_memory=None):
|
||||
super().__init__(config)
|
||||
|
||||
async def save_memory(self, msgs):
|
||||
async def save_memory(self, msgs, session_id=None):
|
||||
logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
|
||||
return None
|
||||
|
||||
|
||||
@@ -116,14 +116,14 @@ async def send_mcp_message(conn, payload: dict, transport=None):
|
||||
await conn.transport.send(message)
|
||||
else:
|
||||
raise AttributeError("无法找到可用的传输层接口")
|
||||
logger.bind(tag=TAG).info(f"成功发送MCP消息: {message}")
|
||||
logger.bind(tag=TAG).debug(f"成功发送MCP消息: {message}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送MCP消息失败: {e}")
|
||||
|
||||
|
||||
async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict, transport=None):
|
||||
"""处理MCP消息,包括初始化、工具列表和工具调用响应等"""
|
||||
logger.bind(tag=TAG).info(f"处理MCP消息: {str(payload)[:100]}")
|
||||
logger.bind(tag=TAG).debug(f"处理MCP消息: {str(payload)[:100]}")
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
logger.bind(tag=TAG).error("MCP消息缺少payload字段或格式错误")
|
||||
@@ -148,7 +148,7 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict, transpo
|
||||
if isinstance(server_info, dict):
|
||||
name = server_info.get("name")
|
||||
version = server_info.get("version")
|
||||
logger.bind(tag=TAG).info(
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"客户端MCP服务器信息: name={name}, version={version}"
|
||||
)
|
||||
return
|
||||
@@ -205,11 +205,11 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict, transpo
|
||||
|
||||
next_cursor = result.get("nextCursor", "")
|
||||
if next_cursor:
|
||||
logger.bind(tag=TAG).info(f"有更多工具，nextCursor: {next_cursor}")
|
||||
logger.bind(tag=TAG).debug(f"有更多工具，nextCursor: {next_cursor}")
|
||||
await send_mcp_tools_list_continue_request(conn, next_cursor, transport)
|
||||
else:
|
||||
await mcp_client.set_ready(True)
|
||||
logger.bind(tag=TAG).info("所有工具已获取，MCP客户端准备就绪")
|
||||
logger.bind(tag=TAG).debug("所有工具已获取，MCP客户端准备就绪")
|
||||
|
||||
# 刷新工具缓存，确保MCP工具被包含在函数列表中
|
||||
if hasattr(conn, "func_handler") and conn.func_handler:
|
||||
@@ -265,7 +265,7 @@ async def send_mcp_initialize_message(conn, transport=None):
|
||||
},
|
||||
},
|
||||
}
|
||||
logger.bind(tag=TAG).info("发送MCP初始化消息")
|
||||
logger.bind(tag=TAG).debug("发送MCP初始化消息")
|
||||
await send_mcp_message(conn, payload, transport)
|
||||
|
||||
|
||||
|
||||
@@ -358,7 +358,9 @@ async def call_mcp_endpoint_tool(
|
||||
}
|
||||
|
||||
message = json.dumps(payload)
|
||||
logger.bind(tag=TAG).info(f"发送MCP接入点工具调用请求: {actual_name}，参数: {args}")
|
||||
logger.bind(tag=TAG).info(
|
||||
f"发送MCP接入点工具调用请求: {actual_name}，参数: {json.dumps(arguments, ensure_ascii=False)}"
|
||||
)
|
||||
await mcp_client.send_message(message)
|
||||
|
||||
try:
|
||||
|
||||
@@ -10,9 +10,13 @@ import concurrent.futures
|
||||
from contextlib import AsyncExitStack
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
from mcp import ClientSession, StdioServerParameters
|
||||
from mcp import ClientSession, StdioServerParameters, Implementation
|
||||
from mcp.client.session import SamplingFnT, ElicitationFnT, ListRootsFnT, LoggingFnT, MessageHandlerFnT
|
||||
from mcp.client.stdio import stdio_client
|
||||
from mcp.client.sse import sse_client
|
||||
from mcp.client.streamable_http import streamablehttp_client
|
||||
from mcp.shared.session import ProgressFnT
|
||||
|
||||
from config.logger import setup_logging
|
||||
from core.utils.util import sanitize_tool_name
|
||||
|
||||
@@ -40,13 +44,25 @@ class ServerMCPClient:
|
||||
self.tools_dict: Dict[str, Any] = {}
|
||||
self.name_mapping: Dict[str, str] = {}
|
||||
|
||||
async def initialize(self):
|
||||
async def initialize(self, read_timeout_seconds: timedelta | None = None,
|
||||
sampling_callback: SamplingFnT | None = None,
|
||||
elicitation_callback: ElicitationFnT | None = None,
|
||||
list_roots_callback: ListRootsFnT | None = None,
|
||||
logging_callback: LoggingFnT | None = None,
|
||||
message_handler: MessageHandlerFnT | None = None,
|
||||
client_info: Implementation | None = None):
|
||||
"""初始化MCP客户端连接"""
|
||||
if self._worker_task:
|
||||
return
|
||||
|
||||
self._worker_task = asyncio.create_task(
|
||||
self._worker(), name="ServerMCPClientWorker"
|
||||
self._worker(read_timeout_seconds=read_timeout_seconds,
|
||||
sampling_callback=sampling_callback,
|
||||
elicitation_callback=elicitation_callback,
|
||||
list_roots_callback=list_roots_callback,
|
||||
logging_callback=logging_callback,
|
||||
message_handler=message_handler,
|
||||
client_info=client_info), name="ServerMCPClientWorker"
|
||||
)
|
||||
await self._ready_evt.wait()
|
||||
|
||||
@@ -96,12 +112,15 @@ class ServerMCPClient:
|
||||
for name, tool in self.tools_dict.items()
|
||||
]
|
||||
|
||||
async def call_tool(self, name: str, args: dict) -> Any:
|
||||
async def call_tool(self, name: str, arguments: dict, read_timeout_seconds: timedelta | None = None, progress_callback: ProgressFnT | None = None, *, meta: dict[str, Any] | None = None) -> Any:
|
||||
"""调用指定工具
|
||||
|
||||
Args:
|
||||
name: 工具名称
|
||||
args: 工具参数
|
||||
arguments: 工具参数
|
||||
read_timeout_seconds:
|
||||
progress_callback: 进度回调函数
|
||||
meta:
|
||||
|
||||
Returns:
|
||||
Any: 工具执行结果
|
||||
@@ -114,7 +133,7 @@ class ServerMCPClient:
|
||||
|
||||
real_name = self.name_mapping.get(name, name)
|
||||
loop = self._worker_task.get_loop()
|
||||
coro = self.session.call_tool(real_name, args)
|
||||
coro = self.session.call_tool(real_name, arguments=arguments, read_timeout_seconds=read_timeout_seconds, progress_callback=progress_callback, meta=meta)
|
||||
|
||||
if loop is asyncio.get_running_loop():
|
||||
return await coro
|
||||
@@ -143,7 +162,13 @@ class ServerMCPClient:
|
||||
# 所有检查都通过，连接正常
|
||||
return True
|
||||
|
||||
async def _worker(self):
|
||||
async def _worker(self, read_timeout_seconds: timedelta | None = None,
|
||||
sampling_callback: SamplingFnT | None = None,
|
||||
elicitation_callback: ElicitationFnT | None = None,
|
||||
list_roots_callback: ListRootsFnT | None = None,
|
||||
logging_callback: LoggingFnT | None = None,
|
||||
message_handler: MessageHandlerFnT | None = None,
|
||||
client_info: Implementation | None = None):
|
||||
"""MCP客户端工作协程"""
|
||||
async with AsyncExitStack() as stack:
|
||||
try:
|
||||
@@ -167,16 +192,38 @@ class ServerMCPClient:
|
||||
|
||||
# 建立SSEClient
|
||||
elif "url" in self.config:
|
||||
headers = dict(self.config.get("headers", {}))
|
||||
# TODO 兼容旧版本
|
||||
if "API_ACCESS_TOKEN" in self.config:
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.config['API_ACCESS_TOKEN']}"
|
||||
}
|
||||
headers["Authorization"] = f"Bearer {self.config['API_ACCESS_TOKEN']}"
|
||||
self.logger.bind(tag=TAG).warning(f"你正在使用旧过时的配置 API_ACCESS_TOKEN ，请在.mcp_server_settings.json中将API_ACCESS_TOKEN直接设置在headers中，例如 'Authorization': 'Bearer API_ACCESS_TOKEN'")
|
||||
|
||||
# 根据transport类型选择不同的客户端，默认为SSE
|
||||
transport_type = self.config.get("transport", "sse")
|
||||
|
||||
if transport_type == "streamable-http" or transport_type == "http":
|
||||
# 使用 Streamable HTTP 传输
|
||||
http_r, http_w, get_session_id = await stack.enter_async_context(
|
||||
streamablehttp_client(
|
||||
url=self.config["url"],
|
||||
headers=headers,
|
||||
timeout=self.config.get("timeout", 30),
|
||||
sse_read_timeout=self.config.get("sse_read_timeout", 60 * 5),
|
||||
terminate_on_close=self.config.get("terminate_on_close", True)
|
||||
)
|
||||
)
|
||||
read_stream, write_stream = http_r, http_w
|
||||
else:
|
||||
headers = {}
|
||||
sse_r, sse_w = await stack.enter_async_context(
|
||||
sse_client(self.config["url"], headers=headers)
|
||||
)
|
||||
read_stream, write_stream = sse_r, sse_w
|
||||
# 使用传统的 SSE 传输
|
||||
sse_r, sse_w = await stack.enter_async_context(
|
||||
sse_client(
|
||||
url=self.config["url"],
|
||||
headers=headers,
|
||||
timeout=self.config.get("timeout", 5),
|
||||
sse_read_timeout=self.config.get("sse_read_timeout", 60 * 5)
|
||||
)
|
||||
)
|
||||
read_stream, write_stream = sse_r, sse_w
|
||||
|
||||
else:
|
||||
raise ValueError("MCP客户端配置必须包含'command'或'url'")
|
||||
@@ -185,7 +232,13 @@ class ServerMCPClient:
|
||||
ClientSession(
|
||||
read_stream=read_stream,
|
||||
write_stream=write_stream,
|
||||
read_timeout_seconds=timedelta(seconds=15),
|
||||
read_timeout_seconds=read_timeout_seconds,
|
||||
sampling_callback=sampling_callback,
|
||||
elicitation_callback=elicitation_callback,
|
||||
list_roots_callback=list_roots_callback,
|
||||
logging_callback=logging_callback,
|
||||
message_handler=message_handler,
|
||||
client_info=client_info
|
||||
)
|
||||
)
|
||||
await self.session.initialize()
|
||||
|
||||
@@ -4,6 +4,9 @@ import asyncio
|
||||
import os
|
||||
import json
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from mcp.types import LoggingMessageNotificationParams
|
||||
|
||||
from config.config_loader import get_project_dir
|
||||
from config.logger import setup_logging
|
||||
from .mcp_client import ServerMCPClient
|
||||
@@ -26,6 +29,7 @@ class ServerMCPManager:
|
||||
)
|
||||
self.clients: Dict[str, ServerMCPClient] = {}
|
||||
self.tools = []
|
||||
self._init_lock = asyncio.Lock()
|
||||
|
||||
def load_config(self) -> Dict[str, Any]:
|
||||
"""加载MCP服务配置"""
|
||||
@@ -42,29 +46,50 @@ class ServerMCPManager:
|
||||
)
|
||||
return {}
|
||||
|
||||
async def _init_server(self, name: str, srv_config: Dict[str, Any]):
|
||||
"""初始化单个MCP服务"""
|
||||
client = None
|
||||
try:
|
||||
# 初始化服务端MCP客户端
|
||||
logger.bind(tag=TAG).info(f"初始化服务端MCP客户端: {name}")
|
||||
client = ServerMCPClient(srv_config)
|
||||
# 设置超时时间10秒
|
||||
await asyncio.wait_for(client.initialize(logging_callback=self.logging_callback), timeout=10)
|
||||
|
||||
# 使用锁保护共享状态的修改
|
||||
async with self._init_lock:
|
||||
self.clients[name] = client
|
||||
client_tools = client.get_available_tools()
|
||||
self.tools.extend(client_tools)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"Failed to initialize MCP server {name}: Timeout"
|
||||
)
|
||||
if client:
|
||||
await client.cleanup()
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"Failed to initialize MCP server {name}: {e}"
|
||||
)
|
||||
if client:
|
||||
await client.cleanup()
|
||||
|
||||
async def initialize_servers(self) -> None:
|
||||
"""初始化所有MCP服务"""
|
||||
config = self.load_config()
|
||||
tasks = []
|
||||
for name, srv_config in config.items():
|
||||
if not srv_config.get("command") and not srv_config.get("url"):
|
||||
logger.bind(tag=TAG).warning(
|
||||
f"Skipping server {name}: neither command nor url specified"
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
# 初始化服务端MCP客户端
|
||||
logger.bind(tag=TAG).info(f"初始化服务端MCP客户端: {name}")
|
||||
client = ServerMCPClient(srv_config)
|
||||
await client.initialize()
|
||||
self.clients[name] = client
|
||||
client_tools = client.get_available_tools()
|
||||
self.tools.extend(client_tools)
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"Failed to initialize MCP server {name}: {e}"
|
||||
)
|
||||
|
||||
tasks.append(self._init_server(name, srv_config))
|
||||
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
# 输出当前支持的服务端MCP工具列表
|
||||
if hasattr(self.conn, "func_handler") and self.conn.func_handler:
|
||||
@@ -109,7 +134,7 @@ class ServerMCPManager:
|
||||
# 带重试机制的工具调用
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return await target_client.call_tool(tool_name, arguments)
|
||||
return await target_client.call_tool(tool_name, arguments, progress_callback=self.progress_callback)
|
||||
except Exception as e:
|
||||
# 最后一次尝试失败时直接抛出异常
|
||||
if attempt == max_retries - 1:
|
||||
@@ -131,7 +156,7 @@ class ServerMCPManager:
|
||||
config = self.load_config()
|
||||
if client_name in config:
|
||||
client = ServerMCPClient(config[client_name])
|
||||
await client.initialize()
|
||||
await client.initialize(logging_callback=self.logging_callback)
|
||||
self.clients[client_name] = client
|
||||
target_client = client
|
||||
logger.bind(tag=TAG).info(
|
||||
@@ -159,3 +184,11 @@ class ServerMCPManager:
|
||||
except (asyncio.TimeoutError, Exception) as e:
|
||||
logger.bind(tag=TAG).error(f"关闭服务端MCP客户端 {name} 时出错: {e}")
|
||||
self.clients.clear()
|
||||
|
||||
# 可选回调方法
|
||||
|
||||
async def logging_callback(self, params: LoggingMessageNotificationParams):
|
||||
logger.bind(tag=TAG).info(f"[Server Log - {params.level.upper()}] {params.data}")
|
||||
|
||||
async def progress_callback(self, progress: float, total: float | None, message: str | None) -> None:
|
||||
logger.bind(tag=TAG).info(f"[Progress {progress}/{total}]: {message}")
|
||||
@@ -71,6 +71,19 @@ class ServerPluginExecutor(ToolExecutor):
|
||||
for func_name in all_required_functions:
|
||||
func_item = all_function_registry.get(func_name)
|
||||
if func_item:
|
||||
# 从函数注册中获取描述
|
||||
fun_description = (
|
||||
self.config.get("plugins", {})
|
||||
.get(func_name, {})
|
||||
.get("description", "")
|
||||
)
|
||||
if fun_description is not None and len(fun_description) > 0:
|
||||
if "function" in func_item.description and isinstance(
|
||||
func_item.description["function"], dict
|
||||
):
|
||||
func_item.description["function"][
|
||||
"description"
|
||||
] = fun_description
|
||||
tools[func_name] = ToolDefinition(
|
||||
name=func_name,
|
||||
description=func_item.description,
|
||||
|
||||
@@ -69,7 +69,7 @@ class UnifiedToolHandler:
|
||||
self._initialize_home_assistant()
|
||||
|
||||
self.finish_init = True
|
||||
self.logger.info("统一工具处理器初始化完成")
|
||||
self.logger.debug("统一工具处理器初始化完成")
|
||||
|
||||
# 输出当前支持的所有工具列表
|
||||
self.current_support_functions()
|
||||
|
||||
@@ -20,7 +20,7 @@ class ToolManager:
|
||||
"""注册工具执行器"""
|
||||
self.executors[tool_type] = executor
|
||||
self._invalidate_cache()
|
||||
self.logger.info(f"注册工具执行器: {tool_type.value}")
|
||||
self.logger.debug(f"注册工具执行器: {tool_type.value}")
|
||||
|
||||
def _invalidate_cache(self):
|
||||
"""使缓存失效"""
|
||||
@@ -109,7 +109,7 @@ class ToolManager:
|
||||
def refresh_tools(self):
|
||||
"""刷新工具缓存"""
|
||||
self._invalidate_cache()
|
||||
self.logger.info("工具缓存已刷新")
|
||||
self.logger.debug("工具缓存已刷新")
|
||||
|
||||
def get_tool_statistics(self) -> Dict[str, int]:
|
||||
"""获取工具统计信息"""
|
||||
|
||||
@@ -0,0 +1,522 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import time
|
||||
import queue
|
||||
import asyncio
|
||||
import traceback
|
||||
import websockets
|
||||
from asyncio import Task
|
||||
from config.logger import setup_logging
|
||||
from core.utils import opus_encoder_utils
|
||||
from core.utils.tts import MarkdownCleaner
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
|
||||
self.interface_type = InterfaceType.DUAL_STREAM
|
||||
# 基础配置
|
||||
self.api_key = config.get("api_key")
|
||||
if not self.api_key:
|
||||
raise ValueError("api_key is required for CosyVoice TTS")
|
||||
|
||||
# WebSocket配置
|
||||
self.ws_url = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/"
|
||||
self.ws = None
|
||||
self._monitor_task = None
|
||||
self.last_active_time = None
|
||||
|
||||
# 模型和音色配置
|
||||
self.model = config.get("model", "cosyvoice-v2")
|
||||
self.voice = config.get("voice", "longxiaochun_v2") # 默认音色
|
||||
if config.get("private_voice"):
|
||||
self.voice = config.get("private_voice")
|
||||
|
||||
# 音频参数配置
|
||||
self.format = config.get("format", "pcm")
|
||||
sample_rate = config.get("sample_rate", "24000")
|
||||
self.sample_rate = int(sample_rate) if sample_rate else 24000
|
||||
|
||||
volume = config.get("volume", "50")
|
||||
self.volume = int(volume) if volume else 50
|
||||
|
||||
rate = config.get("rate", "1.0")
|
||||
self.rate = float(rate) if rate else 1.0
|
||||
|
||||
pitch = config.get("pitch", "1.0")
|
||||
self.pitch = float(pitch) if pitch else 1.0
|
||||
|
||||
self.header = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
# "user-agent": "your_platform_info", // 可选
|
||||
# "X-DashScope-WorkSpace": workspace, // 可选，阿里云百炼业务空间ID
|
||||
"X-DashScope-DataInspection": "enable",
|
||||
}
|
||||
|
||||
# 创建Opus编码器
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
async def _ensure_connection(self):
|
||||
"""确保WebSocket连接可用，支持60秒内连接复用"""
|
||||
try:
|
||||
current_time = time.time()
|
||||
if self.ws and current_time - self.last_active_time < 60:
|
||||
# 一分钟内才可以复用链接进行连续对话
|
||||
logger.bind(tag=TAG).info(f"使用已有链接...")
|
||||
return self.ws
|
||||
logger.bind(tag=TAG).info("开始建立新连接...")
|
||||
|
||||
self.ws = await websockets.connect(
|
||||
self.ws_url,
|
||||
additional_headers=self.header,
|
||||
ping_interval=30,
|
||||
ping_timeout=10,
|
||||
close_timeout=10,
|
||||
)
|
||||
|
||||
logger.bind(tag=TAG).info("WebSocket连接建立成功")
|
||||
self.last_active_time = current_time
|
||||
return self.ws
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}")
|
||||
self.ws = None
|
||||
self.last_active_time = None
|
||||
raise
|
||||
|
||||
def tts_text_priority_thread(self):
|
||||
"""流式TTS文本处理线程"""
|
||||
while not self.conn.stop_event.is_set():
|
||||
try:
|
||||
message = self.tts_text_queue.get(timeout=1)
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}"
|
||||
)
|
||||
|
||||
if message.sentence_type == SentenceType.FIRST:
|
||||
self.conn.client_abort = False
|
||||
|
||||
if self.conn.client_abort:
|
||||
try:
|
||||
logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"取消TTS会话失败: {str(e)}")
|
||||
continue
|
||||
|
||||
if message.sentence_type == SentenceType.FIRST:
|
||||
# 初始化会话
|
||||
try:
|
||||
if not getattr(self.conn, "sentence_id", None):
|
||||
self.conn.sentence_id = uuid.uuid4().hex
|
||||
logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}")
|
||||
|
||||
logger.bind(tag=TAG).info("开始启动TTS会话...")
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.start_session(self.conn.sentence_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
self.before_stop_play_files.clear()
|
||||
logger.bind(tag=TAG).info("TTS会话启动成功")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
|
||||
continue
|
||||
|
||||
elif ContentType.TEXT == message.content_type:
|
||||
if message.content_detail:
|
||||
try:
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"开始发送TTS文本: {message.content_detail}"
|
||||
)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.text_to_speak(message.content_detail, None),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
logger.bind(tag=TAG).debug("TTS文本发送成功")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
|
||||
continue
|
||||
|
||||
elif ContentType.FILE == message.content_type:
|
||||
logger.bind(tag=TAG).info(
|
||||
f"添加音频文件到待播放列表: {message.content_file}"
|
||||
)
|
||||
if message.content_file and os.path.exists(message.content_file):
|
||||
# 先处理文件音频数据
|
||||
self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
|
||||
|
||||
if message.sentence_type == SentenceType.LAST:
|
||||
try:
|
||||
logger.bind(tag=TAG).info("开始结束TTS会话...")
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.finish_session(self.conn.sentence_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"结束TTS会话失败: {str(e)}")
|
||||
continue
|
||||
|
||||
except queue.Empty:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
|
||||
)
|
||||
continue
|
||||
|
||||
async def text_to_speak(self, text, _):
|
||||
"""发送文本到TTS服务进行合成"""
|
||||
try:
|
||||
if self.ws is None:
|
||||
logger.bind(tag=TAG).warning("WebSocket连接不存在，终止发送文本")
|
||||
return
|
||||
|
||||
# 过滤Markdown
|
||||
filtered_text = MarkdownCleaner.clean_markdown(text)
|
||||
|
||||
# 发送continue-task消息
|
||||
continue_task_message = {
|
||||
"header": {
|
||||
"action": "continue-task",
|
||||
"task_id": self.conn.sentence_id,
|
||||
"streaming": "duplex",
|
||||
},
|
||||
"payload": {"input": {"text": filtered_text}},
|
||||
}
|
||||
|
||||
await self.ws.send(json.dumps(continue_task_message))
|
||||
self.last_active_time = time.time()
|
||||
logger.bind(tag=TAG).debug(f"已发送文本: {filtered_text}")
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
|
||||
if self.ws:
|
||||
try:
|
||||
await self.ws.close()
|
||||
except:
|
||||
pass
|
||||
self.ws = None
|
||||
raise
|
||||
|
||||
async def start_session(self, session_id):
|
||||
"""启动TTS会话"""
|
||||
logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
|
||||
try:
|
||||
# 检查并清理上一个会话的监听任务
|
||||
if (
|
||||
self._monitor_task is not None
|
||||
and isinstance(self._monitor_task, Task)
|
||||
and not self._monitor_task.done()
|
||||
):
|
||||
logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务...")
|
||||
await self.close()
|
||||
|
||||
# 确保连接可用
|
||||
await self._ensure_connection()
|
||||
|
||||
# 启动监听任务
|
||||
self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
|
||||
|
||||
# 发送run-task消息启动会话
|
||||
run_task_message = {
|
||||
"header": {
|
||||
"action": "run-task",
|
||||
"task_id": session_id,
|
||||
"streaming": "duplex",
|
||||
},
|
||||
"payload": {
|
||||
"task_group": "audio",
|
||||
"task": "tts",
|
||||
"function": "SpeechSynthesizer",
|
||||
"model": self.model,
|
||||
"parameters": {
|
||||
"text_type": "PlainText",
|
||||
"voice": self.voice,
|
||||
"format": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"volume": self.volume,
|
||||
"rate": self.rate,
|
||||
"pitch": self.pitch,
|
||||
},
|
||||
"input": {}
|
||||
},
|
||||
}
|
||||
|
||||
await self.ws.send(json.dumps(run_task_message))
|
||||
self.last_active_time = time.time()
|
||||
logger.bind(tag=TAG).info("会话启动请求已发送")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
|
||||
await self.close()
|
||||
raise
|
||||
|
||||
async def finish_session(self, session_id):
|
||||
"""结束TTS会话"""
|
||||
logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
|
||||
try:
|
||||
if self.ws and session_id:
|
||||
# 发送finish-task消息
|
||||
finish_task_message = {
|
||||
"header": {
|
||||
"action": "finish-task",
|
||||
"task_id": session_id,
|
||||
"streaming": "duplex",
|
||||
},
|
||||
"payload": {
|
||||
"input": {}
|
||||
}
|
||||
}
|
||||
|
||||
await self.ws.send(json.dumps(finish_task_message))
|
||||
self.last_active_time = time.time()
|
||||
logger.bind(tag=TAG).info("会话结束请求已发送")
|
||||
# 等待监听任务完成
|
||||
if self._monitor_task:
|
||||
try:
|
||||
await self._monitor_task
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"等待监听任务完成时发生错误: {str(e)}"
|
||||
)
|
||||
finally:
|
||||
self._monitor_task = None
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
|
||||
await self.close()
|
||||
raise
|
||||
|
||||
async def close(self):
|
||||
"""清理资源"""
|
||||
# 取消监听任务
|
||||
if self._monitor_task:
|
||||
try:
|
||||
self._monitor_task.cancel()
|
||||
await self._monitor_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}")
|
||||
self._monitor_task = None
|
||||
|
||||
# 关闭WebSocket连接
|
||||
if self.ws:
|
||||
try:
|
||||
await self.ws.close()
|
||||
except:
|
||||
pass
|
||||
self.ws = None
|
||||
self.last_active_time = None
|
||||
|
||||
async def _start_monitor_tts_response(self):
|
||||
"""监听TTS响应"""
|
||||
try:
|
||||
session_finished = False
|
||||
while not self.conn.stop_event.is_set():
|
||||
try:
|
||||
msg = await self.ws.recv()
|
||||
self.last_active_time = time.time()
|
||||
|
||||
# 检查客户端是否中止
|
||||
if self.conn.client_abort:
|
||||
logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应")
|
||||
break
|
||||
|
||||
if isinstance(msg, str): # JSON控制消息
|
||||
try:
|
||||
data = json.loads(msg)
|
||||
event = data["header"].get("event")
|
||||
|
||||
if event == "task-started":
|
||||
logger.bind(tag=TAG).debug("TTS任务启动成功~")
|
||||
self.tts_audio_queue.put((SentenceType.FIRST, [], None))
|
||||
elif event == "result-generated":
|
||||
# 发送缓存的数据
|
||||
if self.conn.tts_MessageText:
|
||||
logger.bind(tag=TAG).info(
|
||||
f"句子语音生成成功： {self.conn.tts_MessageText}"
|
||||
)
|
||||
self.tts_audio_queue.put(
|
||||
(SentenceType.FIRST, [], self.conn.tts_MessageText)
|
||||
)
|
||||
self.conn.tts_MessageText = None
|
||||
elif event == "task-finished":
|
||||
logger.bind(tag=TAG).debug("TTS任务完成~")
|
||||
self._process_before_stop_play_files()
|
||||
session_finished = True
|
||||
break
|
||||
elif event == "task-failed":
|
||||
error_code = data["header"].get("error_code", "unknown")
|
||||
error_message = data["header"].get("error_message", "未知错误")
|
||||
logger.bind(tag=TAG).error(
|
||||
f"TTS任务失败: {error_code} - {error_message}"
|
||||
)
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
logger.bind(tag=TAG).warning("收到无效的JSON消息")
|
||||
elif isinstance(msg, (bytes, bytearray)):
|
||||
self.opus_encoder.encode_pcm_to_opus_stream(
|
||||
msg, False, callback=self.handle_opus
|
||||
)
|
||||
except websockets.ConnectionClosed:
|
||||
logger.bind(tag=TAG).warning("WebSocket连接已关闭")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"处理TTS响应时出错: {e}\n{traceback.format_exc()}"
|
||||
)
|
||||
break
|
||||
|
||||
# 仅在连接异常且非正常结束时才关闭连接
|
||||
if not session_finished and self.ws:
|
||||
try:
|
||||
await self.ws.close()
|
||||
except:
|
||||
pass
|
||||
self.ws = None
|
||||
# 监听任务退出时清理引用
|
||||
finally:
|
||||
self._monitor_task = None
|
||||
|
||||
def to_tts(self, text: str) -> list:
|
||||
"""非流式生成音频数据，用于生成音频及测试场景"""
|
||||
try:
|
||||
# 创建事件循环
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
# 生成会话ID
|
||||
session_id = uuid.uuid4().hex
|
||||
# 存储音频数据
|
||||
audio_data = []
|
||||
|
||||
async def _generate_audio():
|
||||
ws = await websockets.connect(
|
||||
self.ws_url,
|
||||
additional_headers=self.header,
|
||||
ping_interval=30,
|
||||
ping_timeout=10,
|
||||
close_timeout=10,
|
||||
max_size=10 * 1024 * 1024,
|
||||
)
|
||||
|
||||
try:
|
||||
# 发送run-task消息启动会话
|
||||
run_task_message = {
|
||||
"header": {
|
||||
"action": "run-task",
|
||||
"task_id": session_id,
|
||||
"streaming": "duplex",
|
||||
},
|
||||
"payload": {
|
||||
"task_group": "audio",
|
||||
"task": "tts",
|
||||
"function": "SpeechSynthesizer",
|
||||
"model": self.model,
|
||||
"parameters": {
|
||||
"text_type": "PlainText",
|
||||
"voice": self.voice,
|
||||
"format": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"volume": self.volume,
|
||||
"rate": self.rate,
|
||||
"pitch": self.pitch,
|
||||
},
|
||||
"input": {}
|
||||
},
|
||||
}
|
||||
await ws.send(json.dumps(run_task_message))
|
||||
|
||||
# 等待任务启动
|
||||
task_started = False
|
||||
while not task_started:
|
||||
msg = await ws.recv()
|
||||
if isinstance(msg, str):
|
||||
data = json.loads(msg)
|
||||
header = data.get("header", {})
|
||||
if header.get("event") == "task-started":
|
||||
task_started = True
|
||||
logger.bind(tag=TAG).debug("TTS任务已启动")
|
||||
elif header.get("event") == "task-failed":
|
||||
error_code = header.get("error_code", "unknown")
|
||||
error_message = header.get("error_message", "未知错误")
|
||||
raise Exception(
|
||||
f"启动任务失败: {error_code} - {error_message}"
|
||||
)
|
||||
|
||||
# 发送文本
|
||||
filtered_text = MarkdownCleaner.clean_markdown(text)
|
||||
# 发送continue-task消息
|
||||
continue_task_message = {
|
||||
"header": {
|
||||
"action": "continue-task",
|
||||
"task_id": session_id,
|
||||
"streaming": "duplex",
|
||||
},
|
||||
"payload": {"input": {"text": filtered_text}},
|
||||
}
|
||||
await ws.send(json.dumps(continue_task_message))
|
||||
|
||||
# 发送finish-task消息
|
||||
finish_task_message = {
|
||||
"header": {
|
||||
"action": "finish-task",
|
||||
"task_id": session_id,
|
||||
"streaming": "duplex",
|
||||
},
|
||||
"payload": {
|
||||
"input": {}
|
||||
}
|
||||
}
|
||||
await ws.send(json.dumps(finish_task_message))
|
||||
|
||||
# 接收音频数据
|
||||
task_finished = False
|
||||
while not task_finished:
|
||||
msg = await ws.recv()
|
||||
if isinstance(msg, (bytes, bytearray)):
|
||||
self.opus_encoder.encode_pcm_to_opus_stream(
|
||||
msg,
|
||||
end_of_stream=False,
|
||||
callback=lambda opus: audio_data.append(opus)
|
||||
)
|
||||
elif isinstance(msg, str):
|
||||
data = json.loads(msg)
|
||||
header = data.get("header", {})
|
||||
if header.get("event") == "task-finished":
|
||||
task_finished = True
|
||||
logger.bind(tag=TAG).debug("TTS任务完成")
|
||||
elif header.get("event") == "task-failed":
|
||||
error_code = header.get("error_code", "unknown")
|
||||
error_message = header.get("error_message", "未知错误")
|
||||
raise Exception(
|
||||
f"合成失败: {error_code} - {error_message}"
|
||||
)
|
||||
|
||||
finally:
|
||||
# 清理资源
|
||||
try:
|
||||
await ws.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
# 运行异步任务
|
||||
loop.run_until_complete(_generate_audio())
|
||||
loop.close()
|
||||
|
||||
return audio_data
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}")
|
||||
return []
|
||||
@@ -1,3 +1,4 @@
|
||||
import random
|
||||
import uuid
|
||||
import json
|
||||
import hmac
|
||||
@@ -131,7 +132,7 @@ class TTSProvider(TTSProviderBase):
|
||||
self.last_active_time = None
|
||||
|
||||
# 专属tts设置
|
||||
self.message_id = ""
|
||||
self.task_id = uuid.uuid4().hex
|
||||
|
||||
# 创建Opus编码器
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
@@ -185,9 +186,10 @@ class TTSProvider(TTSProviderBase):
|
||||
current_time = time.time()
|
||||
if self.ws and current_time - self.last_active_time < 10:
|
||||
# 10秒内才可以复用链接进行连续对话
|
||||
logger.bind(tag=TAG).info(f"使用已有链接...")
|
||||
self.task_id = uuid.uuid4().hex
|
||||
logger.bind(tag=TAG).info(f"使用已有链接..., task_id: {self.task_id}")
|
||||
return self.ws
|
||||
logger.bind(tag=TAG).info("开始建立新连接...")
|
||||
logger.bind(tag=TAG).debug("开始建立新连接...")
|
||||
|
||||
self.ws = await websockets.connect(
|
||||
self.ws_url,
|
||||
@@ -196,7 +198,8 @@ class TTSProvider(TTSProviderBase):
|
||||
ping_timeout=10,
|
||||
close_timeout=10,
|
||||
)
|
||||
logger.bind(tag=TAG).info("WebSocket连接建立成功")
|
||||
self.task_id = uuid.uuid4().hex
|
||||
logger.bind(tag=TAG).debug(f"WebSocket连接建立成功, task_id: {self.task_id}")
|
||||
self.last_active_time = time.time()
|
||||
return self.ws
|
||||
except Exception as e:
|
||||
@@ -224,23 +227,14 @@ class TTSProvider(TTSProviderBase):
|
||||
if message.sentence_type == SentenceType.FIRST:
|
||||
# 初始化参数
|
||||
try:
|
||||
if not getattr(self.conn, "sentence_id", None):
|
||||
self.conn.sentence_id = uuid.uuid4().hex
|
||||
logger.bind(tag=TAG).info(
|
||||
f"自动生成新的 会话ID: {self.conn.sentence_id}"
|
||||
)
|
||||
|
||||
# aliyunStream独有的参数生成
|
||||
self.message_id = str(uuid.uuid4().hex)
|
||||
|
||||
logger.bind(tag=TAG).info("开始启动TTS会话...")
|
||||
logger.bind(tag=TAG).debug("开始启动TTS会话...")
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.start_session(self.conn.sentence_id),
|
||||
self.start_session(self.task_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
self.before_stop_play_files.clear()
|
||||
logger.bind(tag=TAG).info("TTS会话启动成功")
|
||||
logger.bind(tag=TAG).debug("TTS会话启动成功")
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
|
||||
@@ -271,9 +265,9 @@ class TTSProvider(TTSProviderBase):
|
||||
self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
|
||||
if message.sentence_type == SentenceType.LAST:
|
||||
try:
|
||||
logger.bind(tag=TAG).info("开始结束TTS会话...")
|
||||
logger.bind(tag=TAG).debug("开始结束TTS会话...")
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.finish_session(self.conn.sentence_id),
|
||||
self.finish_session(self.task_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
@@ -296,8 +290,8 @@ class TTSProvider(TTSProviderBase):
|
||||
filtered_text = MarkdownCleaner.clean_markdown(text)
|
||||
run_request = {
|
||||
"header": {
|
||||
"message_id": self.message_id,
|
||||
"task_id": self.conn.sentence_id,
|
||||
"message_id": uuid.uuid4().hex,
|
||||
"task_id": self.task_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "RunSynthesis",
|
||||
"appkey": self.appkey,
|
||||
@@ -318,8 +312,8 @@ class TTSProvider(TTSProviderBase):
|
||||
self.ws = None
|
||||
raise
|
||||
|
||||
async def start_session(self, session_id):
|
||||
logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
|
||||
async def start_session(self, task_id):
|
||||
logger.bind(tag=TAG).debug("开始会话～～")
|
||||
try:
|
||||
# 会话开始时检测上个会话的监听状态
|
||||
if (
|
||||
@@ -340,8 +334,8 @@ class TTSProvider(TTSProviderBase):
|
||||
|
||||
start_request = {
|
||||
"header": {
|
||||
"message_id": self.message_id,
|
||||
"task_id": self.conn.sentence_id,
|
||||
"message_id": uuid.uuid4().hex,
|
||||
"task_id": self.task_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "StartSynthesis",
|
||||
"appkey": self.appkey,
|
||||
@@ -358,28 +352,28 @@ class TTSProvider(TTSProviderBase):
|
||||
}
|
||||
await self.ws.send(json.dumps(start_request))
|
||||
self.last_active_time = time.time()
|
||||
logger.bind(tag=TAG).info("会话启动请求已发送")
|
||||
logger.bind(tag=TAG).debug("会话启动请求已发送")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
|
||||
# 确保清理资源
|
||||
await self.close()
|
||||
raise
|
||||
|
||||
async def finish_session(self, session_id):
|
||||
logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
|
||||
async def finish_session(self, task_id):
|
||||
logger.bind(tag=TAG).debug(f"关闭会话～～{task_id}")
|
||||
try:
|
||||
if self.ws:
|
||||
stop_request = {
|
||||
"header": {
|
||||
"message_id": self.message_id,
|
||||
"task_id": self.conn.sentence_id,
|
||||
"message_id": uuid.uuid4().hex,
|
||||
"task_id": self.task_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "StopSynthesis",
|
||||
"appkey": self.appkey,
|
||||
}
|
||||
}
|
||||
await self.ws.send(json.dumps(stop_request))
|
||||
logger.bind(tag=TAG).info("会话结束请求已发送")
|
||||
logger.bind(tag=TAG).debug("会话结束请求已发送")
|
||||
self.last_active_time = time.time()
|
||||
if self._monitor_task:
|
||||
try:
|
||||
@@ -457,7 +451,6 @@ class TTSProvider(TTSProviderBase):
|
||||
logger.bind(tag=TAG).warning("收到无效的JSON消息")
|
||||
# 二进制消息（音频数据）
|
||||
elif isinstance(msg, (bytes, bytearray)):
|
||||
logger.bind(tag=TAG).debug(f"推送数据到队列里面～～")
|
||||
self.opus_encoder.encode_pcm_to_opus_stream(msg, False, self.handle_opus)
|
||||
except websockets.ConnectionClosed:
|
||||
logger.bind(tag=TAG).warning("WebSocket连接已关闭")
|
||||
@@ -485,8 +478,6 @@ class TTSProvider(TTSProviderBase):
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
# 生成会话ID
|
||||
session_id = uuid.uuid4().hex
|
||||
# 存储音频数据
|
||||
audio_data = []
|
||||
|
||||
@@ -505,11 +496,10 @@ class TTSProvider(TTSProviderBase):
|
||||
)
|
||||
try:
|
||||
# 发送StartSynthesis请求
|
||||
start_message_id = str(uuid.uuid4().hex)
|
||||
start_request = {
|
||||
"header": {
|
||||
"message_id": start_message_id,
|
||||
"task_id": session_id,
|
||||
"message_id": uuid.uuid4().hex,
|
||||
"task_id": self.task_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "StartSynthesis",
|
||||
"appkey": self.appkey,
|
||||
@@ -550,11 +540,10 @@ class TTSProvider(TTSProviderBase):
|
||||
|
||||
# 发送文本合成请求
|
||||
filtered_text = MarkdownCleaner.clean_markdown(text)
|
||||
run_message_id = str(uuid.uuid4().hex)
|
||||
run_request = {
|
||||
"header": {
|
||||
"message_id": run_message_id,
|
||||
"task_id": session_id,
|
||||
"message_id": uuid.uuid4().hex,
|
||||
"task_id": self.task_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "RunSynthesis",
|
||||
"appkey": self.appkey,
|
||||
@@ -564,11 +553,10 @@ class TTSProvider(TTSProviderBase):
|
||||
await ws.send(json.dumps(run_request))
|
||||
|
||||
# 发送停止合成请求
|
||||
stop_message_id = str(uuid.uuid4().hex)
|
||||
stop_request = {
|
||||
"header": {
|
||||
"message_id": stop_message_id,
|
||||
"task_id": session_id,
|
||||
"message_id": uuid.uuid4().hex,
|
||||
"task_id": self.task_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "StopSynthesis",
|
||||
"appkey": self.appkey,
|
||||
|
||||
@@ -395,14 +395,6 @@ class TTSProviderBase(ABC):
|
||||
|
||||
# 收到下一个文本开始或会话结束时进行上报
|
||||
if sentence_type is not SentenceType.MIDDLE:
|
||||
# 重置音频流控状态（新句子开始或者结束）
|
||||
if hasattr(self.conn, 'audio_flow_control'):
|
||||
self.conn.audio_flow_control = {
|
||||
'last_send_time': 0,
|
||||
'packet_count': 0,
|
||||
'start_time': time.perf_counter()
|
||||
}
|
||||
|
||||
# 上报TTS数据
|
||||
if enqueue_text is not None and enqueue_audio is not None:
|
||||
enqueue_tts_report(self.conn, enqueue_text, enqueue_audio)
|
||||
|
||||
@@ -149,6 +149,7 @@ class TTSProvider(TTSProviderBase):
|
||||
self.access_token = config.get("access_token")
|
||||
self.cluster = config.get("cluster")
|
||||
self.resource_id = config.get("resource_id")
|
||||
self.activate_session = False
|
||||
if config.get("private_voice"):
|
||||
self.voice = config.get("private_voice")
|
||||
else:
|
||||
@@ -162,7 +163,8 @@ class TTSProvider(TTSProviderBase):
|
||||
self.ws_url = config.get("ws_url")
|
||||
self.authorization = config.get("authorization")
|
||||
self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
|
||||
self.enable_two_way = True
|
||||
enable_ws_reuse_value = config.get("enable_ws_reuse", True)
|
||||
self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() in ('false', 'False') else True
|
||||
self.tts_text = ""
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=16000, channels=1, frame_size_ms=60
|
||||
@@ -180,12 +182,18 @@ class TTSProvider(TTSProviderBase):
|
||||
raise
|
||||
|
||||
async def _ensure_connection(self):
|
||||
"""建立新的WebSocket连接"""
|
||||
"""建立新的WebSocket连接，并启动监听任务（仅第一次）"""
|
||||
try:
|
||||
if self.ws:
|
||||
logger.bind(tag=TAG).info(f"使用已有链接...")
|
||||
return self.ws
|
||||
logger.bind(tag=TAG).info("开始建立新连接...")
|
||||
if self.enable_ws_reuse:
|
||||
logger.bind(tag=TAG).info(f"使用已有链接...")
|
||||
return self.ws
|
||||
else:
|
||||
try:
|
||||
await self.finish_connection()
|
||||
except:
|
||||
pass
|
||||
logger.bind(tag=TAG).debug("开始建立新连接...")
|
||||
ws_header = {
|
||||
"X-Api-App-Key": self.appId,
|
||||
"X-Api-Access-Key": self.access_token,
|
||||
@@ -195,12 +203,34 @@ class TTSProvider(TTSProviderBase):
|
||||
self.ws = await websockets.connect(
|
||||
self.ws_url, additional_headers=ws_header, max_size=1000000000
|
||||
)
|
||||
logger.bind(tag=TAG).info("WebSocket连接建立成功")
|
||||
logger.bind(tag=TAG).debug("WebSocket连接建立成功")
|
||||
|
||||
# 连接建立成功后，启动监听任务
|
||||
if self._monitor_task is None or self._monitor_task.done():
|
||||
logger.bind(tag=TAG).debug("启动监听任务...")
|
||||
self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
|
||||
|
||||
return self.ws
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}")
|
||||
self.ws = None
|
||||
raise
|
||||
|
||||
async def finish_connection(self):
|
||||
"""发送 FinishConnection 事件，等待服务端返回 EVENT_ConnectionFinished"""
|
||||
try:
|
||||
if self.ws:
|
||||
logger.bind(tag=TAG).debug("开始关闭连接...")
|
||||
header = Header(
|
||||
message_type=FULL_CLIENT_REQUEST,
|
||||
message_type_specific_flags=MsgTypeFlagWithEvent,
|
||||
serial_method=JSON,
|
||||
).as_bytes()
|
||||
optional = Optional(event=EVENT_FinishConnection).as_bytes()
|
||||
payload = str.encode("{}")
|
||||
await self.send_event(self.ws, header, optional, payload)
|
||||
except:
|
||||
pass
|
||||
|
||||
def tts_text_priority_thread(self):
|
||||
"""火山引擎双流式TTS的文本处理线程"""
|
||||
@@ -217,10 +247,16 @@ class TTSProvider(TTSProviderBase):
|
||||
if self.conn.client_abort:
|
||||
try:
|
||||
logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.cancel_session(self.conn.sentence_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
if self.enable_ws_reuse:
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.cancel_session(self.conn.sentence_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
else:
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.finish_connection(),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"取消TTS会话失败: {str(e)}")
|
||||
@@ -231,16 +267,16 @@ class TTSProvider(TTSProviderBase):
|
||||
try:
|
||||
if not getattr(self.conn, "sentence_id", None):
|
||||
self.conn.sentence_id = uuid.uuid4().hex
|
||||
logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}")
|
||||
logger.bind(tag=TAG).debug(f"自动生成新的 会话ID: {self.conn.sentence_id}")
|
||||
|
||||
logger.bind(tag=TAG).info("开始启动TTS会话...")
|
||||
logger.bind(tag=TAG).debug("开始启动TTS会话...")
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.start_session(self.conn.sentence_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
self.before_stop_play_files.clear()
|
||||
logger.bind(tag=TAG).info("TTS会话启动成功")
|
||||
logger.bind(tag=TAG).debug("TTS会话启动成功")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
|
||||
continue
|
||||
@@ -270,7 +306,7 @@ class TTSProvider(TTSProviderBase):
|
||||
self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
|
||||
if message.sentence_type == SentenceType.LAST:
|
||||
try:
|
||||
logger.bind(tag=TAG).info("开始结束TTS会话...")
|
||||
logger.bind(tag=TAG).debug("开始结束TTS会话...")
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.finish_session(self.conn.sentence_id),
|
||||
loop=self.conn.loop,
|
||||
@@ -313,23 +349,25 @@ class TTSProvider(TTSProviderBase):
|
||||
raise
|
||||
|
||||
async def start_session(self, session_id):
|
||||
logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
|
||||
try:
|
||||
# 会话开始时检测上个会话的监听状态
|
||||
if (
|
||||
self._monitor_task is not None
|
||||
and isinstance(self._monitor_task, Task)
|
||||
and not self._monitor_task.done()
|
||||
):
|
||||
logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务和连接...")
|
||||
logger.bind(tag=TAG).debug(f"开始会话～～{session_id}")
|
||||
try:
|
||||
# 等待上一个会话结束，最多等待3次
|
||||
for _ in range(3):
|
||||
if not self.activate_session:
|
||||
break
|
||||
logger.bind(tag=TAG).debug(f"等待上一个会话结束...")
|
||||
await asyncio.sleep(0.1)
|
||||
else:
|
||||
# 等待超时，强制清除连接状态
|
||||
logger.bind(tag=TAG).debug("等待上一个会话超时，清除连接状态...")
|
||||
await self.close()
|
||||
|
||||
# 建立新连接
|
||||
|
||||
# 设置会话激活标志
|
||||
self.activate_session = True
|
||||
|
||||
# 确保连接建立
|
||||
await self._ensure_connection()
|
||||
|
||||
# 启动监听任务
|
||||
self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
|
||||
|
||||
header = Header(
|
||||
message_type=FULL_CLIENT_REQUEST,
|
||||
message_type_specific_flags=MsgTypeFlagWithEvent,
|
||||
@@ -342,7 +380,7 @@ class TTSProvider(TTSProviderBase):
|
||||
event=EVENT_StartSession, speaker=self.voice
|
||||
)
|
||||
await self.send_event(self.ws, header, optional, payload)
|
||||
logger.bind(tag=TAG).info("会话启动请求已发送")
|
||||
logger.bind(tag=TAG).debug("会话启动请求已发送")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
|
||||
# 确保清理资源
|
||||
@@ -350,7 +388,7 @@ class TTSProvider(TTSProviderBase):
|
||||
raise
|
||||
|
||||
async def finish_session(self, session_id):
|
||||
logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
|
||||
logger.bind(tag=TAG).debug(f"关闭会话～～{session_id}")
|
||||
try:
|
||||
if self.ws:
|
||||
header = Header(
|
||||
@@ -363,18 +401,7 @@ class TTSProvider(TTSProviderBase):
|
||||
).as_bytes()
|
||||
payload = str.encode("{}")
|
||||
await self.send_event(self.ws, header, optional, payload)
|
||||
logger.bind(tag=TAG).info("会话结束请求已发送")
|
||||
|
||||
# 等待监听任务完成
|
||||
if self._monitor_task:
|
||||
try:
|
||||
await self._monitor_task
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"等待监听任务完成时发生错误: {str(e)}"
|
||||
)
|
||||
finally:
|
||||
self._monitor_task = None
|
||||
logger.bind(tag=TAG).debug("会话结束请求已发送")
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
|
||||
@@ -383,7 +410,7 @@ class TTSProvider(TTSProviderBase):
|
||||
raise
|
||||
|
||||
async def cancel_session(self,session_id):
|
||||
logger.bind(tag=TAG).info(f"取消会话，释放服务端资源～～{session_id}")
|
||||
logger.bind(tag=TAG).debug(f"取消会话，释放服务端资源～～{session_id}")
|
||||
try:
|
||||
if self.ws:
|
||||
header = Header(
|
||||
@@ -396,7 +423,7 @@ class TTSProvider(TTSProviderBase):
|
||||
).as_bytes()
|
||||
payload = str.encode("{}")
|
||||
await self.send_event(self.ws, header, optional, payload)
|
||||
logger.bind(tag=TAG).info("会话取消请求已发送")
|
||||
logger.bind(tag=TAG).debug("会话取消请求已发送")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"取消会话失败: {str(e)}")
|
||||
# 确保清理资源
|
||||
@@ -405,6 +432,7 @@ class TTSProvider(TTSProviderBase):
|
||||
|
||||
async def close(self):
|
||||
"""资源清理方法"""
|
||||
self.activate_session = False
|
||||
# 取消监听任务
|
||||
if self._monitor_task:
|
||||
try:
|
||||
@@ -424,9 +452,8 @@ class TTSProvider(TTSProviderBase):
|
||||
self.ws = None
|
||||
|
||||
async def _start_monitor_tts_response(self):
|
||||
"""监听TTS响应"""
|
||||
"""监听TTS响应 - 长期运行"""
|
||||
try:
|
||||
session_finished = False # 标记会话是否正常结束
|
||||
while not self.conn.stop_event.is_set():
|
||||
try:
|
||||
# 确保 `recv()` 运行在同一个 event loop
|
||||
@@ -434,10 +461,22 @@ class TTSProvider(TTSProviderBase):
|
||||
res = self.parser_response(msg)
|
||||
self.print_response(res, "send_text res:")
|
||||
|
||||
# 优先处理连接级别事件
|
||||
if res.optional.event == EVENT_ConnectionFinished:
|
||||
logger.bind(tag=TAG).debug(f"链接关闭成功～～")
|
||||
break
|
||||
|
||||
# 只处理当前活跃会话的响应
|
||||
if res.optional.sessionId and self.conn.sentence_id != res.optional.sessionId:
|
||||
# 如果是会话结束相关事件，即使会话ID不匹配也要重置状态
|
||||
if res.optional.event in [EVENT_SessionCanceled, EVENT_SessionFailed, EVENT_SessionFinished]:
|
||||
logger.bind(tag=TAG).debug(f"收到残余下行结束响应重置会话状态～～")
|
||||
self.activate_session = False
|
||||
continue
|
||||
|
||||
if res.optional.event == EVENT_SessionCanceled:
|
||||
logger.bind(tag=TAG).debug(f"释放服务端资源成功～～")
|
||||
session_finished = True
|
||||
break
|
||||
self.activate_session = False
|
||||
elif res.optional.event == EVENT_TTSSentenceStart:
|
||||
json_data = json.loads(res.payload.decode("utf-8"))
|
||||
self.tts_text = json_data.get("text", "")
|
||||
@@ -449,15 +488,16 @@ class TTSProvider(TTSProviderBase):
|
||||
res.optional.event == EVENT_TTSResponse
|
||||
and res.header.message_type == AUDIO_ONLY_RESPONSE
|
||||
):
|
||||
logger.bind(tag=TAG).debug(f"推送数据到队列里面～～")
|
||||
self.wav_to_opus_data_audio_raw_stream(res.payload, callback=self.handle_opus)
|
||||
elif res.optional.event == EVENT_TTSSentenceEnd:
|
||||
logger.bind(tag=TAG).info(f"句子语音生成成功：{self.tts_text}")
|
||||
elif res.optional.event == EVENT_SessionFinished:
|
||||
logger.bind(tag=TAG).debug(f"会话结束～～")
|
||||
self.activate_session = False
|
||||
self._process_before_stop_play_files()
|
||||
session_finished = True
|
||||
break
|
||||
# 非复用模式下，会话结束后发送 FinishConnection
|
||||
if not self.enable_ws_reuse:
|
||||
await self.finish_connection()
|
||||
except websockets.ConnectionClosed:
|
||||
logger.bind(tag=TAG).warning("WebSocket连接已关闭")
|
||||
break
|
||||
@@ -467,8 +507,8 @@ class TTSProvider(TTSProviderBase):
|
||||
)
|
||||
traceback.print_exc()
|
||||
break
|
||||
# 仅在连接异常时才关闭
|
||||
if not session_finished and self.ws:
|
||||
# 连接异常时关闭WebSocket
|
||||
if self.ws:
|
||||
try:
|
||||
await self.ws.close()
|
||||
except:
|
||||
@@ -476,6 +516,7 @@ class TTSProvider(TTSProviderBase):
|
||||
self.ws = None
|
||||
# 监听任务退出时清理引用
|
||||
finally:
|
||||
self.activate_session = False
|
||||
self._monitor_task = None
|
||||
|
||||
async def send_event(
|
||||
@@ -514,7 +555,7 @@ class TTSProvider(TTSProviderBase):
|
||||
def read_res_content(self, res: bytes, offset: int):
|
||||
content_size = int.from_bytes(res[offset : offset + 4], "big", signed=True)
|
||||
offset += 4
|
||||
content = str(res[offset : offset + content_size])
|
||||
content = res[offset : offset + content_size].decode('utf-8')
|
||||
offset += content_size
|
||||
return content, offset
|
||||
|
||||
@@ -616,12 +657,13 @@ class TTSProvider(TTSProviderBase):
|
||||
"speech_rate": self.speech_rate,
|
||||
"loudness_rate": self.loudness_rate
|
||||
},
|
||||
"additions": json.dumps({
|
||||
"post_process": {
|
||||
"pitch": self.pitch
|
||||
}
|
||||
})
|
||||
},
|
||||
"additions": {
|
||||
"post_process": {
|
||||
"pitch": self.pitch
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
@@ -190,7 +190,7 @@ class TTSProvider(TTSProviderBase):
|
||||
start_time = time.time()
|
||||
text = MarkdownCleaner.clean_markdown(text)
|
||||
|
||||
payload = {"text": text, "character": self.character}
|
||||
payload = {"text": text, "character": self.voice}
|
||||
|
||||
try:
|
||||
with requests.post(self.api_url, json=payload, timeout=5) as response:
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.utils.util import parse_string_to_list
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.group_id = config.get("group_id")
|
||||
self.api_key = config.get("api_key")
|
||||
self.model = config.get("model")
|
||||
if config.get("private_voice"):
|
||||
self.voice = config.get("private_voice")
|
||||
else:
|
||||
self.voice = config.get("voice_id")
|
||||
|
||||
default_voice_setting = {
|
||||
"voice_id": "female-shaonv",
|
||||
"speed": 1,
|
||||
"vol": 1,
|
||||
"pitch": 0,
|
||||
"emotion": "happy",
|
||||
}
|
||||
default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}
|
||||
defult_audio_setting = {
|
||||
"sample_rate": 32000,
|
||||
"bitrate": 128000,
|
||||
"format": "mp3",
|
||||
"channel": 1,
|
||||
}
|
||||
self.voice_setting = {
|
||||
**default_voice_setting,
|
||||
**config.get("voice_setting", {}),
|
||||
}
|
||||
self.pronunciation_dict = {
|
||||
**default_pronunciation_dict,
|
||||
**config.get("pronunciation_dict", {}),
|
||||
}
|
||||
self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
|
||||
self.timber_weights = parse_string_to_list(config.get("timber_weights"))
|
||||
|
||||
if self.voice:
|
||||
self.voice_setting["voice_id"] = self.voice
|
||||
|
||||
self.host = "api.minimax.chat"
|
||||
self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
|
||||
self.header = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
}
|
||||
self.audio_file_type = defult_audio_setting.get("format", "mp3")
|
||||
|
||||
def generate_filename(self, extension=".mp3"):
|
||||
return os.path.join(
|
||||
self.output_file,
|
||||
f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"model": self.model,
|
||||
"text": text,
|
||||
"stream": False,
|
||||
"voice_setting": self.voice_setting,
|
||||
"pronunciation_dict": self.pronunciation_dict,
|
||||
"audio_setting": self.audio_setting,
|
||||
}
|
||||
|
||||
if type(self.timber_weights) is list and len(self.timber_weights) > 0:
|
||||
request_json["timber_weights"] = self.timber_weights
|
||||
request_json["voice_setting"]["voice_id"] = ""
|
||||
|
||||
try:
|
||||
resp = requests.post(
|
||||
self.api_url, json.dumps(request_json), headers=self.header
|
||||
)
|
||||
# 检查返回请求数据的status_code是否为0
|
||||
if resp.json()["base_resp"]["status_code"] == 0:
|
||||
data = resp.json()["data"]["audio"]
|
||||
audio_bytes = bytes.fromhex(data)
|
||||
if output_file:
|
||||
with open(output_file, "wb") as file_to_save:
|
||||
file_to_save.write(audio_bytes)
|
||||
else:
|
||||
return audio_bytes
|
||||
else:
|
||||
raise Exception(
|
||||
f"{__name__} status_code: {resp.status_code} response: {resp.content}"
|
||||
)
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
@@ -1,11 +1,20 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import time
|
||||
import queue
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from typing import Iterator, Optional, Union
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
import traceback
|
||||
from config.logger import setup_logging
|
||||
from core.utils.tts import MarkdownCleaner
|
||||
from core.utils.util import parse_string_to_list
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.utils import opus_encoder_utils, textUtils
|
||||
from core.providers.tts.dto.dto import SentenceType, ContentType
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
@@ -28,9 +37,9 @@ class TTSProvider(TTSProviderBase):
|
||||
}
|
||||
default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}
|
||||
defult_audio_setting = {
|
||||
"sample_rate": 32000,
|
||||
"sample_rate": 24000,
|
||||
"bitrate": 128000,
|
||||
"format": "mp3",
|
||||
"format": "pcm",
|
||||
"channel": 1,
|
||||
}
|
||||
self.voice_setting = {
|
||||
@@ -47,66 +56,101 @@ class TTSProvider(TTSProviderBase):
|
||||
if self.voice:
|
||||
self.voice_setting["voice_id"] = self.voice
|
||||
|
||||
self.host = "api.minimax.chat"
|
||||
self.host = "api.minimaxi.com" # 备用地址：api-bj.minimaxi.com
|
||||
self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
|
||||
self.header = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
}
|
||||
self.audio_file_type = defult_audio_setting.get("format", "mp3")
|
||||
self.audio_file_type = defult_audio_setting.get("format", "pcm")
|
||||
|
||||
def generate_filename(self, extension=".mp3"):
|
||||
return os.path.join(
|
||||
self.output_file,
|
||||
f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=24000, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
"""非流式语音合成（保留原有实现）"""
|
||||
request_json = {
|
||||
"model": self.model,
|
||||
"text": text,
|
||||
"stream": False,
|
||||
"voice_setting": self.voice_setting,
|
||||
"pronunciation_dict": self.pronunciation_dict,
|
||||
"audio_setting": self.audio_setting,
|
||||
}
|
||||
# PCM缓冲区
|
||||
self.pcm_buffer = bytearray()
|
||||
|
||||
if type(self.timber_weights) is list and len(self.timber_weights) > 0:
|
||||
request_json["timber_weights"] = self.timber_weights
|
||||
request_json["voice_setting"]["voice_id"] = ""
|
||||
def tts_text_priority_thread(self):
|
||||
"""流式文本处理线程"""
|
||||
while not self.conn.stop_event.is_set():
|
||||
try:
|
||||
message = self.tts_text_queue.get(timeout=1)
|
||||
if message.sentence_type == SentenceType.FIRST:
|
||||
# 初始化参数
|
||||
self.tts_stop_request = False
|
||||
self.processed_chars = 0
|
||||
self.tts_text_buff = []
|
||||
self.before_stop_play_files.clear()
|
||||
elif ContentType.TEXT == message.content_type:
|
||||
self.tts_text_buff.append(message.content_detail)
|
||||
segment_text = self._get_segment_text()
|
||||
if segment_text:
|
||||
self.to_tts_single_stream(segment_text)
|
||||
|
||||
try:
|
||||
resp = requests.post(
|
||||
self.api_url, json.dumps(request_json), headers=self.header
|
||||
)
|
||||
if resp.json()["base_resp"]["status_code"] == 0:
|
||||
data = resp.json()["data"]["audio"]
|
||||
audio_bytes = bytes.fromhex(data)
|
||||
if output_file:
|
||||
with open(output_file, "wb") as file_to_save:
|
||||
file_to_save.write(audio_bytes)
|
||||
else:
|
||||
return audio_bytes
|
||||
elif ContentType.FILE == message.content_type:
|
||||
logger.bind(tag=TAG).info(
|
||||
f"添加音频文件到待播放列表: {message.content_file}"
|
||||
)
|
||||
if message.content_file and os.path.exists(message.content_file):
|
||||
# 先处理文件音频数据
|
||||
self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
|
||||
if message.sentence_type == SentenceType.LAST:
|
||||
# 处理剩余的文本
|
||||
self._process_remaining_text_stream(True)
|
||||
|
||||
except queue.Empty:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
|
||||
)
|
||||
|
||||
def _process_remaining_text_stream(self, is_last=False):
|
||||
"""处理剩余的文本并生成语音
|
||||
Returns:
|
||||
bool: 是否成功处理了文本
|
||||
"""
|
||||
full_text = "".join(self.tts_text_buff)
|
||||
remaining_text = full_text[self.processed_chars :]
|
||||
if remaining_text:
|
||||
segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text)
|
||||
if segment_text:
|
||||
self.to_tts_single_stream(segment_text, is_last)
|
||||
self.processed_chars += len(full_text)
|
||||
else:
|
||||
raise Exception(
|
||||
f"{__name__} status_code: {resp.status_code} response: {resp.content}"
|
||||
self._process_before_stop_play_files()
|
||||
else:
|
||||
self._process_before_stop_play_files()
|
||||
|
||||
def to_tts_single_stream(self, text, is_last=False):
|
||||
try:
|
||||
max_repeat_time = 5
|
||||
text = MarkdownCleaner.clean_markdown(text)
|
||||
try:
|
||||
asyncio.run(self.text_to_speak(text, is_last))
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(
|
||||
f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}"
|
||||
)
|
||||
max_repeat_time -= 1
|
||||
|
||||
if max_repeat_time > 0:
|
||||
logger.bind(tag=TAG).info(
|
||||
f"语音生成成功: {text}，重试{5 - max_repeat_time}次"
|
||||
)
|
||||
else:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"语音生成失败: {text}，请检查网络或服务是否正常"
|
||||
)
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
|
||||
finally:
|
||||
return None
|
||||
|
||||
def text_to_speak_stream(
|
||||
self,
|
||||
text: str,
|
||||
chunk_callback: Optional[callable] = None
|
||||
) -> Iterator[bytes]:
|
||||
"""
|
||||
流式语音合成方法
|
||||
:param text: 要合成的文本
|
||||
:param chunk_callback: 可选的回调函数，用于处理每个音频块
|
||||
:return: 生成器，每次产生一个音频数据块(bytes)
|
||||
"""
|
||||
request_json = {
|
||||
async def text_to_speak(self, text, is_last):
|
||||
"""流式处理TTS音频，每句只推送一次音频列表"""
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"text": text,
|
||||
"stream": True,
|
||||
@@ -115,116 +159,183 @@ class TTSProvider(TTSProviderBase):
|
||||
"audio_setting": self.audio_setting,
|
||||
}
|
||||
|
||||
if isinstance(self.timber_weights, list) and len(self.timber_weights) > 0:
|
||||
request_json["timber_weights"] = self.timber_weights
|
||||
request_json["voice_setting"]["voice_id"] = ""
|
||||
if type(self.timber_weights) is list and len(self.timber_weights) > 0:
|
||||
payload["timber_weights"] = self.timber_weights
|
||||
payload["voice_setting"]["voice_id"] = ""
|
||||
|
||||
frame_bytes = int(
|
||||
self.opus_encoder.sample_rate
|
||||
* self.opus_encoder.channels # 1
|
||||
* self.opus_encoder.frame_size_ms
|
||||
/ 1000
|
||||
* 2
|
||||
) # 16-bit = 2 bytes
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(
|
||||
self.api_url,
|
||||
headers=self.header,
|
||||
data=json.dumps(payload),
|
||||
timeout=10,
|
||||
) as resp:
|
||||
|
||||
if resp.status != 200:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"TTS请求失败: {resp.status}, {await resp.text()}"
|
||||
)
|
||||
self.tts_audio_queue.put((SentenceType.LAST, [], None))
|
||||
return
|
||||
|
||||
self.pcm_buffer.clear()
|
||||
self.tts_audio_queue.put((SentenceType.FIRST, [], text))
|
||||
|
||||
# 处理音频流数据
|
||||
buffer = b""
|
||||
async for chunk in resp.content.iter_any():
|
||||
if not chunk:
|
||||
continue
|
||||
|
||||
buffer += chunk
|
||||
while True:
|
||||
# 查找数据块分隔符
|
||||
header_pos = buffer.find(b"data: ")
|
||||
if header_pos == -1:
|
||||
break
|
||||
|
||||
end_pos = buffer.find(b"\n\n", header_pos)
|
||||
if end_pos == -1:
|
||||
break
|
||||
|
||||
# 提取单个完整JSON块
|
||||
json_str = buffer[header_pos + 6 : end_pos].decode("utf-8")
|
||||
buffer = buffer[end_pos + 2 :]
|
||||
|
||||
try:
|
||||
data = json.loads(json_str)
|
||||
status = data.get("data", {}).get("status", 1)
|
||||
audio_hex = data.get("data", {}).get("audio")
|
||||
|
||||
# 仅处理status=1的有效音频块 忽略status=2的结束汇总块
|
||||
if status == 1 and audio_hex:
|
||||
pcm_data = bytes.fromhex(audio_hex)
|
||||
self.pcm_buffer.extend(pcm_data)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.bind(tag=TAG).error(f"JSON解析失败: {e}")
|
||||
continue
|
||||
|
||||
while len(self.pcm_buffer) >= frame_bytes:
|
||||
frame = bytes(self.pcm_buffer[:frame_bytes])
|
||||
del self.pcm_buffer[:frame_bytes]
|
||||
|
||||
self.opus_encoder.encode_pcm_to_opus_stream(
|
||||
frame, end_of_stream=False, callback=self.handle_opus
|
||||
)
|
||||
|
||||
# flush 剩余不足一帧的数据
|
||||
if self.pcm_buffer:
|
||||
self.opus_encoder.encode_pcm_to_opus_stream(
|
||||
bytes(self.pcm_buffer),
|
||||
end_of_stream=True,
|
||||
callback=self.handle_opus,
|
||||
)
|
||||
self.pcm_buffer.clear()
|
||||
|
||||
# 如果是最后一段，输出音频获取完毕
|
||||
if is_last:
|
||||
self._process_before_stop_play_files()
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
|
||||
self.tts_audio_queue.put((SentenceType.LAST, [], None))
|
||||
|
||||
async def close(self):
|
||||
"""资源清理"""
|
||||
await super().close()
|
||||
if hasattr(self, "opus_encoder"):
|
||||
self.opus_encoder.close()
|
||||
|
||||
def to_tts(self, text: str) -> list:
|
||||
"""非流式TTS处理，用于测试及保存音频文件的场景
|
||||
Args:
|
||||
text: 要转换的文本
|
||||
Returns:
|
||||
list: 返回opus编码后的音频数据列表
|
||||
"""
|
||||
start_time = time.time()
|
||||
text = MarkdownCleaner.clean_markdown(text)
|
||||
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"text": text,
|
||||
"stream": True,
|
||||
"voice_setting": self.voice_setting,
|
||||
"pronunciation_dict": self.pronunciation_dict,
|
||||
"audio_setting": self.audio_setting,
|
||||
}
|
||||
|
||||
if type(self.timber_weights) is list and len(self.timber_weights) > 0:
|
||||
payload["timber_weights"] = self.timber_weights
|
||||
payload["voice_setting"]["voice_id"] = ""
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
}
|
||||
|
||||
try:
|
||||
with requests.post(
|
||||
self.api_url,
|
||||
data=json.dumps(request_json),
|
||||
headers=self.header,
|
||||
stream=True
|
||||
self.api_url, data=json.dumps(payload), headers=headers, timeout=5
|
||||
) as response:
|
||||
|
||||
# 检查HTTP状态码
|
||||
if response.status_code != 200:
|
||||
raise Exception(
|
||||
f"HTTP error: {response.status_code}, response: {response.text}"
|
||||
logger.bind(tag=TAG).error(
|
||||
f"TTS请求失败: {response.status_code}, {response.text}"
|
||||
)
|
||||
|
||||
# 处理流式响应
|
||||
for line in response.iter_lines():
|
||||
if line: # 过滤空行
|
||||
# 检查是否为数据行 (SSE格式)
|
||||
if line.startswith(b'data:'):
|
||||
try:
|
||||
data = json.loads(line[5:].strip()) # 去掉"data:"前缀
|
||||
|
||||
# 检查API状态码
|
||||
if data.get("base_resp", {}).get("status_code", -1) != 0:
|
||||
raise Exception(
|
||||
f"API error: {data.get('base_resp', {}).get('status_msg')}"
|
||||
)
|
||||
|
||||
# 跳过非音频数据块
|
||||
if "extra_info" in data:
|
||||
continue
|
||||
|
||||
# 提取音频数据
|
||||
audio_hex = data.get("data", {}).get("audio")
|
||||
if audio_hex:
|
||||
audio_chunk = bytes.fromhex(audio_hex)
|
||||
if chunk_callback:
|
||||
chunk_callback(audio_chunk)
|
||||
yield audio_chunk
|
||||
|
||||
except json.JSONDecodeError:
|
||||
# 忽略JSON解析错误（可能是心跳包等）
|
||||
continue
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} stream error: {e}")
|
||||
return []
|
||||
|
||||
def save_stream_to_file(
|
||||
self,
|
||||
text: str,
|
||||
output_file: Optional[str] = None,
|
||||
progress_callback: Optional[callable] = None
|
||||
) -> str:
|
||||
"""
|
||||
流式合成并保存到文件
|
||||
:param text: 要合成的文本
|
||||
:param output_file: 输出文件路径，如果为None则自动生成
|
||||
:param progress_callback: 可选的回调函数，接收已写入的字节数
|
||||
:return: 保存的文件路径
|
||||
"""
|
||||
if not output_file:
|
||||
output_file = self.generate_filename(extension=f".{self.audio_file_type}")
|
||||
|
||||
os.makedirs(os.path.dirname(output_file), exist_ok=True)
|
||||
|
||||
total_bytes = 0
|
||||
try:
|
||||
with open(output_file, "wb") as audio_file:
|
||||
for audio_chunk in self.text_to_speak_stream(text):
|
||||
audio_file.write(audio_chunk)
|
||||
audio_file.flush()
|
||||
total_bytes += len(audio_chunk)
|
||||
if progress_callback:
|
||||
progress_callback(total_bytes)
|
||||
return output_file
|
||||
except Exception as e:
|
||||
# 清理可能创建的不完整文件
|
||||
if os.path.exists(output_file):
|
||||
os.remove(output_file)
|
||||
raise e
|
||||
logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒")
|
||||
|
||||
# 使用opus编码器处理PCM数据
|
||||
opus_datas = []
|
||||
full_content = response.content.decode('utf-8')
|
||||
pcm_data = bytearray()
|
||||
for data_block in full_content.split('\n\n'):
|
||||
if not data_block.startswith('data: '):
|
||||
continue
|
||||
|
||||
try:
|
||||
json_str = data_block[6:] # 去除'data: '前缀
|
||||
data = json.loads(json_str)
|
||||
if data.get('data', {}).get('status') == 1:
|
||||
audio_hex = data['data']['audio']
|
||||
pcm_data.extend(bytes.fromhex(audio_hex))
|
||||
except (json.JSONDecodeError, KeyError) as e:
|
||||
logger.bind(tag=TAG).warning(f"无效数据块: {e}")
|
||||
continue
|
||||
|
||||
# 计算每帧的字节数
|
||||
frame_bytes = int(
|
||||
self.opus_encoder.sample_rate
|
||||
* self.opus_encoder.channels
|
||||
* self.opus_encoder.frame_size_ms
|
||||
/ 1000
|
||||
* 2
|
||||
)
|
||||
|
||||
# 分帧处理合并后的PCM数据
|
||||
for i in range(0, len(pcm_data), frame_bytes):
|
||||
frame = bytes(pcm_data[i:i+frame_bytes])
|
||||
if len(frame) < frame_bytes:
|
||||
frame += b"\x00" * (frame_bytes - len(frame))
|
||||
|
||||
self.opus_encoder.encode_pcm_to_opus_stream(
|
||||
frame,
|
||||
end_of_stream=(i + frame_bytes >= len(pcm_data)),
|
||||
callback=lambda opus: opus_datas.append(opus)
|
||||
)
|
||||
|
||||
return opus_datas
|
||||
|
||||
def stream_to_audio_player(self, text: str, player_command: list = None):
|
||||
"""
|
||||
流式合成并直接播放音频
|
||||
:param text: 要合成的文本
|
||||
:param player_command: 音频播放器命令，默认使用mpv
|
||||
"""
|
||||
if player_command is None:
|
||||
player_command = ["mpv", "--no-cache", "--no-terminal", "--", "fd://0"]
|
||||
|
||||
try:
|
||||
import subprocess
|
||||
player_process = subprocess.Popen(
|
||||
player_command,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
for audio_chunk in self.text_to_speak_stream(text):
|
||||
player_process.stdin.write(audio_chunk)
|
||||
player_process.stdin.flush()
|
||||
|
||||
player_process.stdin.close()
|
||||
player_process.wait()
|
||||
except Exception as e:
|
||||
raise Exception(f"Audio player error: {e}")
|
||||
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
|
||||
return []
|
||||
|
||||
@@ -1,180 +0,0 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import asyncio
|
||||
import websockets
|
||||
import ssl
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.utils.util import parse_string_to_list
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.group_id = config.get("group_id")
|
||||
self.api_key = config.get("api_key")
|
||||
self.model = config.get("model")
|
||||
|
||||
# 初始化语音设置
|
||||
default_voice_setting = {
|
||||
"voice_id": "female-shaonv",
|
||||
"speed": 1,
|
||||
"vol": 1,
|
||||
"pitch": 0,
|
||||
"emotion": "happy",
|
||||
}
|
||||
default_pronunciation_dict = {"tone": ["处理/(chu3)(li3)", "危险/dangerous"]}
|
||||
default_audio_setting = {
|
||||
"sample_rate": 32000,
|
||||
"bitrate": 128000,
|
||||
"format": "mp3",
|
||||
"channel": 1,
|
||||
}
|
||||
|
||||
# 合并配置
|
||||
self.voice_setting = {
|
||||
**default_voice_setting,
|
||||
**config.get("voice_setting", {}),
|
||||
}
|
||||
self.pronunciation_dict = {
|
||||
**default_pronunciation_dict,
|
||||
**config.get("pronunciation_dict", {}),
|
||||
}
|
||||
self.audio_setting = {
|
||||
**default_audio_setting,
|
||||
**config.get("audio_setting", {})
|
||||
}
|
||||
self.timber_weights = parse_string_to_list(config.get("timber_weights"))
|
||||
|
||||
# 设置语音ID
|
||||
if config.get("private_voice"):
|
||||
self.voice_setting["voice_id"] = config.get("private_voice")
|
||||
elif config.get("voice_id"):
|
||||
self.voice_setting["voice_id"] = config.get("voice_id")
|
||||
|
||||
# WebSocket配置
|
||||
self.ws_url = "wss://api.minimaxi.com/ws/v1/t2a_v2"
|
||||
self.headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"GroupId": self.group_id
|
||||
}
|
||||
self.audio_file_type = self.audio_setting.get("format", "mp3")
|
||||
|
||||
def generate_filename(self, extension=".mp3"):
|
||||
"""生成唯一的音频文件名"""
|
||||
return os.path.join(
|
||||
self.output_file,
|
||||
f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
|
||||
)
|
||||
|
||||
async def _establish_connection(self):
|
||||
"""建立WebSocket连接"""
|
||||
ssl_context = ssl.create_default_context()
|
||||
ssl_context.check_hostname = False
|
||||
ssl_context.verify_mode = ssl.CERT_NONE
|
||||
|
||||
try:
|
||||
ws = await websockets.connect(
|
||||
self.ws_url,
|
||||
additional_headers=self.headers,
|
||||
ssl=ssl_context
|
||||
)
|
||||
connected = json.loads(await ws.recv())
|
||||
if connected.get("event") == "connected_success":
|
||||
print("连接成功")
|
||||
return ws
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"连接失败: {e}")
|
||||
return None
|
||||
|
||||
async def _start_task(self, websocket):
|
||||
"""发送任务开始请求"""
|
||||
start_msg = {
|
||||
"event": "task_start",
|
||||
"model": self.model,
|
||||
"voice_setting": self.voice_setting,
|
||||
"pronunciation_dict": self.pronunciation_dict,
|
||||
"audio_setting": self.audio_setting
|
||||
}
|
||||
|
||||
if self.timber_weights and len(self.timber_weights) > 0:
|
||||
start_msg["timber_weights"] = self.timber_weights
|
||||
start_msg["voice_setting"]["voice_id"] = ""
|
||||
|
||||
await websocket.send(json.dumps(start_msg))
|
||||
response = json.loads(await websocket.recv())
|
||||
return response.get("event") == "task_started"
|
||||
|
||||
async def _continue_task(self, websocket, text):
|
||||
"""发送继续请求并收集音频数据"""
|
||||
await websocket.send(json.dumps({
|
||||
"event": "task_continue",
|
||||
"text": text
|
||||
}))
|
||||
|
||||
audio_chunks = []
|
||||
while True:
|
||||
response = json.loads(await websocket.recv())
|
||||
if "data" in response and "audio" in response["data"]:
|
||||
audio_chunks.append(response["data"]["audio"])
|
||||
if response.get("is_final"):
|
||||
break
|
||||
return "".join(audio_chunks)
|
||||
|
||||
async def _close_connection(self, websocket):
|
||||
"""关闭连接"""
|
||||
if websocket:
|
||||
await websocket.send(json.dumps({"event": "task_finish"}))
|
||||
await websocket.close()
|
||||
print("连接已关闭")
|
||||
|
||||
async def text_to_speak(self, text, output_file=None):
|
||||
"""主方法：文本转语音"""
|
||||
ws = await self._establish_connection()
|
||||
if not ws:
|
||||
raise Exception("无法建立WebSocket连接")
|
||||
|
||||
try:
|
||||
if not await self._start_task(ws):
|
||||
raise Exception("任务启动失败")
|
||||
|
||||
hex_audio = await self._continue_task(ws, text)
|
||||
audio_bytes = bytes.fromhex(hex_audio)
|
||||
|
||||
# 保存到文件或返回二进制数据
|
||||
if output_file:
|
||||
with open(output_file, "wb") as f:
|
||||
f.write(audio_bytes)
|
||||
print(f"音频已保存为{output_file}")
|
||||
return output_file
|
||||
else:
|
||||
# 返回音频二进制数据（不播放）
|
||||
return audio_bytes
|
||||
|
||||
finally:
|
||||
await self._close_connection(ws)
|
||||
|
||||
|
||||
async def main():
|
||||
"""测试用主函数"""
|
||||
# 示例配置
|
||||
config = {
|
||||
"group_id": "YOUR_GROUP_ID", # 替换为实际的group_id
|
||||
"api_key": "YOUR_API_KEY", # 替换为实际的api_key
|
||||
"model": "your-model", # 替换为实际的模型名称
|
||||
"voice_id": "male-qn-qingse",
|
||||
"voice_setting": {
|
||||
"speed": 1.2,
|
||||
"emotion": "happy"
|
||||
}
|
||||
}
|
||||
|
||||
tts = TTSProvider(config, delete_audio_file=True)
|
||||
output_file = tts.generate_filename()
|
||||
await tts.text_to_speak("这是一个测试文本，用于验证流式语音合成功能", output_file)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,527 @@
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import json
|
||||
import hmac
|
||||
import queue
|
||||
import base64
|
||||
import hashlib
|
||||
import asyncio
|
||||
import traceback
|
||||
import websockets
|
||||
from asyncio import Task
|
||||
from config.logger import setup_logging
|
||||
from core.utils import opus_encoder_utils
|
||||
from core.utils.tts import MarkdownCleaner
|
||||
from urllib.parse import urlencode, urlparse
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class XunfeiWSAuth:
|
||||
@staticmethod
|
||||
def create_auth_url(api_key, api_secret, api_url):
|
||||
"""生成讯飞WebSocket认证URL"""
|
||||
parsed_url = urlparse(api_url)
|
||||
host = parsed_url.netloc
|
||||
path = parsed_url.path
|
||||
|
||||
# 获取UTC时间，讯飞要求使用RFC1123格式
|
||||
now = time.gmtime()
|
||||
date = time.strftime('%a, %d %b %Y %H:%M:%S GMT', now)
|
||||
|
||||
# 构造签名字符串
|
||||
signature_origin = f"host: {host}\ndate: {date}\nGET {path} HTTP/1.1"
|
||||
|
||||
# 计算签名
|
||||
signature_sha = hmac.new(
|
||||
api_secret.encode('utf-8'),
|
||||
signature_origin.encode('utf-8'),
|
||||
digestmod=hashlib.sha256
|
||||
).digest()
|
||||
signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding='utf-8')
|
||||
|
||||
# 构造authorization
|
||||
authorization_origin = f'api_key="{api_key}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha_base64}"'
|
||||
authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8')
|
||||
|
||||
# 构造最终的WebSocket URL
|
||||
v = {
|
||||
"authorization": authorization,
|
||||
"date": date,
|
||||
"host": host
|
||||
}
|
||||
url = api_url + '?' + urlencode(v)
|
||||
return url
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
|
||||
# 设置为流式接口类型
|
||||
self.interface_type = InterfaceType.DUAL_STREAM
|
||||
|
||||
# 基础配置
|
||||
self.app_id = config.get("app_id")
|
||||
self.api_key = config.get("api_key")
|
||||
self.api_secret = config.get("api_secret")
|
||||
|
||||
# 接口地址
|
||||
self.api_url = config.get("api_url", "wss://cbm01.cn-huabei-1.xf-yun.com/v1/private/mcd9m97e6")
|
||||
|
||||
# 音色配置
|
||||
self.voice = config.get("voice", "x5_lingxiaoxuan_flow")
|
||||
if config.get("private_voice"):
|
||||
self.voice = config.get("private_voice")
|
||||
|
||||
# 音频参数配置
|
||||
speed = config.get("speed", "50")
|
||||
self.speed = int(speed) if speed else 50
|
||||
|
||||
volume = config.get("volume", "50")
|
||||
self.volume = int(volume) if volume else 50
|
||||
|
||||
pitch = config.get("pitch", "50")
|
||||
self.pitch = int(pitch) if pitch else 50
|
||||
|
||||
# 音频编码配置
|
||||
self.format = config.get("format", "raw")
|
||||
|
||||
sample_rate = config.get("sample_rate", "24000")
|
||||
self.sample_rate = int(sample_rate) if sample_rate else 24000
|
||||
|
||||
# 口语化配置
|
||||
self.oral_level = config.get("oral_level", "mid")
|
||||
|
||||
spark_assist = config.get("spark_assist", "1")
|
||||
self.spark_assist = int(spark_assist) if spark_assist else 1
|
||||
|
||||
stop_split = config.get("stop_split", "0")
|
||||
self.stop_split = int(stop_split) if stop_split else 0
|
||||
|
||||
remain = config.get("remain", "0")
|
||||
self.remain = int(remain) if remain else 0
|
||||
|
||||
# WebSocket配置
|
||||
self.ws = None
|
||||
self._monitor_task = None
|
||||
|
||||
# 序列号管理
|
||||
self.text_seq = 0
|
||||
|
||||
# 创建Opus编码器
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
# 验证必需参数
|
||||
if not all([self.app_id, self.api_key, self.api_secret]):
|
||||
raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
|
||||
|
||||
async def _ensure_connection(self):
|
||||
"""确保WebSocket连接可用"""
|
||||
try:
|
||||
logger.bind(tag=TAG).info("开始建立新连接...")
|
||||
|
||||
# 生成认证URL
|
||||
auth_url = XunfeiWSAuth.create_auth_url(
|
||||
self.api_key, self.api_secret, self.api_url
|
||||
)
|
||||
|
||||
self.ws = await websockets.connect(
|
||||
auth_url,
|
||||
ping_interval=30,
|
||||
ping_timeout=10,
|
||||
close_timeout=10,
|
||||
)
|
||||
logger.bind(tag=TAG).info("WebSocket连接建立成功")
|
||||
return self.ws
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}")
|
||||
self.ws = None
|
||||
raise
|
||||
|
||||
def tts_text_priority_thread(self):
|
||||
"""流式文本处理线程"""
|
||||
while not self.conn.stop_event.is_set():
|
||||
try:
|
||||
message = self.tts_text_queue.get(timeout=1)
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}"
|
||||
)
|
||||
|
||||
if message.sentence_type == SentenceType.FIRST:
|
||||
# 重置序列号
|
||||
self.text_seq = 0
|
||||
self.conn.client_abort = False
|
||||
# 增加序列号
|
||||
self.text_seq += 1
|
||||
if self.conn.client_abort:
|
||||
logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
|
||||
continue
|
||||
|
||||
if message.sentence_type == SentenceType.FIRST:
|
||||
# 初始化参数
|
||||
try:
|
||||
if not getattr(self.conn, "sentence_id", None):
|
||||
self.conn.sentence_id = uuid.uuid4().hex
|
||||
logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}")
|
||||
|
||||
logger.bind(tag=TAG).info("开始启动TTS会话...")
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.start_session(self.conn.sentence_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
self.before_stop_play_files.clear()
|
||||
logger.bind(tag=TAG).info("TTS会话启动成功")
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
|
||||
continue
|
||||
|
||||
# 处理文本内容
|
||||
if ContentType.TEXT == message.content_type:
|
||||
if message.content_detail:
|
||||
try:
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"开始发送TTS文本: {message.content_detail}"
|
||||
)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.text_to_speak(message.content_detail, None),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
logger.bind(tag=TAG).debug("TTS文本发送成功")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
|
||||
# 不使用continue，确保后续处理不被中断
|
||||
|
||||
# 处理文件内容
|
||||
if ContentType.FILE == message.content_type:
|
||||
logger.bind(tag=TAG).info(
|
||||
f"添加音频文件到待播放列表: {message.content_file}"
|
||||
)
|
||||
if message.content_file and os.path.exists(message.content_file):
|
||||
# 先处理文件音频数据
|
||||
self._process_audio_file_stream(message.content_file, callback=lambda audio_data: self.handle_audio_file(audio_data, message.content_detail))
|
||||
|
||||
# 处理会话结束
|
||||
if message.sentence_type == SentenceType.LAST:
|
||||
try:
|
||||
logger.bind(tag=TAG).info("开始结束TTS会话...")
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.finish_session(self.conn.sentence_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"结束TTS会话失败: {str(e)}")
|
||||
continue
|
||||
|
||||
except queue.Empty:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
|
||||
)
|
||||
|
||||
async def text_to_speak(self, text, _):
|
||||
"""发送文本到TTS服务进行合成"""
|
||||
try:
|
||||
if self.ws is None:
|
||||
logger.bind(tag=TAG).warning(f"WebSocket连接不存在，终止发送文本")
|
||||
return
|
||||
|
||||
filtered_text = MarkdownCleaner.clean_markdown(text)
|
||||
|
||||
# 发送文本合成请求
|
||||
run_request = self._build_base_request(status=1,text=filtered_text)
|
||||
await self.ws.send(json.dumps(run_request))
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
|
||||
if self.ws:
|
||||
try:
|
||||
await self.ws.close()
|
||||
except:
|
||||
pass
|
||||
self.ws = None
|
||||
raise
|
||||
|
||||
async def start_session(self, session_id):
|
||||
logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
|
||||
try:
|
||||
# 会话开始时检测上个会话的监听状态
|
||||
if (
|
||||
self._monitor_task is not None
|
||||
and isinstance(self._monitor_task, Task)
|
||||
and not self._monitor_task.done()
|
||||
):
|
||||
logger.bind(tag=TAG).info(
|
||||
"检测到未完成的上个会话，关闭监听任务和连接..."
|
||||
)
|
||||
await self.close()
|
||||
|
||||
# 建立新连接
|
||||
await self._ensure_connection()
|
||||
|
||||
# 启动监听任务
|
||||
self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
|
||||
|
||||
# 发送会话启动请求
|
||||
start_request = self._build_base_request(status=0)
|
||||
|
||||
await self.ws.send(json.dumps(start_request))
|
||||
logger.bind(tag=TAG).info("会话启动请求已发送")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
|
||||
# 确保清理资源
|
||||
await self.close()
|
||||
raise
|
||||
|
||||
async def finish_session(self, session_id):
|
||||
logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
|
||||
try:
|
||||
if self.ws:
|
||||
# 发送会话结束请求
|
||||
stop_request = self._build_base_request(status=2)
|
||||
await self.ws.send(json.dumps(stop_request))
|
||||
logger.bind(tag=TAG).info("会话结束请求已发送")
|
||||
|
||||
if self._monitor_task:
|
||||
try:
|
||||
await self._monitor_task
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"等待监听任务完成时发生错误: {str(e)}")
|
||||
finally:
|
||||
self._monitor_task = None
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
|
||||
await self.close()
|
||||
raise
|
||||
|
||||
async def close(self):
|
||||
"""资源清理"""
|
||||
if self._monitor_task:
|
||||
try:
|
||||
self._monitor_task.cancel()
|
||||
await self._monitor_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}")
|
||||
self._monitor_task = None
|
||||
|
||||
if self.ws:
|
||||
try:
|
||||
await self.ws.close()
|
||||
except:
|
||||
pass
|
||||
self.ws = None
|
||||
|
||||
async def _start_monitor_tts_response(self):
|
||||
"""监听TTS响应"""
|
||||
try:
|
||||
while not self.conn.stop_event.is_set():
|
||||
try:
|
||||
msg = await self.ws.recv()
|
||||
|
||||
# 检查客户端是否中止
|
||||
if self.conn.client_abort:
|
||||
logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应")
|
||||
break
|
||||
|
||||
try:
|
||||
data = json.loads(msg)
|
||||
header = data.get("header", {})
|
||||
code = header.get("code")
|
||||
|
||||
if code == 0:
|
||||
payload = data.get("payload", {})
|
||||
audio_payload = payload.get("audio", {})
|
||||
|
||||
if audio_payload:
|
||||
status = audio_payload.get("status", 0)
|
||||
audio_data = audio_payload.get("audio", "")
|
||||
if status == 0:
|
||||
logger.bind(tag=TAG).debug("TTS合成已启动")
|
||||
self.tts_audio_queue.put(
|
||||
(SentenceType.FIRST, [], None)
|
||||
)
|
||||
elif status == 2:
|
||||
logger.bind(tag=TAG).debug("收到结束状态的音频数据，TTS合成完成")
|
||||
self._process_before_stop_play_files()
|
||||
break
|
||||
else:
|
||||
if self.conn.tts_MessageText:
|
||||
logger.bind(tag=TAG).info(
|
||||
f"句子语音生成成功： {self.conn.tts_MessageText}"
|
||||
)
|
||||
self.tts_audio_queue.put(
|
||||
(SentenceType.FIRST, [], self.conn.tts_MessageText)
|
||||
)
|
||||
self.conn.tts_MessageText = None
|
||||
try:
|
||||
audio_bytes = base64.b64decode(audio_data)
|
||||
self.opus_encoder.encode_pcm_to_opus_stream(
|
||||
audio_bytes, False, self.handle_opus
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理音频数据失败: {e}")
|
||||
|
||||
else:
|
||||
message = header.get("message", "未知错误")
|
||||
logger.bind(tag=TAG).error(f"TTS合成错误: {code} - {message}")
|
||||
break
|
||||
|
||||
except json.JSONDecodeError:
|
||||
logger.bind(tag=TAG).warning("收到无效的JSON消息")
|
||||
|
||||
except websockets.ConnectionClosed:
|
||||
logger.bind(tag=TAG).warning("WebSocket连接已关闭")
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"处理TTS响应时出错: {e}\n{traceback.format_exc()}"
|
||||
)
|
||||
break
|
||||
|
||||
# 链接不可复用
|
||||
if self.ws:
|
||||
try:
|
||||
await self.ws.close()
|
||||
except:
|
||||
pass
|
||||
self.ws = None
|
||||
# 监听任务退出时清理引用
|
||||
finally:
|
||||
self._monitor_task = None
|
||||
|
||||
def to_tts(self, text: str) -> list:
|
||||
"""非流式TTS处理，用于测试及保存音频文件的场景"""
|
||||
try:
|
||||
# 创建新的事件循环
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
# 存储音频数据
|
||||
audio_data = []
|
||||
|
||||
async def _generate_audio():
|
||||
# 生成认证URL
|
||||
auth_url = XunfeiWSAuth.create_auth_url(
|
||||
self.api_key, self.api_secret, self.api_url
|
||||
)
|
||||
|
||||
# 建立WebSocket连接
|
||||
ws = await websockets.connect(
|
||||
auth_url,
|
||||
ping_interval=30,
|
||||
ping_timeout=10,
|
||||
close_timeout=10,
|
||||
)
|
||||
|
||||
try:
|
||||
filtered_text = MarkdownCleaner.clean_markdown(text)
|
||||
|
||||
text_request = self._build_base_request(status=2,text=filtered_text)
|
||||
|
||||
await ws.send(json.dumps(text_request))
|
||||
|
||||
task_finished = False
|
||||
while not task_finished:
|
||||
msg = await ws.recv()
|
||||
|
||||
data = json.loads(msg)
|
||||
header = data.get("header", {})
|
||||
code = header.get("code")
|
||||
|
||||
if code == 0:
|
||||
payload = data.get("payload", {})
|
||||
audio_payload = payload.get("audio", {})
|
||||
if audio_payload:
|
||||
status = audio_payload.get("status", 0)
|
||||
audio_base64 = audio_payload.get("audio", "")
|
||||
if status == 1:
|
||||
try:
|
||||
audio_bytes = base64.b64decode(audio_base64)
|
||||
self.opus_encoder.encode_pcm_to_opus_stream(
|
||||
audio_bytes,
|
||||
end_of_stream=False,
|
||||
callback=lambda opus: audio_data.append(opus)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理音频数据失败: {e}")
|
||||
elif status == 2:
|
||||
task_finished = True
|
||||
logger.bind(tag=TAG).debug("TTS任务完成")
|
||||
|
||||
else:
|
||||
message = header.get("message", "未知错误")
|
||||
raise Exception(f"合成失败: {code} - {message}")
|
||||
|
||||
finally:
|
||||
# 清理资源
|
||||
try:
|
||||
await ws.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
loop.run_until_complete(_generate_audio())
|
||||
loop.close()
|
||||
|
||||
return audio_data
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}")
|
||||
return []
|
||||
|
||||
def _build_base_request(self, status,text=" "):
|
||||
"""构建基础请求结构"""
|
||||
return {
|
||||
"header": {
|
||||
"app_id": self.app_id,
|
||||
"status": status,
|
||||
},
|
||||
"parameter": {
|
||||
"oral": {
|
||||
"oral_level": self.oral_level,
|
||||
"spark_assist": self.spark_assist,
|
||||
"stop_split": self.stop_split,
|
||||
"remain": self.remain
|
||||
},
|
||||
"tts": {
|
||||
"vcn": self.voice,
|
||||
"speed": self.speed,
|
||||
"volume": self.volume,
|
||||
"pitch": self.pitch,
|
||||
"bgs": 0,
|
||||
"reg": 0,
|
||||
"rdn": 0,
|
||||
"rhy": 0,
|
||||
"audio": {
|
||||
"encoding": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"channels": 1,
|
||||
"bit_depth": 16,
|
||||
"frame_size": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"payload": {
|
||||
"text": {
|
||||
"encoding": "utf8",
|
||||
"compress": "raw",
|
||||
"format": "plain",
|
||||
"status": status,
|
||||
"seq": self.text_seq,
|
||||
"text": base64.b64encode(text.encode('utf-8')).decode('utf-8')
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,18 @@ class VADProvider(VADProviderBase):
|
||||
# 至少要多少帧才算有语音
|
||||
self.frame_window_threshold = 3
|
||||
|
||||
def __del__(self):
|
||||
if hasattr(self, 'decoder') and self.decoder is not None:
|
||||
try:
|
||||
del self.decoder
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def is_vad(self, conn, opus_packet):
|
||||
# 手动模式：直接返回True，不进行实时VAD检测，所有音频都缓存
|
||||
if conn.client_listen_mode == "manual":
|
||||
return True
|
||||
|
||||
try:
|
||||
pcm_frame = self.decoder.decode(opus_packet, 960)
|
||||
conn.client_audio_buffer.extend(pcm_frame) # 将新数据加入缓冲区
|
||||
@@ -70,7 +81,9 @@ class VADProvider(VADProviderBase):
|
||||
|
||||
# 更新滑动窗口
|
||||
conn.client_voice_window.append(is_voice)
|
||||
client_have_voice = (conn.client_voice_window.count(True) >= self.frame_window_threshold)
|
||||
client_have_voice = (
|
||||
conn.client_voice_window.count(True) >= self.frame_window_threshold
|
||||
)
|
||||
|
||||
# 如果之前有声音，但本次没有声音，且与上次有声音的时间差已经超过了静默阈值，则认为已经说完一句话
|
||||
if conn.client_have_voice and not client_have_voice:
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import time
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AudioRateController:
|
||||
"""
|
||||
音频速率控制器 - 按照60ms帧时长精确控制音频发送
|
||||
解决高并发下的时间累积误差问题
|
||||
"""
|
||||
|
||||
def __init__(self, frame_duration=60):
|
||||
"""
|
||||
Args:
|
||||
frame_duration: 单个音频帧时长（毫秒），默认60ms
|
||||
"""
|
||||
self.frame_duration = frame_duration
|
||||
self.queue = deque()
|
||||
self.play_position = 0 # 虚拟播放位置（毫秒）
|
||||
self.start_timestamp = None # 开始时间戳（只读，不修改）
|
||||
self.pending_send_task = None
|
||||
self.logger = logger
|
||||
self.queue_empty_event = asyncio.Event() # 队列清空事件
|
||||
self.queue_empty_event.set() # 初始为空状态
|
||||
self.queue_has_data_event = asyncio.Event() # 队列数据事件
|
||||
|
||||
def reset(self):
|
||||
"""重置控制器状态"""
|
||||
if self.pending_send_task and not self.pending_send_task.done():
|
||||
self.pending_send_task.cancel()
|
||||
# 取消任务后，任务会在下次事件循环时清理，无需阻塞等待
|
||||
|
||||
self.queue.clear()
|
||||
self.play_position = 0
|
||||
self.start_timestamp = None # 由首个音频包设置
|
||||
# 相关事件处理
|
||||
self.queue_empty_event.set()
|
||||
self.queue_has_data_event.clear()
|
||||
|
||||
def add_audio(self, opus_packet):
|
||||
"""添加音频包到队列"""
|
||||
self.queue.append(("audio", opus_packet))
|
||||
# 相关事件处理
|
||||
self.queue_empty_event.clear()
|
||||
self.queue_has_data_event.set()
|
||||
|
||||
def add_message(self, message_callback):
|
||||
"""
|
||||
添加消息到队列（立即发送，不占用播放时间）
|
||||
|
||||
Args:
|
||||
message_callback: 消息发送回调函数 async def()
|
||||
"""
|
||||
self.queue.append(("message", message_callback))
|
||||
# 相关事件处理
|
||||
self.queue_empty_event.clear()
|
||||
self.queue_has_data_event.set()
|
||||
|
||||
def _get_elapsed_ms(self):
|
||||
"""获取已经过的时间（毫秒）"""
|
||||
if self.start_timestamp is None:
|
||||
return 0
|
||||
return (time.monotonic() - self.start_timestamp) * 1000
|
||||
|
||||
async def check_queue(self, send_audio_callback):
|
||||
"""
|
||||
检查队列并按时发送音频/消息
|
||||
|
||||
Args:
|
||||
send_audio_callback: 发送音频的回调函数 async def(opus_packet)
|
||||
"""
|
||||
while self.queue:
|
||||
item = self.queue[0]
|
||||
item_type = item[0]
|
||||
|
||||
if item_type == "message":
|
||||
# 消息类型：立即发送，不占用播放时间
|
||||
_, message_callback = item
|
||||
self.queue.popleft()
|
||||
try:
|
||||
await message_callback()
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"发送消息失败: {e}")
|
||||
raise
|
||||
|
||||
elif item_type == "audio":
|
||||
if self.start_timestamp is None:
|
||||
self.start_timestamp = time.monotonic()
|
||||
|
||||
_, opus_packet = item
|
||||
|
||||
# 循环等待直到时间到达
|
||||
while True:
|
||||
# 计算时间差
|
||||
elapsed_ms = self._get_elapsed_ms()
|
||||
output_ms = self.play_position
|
||||
|
||||
if elapsed_ms < output_ms:
|
||||
# 还不到发送时间，计算等待时长
|
||||
wait_ms = output_ms - elapsed_ms
|
||||
|
||||
# 等待后继续检查（允许被中断）
|
||||
try:
|
||||
await asyncio.sleep(wait_ms / 1000)
|
||||
except asyncio.CancelledError:
|
||||
self.logger.bind(tag=TAG).debug("音频发送任务被取消")
|
||||
raise
|
||||
# 等待结束后重新检查时间（循环回到 while True）
|
||||
else:
|
||||
# 时间已到，跳出等待循环
|
||||
break
|
||||
|
||||
# 时间已到，从队列移除并发送
|
||||
self.queue.popleft()
|
||||
self.play_position += self.frame_duration
|
||||
try:
|
||||
await send_audio_callback(opus_packet)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"发送音频失败: {e}")
|
||||
raise
|
||||
|
||||
# 队列处理完后清除事件
|
||||
self.queue_empty_event.set()
|
||||
self.queue_has_data_event.clear()
|
||||
|
||||
def start_sending(self, send_audio_callback):
|
||||
"""
|
||||
启动异步发送任务
|
||||
|
||||
Args:
|
||||
send_audio_callback: 发送音频的回调函数
|
||||
|
||||
Returns:
|
||||
asyncio.Task: 发送任务
|
||||
"""
|
||||
|
||||
async def _send_loop():
|
||||
try:
|
||||
while True:
|
||||
# 等待队列数据事件，不轮询等待占用CPU
|
||||
await self.queue_has_data_event.wait()
|
||||
|
||||
await self.check_queue(send_audio_callback)
|
||||
except asyncio.CancelledError:
|
||||
self.logger.bind(tag=TAG).debug("音频发送循环已停止")
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"音频发送循环异常: {e}")
|
||||
|
||||
self.pending_send_task = asyncio.create_task(_send_loop())
|
||||
return self.pending_send_task
|
||||
|
||||
def stop_sending(self):
|
||||
"""停止发送任务"""
|
||||
if self.pending_send_task and not self.pending_send_task.done():
|
||||
self.pending_send_task.cancel()
|
||||
self.logger.bind(tag=TAG).debug("已取消音频发送任务")
|
||||
@@ -19,6 +19,7 @@ class CacheType(Enum):
|
||||
CONFIG = "config"
|
||||
DEVICE_PROMPT = "device_prompt"
|
||||
VOICEPRINT_HEALTH = "voiceprint_health" # 声纹识别健康检查
|
||||
AUDIO_DATA = "audio_data" # 音频数据缓存
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -58,5 +59,8 @@ class CacheConfig:
|
||||
CacheType.VOICEPRINT_HEALTH: cls(
|
||||
strategy=CacheStrategy.TTL, ttl=600, max_size=100 # 10分钟过期
|
||||
),
|
||||
CacheType.AUDIO_DATA: cls(
|
||||
strategy=CacheStrategy.TTL, ttl=600, max_size=100 # 10分钟过期
|
||||
),
|
||||
}
|
||||
return configs.get(cache_type, cls())
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import httpx
|
||||
from typing import Dict, Any, List
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
|
||||
class ContextDataProvider:
|
||||
"""数据上下文填充，负责从配置的API获取数据"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any], logger=None):
|
||||
self.config = config
|
||||
self.logger = logger or setup_logging()
|
||||
self.context_data = ""
|
||||
|
||||
def fetch_all(self, device_id: str) -> str:
|
||||
"""获取所有配置的上下文数据"""
|
||||
context_providers = self.config.get("context_providers", [])
|
||||
if not context_providers:
|
||||
return ""
|
||||
|
||||
formatted_lines = []
|
||||
for provider in context_providers:
|
||||
url = provider.get("url")
|
||||
headers = provider.get("headers", {})
|
||||
|
||||
if not url:
|
||||
continue
|
||||
|
||||
try:
|
||||
headers = headers.copy() if isinstance(headers, dict) else {}
|
||||
# 将 device_id 添加到请求头
|
||||
headers["device-id"] = device_id
|
||||
|
||||
# 发送请求
|
||||
response = httpx.get(url, headers=headers, timeout=3)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
if isinstance(result, dict):
|
||||
if result.get("code") == 0:
|
||||
data = result.get("data")
|
||||
# 格式化数据
|
||||
if isinstance(data, dict):
|
||||
for k, v in data.items():
|
||||
formatted_lines.append(f"- **{k}：** {v}")
|
||||
elif isinstance(data, list):
|
||||
for item in data:
|
||||
formatted_lines.append(f"- {item}")
|
||||
else:
|
||||
formatted_lines.append(f"- {data}")
|
||||
else:
|
||||
self.logger.bind(tag=TAG).warning(f"API {url} 返回错误码: {result.get('msg')}")
|
||||
else:
|
||||
self.logger.bind(tag=TAG).warning(f"API {url} 返回的不是JSON字典")
|
||||
else:
|
||||
self.logger.bind(tag=TAG).warning(f"API {url} 请求失败: {response.status_code}")
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"获取上下文数据 {url} 失败: {e}")
|
||||
|
||||
# 将所有格式化后的行拼接成一个字符串
|
||||
self.context_data = "\n".join(formatted_lines)
|
||||
if self.context_data:
|
||||
self.logger.bind(tag=TAG).debug(f"已注入动态上下文数据:\n{self.context_data}")
|
||||
return self.context_data
|
||||
@@ -0,0 +1,68 @@
|
||||
"""
|
||||
时间工具模块
|
||||
提供统一的时间获取功能
|
||||
"""
|
||||
|
||||
import cnlunar
|
||||
from datetime import datetime
|
||||
|
||||
WEEKDAY_MAP = {
|
||||
"Monday": "星期一",
|
||||
"Tuesday": "星期二",
|
||||
"Wednesday": "星期三",
|
||||
"Thursday": "星期四",
|
||||
"Friday": "星期五",
|
||||
"Saturday": "星期六",
|
||||
"Sunday": "星期日",
|
||||
}
|
||||
|
||||
|
||||
def get_current_time() -> str:
|
||||
"""
|
||||
获取当前时间字符串 (格式: HH:MM)
|
||||
"""
|
||||
return datetime.now().strftime("%H:%M")
|
||||
|
||||
|
||||
def get_current_date() -> str:
|
||||
"""
|
||||
获取今天日期字符串 (格式: YYYY-MM-DD)
|
||||
"""
|
||||
return datetime.now().strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def get_current_weekday() -> str:
|
||||
"""
|
||||
获取今天星期几
|
||||
"""
|
||||
now = datetime.now()
|
||||
return WEEKDAY_MAP[now.strftime("%A")]
|
||||
|
||||
|
||||
def get_current_lunar_date() -> str:
|
||||
"""
|
||||
获取农历日期字符串
|
||||
"""
|
||||
try:
|
||||
now = datetime.now()
|
||||
today_lunar = cnlunar.Lunar(now, godType="8char")
|
||||
return "%s年%s%s" % (
|
||||
today_lunar.lunarYearCn,
|
||||
today_lunar.lunarMonthCn[:-1],
|
||||
today_lunar.lunarDayCn,
|
||||
)
|
||||
except Exception:
|
||||
return "农历获取失败"
|
||||
|
||||
|
||||
def get_current_time_info() -> tuple:
|
||||
"""
|
||||
获取当前时间信息
|
||||
返回: (当前时间字符串, 今天日期, 今天星期, 农历日期)
|
||||
"""
|
||||
current_time = get_current_time()
|
||||
today_date = get_current_date()
|
||||
today_weekday = get_current_weekday()
|
||||
lunar_date = get_current_lunar_date()
|
||||
|
||||
return current_time, today_date, today_weekday, lunar_date
|
||||
@@ -0,0 +1,122 @@
|
||||
"""
|
||||
全局GC管理模块
|
||||
定期执行垃圾回收，避免频繁触发GC导致的GIL锁问题
|
||||
"""
|
||||
|
||||
import gc
|
||||
import asyncio
|
||||
import threading
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class GlobalGCManager:
|
||||
"""全局垃圾回收管理器"""
|
||||
|
||||
def __init__(self, interval_seconds=300):
|
||||
"""
|
||||
初始化GC管理器
|
||||
|
||||
Args:
|
||||
interval_seconds: GC执行间隔（秒），默认300秒（5分钟）
|
||||
"""
|
||||
self.interval_seconds = interval_seconds
|
||||
self._task = None
|
||||
self._stop_event = asyncio.Event()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
async def start(self):
|
||||
"""启动定时GC任务"""
|
||||
if self._task is not None:
|
||||
logger.bind(tag=TAG).warning("GC管理器已经在运行")
|
||||
return
|
||||
|
||||
logger.bind(tag=TAG).info(f"启动全局GC管理器，间隔{self.interval_seconds}秒")
|
||||
self._stop_event.clear()
|
||||
self._task = asyncio.create_task(self._gc_loop())
|
||||
|
||||
async def stop(self):
|
||||
"""停止定时GC任务"""
|
||||
if self._task is None:
|
||||
return
|
||||
|
||||
logger.bind(tag=TAG).info("停止全局GC管理器")
|
||||
self._stop_event.set()
|
||||
|
||||
if self._task and not self._task.done():
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
self._task = None
|
||||
|
||||
async def _gc_loop(self):
|
||||
"""GC循环任务"""
|
||||
try:
|
||||
while not self._stop_event.is_set():
|
||||
# 等待指定间隔
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._stop_event.wait(), timeout=self.interval_seconds
|
||||
)
|
||||
# 如果stop_event被设置，退出循环
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
# 超时表示到了执行GC的时间
|
||||
pass
|
||||
|
||||
# 执行GC
|
||||
await self._run_gc()
|
||||
|
||||
except asyncio.CancelledError:
|
||||
logger.bind(tag=TAG).info("GC循环任务被取消")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"GC循环任务异常: {e}")
|
||||
finally:
|
||||
logger.bind(tag=TAG).info("GC循环任务已退出")
|
||||
|
||||
async def _run_gc(self):
|
||||
"""执行垃圾回收"""
|
||||
try:
|
||||
# 在线程池中执行GC，避免阻塞事件循环
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def do_gc():
|
||||
with self._lock:
|
||||
before = len(gc.get_objects())
|
||||
collected = gc.collect()
|
||||
after = len(gc.get_objects())
|
||||
return before, collected, after
|
||||
|
||||
before, collected, after = await loop.run_in_executor(None, do_gc)
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"全局GC执行完成 - 回收对象: {collected}, "
|
||||
f"对象数量: {before} -> {after}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"执行GC时出错: {e}")
|
||||
|
||||
|
||||
# 全局单例
|
||||
_gc_manager_instance = None
|
||||
|
||||
|
||||
def get_gc_manager(interval_seconds=300):
|
||||
"""
|
||||
获取全局GC管理器实例（单例模式）
|
||||
|
||||
Args:
|
||||
interval_seconds: GC执行间隔（秒），默认300秒（5分钟）
|
||||
|
||||
Returns:
|
||||
GlobalGCManager实例
|
||||
"""
|
||||
global _gc_manager_instance
|
||||
if _gc_manager_instance is None:
|
||||
_gc_manager_instance = GlobalGCManager(interval_seconds)
|
||||
return _gc_manager_instance
|
||||
@@ -102,6 +102,9 @@ class OpusEncoderUtils:
|
||||
def _encode(self, frame: np.ndarray) -> Optional[bytes]:
|
||||
"""编码一帧音频数据"""
|
||||
try:
|
||||
# 编码器已释放，跳过编码
|
||||
if not hasattr(self, 'encoder') or self.encoder is None:
|
||||
return None
|
||||
# 将numpy数组转换为bytes
|
||||
frame_bytes = frame.tobytes()
|
||||
# opuslib要求输入字节数必须是channels*2的倍数
|
||||
@@ -128,5 +131,9 @@ class OpusEncoderUtils:
|
||||
|
||||
def close(self):
|
||||
"""关闭编码器并释放资源"""
|
||||
# opuslib没有明确的关闭方法，Python的垃圾回收会处理
|
||||
pass
|
||||
if hasattr(self, 'encoder') and self.encoder:
|
||||
try:
|
||||
del self.encoder
|
||||
self.encoder = None
|
||||
except Exception as e:
|
||||
logging.error(f"Error releasing Opus encoder: {e}")
|
||||
@@ -4,7 +4,6 @@
|
||||
"""
|
||||
|
||||
import os
|
||||
import cnlunar
|
||||
from typing import Dict, Any
|
||||
from config.logger import setup_logging
|
||||
from jinja2 import Template
|
||||
@@ -60,13 +59,20 @@ class PromptManager:
|
||||
|
||||
self.cache_manager = cache_manager
|
||||
self.CacheType = CacheType
|
||||
|
||||
# 初始化上下文源
|
||||
from core.utils.context_provider import ContextDataProvider
|
||||
self.context_provider = ContextDataProvider(config, self.logger)
|
||||
self.context_data = {}
|
||||
|
||||
self._load_base_template()
|
||||
|
||||
def _load_base_template(self):
|
||||
"""加载基础提示词模板"""
|
||||
try:
|
||||
template_path = "agent-base-prompt.txt"
|
||||
template_path = self.config.get("prompt_template", None)
|
||||
if not template_path:
|
||||
template_path = "agent-base-prompt.txt"
|
||||
cache_key = f"prompt_template:{template_path}"
|
||||
|
||||
# 先从缓存获取
|
||||
@@ -88,7 +94,7 @@ class PromptManager:
|
||||
self.base_prompt_template = template_content
|
||||
self.logger.bind(tag=TAG).debug("成功加载基础提示词模板并缓存")
|
||||
else:
|
||||
self.logger.bind(tag=TAG).warning("未找到agent-base-prompt.txt文件")
|
||||
self.logger.bind(tag=TAG).warning(f"未找到{template_path}文件")
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"加载提示词模板失败: {e}")
|
||||
|
||||
@@ -117,18 +123,16 @@ class PromptManager:
|
||||
|
||||
def _get_current_time_info(self) -> tuple:
|
||||
"""获取当前时间信息"""
|
||||
from datetime import datetime
|
||||
|
||||
now = datetime.now()
|
||||
today_date = now.strftime("%Y-%m-%d")
|
||||
today_weekday = WEEKDAY_MAP[now.strftime("%A")]
|
||||
today_lunar = cnlunar.Lunar(now, godType="8char")
|
||||
lunar_date = "%s年%s%s\n" % (
|
||||
today_lunar.lunarYearCn,
|
||||
today_lunar.lunarMonthCn[:-1],
|
||||
today_lunar.lunarDayCn,
|
||||
from .current_time import (
|
||||
get_current_date,
|
||||
get_current_weekday,
|
||||
get_current_lunar_date,
|
||||
)
|
||||
|
||||
today_date = get_current_date()
|
||||
today_weekday = get_current_weekday()
|
||||
lunar_date = get_current_lunar_date() + "\n"
|
||||
|
||||
return today_date, today_weekday, lunar_date
|
||||
|
||||
def _get_location_info(self, client_ip: str) -> str:
|
||||
@@ -180,17 +184,40 @@ class PromptManager:
|
||||
def update_context_info(self, conn, client_ip: str):
|
||||
"""同步更新上下文信息"""
|
||||
try:
|
||||
# 获取位置信息（使用全局缓存）
|
||||
local_address = self._get_location_info(client_ip)
|
||||
# 获取天气信息（使用全局缓存）
|
||||
self._get_weather_info(conn, local_address)
|
||||
self.logger.bind(tag=TAG).info(f"上下文信息更新完成")
|
||||
local_address = ""
|
||||
if (
|
||||
client_ip
|
||||
and self.base_prompt_template
|
||||
and (
|
||||
"local_address" in self.base_prompt_template
|
||||
or "weather_info" in self.base_prompt_template
|
||||
)
|
||||
):
|
||||
# 获取位置信息（使用全局缓存）
|
||||
local_address = self._get_location_info(client_ip)
|
||||
|
||||
if (
|
||||
self.base_prompt_template
|
||||
and "weather_info" in self.base_prompt_template
|
||||
and local_address
|
||||
):
|
||||
# 获取天气信息（使用全局缓存）
|
||||
self._get_weather_info(conn, local_address)
|
||||
|
||||
# 获取配置的上下文数据
|
||||
if hasattr(conn, "device_id") and conn.device_id:
|
||||
if self.base_prompt_template and "dynamic_context" in self.base_prompt_template:
|
||||
self.context_data = self.context_provider.fetch_all(conn.device_id)
|
||||
else:
|
||||
self.context_data = ""
|
||||
|
||||
self.logger.bind(tag=TAG).debug(f"上下文信息更新完成")
|
||||
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"更新上下文信息失败: {e}")
|
||||
|
||||
def build_enhanced_prompt(
|
||||
self, user_prompt: str, device_id: str, client_ip: str = None
|
||||
self, user_prompt: str, device_id: str, client_ip: str = None, *args, **kwargs
|
||||
) -> str:
|
||||
"""构建增强的系统提示词"""
|
||||
if not self.base_prompt_template:
|
||||
@@ -198,9 +225,7 @@ class PromptManager:
|
||||
|
||||
try:
|
||||
# 获取最新的时间信息（不缓存）
|
||||
today_date, today_weekday, lunar_date = (
|
||||
self._get_current_time_info()
|
||||
)
|
||||
today_date, today_weekday, lunar_date = self._get_current_time_info()
|
||||
|
||||
# 获取缓存的上下文信息
|
||||
local_address = ""
|
||||
@@ -230,6 +255,11 @@ class PromptManager:
|
||||
local_address=local_address,
|
||||
weather_info=weather_info,
|
||||
emojiList=EMOJI_List,
|
||||
device_id=device_id,
|
||||
client_ip=client_ip,
|
||||
dynamic_context=self.context_data,
|
||||
*args,
|
||||
**kwargs,
|
||||
)
|
||||
device_cache_key = f"device_prompt:{device_id}"
|
||||
self.cache_manager.set(
|
||||
|
||||
@@ -6,6 +6,27 @@ import importlib
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
punctuation_set = {
|
||||
"，",
|
||||
",", # 中文逗号 + 英文逗号
|
||||
"。",
|
||||
".", # 中文句号 + 英文句号
|
||||
"！",
|
||||
"!", # 中文感叹号 + 英文感叹号
|
||||
"“",
|
||||
"”",
|
||||
'"', # 中文双引号 + 英文引号
|
||||
"：",
|
||||
":", # 中文冒号 + 英文冒号
|
||||
"-",
|
||||
"－", # 英文连字符 + 中文全角横线
|
||||
"、", # 中文顿号
|
||||
"[",
|
||||
"]", # 方括号
|
||||
"【",
|
||||
"】", # 中文方括号
|
||||
"~", # 波浪号
|
||||
}
|
||||
|
||||
def create_instance(class_name, *args, **kwargs):
|
||||
# 创建TTS实例
|
||||
@@ -107,6 +128,11 @@ class MarkdownCleaner:
|
||||
"""
|
||||
主入口方法：依序执行所有正则，移除或替换 Markdown 元素
|
||||
"""
|
||||
# 检查文本是否全为英文和基本标点符号
|
||||
if text and all((c.isascii() or c.isspace() or c in punctuation_set) for c in text):
|
||||
# 保留原始空格，直接返回
|
||||
return text
|
||||
|
||||
for regex, replacement in MarkdownCleaner.REGEXES:
|
||||
text = regex.sub(replacement, text)
|
||||
return text.strip()
|
||||
@@ -4,6 +4,7 @@ import json
|
||||
import copy
|
||||
import wave
|
||||
import socket
|
||||
import asyncio
|
||||
import requests
|
||||
import subprocess
|
||||
import numpy as np
|
||||
@@ -176,31 +177,67 @@ def parse_string_to_list(value, separator=";"):
|
||||
return []
|
||||
|
||||
|
||||
def check_ffmpeg_installed():
|
||||
ffmpeg_installed = False
|
||||
def check_ffmpeg_installed() -> bool:
|
||||
"""
|
||||
检查当前环境中是否已正确安装并可执行 ffmpeg。
|
||||
|
||||
Returns:
|
||||
bool: 如果 ffmpeg 正常可用，返回 True；否则抛出 ValueError 异常。
|
||||
|
||||
Raises:
|
||||
ValueError: 当检测到 ffmpeg 未安装或依赖缺失时，抛出详细的提示信息。
|
||||
"""
|
||||
try:
|
||||
# 执行ffmpeg -version命令，并捕获输出
|
||||
# 尝试执行 ffmpeg 命令
|
||||
result = subprocess.run(
|
||||
["ffmpeg", "-version"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True, # 如果返回码非零则抛出异常
|
||||
check=True, # 非零退出码会触发 CalledProcessError
|
||||
)
|
||||
# 检查输出中是否包含版本信息（可选）
|
||||
output = result.stdout + result.stderr
|
||||
if "ffmpeg version" in output.lower():
|
||||
ffmpeg_installed = True
|
||||
return False
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
# 命令执行失败或未找到
|
||||
ffmpeg_installed = False
|
||||
if not ffmpeg_installed:
|
||||
error_msg = "您的电脑还没正确安装ffmpeg\n"
|
||||
error_msg += "\n建议您：\n"
|
||||
error_msg += "1、按照项目的安装文档，正确进入conda环境\n"
|
||||
error_msg += "2、查阅安装文档，如何在conda环境中安装ffmpeg\n"
|
||||
raise ValueError(error_msg)
|
||||
|
||||
output = (result.stdout + result.stderr).lower()
|
||||
if "ffmpeg version" in output:
|
||||
return True
|
||||
|
||||
# 如果未检测到版本信息，也视为异常情况
|
||||
raise ValueError("未检测到有效的 ffmpeg 版本输出。")
|
||||
|
||||
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
||||
# 提取错误输出
|
||||
stderr_output = ""
|
||||
if isinstance(e, subprocess.CalledProcessError):
|
||||
stderr_output = (e.stderr or "").strip()
|
||||
else:
|
||||
stderr_output = str(e).strip()
|
||||
|
||||
# 构建基础错误提示
|
||||
error_msg = [
|
||||
"❌ 检测到 ffmpeg 无法正常运行。\n",
|
||||
"建议您：",
|
||||
"1. 确认已正确激活 conda 环境；",
|
||||
"2. 查阅项目安装文档，了解如何在 conda 环境中安装 ffmpeg。\n",
|
||||
]
|
||||
|
||||
# 🎯 针对具体错误信息提供额外提示
|
||||
if "libiconv.so.2" in stderr_output:
|
||||
error_msg.append("⚠️ 发现缺少依赖库：libiconv.so.2")
|
||||
error_msg.append("解决方法：在当前 conda 环境中执行：")
|
||||
error_msg.append(" conda install -c conda-forge libiconv\n")
|
||||
elif (
|
||||
"no such file or directory" in stderr_output
|
||||
and "ffmpeg" in stderr_output.lower()
|
||||
):
|
||||
error_msg.append("⚠️ 系统未找到 ffmpeg 可执行文件。")
|
||||
error_msg.append("解决方法：在当前 conda 环境中执行：")
|
||||
error_msg.append(" conda install -c conda-forge ffmpeg\n")
|
||||
else:
|
||||
error_msg.append("错误详情：")
|
||||
error_msg.append(stderr_output or "未知错误。")
|
||||
|
||||
# 抛出详细异常信息
|
||||
raise ValueError("\n".join(error_msg)) from e
|
||||
|
||||
|
||||
def extract_json_from_string(input_string):
|
||||
@@ -212,7 +249,9 @@ def extract_json_from_string(input_string):
|
||||
return None
|
||||
|
||||
|
||||
def audio_to_data_stream(audio_file_path, is_opus=True, callback: Callable[[Any], Any]=None) -> None:
|
||||
def audio_to_data_stream(
|
||||
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None
|
||||
) -> None:
|
||||
# 获取文件后缀名
|
||||
file_type = os.path.splitext(audio_file_path)[1]
|
||||
if file_type:
|
||||
@@ -229,58 +268,88 @@ def audio_to_data_stream(audio_file_path, is_opus=True, callback: Callable[[Any]
|
||||
raw_data = audio.raw_data
|
||||
pcm_to_data_stream(raw_data, is_opus, callback)
|
||||
|
||||
def audio_to_data(audio_file_path: str, is_opus: bool = True) -> list[bytes]:
|
||||
|
||||
async def audio_to_data(
|
||||
audio_file_path: str, is_opus: bool = True, use_cache: bool = True
|
||||
) -> list[bytes]:
|
||||
"""
|
||||
将音频文件转换为Opus/PCM编码的帧列表
|
||||
Args:
|
||||
audio_file_path: 音频文件路径
|
||||
is_opus: 是否进行Opus编码
|
||||
use_cache: 是否使用缓存
|
||||
"""
|
||||
# 获取文件后缀名
|
||||
file_type = os.path.splitext(audio_file_path)[1]
|
||||
if file_type:
|
||||
file_type = file_type.lstrip(".")
|
||||
# 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
|
||||
audio = AudioSegment.from_file(
|
||||
audio_file_path, format=file_type, parameters=["-nostdin"]
|
||||
)
|
||||
from core.utils.cache.manager import cache_manager
|
||||
from core.utils.cache.config import CacheType
|
||||
|
||||
# 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
|
||||
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
|
||||
# 生成缓存键，包含文件路径和编码类型
|
||||
cache_key = f"{audio_file_path}:{is_opus}"
|
||||
|
||||
# 获取原始PCM数据（16位小端）
|
||||
raw_data = audio.raw_data
|
||||
# 尝试从缓存获取结果
|
||||
if use_cache:
|
||||
cached_result = cache_manager.get(CacheType.AUDIO_DATA, cache_key)
|
||||
if cached_result is not None:
|
||||
return cached_result
|
||||
|
||||
# 初始化Opus编码器
|
||||
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
|
||||
def _sync_audio_to_data():
|
||||
# 获取文件后缀名
|
||||
file_type = os.path.splitext(audio_file_path)[1]
|
||||
if file_type:
|
||||
file_type = file_type.lstrip(".")
|
||||
# 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
|
||||
audio = AudioSegment.from_file(
|
||||
audio_file_path, format=file_type, parameters=["-nostdin"]
|
||||
)
|
||||
|
||||
# 编码参数
|
||||
frame_duration = 60 # 60ms per frame
|
||||
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
|
||||
# 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
|
||||
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
|
||||
|
||||
datas = []
|
||||
# 按帧处理所有音频数据（包括最后一帧可能补零）
|
||||
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
|
||||
# 获取当前帧的二进制数据
|
||||
chunk = raw_data[i : i + frame_size * 2]
|
||||
# 获取原始PCM数据（16位小端）
|
||||
raw_data = audio.raw_data
|
||||
|
||||
# 如果最后一帧不足，补零
|
||||
if len(chunk) < frame_size * 2:
|
||||
chunk += b"\x00" * (frame_size * 2 - len(chunk))
|
||||
# 初始化Opus编码器
|
||||
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
|
||||
|
||||
if is_opus:
|
||||
# 转换为numpy数组处理
|
||||
np_frame = np.frombuffer(chunk, dtype=np.int16)
|
||||
# 编码Opus数据
|
||||
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
|
||||
else:
|
||||
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
|
||||
# 编码参数
|
||||
frame_duration = 60 # 60ms per frame
|
||||
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
|
||||
|
||||
datas.append(frame_data)
|
||||
datas = []
|
||||
# 按帧处理所有音频数据（包括最后一帧可能补零）
|
||||
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
|
||||
# 获取当前帧的二进制数据
|
||||
chunk = raw_data[i : i + frame_size * 2]
|
||||
|
||||
return datas
|
||||
# 如果最后一帧不足，补零
|
||||
if len(chunk) < frame_size * 2:
|
||||
chunk += b"\x00" * (frame_size * 2 - len(chunk))
|
||||
|
||||
def audio_bytes_to_data_stream(audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]) -> None:
|
||||
if is_opus:
|
||||
# 转换为numpy数组处理
|
||||
np_frame = np.frombuffer(chunk, dtype=np.int16)
|
||||
# 编码Opus数据
|
||||
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
|
||||
else:
|
||||
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
|
||||
|
||||
datas.append(frame_data)
|
||||
|
||||
return datas
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
# 在单独的线程中执行同步的音频处理操作
|
||||
result = await loop.run_in_executor(None, _sync_audio_to_data)
|
||||
|
||||
# 将结果存入缓存，使用配置中定义的TTL（10分钟）
|
||||
if use_cache:
|
||||
cache_manager.set(CacheType.AUDIO_DATA, cache_key, result)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def audio_bytes_to_data_stream(
|
||||
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]
|
||||
) -> None:
|
||||
"""
|
||||
直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3
|
||||
"""
|
||||
@@ -324,31 +393,40 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] =
|
||||
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
|
||||
callback(frame_data)
|
||||
|
||||
|
||||
def opus_datas_to_wav_bytes(opus_datas, sample_rate=16000, channels=1):
|
||||
"""
|
||||
将opus帧列表解码为wav字节流
|
||||
"""
|
||||
decoder = opuslib_next.Decoder(sample_rate, channels)
|
||||
pcm_datas = []
|
||||
try:
|
||||
pcm_datas = []
|
||||
|
||||
frame_duration = 60 # ms
|
||||
frame_size = int(sample_rate * frame_duration / 1000) # 960
|
||||
frame_duration = 60 # ms
|
||||
frame_size = int(sample_rate * frame_duration / 1000) # 960
|
||||
|
||||
for opus_frame in opus_datas:
|
||||
# 解码为PCM（返回bytes，2字节/采样点）
|
||||
pcm = decoder.decode(opus_frame, frame_size)
|
||||
pcm_datas.append(pcm)
|
||||
for opus_frame in opus_datas:
|
||||
# 解码为PCM（返回bytes，2字节/采样点）
|
||||
pcm = decoder.decode(opus_frame, frame_size)
|
||||
pcm_datas.append(pcm)
|
||||
|
||||
pcm_bytes = b"".join(pcm_datas)
|
||||
pcm_bytes = b"".join(pcm_datas)
|
||||
|
||||
# 写入wav字节流
|
||||
wav_buffer = BytesIO()
|
||||
with wave.open(wav_buffer, "wb") as wf:
|
||||
wf.setnchannels(channels)
|
||||
wf.setsampwidth(2) # 16bit
|
||||
wf.setframerate(sample_rate)
|
||||
wf.writeframes(pcm_bytes)
|
||||
return wav_buffer.getvalue()
|
||||
finally:
|
||||
if decoder is not None:
|
||||
try:
|
||||
del decoder
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 写入wav字节流
|
||||
wav_buffer = BytesIO()
|
||||
with wave.open(wav_buffer, "wb") as wf:
|
||||
wf.setnchannels(channels)
|
||||
wf.setsampwidth(2) # 16bit
|
||||
wf.setframerate(sample_rate)
|
||||
wf.writeframes(pcm_bytes)
|
||||
return wav_buffer.getvalue()
|
||||
|
||||
def check_vad_update(before_config, new_config):
|
||||
if (
|
||||
@@ -423,6 +501,17 @@ def filter_sensitive_info(config: dict) -> dict:
|
||||
filtered[k] = _filter_dict(v)
|
||||
elif isinstance(v, list):
|
||||
filtered[k] = [_filter_dict(i) if isinstance(i, dict) else i for i in v]
|
||||
elif isinstance(v, str):
|
||||
try:
|
||||
json_data = json.loads(v)
|
||||
if isinstance(json_data, dict):
|
||||
filtered[k] = json.dumps(
|
||||
_filter_dict(json_data), ensure_ascii=False
|
||||
)
|
||||
else:
|
||||
filtered[k] = v
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
filtered[k] = v
|
||||
else:
|
||||
filtered[k] = v
|
||||
return filtered
|
||||
|
||||
@@ -19,6 +19,8 @@ class VoiceprintProvider:
|
||||
self.original_url = config.get("url", "")
|
||||
self.speakers = config.get("speakers", [])
|
||||
self.speaker_map = self._parse_speakers()
|
||||
# 声纹识别相似度阈值，默认0.4
|
||||
self.similarity_threshold = float(config.get("similarity_threshold", 0.4))
|
||||
|
||||
# 解析API地址和密钥
|
||||
self.api_url = None
|
||||
@@ -62,7 +64,7 @@ class VoiceprintProvider:
|
||||
# 进行健康检查，验证服务器是否可用
|
||||
if self._check_server_health():
|
||||
self.enabled = True
|
||||
logger.bind(tag=TAG).info(f"声纹识别已启用: API={self.api_url}, 说话人={len(self.speaker_ids)}个")
|
||||
logger.bind(tag=TAG).info(f"声纹识别已启用: API={self.api_url}, 说话人={len(self.speaker_ids)}个, 相似度阈值={self.similarity_threshold}")
|
||||
else:
|
||||
self.enabled = False
|
||||
logger.bind(tag=TAG).warning(f"声纹识别服务器不可用，声纹识别已禁用: {self.api_url}")
|
||||
@@ -169,12 +171,14 @@ class VoiceprintProvider:
|
||||
|
||||
logger.bind(tag=TAG).info(f"声纹识别耗时: {total_elapsed_time:.3f}s")
|
||||
|
||||
# 置信度检查
|
||||
if score < 0.5:
|
||||
logger.bind(tag=TAG).warning(f"声纹识别置信度较低: {score:.3f}")
|
||||
# 相似度阈值检查
|
||||
if score < self.similarity_threshold:
|
||||
logger.bind(tag=TAG).warning(f"声纹识别相似度{score:.3f}低于阈值{self.similarity_threshold}")
|
||||
return "未知说话人"
|
||||
|
||||
if speaker_id and speaker_id in self.speaker_map:
|
||||
result_name = self.speaker_map[speaker_id]["name"]
|
||||
logger.bind(tag=TAG).info(f"声纹识别成功: {result_name} (相似度: {score:.3f})")
|
||||
return result_name
|
||||
else:
|
||||
logger.bind(tag=TAG).warning(f"未识别的说话人ID: {speaker_id}")
|
||||
|
||||
@@ -55,7 +55,7 @@ class WakeupWordsConfig:
|
||||
return self._config_cache
|
||||
|
||||
try:
|
||||
with open(self.config_file, "a+") as f:
|
||||
with open(self.config_file, "a+", encoding="utf-8") as f:
|
||||
with FileLock(f, timeout=self._lock_timeout):
|
||||
f.seek(0)
|
||||
content = f.read()
|
||||
@@ -73,7 +73,7 @@ class WakeupWordsConfig:
|
||||
def _save_config(self, config: Dict):
|
||||
"""保存配置到文件，使用文件锁保护"""
|
||||
try:
|
||||
with open(self.config_file, "w") as f:
|
||||
with open(self.config_file, "w", encoding="utf-8") as f:
|
||||
with FileLock(f, timeout=self._lock_timeout):
|
||||
yaml.dump(config, f, allow_unicode=True)
|
||||
self._config_cache = config
|
||||
|
||||
@@ -1,8 +1,38 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import websockets
|
||||
from config.logger import setup_logging
|
||||
|
||||
|
||||
class SuppressInvalidHandshakeFilter(logging.Filter):
|
||||
"""过滤掉无效握手错误日志（如HTTPS访问WS端口）"""
|
||||
|
||||
def filter(self, record):
|
||||
msg = record.getMessage()
|
||||
suppress_keywords = [
|
||||
"opening handshake failed",
|
||||
"did not receive a valid HTTP request",
|
||||
"connection closed while reading HTTP request",
|
||||
"line without CRLF",
|
||||
]
|
||||
return not any(keyword in msg for keyword in suppress_keywords)
|
||||
|
||||
|
||||
def _setup_websockets_logger():
|
||||
"""配置 websockets 相关的所有 logger，过滤无效握手错误"""
|
||||
filter_instance = SuppressInvalidHandshakeFilter()
|
||||
for logger_name in ["websockets", "websockets.server", "websockets.client"]:
|
||||
logger = logging.getLogger(logger_name)
|
||||
logger.addFilter(filter_instance)
|
||||
|
||||
|
||||
_setup_websockets_logger()
|
||||
|
||||
|
||||
from core.connection import ConnectionHandler
|
||||
from config.config_loader import get_config_from_api
|
||||
from config.config_loader import get_config_from_api_async
|
||||
from core.auth import AuthManager, AuthenticationError
|
||||
from core.utils.modules_initialize import initialize_modules
|
||||
from core.utils.util import check_vad_update, check_asr_update
|
||||
|
||||
@@ -30,7 +60,13 @@ class WebSocketServer:
|
||||
self._intent = modules["intent"] if "intent" in modules else None
|
||||
self._memory = modules["memory"] if "memory" in modules else None
|
||||
|
||||
self.active_connections = set()
|
||||
auth_config = self.config["server"].get("auth", {})
|
||||
self.auth_enable = auth_config.get("enabled", False)
|
||||
# 设备白名单
|
||||
self.allowed_devices = set(auth_config.get("allowed_devices", []))
|
||||
secret_key = self.config["server"]["auth_key"]
|
||||
expire_seconds = auth_config.get("expire_seconds", None)
|
||||
self.auth = AuthManager(secret_key=secret_key, expire_seconds=expire_seconds)
|
||||
|
||||
async def start(self):
|
||||
server_config = self.config["server"]
|
||||
@@ -43,7 +79,40 @@ class WebSocketServer:
|
||||
await asyncio.Future()
|
||||
|
||||
async def _handle_connection(self, websocket):
|
||||
headers = dict(websocket.request.headers)
|
||||
if headers.get("device-id", None) is None:
|
||||
# 尝试从 URL 的查询参数中获取 device-id
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
# 从 WebSocket 请求中获取路径
|
||||
request_path = websocket.request.path
|
||||
if not request_path:
|
||||
self.logger.bind(tag=TAG).error("无法获取请求路径")
|
||||
await websocket.close()
|
||||
return
|
||||
parsed_url = urlparse(request_path)
|
||||
query_params = parse_qs(parsed_url.query)
|
||||
if "device-id" not in query_params:
|
||||
await websocket.send("端口正常，如需测试连接，请使用test_page.html")
|
||||
await websocket.close()
|
||||
return
|
||||
else:
|
||||
websocket.request.headers["device-id"] = query_params["device-id"][0]
|
||||
if "client-id" in query_params:
|
||||
websocket.request.headers["client-id"] = query_params["client-id"][0]
|
||||
if "authorization" in query_params:
|
||||
websocket.request.headers["authorization"] = query_params[
|
||||
"authorization"
|
||||
][0]
|
||||
|
||||
"""处理新连接，每次创建独立的ConnectionHandler"""
|
||||
# 先认证，后建立连接
|
||||
try:
|
||||
await self._handle_auth(websocket)
|
||||
except AuthenticationError:
|
||||
await websocket.send("认证失败")
|
||||
await websocket.close()
|
||||
return
|
||||
# 创建ConnectionHandler时传入当前server实例
|
||||
handler = ConnectionHandler(
|
||||
self.config,
|
||||
@@ -54,14 +123,11 @@ class WebSocketServer:
|
||||
self._intent,
|
||||
self, # 传入server实例
|
||||
)
|
||||
self.active_connections.add(handler)
|
||||
try:
|
||||
await handler.handle_connection(websocket)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"处理连接时出错: {e}")
|
||||
finally:
|
||||
# 确保从活动连接集合中移除
|
||||
self.active_connections.discard(handler)
|
||||
# 强制关闭连接（如果还没有关闭的话）
|
||||
try:
|
||||
# 安全地检查WebSocket状态并关闭
|
||||
@@ -94,8 +160,8 @@ class WebSocketServer:
|
||||
"""
|
||||
try:
|
||||
async with self.config_lock:
|
||||
# 重新获取配置
|
||||
new_config = get_config_from_api(self.config)
|
||||
# 重新获取配置（使用异步版本）
|
||||
new_config = await get_config_from_api_async(self.config)
|
||||
if new_config is None:
|
||||
self.logger.bind(tag=TAG).error("获取新配置失败")
|
||||
return False
|
||||
@@ -136,3 +202,26 @@ class WebSocketServer:
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"更新服务器配置失败: {str(e)}")
|
||||
return False
|
||||
|
||||
async def _handle_auth(self, websocket):
|
||||
# 先认证，后建立连接
|
||||
if self.auth_enable:
|
||||
headers = dict(websocket.request.headers)
|
||||
device_id = headers.get("device-id", None)
|
||||
client_id = headers.get("client-id", None)
|
||||
if self.allowed_devices and device_id in self.allowed_devices:
|
||||
# 如果属于白名单内的设备，不校验token，直接放行
|
||||
return
|
||||
else:
|
||||
# 否则校验token
|
||||
token = headers.get("authorization", "")
|
||||
if token.startswith("Bearer "):
|
||||
token = token[7:] # 移除'Bearer '前缀
|
||||
else:
|
||||
raise AuthenticationError("Missing or invalid Authorization header")
|
||||
# 进行认证
|
||||
auth_success = self.auth.verify_token(
|
||||
token, client_id=client_id, username=device_id
|
||||
)
|
||||
if not auth_success:
|
||||
raise AuthenticationError("Invalid token")
|
||||
|
||||
@@ -76,7 +76,7 @@ services:
|
||||
- MYSQL_INITDB_ARGS="--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci"
|
||||
# redis模块
|
||||
xiaozhi-esp32-server-redis:
|
||||
image: redis
|
||||
image: redis:8.0
|
||||
expose:
|
||||
- 6379
|
||||
container_name: xiaozhi-esp32-server-redis
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"后面不断测试补充好用的mcp服务，欢迎大家一起补充。",
|
||||
"记得删除注释行,des属性仅为说明,不会被解析。",
|
||||
"des和link属性，仅为说明安装方式，方便大家查看原始链接，不是必须项。",
|
||||
"当前支持stdio/sse两种模式。"
|
||||
"当前支持三种传输模式：stdio(标准输入输出), sse(Server-Sent Events), streamable-http(流式HTTP)。"
|
||||
],
|
||||
"mcpServers": {
|
||||
"Home Assistant": {
|
||||
@@ -36,6 +36,21 @@
|
||||
"command": "npx",
|
||||
"args": ["-y", "@simonb97/server-win-cli"],
|
||||
"link": "https://github.com/SimonB97/win-cli-mcp-server"
|
||||
},
|
||||
"sse-mcp-server": {
|
||||
"url": "http://localhost:8080/sse",
|
||||
"headers": {
|
||||
"Authorization": "Bearer YOUR TOKEN"
|
||||
},
|
||||
"des": "使用SSE传输模式（默认）"
|
||||
},
|
||||
"streamable-http-mcp-server": {
|
||||
"url": "http://localhost:8000/mcp",
|
||||
"transport": "streamable-http",
|
||||
"headers": {
|
||||
"Authorization": "Bearer YOUR TOKEN"
|
||||
},
|
||||
"des": "使用Streamable HTTP传输模式，适用于生产环境的Web部署"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ class LLMPerformanceTester:
|
||||
"""加载系统提示词"""
|
||||
try:
|
||||
prompt_file = os.path.join(
|
||||
os.path.dirname(os.path.dirname(__file__)), "agent-base-prompt.txt"
|
||||
os.path.dirname(os.path.dirname(__file__)), self.config.get("prompt_template", "agent-base-prompt.txt")
|
||||
)
|
||||
with open(prompt_file, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
@@ -5,104 +5,154 @@ import uuid
|
||||
import os
|
||||
import websockets
|
||||
import gzip
|
||||
import hmac
|
||||
import base64
|
||||
import hashlib
|
||||
import random
|
||||
from urllib import parse
|
||||
from tabulate import tabulate
|
||||
from config.settings import load_config
|
||||
description = "流式ASR首词耗时测试"
|
||||
import tempfile
|
||||
import wave
|
||||
import hmac
|
||||
import base64
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
from wsgiref.handlers import format_date_time
|
||||
from time import mktime
|
||||
description = "流式ASR首词延迟测试"
|
||||
try:
|
||||
import dashscope
|
||||
except ImportError:
|
||||
dashscope = None
|
||||
|
||||
class AccessToken:
|
||||
@staticmethod
|
||||
def _encode_text(text):
|
||||
encoded_text = parse.quote_plus(text)
|
||||
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
|
||||
|
||||
@staticmethod
|
||||
def _encode_dict(dic):
|
||||
keys = dic.keys()
|
||||
dic_sorted = [(key, dic[key]) for key in sorted(keys)]
|
||||
encoded_text = parse.urlencode(dic_sorted)
|
||||
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
|
||||
|
||||
@staticmethod
|
||||
def create_token(access_key_id, access_key_secret):
|
||||
parameters = {
|
||||
"AccessKeyId": access_key_id,
|
||||
"Action": "CreateToken",
|
||||
"Format": "JSON",
|
||||
"RegionId": "cn-shanghai",
|
||||
"SignatureMethod": "HMAC-SHA1",
|
||||
"SignatureNonce": str(uuid.uuid1()),
|
||||
"SignatureVersion": "1.0",
|
||||
"Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"Version": "2019-02-28",
|
||||
}
|
||||
query_string = AccessToken._encode_dict(parameters)
|
||||
string_to_sign = (
|
||||
"GET" + "&" + AccessToken._encode_text("/") + "&" + AccessToken._encode_text(query_string)
|
||||
)
|
||||
secreted_string = hmac.new(
|
||||
bytes(access_key_secret + "&", encoding="utf-8"),
|
||||
bytes(string_to_sign, encoding="utf-8"),
|
||||
hashlib.sha1,
|
||||
).digest()
|
||||
signature = base64.b64encode(secreted_string)
|
||||
signature = AccessToken._encode_text(signature)
|
||||
full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (signature, query_string)
|
||||
response = requests.get(full_url)
|
||||
if response.ok:
|
||||
root_obj = response.json()
|
||||
if "Token" in root_obj:
|
||||
return root_obj["Token"]["Id"], root_obj["Token"]["ExpireTime"]
|
||||
return None, None
|
||||
|
||||
|
||||
class DoubaoStreamASRPerformanceTester:
|
||||
def __init__(self):
|
||||
class BaseASRTester:
|
||||
def __init__(self, config_key: str):
|
||||
self.config = load_config()
|
||||
self.config_key = config_key
|
||||
self.asr_config = self.config.get("ASR", {}).get(config_key, {})
|
||||
self.test_audio_files = self._load_test_audio_files()
|
||||
self.results = []
|
||||
|
||||
|
||||
def _load_test_audio_files(self):
|
||||
"""加载测试用的音频文件"""
|
||||
audio_root = os.path.join(os.getcwd(), "config", "assets")
|
||||
test_files = []
|
||||
|
||||
if os.path.exists(audio_root):
|
||||
for file_name in os.listdir(audio_root):
|
||||
if file_name.endswith('.wav') or file_name.endswith('.pcm'):
|
||||
with open(os.path.join(audio_root, file_name), 'rb') as f:
|
||||
test_files.append(f.read())
|
||||
if file_name.endswith(('.wav', '.pcm')):
|
||||
file_path = os.path.join(audio_root, file_name)
|
||||
with open(file_path, 'rb') as f:
|
||||
test_files.append({
|
||||
'data': f.read(),
|
||||
'path': file_path,
|
||||
'name': file_name
|
||||
})
|
||||
return test_files
|
||||
|
||||
async def test_doubao_stream_asr(self, test_count=5):
|
||||
"""测试豆包流式ASR首词响应时间"""
|
||||
|
||||
async def test(self, test_count=5):
|
||||
raise NotImplementedError
|
||||
|
||||
def _calculate_result(self, service_name, latencies, test_count):
|
||||
"""计算测试结果（修复：正确处理None值，剔除失败测试）"""
|
||||
# 剔除None值（失败的测试）和无效延迟，只统计有效延迟
|
||||
valid_latencies = [l for l in latencies if l is not None and l > 0]
|
||||
if valid_latencies:
|
||||
avg_latency = sum(valid_latencies) / len(valid_latencies)
|
||||
status = f"成功（{len(valid_latencies)}/{test_count}次有效）"
|
||||
else:
|
||||
avg_latency = 0
|
||||
status = "失败: 所有测试均失败"
|
||||
return {"name": service_name, "latency": avg_latency, "status": status}
|
||||
|
||||
|
||||
class DoubaoStreamASRTester(BaseASRTester):
|
||||
def __init__(self):
|
||||
super().__init__("DoubaoStreamASR")
|
||||
|
||||
def _generate_header(
|
||||
self,
|
||||
version=0x01,
|
||||
message_type=0x01,
|
||||
message_type_specific_flags=0x00,
|
||||
serial_method=0x01,
|
||||
compression_type=0x01,
|
||||
reserved_data=0x00,
|
||||
extension_header: bytes = b"",
|
||||
):
|
||||
"""生成协议头（修复：使用正确的Header格式）"""
|
||||
header = bytearray()
|
||||
header_size = int(len(extension_header) / 4) + 1
|
||||
header.append((version << 4) | header_size)
|
||||
header.append((message_type << 4) | message_type_specific_flags)
|
||||
header.append((serial_method << 4) | compression_type)
|
||||
header.append(reserved_data)
|
||||
header.extend(extension_header)
|
||||
return header
|
||||
|
||||
def _generate_audio_default_header(self):
|
||||
"""生成音频数据Header"""
|
||||
return self._generate_header(
|
||||
version=0x01,
|
||||
message_type=0x02,
|
||||
message_type_specific_flags=0x00, # 普通音频帧
|
||||
serial_method=0x01,
|
||||
compression_type=0x01,
|
||||
)
|
||||
|
||||
def _generate_last_audio_header(self):
|
||||
"""生成最后一帧音频的Header（标记音频结束）"""
|
||||
return self._generate_header(
|
||||
version=0x01,
|
||||
message_type=0x02,
|
||||
message_type_specific_flags=0x02, # 0x02表示这是最后一帧
|
||||
serial_method=0x01,
|
||||
compression_type=0x01,
|
||||
)
|
||||
|
||||
def _parse_response(self, res: bytes) -> dict:
|
||||
try:
|
||||
if len(res) < 4:
|
||||
return {"error": "响应数据长度不足"}
|
||||
header = res[:4]
|
||||
message_type = header[1] >> 4
|
||||
if message_type == 0x0F:
|
||||
code = int.from_bytes(res[4:8], "big", signed=False)
|
||||
msg_length = int.from_bytes(res[8:12], "big", signed=False)
|
||||
error_msg = json.loads(res[12:].decode("utf-8"))
|
||||
return {
|
||||
"code": code,
|
||||
"msg_length": msg_length,
|
||||
"payload_msg": error_msg
|
||||
}
|
||||
try:
|
||||
json_data = res[12:].decode("utf-8")
|
||||
return {"payload_msg": json.loads(json_data)}
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return {"error": "JSON解析失败"}
|
||||
except Exception:
|
||||
return {"error": "解析响应失败"}
|
||||
|
||||
async def test(self, test_count=5):
|
||||
if not self.test_audio_files:
|
||||
print("没有找到测试音频文件")
|
||||
return
|
||||
|
||||
asr_config = self.config["ASR"]["DoubaoStreamASR"]
|
||||
return {"name": "豆包流式ASR", "latency": 0, "status": "失败: 未找到测试音频"}
|
||||
if not self.asr_config:
|
||||
return {"name": "豆包流式ASR", "latency": 0, "status": "失败: 未配置"}
|
||||
|
||||
latencies = []
|
||||
|
||||
for i in range(test_count):
|
||||
try:
|
||||
ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
|
||||
appid = asr_config["appid"]
|
||||
access_token = asr_config["access_token"]
|
||||
uid = asr_config.get("uid", "streaming_asr_service")
|
||||
|
||||
appid = self.asr_config["appid"]
|
||||
access_token = self.asr_config["access_token"]
|
||||
cluster = self.asr_config.get("cluster", "volcengine_input_common")
|
||||
uid = self.asr_config.get("uid", "streaming_asr_service")
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
|
||||
headers = {
|
||||
"X-Api-App-Key": appid,
|
||||
"X-Api-Access-Key": access_token,
|
||||
"X-Api-Resource-Id": "volc.bigasr.sauc.duration",
|
||||
"X-Api-Connect-Id": str(uuid.uuid4())
|
||||
}
|
||||
|
||||
|
||||
async with websockets.connect(
|
||||
ws_url,
|
||||
additional_headers=headers,
|
||||
@@ -111,12 +161,8 @@ class DoubaoStreamASRPerformanceTester:
|
||||
ping_timeout=None,
|
||||
close_timeout=10
|
||||
) as ws:
|
||||
# 发送初始化请求
|
||||
request_params = {
|
||||
"app": {
|
||||
"appid": appid,
|
||||
"token": access_token
|
||||
},
|
||||
"app": {"appid": appid, "cluster": cluster, "token": access_token},
|
||||
"user": {"uid": uid},
|
||||
"request": {
|
||||
"reqid": str(uuid.uuid4()),
|
||||
@@ -135,203 +181,260 @@ class DoubaoStreamASRPerformanceTester:
|
||||
"sample_rate": 16000
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
payload_bytes = str.encode(json.dumps(request_params))
|
||||
payload_bytes = gzip.compress(payload_bytes)
|
||||
full_client_request = self._generate_header()
|
||||
full_client_request.extend((len(payload_bytes)).to_bytes(4, "big"))
|
||||
full_client_request.extend(payload_bytes)
|
||||
|
||||
await ws.send(full_client_request)
|
||||
|
||||
|
||||
init_res = await ws.recv()
|
||||
result = self._parse_response(init_res)
|
||||
|
||||
if "code" in result and result["code"] != 1000:
|
||||
raise Exception(f"ASR服务初始化失败: {result.get('payload_msg', {}).get('error', '未知错误')}")
|
||||
|
||||
# 发送音频数据
|
||||
audio_data = self.test_audio_files[0]
|
||||
raise Exception(f"初始化失败: {result.get('payload_msg', {}).get('error', '未知错误')}")
|
||||
|
||||
audio_data = self.test_audio_files[0]['data']
|
||||
if audio_data.startswith(b'RIFF'):
|
||||
audio_data = audio_data[44:]
|
||||
|
||||
# 直接发送原始音频数据，不进行opus解码
|
||||
|
||||
# 发送音频数据（使用最后一帧标记，告诉服务端音频已结束）
|
||||
payload = gzip.compress(audio_data)
|
||||
audio_request = bytearray(self._generate_audio_default_header())
|
||||
audio_request = bytearray(self._generate_last_audio_header()) # 修复：使用最后一帧Header
|
||||
audio_request.extend(len(payload).to_bytes(4, "big"))
|
||||
audio_request.extend(payload)
|
||||
await ws.send(audio_request)
|
||||
|
||||
# 等待第一个数据块
|
||||
|
||||
first_chunk = await ws.recv()
|
||||
latency = time.time() - start_time
|
||||
latencies.append(latency)
|
||||
print(f"[豆包ASR] 第{i+1}次 首词延迟: {latency:.3f}s")
|
||||
await ws.close()
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(f"第{i+1}次测试: {str(e)}")
|
||||
latencies.append(0)
|
||||
|
||||
print(f"[豆包ASR] 第{i+1}次测试失败: {str(e)}")
|
||||
latencies.append(None)
|
||||
|
||||
return self._calculate_result("豆包流式ASR", latencies, test_count)
|
||||
|
||||
async def test_aliyun_stream_asr(self, test_count=5):
|
||||
"""测试阿里云流式ASR首词响应时间"""
|
||||
|
||||
|
||||
class QwenASRFlashTester(BaseASRTester):
|
||||
def __init__(self):
|
||||
super().__init__("Qwen3ASRFlash")
|
||||
|
||||
async def _test_single(self, audio_file_info):
|
||||
temp_file_path = None
|
||||
|
||||
try:
|
||||
audio_data = audio_file_info['data']
|
||||
|
||||
# 优化：将临时文件准备工作移到计时前，减少磁盘IO对性能测试的影响
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as f:
|
||||
temp_file_path = f.name
|
||||
|
||||
with wave.open(temp_file_path, 'wb') as wav_file:
|
||||
wav_file.setnchannels(1)
|
||||
wav_file.setsampwidth(2)
|
||||
wav_file.setframerate(16000)
|
||||
wav_file.writeframes(audio_data)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"audio": temp_file_path}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
api_key = self.asr_config.get("api_key") or os.getenv("DASHSCOPE_API_KEY")
|
||||
if not api_key:
|
||||
raise ValueError("未配置 api_key")
|
||||
|
||||
if dashscope is None:
|
||||
raise RuntimeError("未安装 dashscope 库")
|
||||
|
||||
dashscope.api_key = api_key
|
||||
|
||||
# 统一计时起点：在API调用前开始计时（但文件准备已完成）
|
||||
start_time = time.time()
|
||||
|
||||
response = dashscope.MultiModalConversation.call(
|
||||
model="qwen3-asr-flash",
|
||||
messages=messages,
|
||||
result_format="message",
|
||||
asr_options={"enable_lid": True, "enable_itn": False},
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
latency = time.time() - start_time
|
||||
return latency
|
||||
|
||||
raise Exception("流式结束，未收到任何响应")
|
||||
|
||||
except Exception as e:
|
||||
raise Exception(f"通义ASR流式失败: {str(e)}")
|
||||
|
||||
finally:
|
||||
if temp_file_path and os.path.exists(temp_file_path):
|
||||
try:
|
||||
os.unlink(temp_file_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
async def test(self, test_count=5):
|
||||
if not self.test_audio_files:
|
||||
print("没有找到测试音频文件")
|
||||
return
|
||||
|
||||
asr_config = self.config["ASR"]["AliyunStreamASR"]
|
||||
return {"name": "通义千问ASR", "latency": 0, "status": "失败: 未找到测试音频"}
|
||||
if not self.asr_config and not os.getenv("DASHSCOPE_API_KEY"):
|
||||
return {"name": "通义千问ASR", "latency": 0, "status": "失败: 未配置 api_key"}
|
||||
|
||||
latencies = []
|
||||
|
||||
for i in range(test_count):
|
||||
try:
|
||||
access_key_id = asr_config["access_key_id"]
|
||||
access_key_secret = asr_config["access_key_secret"]
|
||||
appkey = asr_config["appkey"]
|
||||
host = asr_config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
|
||||
|
||||
# 获取Token
|
||||
token, _ = AccessToken.create_token(access_key_id, access_key_secret)
|
||||
if not token:
|
||||
raise Exception("无法获取阿里云ASR Token")
|
||||
|
||||
# 确定WebSocket URL
|
||||
if "-internal." in host:
|
||||
ws_url = f"ws://{host}/ws/v1"
|
||||
else:
|
||||
ws_url = f"wss://{host}/ws/v1"
|
||||
|
||||
# print(f"\n[通义ASR] 开始第 {i+1} 次测试...")
|
||||
latency = await self._test_single(self.test_audio_files[0])
|
||||
latencies.append(latency)
|
||||
print(f"[通义ASR] 第{i+1}次 首词延迟: {latency:.3f}s")
|
||||
except Exception as e:
|
||||
# print(f"[通义ASR] 第{i+1}次测试失败: {str(e)}")
|
||||
latencies.append(None)
|
||||
|
||||
return self._calculate_result("通义千问ASR", latencies, test_count)
|
||||
|
||||
|
||||
class XunfeiStreamASRTester(BaseASRTester):
|
||||
def __init__(self):
|
||||
super().__init__("XunfeiStreamASR")
|
||||
|
||||
def _create_url(self):
|
||||
url = "wss://iat-api.xfyun.cn/v2/iat"
|
||||
now = datetime.now()
|
||||
date = format_date_time(mktime(now.timetuple()))
|
||||
|
||||
signature_origin = f"host: iat-api.xfyun.cn\ndate: {date}\nGET /v2/iat HTTP/1.1"
|
||||
signature_sha = hmac.new(
|
||||
self.asr_config["api_secret"].encode('utf-8'),
|
||||
signature_origin.encode('utf-8'),
|
||||
hashlib.sha256
|
||||
).digest()
|
||||
signature_sha = base64.b64encode(signature_sha).decode()
|
||||
|
||||
authorization_origin = f'api_key="{self.asr_config["api_key"]}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha}"'
|
||||
authorization = base64.b64encode(authorization_origin.encode()).decode()
|
||||
|
||||
v = {"authorization": authorization, "date": date, "host": "iat-api.xfyun.cn"}
|
||||
return url + "?" + parse.urlencode(v)
|
||||
|
||||
async def test(self, test_count: int = 5):
|
||||
if not self.test_audio_files:
|
||||
return {"name": "讯飞流式ASR", "latency": 0, "status": "失败: 未找到测试音频"}
|
||||
if not self.asr_config:
|
||||
return {"name": "讯飞流式ASR", "latency": 0, "status": "失败: 未配置"}
|
||||
|
||||
required = ["app_id", "api_key", "api_secret"]
|
||||
for k in required:
|
||||
if k not in self.asr_config:
|
||||
return {"name": "讯飞流式ASR", "latency": 0, "status": f"失败: 缺少配置 {k}"}
|
||||
|
||||
latencies = []
|
||||
frame_size = 1280
|
||||
audio_raw = self.test_audio_files[0]['data']
|
||||
if audio_raw.startswith(b'RIFF'):
|
||||
audio_raw = audio_raw[44:]
|
||||
|
||||
for i in range(test_count):
|
||||
try:
|
||||
start_time = time.time()
|
||||
ws_url = self._create_url()
|
||||
|
||||
async with websockets.connect(
|
||||
ws_url,
|
||||
additional_headers={"X-NLS-Token": token},
|
||||
max_size=1000000000,
|
||||
additional_headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"},
|
||||
max_size=1 << 30,
|
||||
ping_interval=None,
|
||||
ping_timeout=None,
|
||||
close_timeout=10
|
||||
close_timeout=30,
|
||||
) as ws:
|
||||
# 发送开始请求
|
||||
start_request = {
|
||||
"header": {
|
||||
"namespace": "SpeechTranscriber",
|
||||
"name": "StartTranscription",
|
||||
"status": 20000000,
|
||||
"message_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"task_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"status_text": "Gateway:SUCCESS:Success.",
|
||||
"appkey": appkey
|
||||
|
||||
# 第一帧：移除 punc 字段，避免未知参数错误
|
||||
await ws.send(json.dumps({
|
||||
"common": {"app_id": self.asr_config["app_id"]},
|
||||
"business": {
|
||||
"domain": "iat",
|
||||
"language": "zh_cn",
|
||||
"accent": "mandarin",
|
||||
"dwa": "wpgs",
|
||||
"vad_eos": 5000
|
||||
# 已移除 "punc": True
|
||||
},
|
||||
"payload": {
|
||||
"format": "pcm",
|
||||
"sample_rate": 16000,
|
||||
"enable_intermediate_result": True,
|
||||
"enable_punctuation_prediction": True,
|
||||
"enable_inverse_text_normalization": True,
|
||||
"max_sentence_silence": asr_config.get("max_sentence_silence", 8000),
|
||||
"enable_voice_detection": False,
|
||||
"data": {
|
||||
"status": 0,
|
||||
"format": "audio/L16;rate=16000",
|
||||
"encoding": "raw",
|
||||
"audio": base64.b64encode(audio_raw[:frame_size]).decode()
|
||||
}
|
||||
}
|
||||
await ws.send(json.dumps(start_request, ensure_ascii=False))
|
||||
|
||||
# 等待服务器准备
|
||||
start_response = await ws.recv()
|
||||
response_data = json.loads(start_response)
|
||||
if response_data["header"]["name"] != "TranscriptionStarted":
|
||||
raise Exception("阿里云ASR服务初始化失败")
|
||||
|
||||
# 发送音频数据
|
||||
audio_data = self.test_audio_files[0]
|
||||
if audio_data.startswith(b'RIFF'):
|
||||
audio_data = audio_data[44:] # 去掉WAV头
|
||||
|
||||
await ws.send(audio_data)
|
||||
|
||||
# 等待第一个结果
|
||||
while True:
|
||||
response = await ws.recv()
|
||||
if isinstance(response, str):
|
||||
result = json.loads(response)
|
||||
if result["header"]["name"] == "TranscriptionResultChanged":
|
||||
}, ensure_ascii=False))
|
||||
|
||||
# 后续所有帧
|
||||
pos = frame_size
|
||||
while pos < len(audio_raw):
|
||||
chunk = audio_raw[pos:pos + frame_size]
|
||||
status = 2 if (pos + frame_size >= len(audio_raw)) else 1
|
||||
await ws.send(json.dumps({
|
||||
"data": {
|
||||
"status": status,
|
||||
"format": "audio/L16;rate=16000",
|
||||
"encoding": "raw",
|
||||
"audio": base64.b64encode(chunk).decode()
|
||||
}
|
||||
}, ensure_ascii=False))
|
||||
if status == 2:
|
||||
break
|
||||
pos += frame_size
|
||||
|
||||
# 接收首词
|
||||
first_token = True
|
||||
async for message in ws:
|
||||
data = json.loads(message)
|
||||
if data.get("code") != 0:
|
||||
raise Exception(f"讯飞错误: {data.get('message')}")
|
||||
|
||||
ws_result = data.get("data", {}).get("result", {}).get("ws")
|
||||
if ws_result:
|
||||
text = "".join(cw.get("w", "") for seg in ws_result for cw in seg.get("cw", []))
|
||||
if text.strip() and first_token:
|
||||
latency = time.time() - start_time
|
||||
latencies.append(latency)
|
||||
print(f"[讯飞ASR] 第{i+1}次 首词延迟: {latency:.3f}s")
|
||||
first_token = False
|
||||
break
|
||||
elif result["header"]["name"] == "TaskFailed":
|
||||
raise Exception(f"阿里云ASR识别失败: {result.get('payload', {}).get('error_info', '未知错误')}")
|
||||
|
||||
# 发送停止请求
|
||||
stop_msg = {
|
||||
"header": {
|
||||
"namespace": "SpeechTranscriber",
|
||||
"name": "StopTranscription",
|
||||
"status": 20000000,
|
||||
"message_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"status_text": "Client:Stop",
|
||||
"appkey": appkey
|
||||
}
|
||||
}
|
||||
await ws.send(json.dumps(stop_msg, ensure_ascii=False))
|
||||
await ws.close()
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(f"第{i+1}次测试: {str(e)}")
|
||||
latencies.append(0)
|
||||
|
||||
return self._calculate_result("阿里云流式ASR", latencies, test_count)
|
||||
|
||||
def _generate_header(self):
|
||||
"""生成请求头"""
|
||||
header = bytearray()
|
||||
header.append((0x01 << 4) | 0x01)
|
||||
header.append((0x01 << 4) | 0x00)
|
||||
header.append((0x01 << 4) | 0x01)
|
||||
header.append(0x00)
|
||||
return header
|
||||
|
||||
def _generate_audio_default_header(self):
|
||||
"""生成音频请求头"""
|
||||
return self._generate_header()
|
||||
|
||||
def _parse_response(self, res: bytes) -> dict:
|
||||
"""解析响应"""
|
||||
print(f"[讯飞ASR] 第{i+1}次测试失败: {str(e)}")
|
||||
latencies.append(None)
|
||||
|
||||
return self._calculate_result("讯飞流式ASR", latencies, test_count)
|
||||
class ASRPerformanceSuite:
|
||||
def __init__(self):
|
||||
self.testers = []
|
||||
self.results = []
|
||||
|
||||
def register_tester(self, tester_class):
|
||||
try:
|
||||
if len(res) < 4:
|
||||
return {"error": "响应数据长度不足"}
|
||||
|
||||
header = res[:4]
|
||||
message_type = header[1] >> 4
|
||||
|
||||
if message_type == 0x0F:
|
||||
code = int.from_bytes(res[4:8], "big", signed=False)
|
||||
msg_length = int.from_bytes(res[8:12], "big", signed=False)
|
||||
error_msg = json.loads(res[12:].decode("utf-8"))
|
||||
return {
|
||||
"code": code,
|
||||
"msg_length": msg_length,
|
||||
"payload_msg": error_msg
|
||||
}
|
||||
|
||||
try:
|
||||
json_data = res[12:].decode("utf-8")
|
||||
return {"payload_msg": json.loads(json_data)}
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return {"error": "JSON解析失败"}
|
||||
|
||||
except Exception:
|
||||
return {"error": "解析响应失败"}
|
||||
|
||||
def _calculate_result(self, service_name, latencies, test_count):
|
||||
"""计算结果"""
|
||||
valid_latencies = [l for l in latencies if l > 0]
|
||||
if valid_latencies:
|
||||
avg_latency = sum(valid_latencies) / len(valid_latencies)
|
||||
status = f"成功（{len(valid_latencies)}/{test_count}次有效）"
|
||||
else:
|
||||
avg_latency = 0
|
||||
status = "失败: 所有测试均失败"
|
||||
return {"name": service_name, "latency": avg_latency, "status": status}
|
||||
|
||||
tester = tester_class()
|
||||
self.testers.append(tester)
|
||||
print(f"已注册测试器: {tester.config_key}")
|
||||
except Exception as e:
|
||||
name_map = {
|
||||
"DoubaoStreamASRTester": "豆包流式ASR",
|
||||
"QwenASRFlashTester": "通义千问ASR",
|
||||
"XunfeiStreamASRTester": "讯飞流式ASR"
|
||||
}
|
||||
name = name_map.get(tester_class.__name__, tester_class.__name__)
|
||||
print(f"跳过 {name}: {str(e)}")
|
||||
|
||||
def _print_results(self, test_count):
|
||||
"""打印测试结果"""
|
||||
if not self.results:
|
||||
print("没有有效的ASR测试结果")
|
||||
return
|
||||
@@ -341,7 +444,6 @@ class DoubaoStreamASRPerformanceTester:
|
||||
print(f"{'='*60}")
|
||||
print(f"测试次数: 每个ASR服务测试 {test_count} 次")
|
||||
|
||||
# 排序结果：成功优先，按延迟升序
|
||||
success_results = sorted(
|
||||
[r for r in self.results if "成功" in r["status"]],
|
||||
key=lambda x: x["latency"]
|
||||
@@ -349,56 +451,43 @@ class DoubaoStreamASRPerformanceTester:
|
||||
failed_results = [r for r in self.results if "成功" not in r["status"]]
|
||||
|
||||
table_data = [
|
||||
[r["name"], f"{r['latency']:.3f}", r["status"]]
|
||||
[r["name"], f"{r['latency']:.3f}s" if r['latency'] > 0 else "N/A", r["status"]]
|
||||
for r in success_results + failed_results
|
||||
]
|
||||
|
||||
print(tabulate(table_data, headers=["ASR服务", "首词延迟(秒)", "状态"], tablefmt="grid"))
|
||||
print("\n测试说明：测量从发送请求到接收第一个识别结果的时间，取多次测试平均值")
|
||||
print("- 超时控制: 单个请求最大等待时间为10秒")
|
||||
print("- 错误处理: 无法连接和超时的列为网络错误")
|
||||
print("- 排序规则: 按平均耗时从快到慢排序")
|
||||
|
||||
print(tabulate(table_data, headers=["ASR服务", "首词延迟", "状态"], tablefmt="grid"))
|
||||
print("\n测试说明：")
|
||||
print("- 计时起点: 建立连接前（包含握手、发送音频、接收首个识别结果全流程）")
|
||||
print("- 通义千问优化: 临时文件准备在计时前完成，减少磁盘IO对测试的影响")
|
||||
print("- 错误处理: 失败的测试不计入平均值，只统计成功测试的延迟")
|
||||
print("- 排序规则: 成功的按延迟升序，失败的排在后面")
|
||||
|
||||
async def run(self, test_count=5):
|
||||
"""执行测试"""
|
||||
print(f"开始流式ASR首词响应时间测试...")
|
||||
print(f"每个ASR服务测试次数: {test_count}次")
|
||||
|
||||
if not self.config.get("ASR"):
|
||||
print("配置文件中未找到ASR配置")
|
||||
return
|
||||
|
||||
# 测试每种ASR服务
|
||||
print(f"每个ASR服务测试次数: {test_count}次\n")
|
||||
|
||||
self.results = []
|
||||
|
||||
# 测试豆包ASR
|
||||
if self.config["ASR"].get("DoubaoStreamASR"):
|
||||
result = await self.test_doubao_stream_asr(test_count)
|
||||
for tester in self.testers:
|
||||
print(f"\n--- 测试 {tester.config_key} ---")
|
||||
result = await tester.test(test_count)
|
||||
self.results.append(result)
|
||||
else:
|
||||
print("配置文件中未找到豆包流式ASR配置，跳过测试")
|
||||
|
||||
# 测试阿里云ASR
|
||||
if self.config["ASR"].get("AliyunStreamASR"):
|
||||
result = await self.test_aliyun_stream_asr(test_count)
|
||||
self.results.append(result)
|
||||
else:
|
||||
print("配置文件中未找到阿里云流式ASR配置，跳过测试")
|
||||
|
||||
# 打印结果
|
||||
|
||||
self._print_results(test_count)
|
||||
|
||||
|
||||
async def main():
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="流式ASR首词响应时间测试工具")
|
||||
parser.add_argument("--count", type=int, default=5, help="测试次数")
|
||||
|
||||
args = parser.parse_args()
|
||||
await DoubaoStreamASRPerformanceTester().run(args.count)
|
||||
|
||||
suite = ASRPerformanceSuite()
|
||||
suite.register_tester(DoubaoStreamASRTester)
|
||||
suite.register_tester(QwenASRFlashTester)
|
||||
suite.register_tester(XunfeiStreamASRTester)
|
||||
|
||||
await suite.run(args.count)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import os
|
||||
import gzip
|
||||
import opuslib_next
|
||||
asyncio.run(main())
|
||||
@@ -4,6 +4,11 @@ import json
|
||||
import uuid
|
||||
import aiohttp
|
||||
import websockets
|
||||
import hmac
|
||||
import base64
|
||||
import hashlib
|
||||
import asyncio
|
||||
from urllib.parse import urlparse, urlencode
|
||||
from tabulate import tabulate
|
||||
from config.settings import load_config
|
||||
|
||||
@@ -30,11 +35,12 @@ class StreamTTSPerformanceTester:
|
||||
host = tts_config["host"]
|
||||
ws_url = f"wss://{host}/ws/v1"
|
||||
|
||||
# 统一计时起点：在建立连接前开始计时
|
||||
start_time = time.time()
|
||||
async with websockets.connect(ws_url, extra_headers={"X-NLS-Token": token}) as ws:
|
||||
task_id = str(uuid.uuid4())
|
||||
message_id = str(uuid.uuid4())
|
||||
|
||||
|
||||
start_request = {
|
||||
"header": {
|
||||
"message_id": message_id,
|
||||
@@ -50,14 +56,15 @@ class StreamTTSPerformanceTester:
|
||||
"volume": 50,
|
||||
"speech_rate": 0,
|
||||
"pitch_rate": 0,
|
||||
"enable_subtitle": True,
|
||||
}
|
||||
}
|
||||
await ws.send(json.dumps(start_request))
|
||||
|
||||
|
||||
start_response = json.loads(await ws.recv())
|
||||
if start_response["header"]["name"] != "SynthesisStarted":
|
||||
raise Exception("启动合成失败")
|
||||
|
||||
|
||||
run_request = {
|
||||
"header": {
|
||||
"message_id": str(uuid.uuid4()),
|
||||
@@ -69,23 +76,142 @@ class StreamTTSPerformanceTester:
|
||||
"payload": {"text": text}
|
||||
}
|
||||
await ws.send(json.dumps(run_request))
|
||||
|
||||
|
||||
while True:
|
||||
response = await ws.recv()
|
||||
if isinstance(response, bytes):
|
||||
latency = time.time() - start_time
|
||||
latencies.append(latency)
|
||||
print(f"[阿里云TTS] 第{i+1}次 首词延迟: {latency:.3f}s")
|
||||
break
|
||||
elif isinstance(response, str):
|
||||
data = json.loads(response)
|
||||
if data["header"]["name"] == "TaskFailed":
|
||||
raise Exception(f"合成失败: {data['payload']['error_info']}")
|
||||
|
||||
|
||||
except Exception as e:
|
||||
latencies.append(0)
|
||||
print(f"[阿里云TTS] 第{i+1}次测试失败: {str(e)}")
|
||||
latencies.append(None)
|
||||
|
||||
return self._calculate_result("阿里云TTS", latencies, test_count)
|
||||
|
||||
async def test_alibl_tts(self, text=None, test_count=5):
|
||||
"""测试阿里云百炼CosyVoice流式TTS首词延迟"""
|
||||
text = text or self.test_texts[0]
|
||||
latencies = []
|
||||
|
||||
for i in range(test_count):
|
||||
try:
|
||||
tts_config = self.config["TTS"]["AliBLTTS"]
|
||||
api_key = tts_config["api_key"]
|
||||
model = tts_config.get("model", "cosyvoice-v2")
|
||||
voice = tts_config.get("voice", "longxiaochun_v2")
|
||||
format_type = tts_config.get("format", "pcm")
|
||||
sample_rate = int(tts_config.get("sample_rate", "24000"))
|
||||
|
||||
ws_url = "wss://dashscope.aliyuncs.com/api-ws/v1/inference/"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"X-DashScope-DataInspection": "enable",
|
||||
}
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
async with websockets.connect(
|
||||
ws_url,
|
||||
additional_headers=headers,
|
||||
ping_interval=30,
|
||||
ping_timeout=10,
|
||||
close_timeout=10,
|
||||
max_size=10 * 1024 * 1024,
|
||||
) as ws:
|
||||
session_id = uuid.uuid4().hex
|
||||
|
||||
# 1. 发送 run-task（启动任务）
|
||||
run_task_message = {
|
||||
"header": {
|
||||
"action": "run-task",
|
||||
"task_id": session_id,
|
||||
"streaming": "duplex",
|
||||
},
|
||||
"payload": {
|
||||
"task_group": "audio",
|
||||
"task": "tts",
|
||||
"function": "SpeechSynthesizer",
|
||||
"model": model,
|
||||
"parameters": {
|
||||
"text_type": "PlainText",
|
||||
"voice": voice,
|
||||
"format": format_type,
|
||||
"sample_rate": sample_rate,
|
||||
"volume": 50,
|
||||
"rate": 1.0,
|
||||
"pitch": 1.0,
|
||||
},
|
||||
"input": {}
|
||||
},
|
||||
}
|
||||
await ws.send(json.dumps(run_task_message))
|
||||
|
||||
# 2. 等待 task-started 事件（关键！必须等这个再发文本）
|
||||
task_started = False
|
||||
while not task_started:
|
||||
msg = await ws.recv()
|
||||
if isinstance(msg, str):
|
||||
data = json.loads(msg)
|
||||
header = data.get("header", {})
|
||||
event = header.get("event")
|
||||
if event == "task-started":
|
||||
task_started = True
|
||||
print(f"[阿里云百炼TTS] 第{i+1}次 任务启动成功")
|
||||
elif event == "task-failed":
|
||||
raise Exception(f"启动失败: {header.get('error_message', '未知错误')}")
|
||||
|
||||
# 3. 发送 continue-task（发送文本！这是正确动作）
|
||||
continue_task_message = {
|
||||
"header": {
|
||||
"action": "continue-task", # 改回 continue-task
|
||||
"task_id": session_id,
|
||||
"streaming": "duplex",
|
||||
},
|
||||
"payload": {"input": {"text": text}},
|
||||
}
|
||||
await ws.send(json.dumps(continue_task_message))
|
||||
|
||||
# 4. 发送 finish-task（结束任务）
|
||||
finish_task_message = {
|
||||
"header": {
|
||||
"action": "finish-task",
|
||||
"task_id": session_id,
|
||||
"streaming": "duplex",
|
||||
},
|
||||
"payload": {"input": {}}
|
||||
}
|
||||
await ws.send(json.dumps(finish_task_message))
|
||||
|
||||
# 5. 等待第一个音频数据块
|
||||
while True:
|
||||
msg = await asyncio.wait_for(ws.recv(), timeout=15.0)
|
||||
if isinstance(msg, (bytes, bytearray)) and len(msg) > 0:
|
||||
latency = time.time() - start_time
|
||||
print(f"[阿里云百炼TTS] 第{i+1}次 首词延迟: {latency:.3f}s")
|
||||
latencies.append(latency)
|
||||
break
|
||||
elif isinstance(msg, str):
|
||||
data = json.loads(msg)
|
||||
event = data.get("header", {}).get("event")
|
||||
if event == "task-failed":
|
||||
raise Exception(f"合成失败: {data}")
|
||||
elif event == "task-finished":
|
||||
if not latencies or latencies[-1] is None:
|
||||
raise Exception("任务结束但未收到音频")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[阿里云百炼TTS] 第{i+1}次失败: {str(e)}")
|
||||
latencies.append(None)
|
||||
|
||||
return self._calculate_result("阿里云百炼TTS", latencies, test_count)
|
||||
|
||||
async def test_doubao_tts(self, text=None, test_count=5):
|
||||
"""测试火山引擎流式TTS首词延迟（测试多次取平均）"""
|
||||
text = text or self.test_texts[0]
|
||||
@@ -109,13 +235,12 @@ class StreamTTSPerformanceTester:
|
||||
}
|
||||
async with websockets.connect(ws_url, additional_headers=ws_header, max_size=1000000000) as ws:
|
||||
session_id = uuid.uuid4().hex
|
||||
|
||||
|
||||
# 发送会话启动请求
|
||||
header = bytes([
|
||||
(0b0001 << 4) | 0b0001,
|
||||
0b0001 << 4 | 0b100,
|
||||
0b0001 << 4 | 0b0000,
|
||||
0
|
||||
(0b0001 << 4) | 0b0001,
|
||||
0b0001 << 4 | 0b1011,
|
||||
0b0001 << 4 | 0b0000,
|
||||
])
|
||||
optional = bytearray()
|
||||
optional.extend((1).to_bytes(4, "big", signed=True))
|
||||
@@ -124,13 +249,13 @@ class StreamTTSPerformanceTester:
|
||||
optional.extend(session_id_bytes)
|
||||
payload = json.dumps({"speaker": speaker}).encode()
|
||||
await ws.send(header + optional + len(payload).to_bytes(4, "big", signed=True) + payload)
|
||||
|
||||
|
||||
# 发送文本
|
||||
header = bytes([
|
||||
(0b0001 << 4) | 0b0001,
|
||||
0b0001 << 4 | 0b100,
|
||||
0b0001 << 4 | 0b0000,
|
||||
0
|
||||
(0b0001 << 4) | 0b0001,
|
||||
0b0001 << 4 | 0b1011,
|
||||
0b0001 << 4 | 0b0000,
|
||||
0
|
||||
])
|
||||
optional = bytearray()
|
||||
optional.extend((200).to_bytes(4, "big", signed=True))
|
||||
@@ -139,13 +264,15 @@ class StreamTTSPerformanceTester:
|
||||
optional.extend(session_id_bytes)
|
||||
payload = json.dumps({"text": text, "speaker": speaker}).encode()
|
||||
await ws.send(header + optional + len(payload).to_bytes(4, "big", signed=True) + payload)
|
||||
|
||||
|
||||
first_chunk = await ws.recv()
|
||||
latency = time.time() - start_time
|
||||
latencies.append(latency)
|
||||
|
||||
print(f"[火山引擎TTS] 第{i+1}次 首词延迟: {latency:.3f}s")
|
||||
|
||||
except Exception as e:
|
||||
latencies.append(0)
|
||||
print(f"[火山引擎TTS] 第{i+1}次测试失败: {str(e)}")
|
||||
latencies.append(None)
|
||||
|
||||
return self._calculate_result("火山引擎TTS", latencies, test_count)
|
||||
|
||||
@@ -186,22 +313,24 @@ class StreamTTSPerformanceTester:
|
||||
first_chunk = await ws.recv()
|
||||
latency = time.time() - start_time
|
||||
latencies.append(latency)
|
||||
|
||||
print(f"[PaddleSpeechTTS] 第{i+1}次 首词延迟: {latency:.3f}s")
|
||||
|
||||
# 发送结束请求
|
||||
end_request = {
|
||||
"task": "tts",
|
||||
"signal": "end"
|
||||
}
|
||||
await ws.send(json.dumps(end_request))
|
||||
|
||||
|
||||
# 确保连接正常关闭
|
||||
try:
|
||||
await ws.recv()
|
||||
except websockets.exceptions.ConnectionClosedOK:
|
||||
pass
|
||||
|
||||
|
||||
except Exception as e:
|
||||
latencies.append(0)
|
||||
print(f"[PaddleSpeechTTS] 第{i+1}次测试失败: {str(e)}")
|
||||
latencies.append(None)
|
||||
|
||||
return self._calculate_result("PaddleSpeechTTS", latencies, test_count)
|
||||
|
||||
@@ -215,29 +344,32 @@ class StreamTTSPerformanceTester:
|
||||
tts_config = self.config["TTS"]["IndexStreamTTS"]
|
||||
api_url = tts_config.get("api_url")
|
||||
voice = tts_config.get("voice")
|
||||
|
||||
|
||||
# 统一计时起点：在建立连接前开始计时
|
||||
start_time = time.time()
|
||||
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
payload = {"text": text, "character": voice}
|
||||
async with session.post(api_url, json=payload, timeout=10) as resp:
|
||||
if resp.status != 200:
|
||||
raise Exception(f"请求失败: {resp.status}, {await resp.text()}")
|
||||
|
||||
|
||||
async for chunk in resp.content.iter_any():
|
||||
data = chunk[0] if isinstance(chunk, (list, tuple)) else chunk
|
||||
if not data:
|
||||
continue
|
||||
|
||||
|
||||
latency = time.time() - start_time
|
||||
latencies.append(latency)
|
||||
print(f"[IndexStreamTTS] 第{i+1}次 首词延迟: {latency:.3f}s")
|
||||
resp.close()
|
||||
break
|
||||
else:
|
||||
latencies.append(0)
|
||||
|
||||
latencies.append(None)
|
||||
|
||||
except Exception as e:
|
||||
latencies.append(0)
|
||||
print(f"[IndexStreamTTS] 第{i+1}次测试失败: {str(e)}")
|
||||
latencies.append(None)
|
||||
|
||||
return self._calculate_result("IndexStreamTTS", latencies, test_count)
|
||||
|
||||
@@ -252,7 +384,8 @@ class StreamTTSPerformanceTester:
|
||||
api_url = tts_config["api_url"]
|
||||
access_token = tts_config["access_token"]
|
||||
voice = tts_config["voice"]
|
||||
|
||||
|
||||
# 统一计时起点：在建立连接前开始计时
|
||||
start_time = time.time()
|
||||
async with aiohttp.ClientSession() as session:
|
||||
params = {
|
||||
@@ -268,28 +401,167 @@ class StreamTTSPerformanceTester:
|
||||
"Authorization": f"Bearer {access_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
async with session.get(api_url, params=params, headers=headers, timeout=10) as resp:
|
||||
if resp.status != 200:
|
||||
raise Exception(f"请求失败: {resp.status}, {await resp.text()}")
|
||||
|
||||
|
||||
# 接收第一个数据块
|
||||
async for _ in resp.content.iter_any():
|
||||
latency = time.time() - start_time
|
||||
latencies.append(latency)
|
||||
print(f"[LinkeraiTTS] 第{i+1}次 首词延迟: {latency:.3f}s")
|
||||
break
|
||||
else:
|
||||
latencies.append(0)
|
||||
|
||||
latencies.append(None)
|
||||
|
||||
except Exception as e:
|
||||
latencies.append(0)
|
||||
print(f"[LinkeraiTTS] 第{i+1}次测试失败: {str(e)}")
|
||||
latencies.append(None)
|
||||
|
||||
return self._calculate_result("LinkeraiTTS", latencies, test_count)
|
||||
|
||||
async def test_xunfei_tts(self, text=None, test_count=5):
|
||||
"""测试讯飞流式TTS首词延迟（测试多次取平均）"""
|
||||
text = text or self.test_texts[0]
|
||||
latencies = []
|
||||
|
||||
for i in range(test_count):
|
||||
try:
|
||||
# 修正配置节点名称，与配置文件中的XunFeiTTS匹配
|
||||
tts_config = self.config["TTS"]["XunFeiTTS"]
|
||||
app_id = tts_config["app_id"]
|
||||
api_key = tts_config["api_key"]
|
||||
api_secret = tts_config["api_secret"]
|
||||
api_url = tts_config.get("api_url", "wss://cbm01.cn-huabei-1.xf-yun.com/v1/private/mcd9m97e6")
|
||||
voice = tts_config.get("voice", "x5_lingxiaoxuan_flow")
|
||||
# 生成认证URL
|
||||
auth_url = self._create_xunfei_auth_url(api_key, api_secret, api_url)
|
||||
start_time = time.time()
|
||||
async with websockets.connect(
|
||||
auth_url,
|
||||
ping_interval=30,
|
||||
ping_timeout=10,
|
||||
close_timeout=10,
|
||||
max_size=1000000000
|
||||
) as ws:
|
||||
# 构造请求
|
||||
request = self._build_xunfei_request(app_id, text, voice)
|
||||
await ws.send(json.dumps(request))
|
||||
# 等待第一个音频数据块
|
||||
first_audio_received = False
|
||||
while not first_audio_received:
|
||||
msg = await asyncio.wait_for(ws.recv(), timeout=10)
|
||||
data = json.loads(msg)
|
||||
header = data.get("header", {})
|
||||
code = header.get("code")
|
||||
|
||||
if code != 0:
|
||||
message = header.get("message", "未知错误")
|
||||
raise Exception(f"合成失败: {code} - {message}")
|
||||
|
||||
payload = data.get("payload", {})
|
||||
audio_payload = payload.get("audio", {})
|
||||
|
||||
if audio_payload:
|
||||
status = audio_payload.get("status", 0)
|
||||
audio_data = audio_payload.get("audio", "")
|
||||
if status == 1 and audio_data:
|
||||
# 收到第一个音频数据块
|
||||
latency = time.time() - start_time
|
||||
latencies.append(latency)
|
||||
print(f"[讯飞TTS] 第{i+1}次 首词延迟: {latency:.3f}s")
|
||||
first_audio_received = True
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"[讯飞TTS] 第{i+1}次测试失败: {str(e)}")
|
||||
latencies.append(None)
|
||||
|
||||
return self._calculate_result("讯飞TTS", latencies, test_count)
|
||||
|
||||
def _create_xunfei_auth_url(self, api_key, api_secret, api_url):
|
||||
"""生成讯飞WebSocket认证URL"""
|
||||
parsed_url = urlparse(api_url)
|
||||
host = parsed_url.netloc
|
||||
path = parsed_url.path
|
||||
|
||||
# 获取UTC时间，讯飞要求使用RFC1123格式
|
||||
now = time.gmtime()
|
||||
date = time.strftime('%a, %d %b %Y %H:%M:%S GMT', now)
|
||||
|
||||
# 构造签名字符串
|
||||
signature_origin = f"host: {host}\ndate: {date}\nGET {path} HTTP/1.1"
|
||||
|
||||
# 计算签名
|
||||
signature_sha = hmac.new(
|
||||
api_secret.encode('utf-8'),
|
||||
signature_origin.encode('utf-8'),
|
||||
digestmod=hashlib.sha256
|
||||
).digest()
|
||||
signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding='utf-8')
|
||||
|
||||
# 构造authorization
|
||||
authorization_origin = f'api_key="{api_key}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha_base64}"'
|
||||
authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8')
|
||||
|
||||
# 构造最终的WebSocket URL
|
||||
v = {
|
||||
"authorization": authorization,
|
||||
"date": date,
|
||||
"host": host
|
||||
}
|
||||
url = api_url + '?' + urlencode(v)
|
||||
return url
|
||||
|
||||
def _build_xunfei_request(self, app_id, text, voice):
|
||||
"""构建讯飞TTS请求结构"""
|
||||
return {
|
||||
"header": {
|
||||
"app_id": app_id,
|
||||
"status": 2,
|
||||
},
|
||||
"parameter": {
|
||||
"oral": {
|
||||
"oral_level": "mid",
|
||||
"spark_assist": 1,
|
||||
"stop_split": 0,
|
||||
"remain": 0
|
||||
},
|
||||
"tts": {
|
||||
"vcn": voice,
|
||||
"speed": 50,
|
||||
"volume": 50,
|
||||
"pitch": 50,
|
||||
"bgs": 0,
|
||||
"reg": 0,
|
||||
"rdn": 0,
|
||||
"rhy": 0,
|
||||
"audio": {
|
||||
"encoding": "raw",
|
||||
"sample_rate": 24000,
|
||||
"channels": 1,
|
||||
"bit_depth": 16,
|
||||
"frame_size": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"payload": {
|
||||
"text": {
|
||||
"encoding": "utf8",
|
||||
"compress": "raw",
|
||||
"format": "plain",
|
||||
"status": 2,
|
||||
"seq": 1,
|
||||
"text": base64.b64encode(text.encode('utf-8')).decode('utf-8')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _calculate_result(self, service_name, latencies, test_count):
|
||||
"""计算测试结果"""
|
||||
valid_latencies = [l for l in latencies if l > 0]
|
||||
"""计算测试结果（正确处理None值，剔除失败测试）"""
|
||||
# 剔除失败的测试（None值和<=0延迟），只统计有效延迟
|
||||
valid_latencies = [l for l in latencies if l is not None and l > 0]
|
||||
if valid_latencies:
|
||||
avg_latency = sum(valid_latencies) / len(valid_latencies)
|
||||
status = f"成功（{len(valid_latencies)}/{test_count}次有效）"
|
||||
@@ -323,9 +595,10 @@ class StreamTTSPerformanceTester:
|
||||
]
|
||||
|
||||
print(tabulate(table_data, headers=["TTS服务", "首词延迟(秒)", "状态"], tablefmt="grid"))
|
||||
print("\n测试说明：测量从发送请求到接收第一个音频数据块的时间，取多次测试平均值")
|
||||
print("\n测试说明：测量从建立连接到接收第一个音频数据块的时间（包含握手、鉴权、发送文本），取多次测试平均值")
|
||||
print("- 计时起点: 建立WebSocket/HTTP连接前（统一包含网络建连、握手、发送文本全流程）")
|
||||
print("- 超时控制: 单个请求最大等待时间为10秒")
|
||||
print("- 错误处理: 无法连接和超时的列为网络错误")
|
||||
print("- 错误处理: 失败的测试不计入平均值，只统计成功测试的延迟")
|
||||
print("- 排序规则: 按平均耗时从快到慢排序")
|
||||
|
||||
|
||||
@@ -351,7 +624,12 @@ class StreamTTSPerformanceTester:
|
||||
# 测试阿里云TTS
|
||||
result = await self.test_aliyun_tts(test_text, test_count)
|
||||
self.results.append(result)
|
||||
|
||||
|
||||
# 测试阿里云百炼TTS
|
||||
if self.config.get("TTS", {}).get("AliBLTTS"):
|
||||
result = await self.test_alibl_tts(test_text, test_count)
|
||||
self.results.append(result)
|
||||
|
||||
# 测试火山引擎TTS
|
||||
result = await self.test_doubao_tts(test_text, test_count)
|
||||
self.results.append(result)
|
||||
@@ -368,6 +646,11 @@ class StreamTTSPerformanceTester:
|
||||
result = await self.test_indexstream_tts(test_text, test_count)
|
||||
self.results.append(result)
|
||||
|
||||
# 测试讯飞TTS
|
||||
if self.config.get("TTS", {}).get("XunFeiTTS"):
|
||||
result = await self.test_xunfei_tts(test_text, test_count)
|
||||
self.results.append(result)
|
||||
|
||||
# 打印结果
|
||||
self._print_results(test_text, test_count)
|
||||
|
||||
|
||||
@@ -195,7 +195,7 @@ def get_news_from_chinanews(
|
||||
|
||||
# 否则，获取新闻列表并随机选择一条
|
||||
# 从配置中获取RSS URL
|
||||
rss_config = conn.config["plugins"]["get_news_from_chinanews"]
|
||||
rss_config = conn.config.get("plugins", {}).get("get_news_from_chinanews", {})
|
||||
default_rss_url = rss_config.get(
|
||||
"default_rss_url", "https://www.chinanews.com.cn/rss/society.xml"
|
||||
)
|
||||
|
||||
@@ -120,10 +120,10 @@ def fetch_news_from_api(conn, source="thepaper"):
|
||||
"""从API获取新闻列表"""
|
||||
try:
|
||||
api_url = f"https://newsnow.busiyi.world/api/s?id={source}"
|
||||
if conn.config["plugins"].get("get_news_from_newsnow") and conn.config[
|
||||
"plugins"
|
||||
]["get_news_from_newsnow"].get("url"):
|
||||
api_url = conn.config["plugins"]["get_news_from_newsnow"]["url"] + source
|
||||
|
||||
news_config = conn.config.get("plugins", {}).get("get_news_from_newsnow", {})
|
||||
if news_config.get("url"):
|
||||
api_url = news_config["url"] + source
|
||||
|
||||
headers = {"User-Agent": "Mozilla/5.0"}
|
||||
response = requests.get(api_url, headers=headers, timeout=10)
|
||||
|
||||
@@ -158,13 +158,10 @@ def parse_weather_info(soup):
|
||||
def get_weather(conn, location: str = None, lang: str = "zh_CN"):
|
||||
from core.utils.cache.manager import cache_manager, CacheType
|
||||
|
||||
api_host = conn.config["plugins"]["get_weather"].get(
|
||||
"api_host", "mj7p3y7naa.re.qweatherapi.com"
|
||||
)
|
||||
api_key = conn.config["plugins"]["get_weather"].get(
|
||||
"api_key", "a861d0d5e7bf4ee1a83d9a9e4f96d4da"
|
||||
)
|
||||
default_location = conn.config["plugins"]["get_weather"]["default_location"]
|
||||
weather_config = conn.config.get("plugins", {}).get("get_weather", {})
|
||||
api_host = weather_config.get("api_host", "mj7p3y7naa.re.qweatherapi.com")
|
||||
api_key = weather_config.get("api_key", "a861d0d5e7bf4ee1a83d9a9e4f96d4da")
|
||||
default_location = weather_config.get("default_location", "广州")
|
||||
client_ip = conn.client_ip
|
||||
|
||||
# 优先使用用户提供的location参数
|
||||
|
||||
@@ -11,15 +11,17 @@ def append_devices_to_prompt(conn):
|
||||
"functions", []
|
||||
)
|
||||
|
||||
# 安全地获取插件配置
|
||||
plugins_config = conn.config.get("plugins", {})
|
||||
config_source = (
|
||||
"home_assistant"
|
||||
if conn.config["plugins"].get("home_assistant")
|
||||
if plugins_config.get("home_assistant")
|
||||
else "hass_get_state"
|
||||
)
|
||||
|
||||
if "hass_get_state" in funcs or "hass_set_state" in funcs:
|
||||
prompt = "\n下面是我家智能设备列表（位置，设备名，entity_id），可以通过homeassistant控制\n"
|
||||
deviceStr = conn.config["plugins"].get(config_source, {}).get("devices", "")
|
||||
deviceStr = plugins_config.get(config_source, {}).get("devices", "")
|
||||
conn.prompt += prompt + deviceStr + "\n"
|
||||
# 更新提示词
|
||||
conn.dialogue.update_system_message(conn.prompt)
|
||||
@@ -30,17 +32,17 @@ def initialize_hass_handler(conn):
|
||||
if not conn.load_function_plugin:
|
||||
return ha_config
|
||||
|
||||
# 安全地获取插件配置
|
||||
plugins_config = conn.config.get("plugins", {})
|
||||
# 确定配置来源
|
||||
config_source = (
|
||||
"home_assistant"
|
||||
if conn.config["plugins"].get("home_assistant")
|
||||
else "hass_get_state"
|
||||
"home_assistant" if plugins_config.get("home_assistant") else "hass_get_state"
|
||||
)
|
||||
if not conn.config["plugins"].get(config_source):
|
||||
if not plugins_config.get(config_source):
|
||||
return ha_config
|
||||
|
||||
# 统一获取配置
|
||||
plugin_config = conn.config["plugins"][config_source]
|
||||
plugin_config = plugins_config[config_source]
|
||||
ha_config["base_url"] = plugin_config.get("base_url")
|
||||
ha_config["api_key"] = plugin_config.get("api_key")
|
||||
|
||||
|
||||
@@ -118,8 +118,9 @@ def get_music_files(music_dir, music_ext):
|
||||
def initialize_music_handler(conn):
|
||||
global MUSIC_CACHE
|
||||
if MUSIC_CACHE == {}:
|
||||
if "play_music" in conn.config["plugins"]:
|
||||
MUSIC_CACHE["music_config"] = conn.config["plugins"]["play_music"]
|
||||
plugins_config = conn.config.get("plugins", {})
|
||||
if "play_music" in plugins_config:
|
||||
MUSIC_CACHE["music_config"] = plugins_config["play_music"]
|
||||
MUSIC_CACHE["music_dir"] = os.path.abspath(
|
||||
MUSIC_CACHE["music_config"].get("music_dir", "./music") # 默认路径修改
|
||||
)
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import requests
|
||||
import sys
|
||||
from config.logger import setup_logging
|
||||
from plugins_func.register import register_function, ToolType, ActionResponse, Action
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
# 定义基础的函数描述模板
|
||||
SEARCH_FROM_RAGFLOW_FUNCTION_DESC = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_from_ragflow",
|
||||
"description": "从知识库中查询信息",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"question": {"type": "string", "description": "查询的问题"}},
|
||||
"required": ["question"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@register_function(
|
||||
"search_from_ragflow", SEARCH_FROM_RAGFLOW_FUNCTION_DESC, ToolType.SYSTEM_CTL
|
||||
)
|
||||
def search_from_ragflow(conn, question=None):
|
||||
# 确保字符串参数正确处理编码
|
||||
if question and isinstance(question, str):
|
||||
# 确保问题参数是UTF-8编码的字符串
|
||||
pass
|
||||
else:
|
||||
question = str(question) if question is not None else ""
|
||||
|
||||
ragflow_config = conn.config.get("plugins", {}).get("search_from_ragflow", {})
|
||||
base_url = ragflow_config.get("base_url", "")
|
||||
api_key = ragflow_config.get("api_key", "")
|
||||
dataset_ids = ragflow_config.get("dataset_ids", [])
|
||||
|
||||
url = base_url + "/api/v1/retrieval"
|
||||
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
|
||||
|
||||
# 确保payload中的字符串都是UTF-8编码
|
||||
payload = {"question": question, "dataset_ids": dataset_ids}
|
||||
|
||||
try:
|
||||
# 使用ensure_ascii=False确保JSON序列化时正确处理中文
|
||||
response = requests.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=5,
|
||||
verify=False,
|
||||
)
|
||||
|
||||
# 显式设置响应的编码为utf-8
|
||||
response.encoding = "utf-8"
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
# 先获取文本内容，然后手动处理JSON解码
|
||||
response_text = response.text
|
||||
import json
|
||||
|
||||
result = json.loads(response_text)
|
||||
|
||||
if result.get("code") != 0:
|
||||
error_detail = response.get("error", {}).get("detail", "")
|
||||
# 安全地记录错误信息
|
||||
logger.bind(tag=TAG).error(
|
||||
"从RAGflow获取信息失败，原因：%s", str(error_detail)
|
||||
)
|
||||
return ActionResponse(Action.RESPONSE, None, "RAG接口返回异常")
|
||||
|
||||
chunks = result.get("data", {}).get("chunks", [])
|
||||
contents = []
|
||||
for chunk in chunks:
|
||||
content = chunk.get("content", "")
|
||||
if content:
|
||||
# 安全地处理内容字符串
|
||||
if isinstance(content, str):
|
||||
contents.append(content)
|
||||
elif isinstance(content, bytes):
|
||||
contents.append(content.decode("utf-8", errors="replace"))
|
||||
else:
|
||||
contents.append(str(content))
|
||||
|
||||
if contents:
|
||||
# 组织知识库内容为引用模式
|
||||
context_text = f"# 关于问题【{question}】查到知识库如下\n"
|
||||
context_text += "```\n\n\n".join(contents[:5])
|
||||
context_text += "\n```"
|
||||
else:
|
||||
context_text = "根据知识库查询结果，没有相关信息。"
|
||||
return ActionResponse(Action.REQLLM, context_text, None)
|
||||
|
||||
except Exception as e:
|
||||
# 使用安全的方式记录异常，避免编码问题
|
||||
logger.bind(tag=TAG).error("从RAGflow获取信息失败，原因：%s", str(e))
|
||||
return ActionResponse(Action.RESPONSE, None, "RAG接口返回异常")
|
||||
Executable → Regular
+31
-26
@@ -1,37 +1,42 @@
|
||||
pyyml==0.0.2
|
||||
# -*- coding:utf-8 -*-
|
||||
#--------- 本项目推荐环境是python3.10，以下暂时不推荐升级的依赖
|
||||
torch==2.2.2
|
||||
silero_vad==5.1.2
|
||||
websockets==14.2
|
||||
opuslib_next==1.1.2
|
||||
numpy==1.26.4
|
||||
pydub==0.25.1
|
||||
funasr==1.2.3
|
||||
torchaudio==2.2.2
|
||||
openai==1.61.0
|
||||
google-generativeai==0.8.4
|
||||
edge_tts==7.0.0
|
||||
httpx==0.27.2
|
||||
aiohttp==3.9.3
|
||||
aiohttp_cors==0.7.0
|
||||
ormsgpack==1.7.0
|
||||
ruamel.yaml==0.18.10
|
||||
numpy==1.26.4
|
||||
#--------- cozepy0.20.0要求websockets<15.0.0，以下暂时不推荐升级的依赖
|
||||
websockets==14.2
|
||||
#--------- 以下是可升级的依赖
|
||||
pyyml==0.0.2
|
||||
silero_vad==6.1.0
|
||||
opuslib_next==1.1.5
|
||||
pydub==0.25.1
|
||||
funasr==1.2.7
|
||||
openai==2.8.1
|
||||
google-generativeai==0.8.5
|
||||
edge_tts==7.2.6
|
||||
httpx==0.28.1
|
||||
aiohttp==3.13.2
|
||||
aiohttp_cors==0.8.1
|
||||
ormsgpack==1.12.0
|
||||
ruamel.yaml==0.18.16
|
||||
loguru==0.7.3
|
||||
requests==2.32.3
|
||||
cozepy==0.12.0
|
||||
mem0ai==0.1.62
|
||||
requests==2.32.5
|
||||
cozepy==0.20.0
|
||||
mem0ai==1.0.0
|
||||
bs4==0.0.2
|
||||
modelscope==1.23.2
|
||||
sherpa_onnx==1.12.8
|
||||
mcp==1.8.1
|
||||
sherpa_onnx==1.12.17
|
||||
mcp==1.20.0
|
||||
cnlunar==0.2.0
|
||||
PySocks==1.7.1
|
||||
dashscope==1.23.1
|
||||
dashscope==1.25.2
|
||||
baidu-aip==4.16.13
|
||||
chardet==5.2.0
|
||||
aioconsole==0.8.1
|
||||
markitdown==0.1.1
|
||||
mcp-proxy==0.8.0
|
||||
PyJWT==2.8.0
|
||||
aioconsole==0.8.2
|
||||
markitdown==0.1.3
|
||||
mcp-proxy==0.10.0
|
||||
PyJWT==2.10.1
|
||||
psutil==7.0.0
|
||||
portalocker==2.10.1
|
||||
portalocker==3.2.0
|
||||
Jinja2==3.1.6
|
||||
vosk==0.3.45
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,503 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>小智语音服务测试</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
}
|
||||
.container {
|
||||
background-color: #f5f5f5;
|
||||
padding: 20px;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
button {
|
||||
padding: 8px 15px;
|
||||
margin-right: 10px;
|
||||
border: none;
|
||||
border-radius: 5px;
|
||||
background-color: #4285f4;
|
||||
color: white;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.2s;
|
||||
}
|
||||
button:hover {
|
||||
background-color: #3367d6;
|
||||
}
|
||||
button:disabled {
|
||||
background-color: #cccccc;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
#status {
|
||||
font-weight: bold;
|
||||
}
|
||||
#scriptStatus {
|
||||
position: fixed;
|
||||
top: 10px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
padding: 15px;
|
||||
border-radius: 5px;
|
||||
display: block;
|
||||
z-index: 100;
|
||||
transition: all 0.3s ease;
|
||||
}
|
||||
#scriptStatus.info {
|
||||
background-color: #e8f0fe;
|
||||
color: #4285f4;
|
||||
border-left: 4px solid #4285f4;
|
||||
}
|
||||
#scriptStatus.success {
|
||||
background-color: #e6f4ea;
|
||||
color: #0f9d58;
|
||||
border-left: 4px solid #0f9d58;
|
||||
}
|
||||
#scriptStatus.error {
|
||||
background-color: #fce8e6;
|
||||
color: #db4437;
|
||||
border-left: 4px solid #db4437;
|
||||
}
|
||||
#debugInfo {
|
||||
margin-top: 20px;
|
||||
border: 1px solid #ccc;
|
||||
border-radius: 5px;
|
||||
padding: 10px;
|
||||
font-family: monospace;
|
||||
font-size: 12px;
|
||||
max-height: 200px;
|
||||
overflow-y: auto;
|
||||
display: none;
|
||||
}
|
||||
#showDebug {
|
||||
margin-top: 10px;
|
||||
background-color: #f5f5f5;
|
||||
color: #444;
|
||||
font-size: 12px;
|
||||
}
|
||||
#audioMeter {
|
||||
margin-top: 10px;
|
||||
height: 20px;
|
||||
background-color: #eee;
|
||||
border-radius: 4px;
|
||||
overflow: hidden;
|
||||
display: none;
|
||||
}
|
||||
#audioLevel {
|
||||
height: 100%;
|
||||
width: 0%;
|
||||
background-color: #4285f4;
|
||||
transition: width 0.1s;
|
||||
}
|
||||
.conversation {
|
||||
max-height: 300px;
|
||||
overflow-y: auto;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 5px;
|
||||
padding: 10px;
|
||||
background-color: white;
|
||||
margin-top: 10px;
|
||||
}
|
||||
.message {
|
||||
margin-bottom: 10px;
|
||||
padding: 8px 12px;
|
||||
border-radius: 8px;
|
||||
max-width: 80%;
|
||||
}
|
||||
.user {
|
||||
background-color: #e2f2ff;
|
||||
margin-left: auto;
|
||||
margin-right: 10px;
|
||||
text-align: right;
|
||||
}
|
||||
.server {
|
||||
background-color: #f0f0f0;
|
||||
margin-right: auto;
|
||||
margin-left: 10px;
|
||||
}
|
||||
#serverUrl {
|
||||
flex-grow: 1;
|
||||
padding: 8px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 5px;
|
||||
width: 60%;
|
||||
margin-right: 10px;
|
||||
}
|
||||
#messageInput {
|
||||
flex-grow: 1;
|
||||
padding: 8px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 5px;
|
||||
width: 70%;
|
||||
margin-right: 10px;
|
||||
}
|
||||
.section {
|
||||
margin-bottom: 15px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h2>小智语音服务测试</h2>
|
||||
|
||||
<div id="scriptStatus" class="info">正在加载Opus库...</div>
|
||||
|
||||
<div class="section">
|
||||
<h3>WebSocket连接 <span id="connectionStatus">未连接</span></h3>
|
||||
<div style="display: flex; align-items: center; margin-bottom: 10px;">
|
||||
<input type="text" id="serverUrl" value="ws://127.0.0.1:8000/xiaozhi/v1/" placeholder="WebSocket服务器地址">
|
||||
<button id="connectButton">连接</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h3>录音测试</h3>
|
||||
<button id="initAudio" style="background-color: #34a853;">初始化音频</button>
|
||||
<button id="testMic" style="background-color: #fbbc05;">测试麦克风</button>
|
||||
<p></p>
|
||||
<button id="start" disabled>开始录音</button>
|
||||
<button id="stop" disabled>停止录音</button>
|
||||
<button id="play" disabled>播放录音</button>
|
||||
<p>录音状态: <span id="status">待机，正在初始化...</span></p>
|
||||
<div id="audioMeter">
|
||||
<div id="audioLevel"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h3>文本消息</h3>
|
||||
<div style="display: flex; align-items: center;">
|
||||
<input type="text" id="messageInput" placeholder="输入消息..." disabled>
|
||||
<button id="sendTextButton" disabled>发送</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h3>会话记录</h3>
|
||||
<div id="conversation" class="conversation"></div>
|
||||
</div>
|
||||
|
||||
<button id="showDebug">显示/隐藏调试信息</button>
|
||||
<div id="debugInfo"></div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// 定义全局变量以跟踪库加载状态
|
||||
window.opusLoaded = false;
|
||||
window.startButton = document.getElementById("start");
|
||||
window.stopButton = document.getElementById("stop");
|
||||
window.playButton = document.getElementById("play");
|
||||
window.statusLabel = document.getElementById("status");
|
||||
window.debugInfo = document.getElementById("debugInfo");
|
||||
window.audioContextReady = false;
|
||||
window.testMicActive = false;
|
||||
|
||||
// 显示/隐藏调试信息
|
||||
document.getElementById("showDebug").addEventListener("click", function() {
|
||||
if (debugInfo.style.display === "none" || !debugInfo.style.display) {
|
||||
debugInfo.style.display = "block";
|
||||
this.textContent = "隐藏调试信息";
|
||||
} else {
|
||||
debugInfo.style.display = "none";
|
||||
this.textContent = "显示调试信息";
|
||||
}
|
||||
});
|
||||
|
||||
// 添加初始化音频按钮事件
|
||||
document.getElementById("initAudio").addEventListener("click", function() {
|
||||
initializeAudioSystem();
|
||||
});
|
||||
|
||||
// 添加测试麦克风按钮事件
|
||||
document.getElementById("testMic").addEventListener("click", function() {
|
||||
if (window.testMicActive) {
|
||||
stopMicTest();
|
||||
} else {
|
||||
startMicTest();
|
||||
}
|
||||
});
|
||||
|
||||
// 初始化音频系统
|
||||
function initializeAudioSystem() {
|
||||
try {
|
||||
log("初始化音频系统...");
|
||||
// 创建临时AudioContext来触发用户授权
|
||||
const tempContext = new (window.AudioContext || window.webkitAudioContext)({
|
||||
sampleRate: 16000,
|
||||
latencyHint: 'interactive'
|
||||
});
|
||||
|
||||
// 创建振荡器并播放短促的声音
|
||||
const oscillator = tempContext.createOscillator();
|
||||
const gain = tempContext.createGain();
|
||||
gain.gain.value = 0.1; // 很小的音量
|
||||
oscillator.connect(gain);
|
||||
gain.connect(tempContext.destination);
|
||||
oscillator.frequency.value = 440; // A4
|
||||
oscillator.start();
|
||||
|
||||
// 0.2秒后停止
|
||||
setTimeout(() => {
|
||||
oscillator.stop();
|
||||
// 关闭上下文
|
||||
tempContext.close().then(() => {
|
||||
log("音频系统初始化成功", "success");
|
||||
updateScriptStatus("音频系统已激活", "success");
|
||||
document.getElementById("initAudio").disabled = true;
|
||||
document.getElementById("initAudio").textContent = "音频已初始化";
|
||||
window.audioContextReady = true;
|
||||
|
||||
// 如果Opus已加载，启用开始录音按钮
|
||||
if (window.opusLoaded) {
|
||||
startButton.disabled = false;
|
||||
}
|
||||
});
|
||||
}, 200);
|
||||
} catch (err) {
|
||||
log("初始化音频系统失败: " + err.message, "error");
|
||||
updateScriptStatus("初始化音频失败: " + err.message, "error");
|
||||
}
|
||||
}
|
||||
|
||||
// 测试麦克风
|
||||
function startMicTest() {
|
||||
const audioMeter = document.getElementById("audioMeter");
|
||||
const audioLevel = document.getElementById("audioLevel");
|
||||
const testMicBtn = document.getElementById("testMic");
|
||||
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
log("浏览器不支持麦克风访问", "error");
|
||||
return;
|
||||
}
|
||||
|
||||
log("开始麦克风测试...");
|
||||
audioMeter.style.display = "block";
|
||||
testMicBtn.textContent = "停止测试";
|
||||
testMicBtn.style.backgroundColor = "#ea4335";
|
||||
window.testMicActive = true;
|
||||
|
||||
// 创建音频上下文
|
||||
const testContext = new (window.AudioContext || window.webkitAudioContext)();
|
||||
window.testContext = testContext;
|
||||
|
||||
// 获取麦克风权限
|
||||
navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true
|
||||
}
|
||||
})
|
||||
.then(stream => {
|
||||
log("已获取麦克风访问权限", "success");
|
||||
|
||||
// 保存流以便稍后关闭
|
||||
window.testStream = stream;
|
||||
|
||||
// 创建音频分析器
|
||||
const source = testContext.createMediaStreamSource(stream);
|
||||
const analyser = testContext.createAnalyser();
|
||||
analyser.fftSize = 256;
|
||||
source.connect(analyser);
|
||||
|
||||
// 创建音量显示更新函数
|
||||
const bufferLength = analyser.frequencyBinCount;
|
||||
const dataArray = new Uint8Array(bufferLength);
|
||||
|
||||
function updateMeter() {
|
||||
if (!window.testMicActive) return;
|
||||
|
||||
analyser.getByteFrequencyData(dataArray);
|
||||
|
||||
// 计算音量级别 (0-100)
|
||||
let sum = 0;
|
||||
for (let i = 0; i < bufferLength; i++) {
|
||||
sum += dataArray[i];
|
||||
}
|
||||
const average = sum / bufferLength;
|
||||
const level = Math.min(100, Math.max(0, average * 2));
|
||||
|
||||
// 更新音量计
|
||||
audioLevel.style.width = level + "%";
|
||||
|
||||
// 如果有声音，记录日志
|
||||
if (level > 10) {
|
||||
log(`检测到声音: ${level.toFixed(1)}%`);
|
||||
}
|
||||
|
||||
// 循环更新
|
||||
window.testMicAnimationFrame = requestAnimationFrame(updateMeter);
|
||||
}
|
||||
|
||||
// 开始更新
|
||||
updateMeter();
|
||||
})
|
||||
.catch(err => {
|
||||
log("麦克风测试失败: " + err.message, "error");
|
||||
window.testMicActive = false;
|
||||
testMicBtn.textContent = "测试麦克风";
|
||||
testMicBtn.style.backgroundColor = "#fbbc05";
|
||||
audioMeter.style.display = "none";
|
||||
});
|
||||
}
|
||||
|
||||
// 停止麦克风测试
|
||||
function stopMicTest() {
|
||||
const audioMeter = document.getElementById("audioMeter");
|
||||
const testMicBtn = document.getElementById("testMic");
|
||||
|
||||
log("停止麦克风测试");
|
||||
window.testMicActive = false;
|
||||
testMicBtn.textContent = "测试麦克风";
|
||||
testMicBtn.style.backgroundColor = "#fbbc05";
|
||||
|
||||
// 停止分析器动画
|
||||
if (window.testMicAnimationFrame) {
|
||||
cancelAnimationFrame(window.testMicAnimationFrame);
|
||||
}
|
||||
|
||||
// 停止麦克风流
|
||||
if (window.testStream) {
|
||||
window.testStream.getTracks().forEach(track => track.stop());
|
||||
}
|
||||
|
||||
// 关闭测试上下文
|
||||
if (window.testContext) {
|
||||
window.testContext.close();
|
||||
}
|
||||
|
||||
// 隐藏音量计
|
||||
audioMeter.style.display = "none";
|
||||
}
|
||||
|
||||
// 添加调试日志
|
||||
function log(message, type = "info") {
|
||||
console.log(message);
|
||||
const time = new Date().toLocaleTimeString();
|
||||
const entry = document.createElement("div");
|
||||
entry.textContent = `[${time}] ${message}`;
|
||||
|
||||
if (type === "error") {
|
||||
entry.style.color = "#db4437";
|
||||
} else if (type === "success") {
|
||||
entry.style.color = "#0f9d58";
|
||||
}
|
||||
|
||||
debugInfo.appendChild(entry);
|
||||
debugInfo.scrollTop = debugInfo.scrollHeight;
|
||||
}
|
||||
|
||||
// 更新脚本状态显示
|
||||
function updateScriptStatus(message, type) {
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) {
|
||||
statusElement.textContent = message;
|
||||
statusElement.className = type;
|
||||
statusElement.style.display = 'block';
|
||||
}
|
||||
log(message, type);
|
||||
}
|
||||
|
||||
// 检查Opus库是否已加载
|
||||
function checkOpusLoaded() {
|
||||
try {
|
||||
// 检查Module是否存在（本地库导出的全局变量）
|
||||
if (typeof Module === 'undefined') {
|
||||
log("Module对象不存在", "error");
|
||||
throw new Error('Opus库未加载，Module对象不存在');
|
||||
}
|
||||
|
||||
// 记录Module对象结构以便调试
|
||||
log("Module对象结构: " + Object.keys(Module).join(", "));
|
||||
|
||||
// 尝试使用全局Module函数
|
||||
if (typeof Module._opus_decoder_get_size === 'function') {
|
||||
window.ModuleInstance = Module;
|
||||
log('Opus库加载成功（使用全局Module）', "success");
|
||||
updateScriptStatus('Opus库加载成功', 'success');
|
||||
|
||||
// 启用开始录音按钮
|
||||
startButton.disabled = false;
|
||||
statusLabel.textContent = "待机";
|
||||
|
||||
// 3秒后隐藏状态
|
||||
setTimeout(() => {
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
}, 3000);
|
||||
|
||||
window.opusLoaded = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
// 尝试使用Module.instance
|
||||
if (typeof Module.instance !== 'undefined' && typeof Module.instance._opus_decoder_get_size === 'function') {
|
||||
window.ModuleInstance = Module.instance;
|
||||
log('Opus库加载成功（使用Module.instance）', "success");
|
||||
updateScriptStatus('Opus库加载成功', 'success');
|
||||
|
||||
// 启用开始录音按钮
|
||||
startButton.disabled = false;
|
||||
statusLabel.textContent = "待机";
|
||||
|
||||
// 3秒后隐藏状态
|
||||
setTimeout(() => {
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
}, 3000);
|
||||
|
||||
window.opusLoaded = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
// 检查是否有其他导出方式
|
||||
log("Module上可用方法: " + Object.getOwnPropertyNames(Module).filter(prop => typeof Module[prop] === 'function').join(", "));
|
||||
|
||||
if (typeof Module.onRuntimeInitialized === 'function' || typeof Module.onRuntimeInitialized === 'undefined') {
|
||||
log("Module.onRuntimeInitialized 尚未执行，等待模块初始化完成...");
|
||||
// Module可能未完成初始化，注册回调
|
||||
Module.onRuntimeInitialized = function() {
|
||||
log("Opus库运行时初始化完成，重新检查");
|
||||
checkOpusLoaded();
|
||||
};
|
||||
return false;
|
||||
}
|
||||
|
||||
throw new Error('Opus解码函数未找到，可能Module结构不正确');
|
||||
} catch (err) {
|
||||
log(`Opus库加载失败: ${err.message}`, "error");
|
||||
updateScriptStatus(`Opus库加载失败: ${err.message}`, 'error');
|
||||
statusLabel.textContent = "错误：Opus库加载失败";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 页面加载完成后检查浏览器能力和Opus库
|
||||
window.addEventListener('load', function() {
|
||||
log("页面加载完成，开始初始化...");
|
||||
updateScriptStatus('正在初始化录音环境...', 'info');
|
||||
|
||||
// 延迟一小段时间再检查，确保库有时间加载
|
||||
setTimeout(function checkAndRetry() {
|
||||
if (!checkOpusLoaded()) {
|
||||
// 如果加载失败，5秒后重试
|
||||
log("Opus库加载失败，5秒后重试...");
|
||||
updateScriptStatus('Opus库加载失败，正在重试...', 'error');
|
||||
setTimeout(checkAndRetry, 5000);
|
||||
}
|
||||
}, 1000);
|
||||
});
|
||||
|
||||
// 防止按钮误操作
|
||||
document.getElementById("stop").disabled = true;
|
||||
document.getElementById("play").disabled = true;
|
||||
</script>
|
||||
<script src="./../libopus.js"></script>
|
||||
<script src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 185 B |
@@ -0,0 +1,999 @@
|
||||
/* ==================== 全局样式 ==================== */
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: 'PingFang SC', 'Microsoft YaHei', sans-serif;
|
||||
margin: 0;
|
||||
padding: 0px;
|
||||
background-image: url('bg.png');
|
||||
}
|
||||
|
||||
/* ==================== 容器布局 ==================== */
|
||||
.container {
|
||||
max-width: 1400px;
|
||||
margin: 0 auto;
|
||||
border-radius: 10px;
|
||||
padding: 0px;
|
||||
}
|
||||
|
||||
/* 两栏布局 */
|
||||
.two-column-layout {
|
||||
display: flex;
|
||||
gap: 20px;
|
||||
align-items: stretch;
|
||||
margin-top: 10px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.two-column-layout>.section {
|
||||
flex: 1;
|
||||
margin-top: 0;
|
||||
margin-bottom: 0;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
/* ==================== 通用区块样式 ==================== */
|
||||
.section {
|
||||
margin-top: 10px;
|
||||
margin-bottom: 10px;
|
||||
padding: 10px;
|
||||
border-radius: 15px;
|
||||
background-color: #fff;
|
||||
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.section h2 {
|
||||
margin: 0;
|
||||
color: #000;
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.02em;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
/* ==================== 通用按钮样式 ==================== */
|
||||
button {
|
||||
padding: 8px 15px;
|
||||
border: none;
|
||||
border-radius: 5px;
|
||||
background-color: #007aff;
|
||||
color: white;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.2s;
|
||||
}
|
||||
|
||||
button:disabled {
|
||||
background-color: #cccccc;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
/* 切换按钮和连接按钮 */
|
||||
.toggle-button,
|
||||
.connect-button {
|
||||
margin-left: auto;
|
||||
padding: 4px 12px;
|
||||
font-size: 14px;
|
||||
border-radius: 20px;
|
||||
cursor: pointer;
|
||||
height: 28px;
|
||||
line-height: 20px;
|
||||
}
|
||||
|
||||
.toggle-button {
|
||||
border: 1px solid #007aff;
|
||||
background-color: #fff;
|
||||
color: #007aff;
|
||||
}
|
||||
|
||||
.connect-button {
|
||||
box-shadow: 0px 0px 10px 2px rgba(0, 122, 255, 0.28);
|
||||
}
|
||||
|
||||
/* ==================== 设备配置面板 ==================== */
|
||||
.config-panel {
|
||||
display: block;
|
||||
transition: all 0.3s ease;
|
||||
}
|
||||
|
||||
/* MCP工具面板默认隐藏 */
|
||||
#mcpToolsPanel {
|
||||
display: none;
|
||||
}
|
||||
|
||||
#mcpToolsPanel.expanded {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.control-panel {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
/* 配置行 */
|
||||
.config-row {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
/* 配置项 */
|
||||
.config-item {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
min-width: 0;
|
||||
margin-bottom: 8px;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.config-item label {
|
||||
white-space: nowrap;
|
||||
min-width: 70px;
|
||||
text-align: left;
|
||||
color: #666;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.config-item input {
|
||||
flex: 1;
|
||||
width: 0;
|
||||
flex-grow: 1;
|
||||
padding: 6px;
|
||||
font-size: 12px;
|
||||
border: 1px solid #b1c9f8;
|
||||
border-radius: 5px;
|
||||
}
|
||||
|
||||
.config-item input:disabled {
|
||||
background-color: #fff;
|
||||
border-color: #fff;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
/* ==================== 连接信息面板 ==================== */
|
||||
.connection-controls {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
width: 100%;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.connection-controls input {
|
||||
width: 100%;
|
||||
padding: 8px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 5px;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
.connection-controls button {
|
||||
white-space: nowrap;
|
||||
padding: 8px 15px;
|
||||
}
|
||||
|
||||
#serverUrl,
|
||||
#otaUrl {
|
||||
flex-grow: 1;
|
||||
padding: 8px;
|
||||
border-radius: 5px;
|
||||
}
|
||||
|
||||
#otaUrl {
|
||||
border: 1px solid #b1c9f8;
|
||||
}
|
||||
|
||||
#serverUrl {
|
||||
border: 1px solid #fafafa;
|
||||
background-color: #fafafa;
|
||||
}
|
||||
|
||||
/* 连接状态 */
|
||||
.connection-status {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 20px;
|
||||
margin-left: 0px;
|
||||
padding: 0 15px;
|
||||
height: 28px;
|
||||
line-height: 28px;
|
||||
}
|
||||
|
||||
.connection-status span {
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.connection-status .status {
|
||||
background-color: #fef2f2;
|
||||
color: #b91c1c;
|
||||
font-size: 12px;
|
||||
padding: 0px 8px;
|
||||
border-radius: 20px;
|
||||
transition: all 0.3s ease;
|
||||
}
|
||||
|
||||
/* 已连接状态 */
|
||||
.connection-status .status.connected {
|
||||
background-color: #ecfdf3;
|
||||
color: #15803d;
|
||||
}
|
||||
|
||||
/* 会话状态 - 离线中 */
|
||||
.connection-status .status.offline {
|
||||
background-color: #fef2f2;
|
||||
color: #b91c1c;
|
||||
}
|
||||
|
||||
/* 会话状态 - 聆听中 */
|
||||
.connection-status .status.listening {
|
||||
background-color: #ecfdf3;
|
||||
color: #15803d;
|
||||
}
|
||||
|
||||
/* 会话状态 - 说话中 */
|
||||
.connection-status .status.speaking {
|
||||
background-color: #fff7ed;
|
||||
color: #c2410c;
|
||||
animation: pulse-speaking 1.5s infinite;
|
||||
}
|
||||
|
||||
@keyframes pulse-speaking {
|
||||
|
||||
0%,
|
||||
100% {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
50% {
|
||||
opacity: 0.7;
|
||||
}
|
||||
}
|
||||
|
||||
/* LLM状态emoji样式 */
|
||||
span.connection-status.llm-emoji {
|
||||
display: block !important;
|
||||
text-align: center !important;
|
||||
width: 100% !important;
|
||||
margin: 0 auto !important;
|
||||
}
|
||||
|
||||
.llm-emoji .status {
|
||||
font-size: 14px !important;
|
||||
padding: 8px 20px !important;
|
||||
line-height: 1.2 !important;
|
||||
}
|
||||
|
||||
.emoji-large {
|
||||
font-size: 2em !important;
|
||||
vertical-align: middle;
|
||||
display: inline-block;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
/* ==================== 标签页样式 ==================== */
|
||||
.tabs {
|
||||
display: flex;
|
||||
margin-bottom: 20px;
|
||||
border-bottom: 1px solid #e0e0e0;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.tab {
|
||||
padding: 10px 20px;
|
||||
cursor: pointer;
|
||||
border: none;
|
||||
background: none;
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
position: relative;
|
||||
transition: all 0.3s ease;
|
||||
}
|
||||
|
||||
.tab:hover {
|
||||
color: #007aff;
|
||||
}
|
||||
|
||||
.tab.active {
|
||||
color: #007aff;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.tab.active::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
bottom: -1px;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 1px;
|
||||
background-color: #007aff;
|
||||
}
|
||||
|
||||
.tab-content {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.tab-content.active {
|
||||
display: block;
|
||||
}
|
||||
|
||||
/* ==================== 文本消息输入 ==================== */
|
||||
.message-input {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
#messageInput {
|
||||
flex-grow: 1;
|
||||
padding: 8px 20px 8px 20px;
|
||||
border: 1px solid #b1c9f8;
|
||||
border-radius: 10px;
|
||||
}
|
||||
|
||||
#messageInput:disabled {
|
||||
border: 1px solid #fafafa;
|
||||
background-color: #fafafa;
|
||||
}
|
||||
|
||||
#sendTextButton,
|
||||
#recordButton {
|
||||
border-radius: 20px;
|
||||
}
|
||||
|
||||
/* ==================== 语音消息控制 ==================== */
|
||||
.audio-controls {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
gap: 10px;
|
||||
margin-top: 20px;
|
||||
}
|
||||
|
||||
.audio-visualizer {
|
||||
height: 60px;
|
||||
width: 100%;
|
||||
margin-top: 10px;
|
||||
margin-bottom: 15px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 5px;
|
||||
background-color: #fafafa;
|
||||
}
|
||||
|
||||
.record-button {
|
||||
background-color: #db4437;
|
||||
}
|
||||
|
||||
.record-button:hover {
|
||||
background-color: #c53929;
|
||||
}
|
||||
|
||||
.record-button.recording {
|
||||
animation: pulse 1.5s infinite;
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0% {
|
||||
background-color: #db4437;
|
||||
}
|
||||
|
||||
50% {
|
||||
background-color: #ff6659;
|
||||
}
|
||||
|
||||
100% {
|
||||
background-color: #db4437;
|
||||
}
|
||||
}
|
||||
|
||||
/* ==================== 会话记录和日志 ==================== */
|
||||
.flex-container {
|
||||
display: flex;
|
||||
margin-top: 20px;
|
||||
background-color: #f9fafb;
|
||||
}
|
||||
|
||||
/* 会话记录容器 */
|
||||
.conversation {
|
||||
max-height: 300px;
|
||||
overflow-y: auto;
|
||||
border: 1px solid #ddd;
|
||||
border-right: none;
|
||||
border-radius: 5px 0px 0px 5px;
|
||||
padding: 10px;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
/* 消息气泡 */
|
||||
.message {
|
||||
margin-bottom: 10px;
|
||||
padding: 8px 12px;
|
||||
border-radius: 8px;
|
||||
width: fit-content;
|
||||
max-width: 80%;
|
||||
}
|
||||
|
||||
.user {
|
||||
background-color: #fff;
|
||||
margin-left: auto;
|
||||
margin-right: 10px;
|
||||
text-align: right;
|
||||
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.server {
|
||||
background-color: #95ec69;
|
||||
margin-right: auto;
|
||||
margin-left: 10px;
|
||||
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
/* 日志容器 */
|
||||
#logContainer {
|
||||
margin-top: 0;
|
||||
padding: 10px;
|
||||
border-radius: 0px 5px 5px 0px;
|
||||
font-family: monospace;
|
||||
height: 300px;
|
||||
overflow-y: auto;
|
||||
flex: 1;
|
||||
border: 1px solid #ddd;
|
||||
}
|
||||
|
||||
.log-entry {
|
||||
margin: 5px 0;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.log-info {
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.log-error {
|
||||
color: #db4437;
|
||||
}
|
||||
|
||||
.log-success {
|
||||
color: #0f9d58;
|
||||
}
|
||||
|
||||
/* ==================== MCP 工具管理 ==================== */
|
||||
.mcp-tools-container {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.mcp-tool-card {
|
||||
background-color: white;
|
||||
border: 1px solid #e0e0e0;
|
||||
border-radius: 8px;
|
||||
padding: 15px;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.mcp-tool-card:hover {
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.mcp-tool-card.disabled {
|
||||
opacity: 0.6;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.mcp-tool-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.mcp-tool-name {
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.mcp-tool-actions {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.mcp-tool-description {
|
||||
color: #666;
|
||||
font-size: 13px;
|
||||
line-height: 1.5;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.mcp-tool-info {
|
||||
background-color: #f9f9f9;
|
||||
border: 1px solid #e0e0e0;
|
||||
border-radius: 4px;
|
||||
padding: 8px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.mcp-tool-info-row {
|
||||
display: flex;
|
||||
gap: 15px;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.mcp-tool-info-label {
|
||||
color: #999;
|
||||
min-width: 60px;
|
||||
}
|
||||
|
||||
.mcp-tool-info-value {
|
||||
color: #333;
|
||||
font-family: 'Courier New', monospace;
|
||||
}
|
||||
|
||||
/* MCP 属性项 */
|
||||
.mcp-property-item {
|
||||
background-color: white;
|
||||
border: 1px solid #e0e0e0;
|
||||
border-radius: 5px;
|
||||
padding: 12px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.mcp-property-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.mcp-property-name {
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.mcp-property-row {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 10px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.mcp-property-row-full {
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.mcp-small-label {
|
||||
display: block;
|
||||
margin-bottom: 4px;
|
||||
font-size: 12px;
|
||||
color: #666;
|
||||
}
|
||||
|
||||
.mcp-small-input {
|
||||
width: 100%;
|
||||
padding: 6px 8px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.mcp-checkbox-label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.mcp-error {
|
||||
background-color: #ffebee;
|
||||
color: #c62828;
|
||||
padding: 10px;
|
||||
border-radius: 5px;
|
||||
margin-bottom: 15px;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.mcp-badge {
|
||||
display: inline-block;
|
||||
padding: 2px 8px;
|
||||
border-radius: 12px;
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
margin-left: 8px;
|
||||
}
|
||||
|
||||
.mcp-badge-required {
|
||||
background-color: #ffebee;
|
||||
color: #c62828;
|
||||
}
|
||||
|
||||
.mcp-badge-optional {
|
||||
background-color: #e3f2fd;
|
||||
color: #1976d2;
|
||||
}
|
||||
|
||||
/* ==================== 脚本加载状态 ==================== */
|
||||
.script-status {
|
||||
display: inline-block;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 50%;
|
||||
margin-right: 5px;
|
||||
}
|
||||
|
||||
.script-loaded {
|
||||
background-color: #0f9d58;
|
||||
}
|
||||
|
||||
.script-loading {
|
||||
background-color: #f4b400;
|
||||
}
|
||||
|
||||
.script-error {
|
||||
background-color: #db4437;
|
||||
}
|
||||
|
||||
.script-list {
|
||||
margin: 10px 0;
|
||||
padding: 10px;
|
||||
background-color: #f9f9f9;
|
||||
border-radius: 5px;
|
||||
font-family: monospace;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
#scriptStatus.success {
|
||||
background-color: #e6f4ea;
|
||||
color: #0f9d58;
|
||||
border-left: 4px solid #0f9d58;
|
||||
}
|
||||
|
||||
#scriptStatus.error {
|
||||
background-color: #fce8e6;
|
||||
color: #db4437;
|
||||
border-left: 4px solid #db4437;
|
||||
}
|
||||
|
||||
#scriptStatus.warning {
|
||||
background-color: #fef7e0;
|
||||
color: #f4b400;
|
||||
border-left: 4px solid #f4b400;
|
||||
}
|
||||
|
||||
/* ==================== File协议警告 ==================== */
|
||||
#fileProtocolWarning {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background-color: rgba(0, 0, 0, 0.8);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 9999;
|
||||
color: white;
|
||||
padding: 20px;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
#fileProtocolWarning h2 {
|
||||
color: #ff4d4d;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
#fileProtocolWarning pre {
|
||||
background-color: green;
|
||||
font-size: 18px;
|
||||
padding: 15px;
|
||||
border-radius: 5px;
|
||||
font-family: monospace;
|
||||
overflow-x: auto;
|
||||
margin: 15px 0;
|
||||
}
|
||||
|
||||
#fileProtocolWarning button {
|
||||
background-color: #4CAF50;
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 10px 20px;
|
||||
text-align: center;
|
||||
text-decoration: none;
|
||||
display: inline-block;
|
||||
font-size: 16px;
|
||||
margin: 10px 2px;
|
||||
cursor: pointer;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
#fileProtocolWarning button:hover {
|
||||
background-color: #45a049;
|
||||
}
|
||||
|
||||
/* ==================== 响应式布局 ==================== */
|
||||
|
||||
/* 大屏幕 (宽度 > 1200px) */
|
||||
@media (min-width: 1201px) {
|
||||
.container {
|
||||
max-width: 80%;
|
||||
}
|
||||
}
|
||||
|
||||
/* 中等屏幕 (宽度 <= 1200px) */
|
||||
@media (max-width: 1200px) {
|
||||
.container {
|
||||
max-width: 95%;
|
||||
padding: 10px;
|
||||
}
|
||||
}
|
||||
|
||||
/* 平板设备 (宽度 <= 1060px) */
|
||||
@media (max-width: 1060px) {
|
||||
|
||||
/* 两栏布局改为单栏 */
|
||||
.two-column-layout {
|
||||
flex-direction: column;
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
.two-column-layout>.section {
|
||||
width: 100%;
|
||||
margin-top: 10px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
/* 连接状态标签调整 */
|
||||
.connection-status {
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-left: 0px;
|
||||
padding: 0 8px;
|
||||
}
|
||||
|
||||
/* 会话记录和日志容器改为上下布局 */
|
||||
.flex-container {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.conversation,
|
||||
#logContainer {
|
||||
border-radius: 5px;
|
||||
border: 1px solid #ddd;
|
||||
max-height: 200px;
|
||||
}
|
||||
|
||||
.conversation {
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
/* MCP 属性行改为单列 */
|
||||
.mcp-property-row {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
|
||||
/* 移动设备 (宽度 <= 768px) */
|
||||
@media (max-width: 768px) {
|
||||
body {
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.container {
|
||||
max-width: 100%;
|
||||
padding: 5px;
|
||||
}
|
||||
|
||||
/* 调整区块内边距 */
|
||||
.section {
|
||||
padding: 8px;
|
||||
margin-top: 8px;
|
||||
margin-bottom: 8px;
|
||||
border-radius: 10px;
|
||||
}
|
||||
|
||||
/* 调整标题字体 */
|
||||
.section h2 {
|
||||
font-size: 13px;
|
||||
flex-wrap: wrap;
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
/* 配置行改为单列 */
|
||||
.config-row {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
/* 按钮调整 */
|
||||
.toggle-button,
|
||||
.connect-button {
|
||||
padding: 3px 10px;
|
||||
font-size: 12px;
|
||||
height: 24px;
|
||||
line-height: 18px;
|
||||
}
|
||||
|
||||
button {
|
||||
padding: 6px 12px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
/* 输入框 */
|
||||
#serverUrl,
|
||||
#otaUrl,
|
||||
#messageInput {
|
||||
font-size: 14px;
|
||||
padding: 6px;
|
||||
}
|
||||
|
||||
/* 配置项标签 */
|
||||
.config-item label {
|
||||
min-width: 60px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.config-item input {
|
||||
font-size: 13px;
|
||||
padding: 5px;
|
||||
}
|
||||
|
||||
/* 标签页 */
|
||||
.tab {
|
||||
padding: 8px 12px;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
/* 消息气泡 */
|
||||
.message {
|
||||
max-width: 90%;
|
||||
font-size: 14px;
|
||||
padding: 6px 10px;
|
||||
}
|
||||
|
||||
/* 日志容器 */
|
||||
#logContainer {
|
||||
font-size: 11px;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
.log-entry {
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
/* 连接状态 */
|
||||
.connection-status {
|
||||
margin-left: 0;
|
||||
padding: 5px 0;
|
||||
height: auto;
|
||||
line-height: normal;
|
||||
}
|
||||
|
||||
.connection-status .status {
|
||||
font-size: 11px;
|
||||
padding: 0px 6px;
|
||||
}
|
||||
|
||||
/* 音频可视化 */
|
||||
.audio-visualizer {
|
||||
height: 50px;
|
||||
}
|
||||
|
||||
/* MCP 工具卡片 */
|
||||
.mcp-tool-card {
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.mcp-tool-name {
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.mcp-tool-description {
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.mcp-tool-info {
|
||||
font-size: 11px;
|
||||
padding: 6px;
|
||||
}
|
||||
|
||||
.mcp-tool-info-row {
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.mcp-tool-info-label {
|
||||
min-width: auto;
|
||||
}
|
||||
|
||||
/* 消息输入框 */
|
||||
.message-input {
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
/* 会话记录高度调整 */
|
||||
.conversation,
|
||||
#logContainer {
|
||||
max-height: 180px;
|
||||
}
|
||||
}
|
||||
|
||||
/* 小型移动设备 (宽度 <= 480px) */
|
||||
@media (max-width: 480px) {
|
||||
.container {
|
||||
padding: 3px;
|
||||
}
|
||||
|
||||
.section {
|
||||
padding: 6px;
|
||||
margin-top: 5px;
|
||||
margin-bottom: 5px;
|
||||
}
|
||||
|
||||
.section h2 {
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
/* 更紧凑的按钮 */
|
||||
.toggle-button,
|
||||
.connect-button {
|
||||
padding: 2px 8px;
|
||||
font-size: 11px;
|
||||
height: 22px;
|
||||
line-height: 16px;
|
||||
}
|
||||
|
||||
button {
|
||||
padding: 5px 10px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.config-item label {
|
||||
min-width: 50px;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.config-item input {
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
/* 标签页 */
|
||||
.tab {
|
||||
padding: 6px 10px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
/* 消息气泡 */
|
||||
.message {
|
||||
max-width: 95%;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
/* 会话记录 */
|
||||
.conversation,
|
||||
#logContainer {
|
||||
max-height: 150px;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
/* 连接状态文字更小 */
|
||||
.connection-status .status {
|
||||
font-size: 10px;
|
||||
}
|
||||
|
||||
/* MCP 工具 */
|
||||
.mcp-tool-name {
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.mcp-tool-description {
|
||||
font-size: 11px;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// 主应用入口
|
||||
import { log } from './utils/logger.js';
|
||||
import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js';
|
||||
import { getUIController } from './ui/controller.js';
|
||||
import { getAudioPlayer } from './core/audio/player.js';
|
||||
import { initMcpTools } from './core/mcp/tools.js';
|
||||
|
||||
// 应用类
|
||||
class App {
|
||||
constructor() {
|
||||
this.uiController = null;
|
||||
this.audioPlayer = null;
|
||||
}
|
||||
|
||||
// 初始化应用
|
||||
async init() {
|
||||
log('正在初始化应用...', 'info');
|
||||
|
||||
// 初始化UI控制器
|
||||
this.uiController = getUIController();
|
||||
this.uiController.init();
|
||||
|
||||
// 检查Opus库
|
||||
checkOpusLoaded();
|
||||
|
||||
// 初始化Opus编码器
|
||||
initOpusEncoder();
|
||||
|
||||
// 初始化音频播放器
|
||||
this.audioPlayer = getAudioPlayer();
|
||||
await this.audioPlayer.start();
|
||||
|
||||
// 初始化MCP工具
|
||||
initMcpTools();
|
||||
|
||||
log('应用初始化完成', 'success');
|
||||
}
|
||||
}
|
||||
|
||||
// 创建并启动应用
|
||||
const app = new App();
|
||||
|
||||
// DOM加载完成后初始化
|
||||
if (document.readyState === 'loading') {
|
||||
document.addEventListener('DOMContentLoaded', () => app.init());
|
||||
} else {
|
||||
app.init();
|
||||
}
|
||||
|
||||
export default app;
|
||||
@@ -0,0 +1,72 @@
|
||||
[
|
||||
{
|
||||
"name": "self.get_device_status",
|
||||
"description": "Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\nUse this tool for: \n1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
},
|
||||
"mockResponse": {
|
||||
"audio_speaker": {
|
||||
"volume": 50,
|
||||
"muted": false
|
||||
},
|
||||
"screen": {
|
||||
"brightness": 80,
|
||||
"theme": "light"
|
||||
},
|
||||
"battery": {
|
||||
"level": 85,
|
||||
"charging": false
|
||||
},
|
||||
"network": {
|
||||
"connected": true,
|
||||
"type": "wifi"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "self.audio_speaker.set_volume",
|
||||
"description": "Set the volume of the audio speaker. If the current volume is unknown, you must call `self.get_device_status` tool first and then call this tool.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"volume": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 100
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"volume"
|
||||
]
|
||||
},
|
||||
"mockResponse": {
|
||||
"success": true,
|
||||
"volume": "${volume}",
|
||||
"message": "音量已设置为 ${volume}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "self.screen.set_brightness",
|
||||
"description": "Set the brightness of the screen.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"brightness": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 100
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"brightness"
|
||||
]
|
||||
},
|
||||
"mockResponse": {
|
||||
"success": true,
|
||||
"brightness": "${brightness}",
|
||||
"message": "亮度已设置为 ${brightness}"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,85 @@
|
||||
// 配置管理模块
|
||||
|
||||
// 生成随机MAC地址
|
||||
function generateRandomMac() {
|
||||
const hexDigits = '0123456789ABCDEF';
|
||||
let mac = '';
|
||||
for (let i = 0; i < 6; i++) {
|
||||
if (i > 0) mac += ':';
|
||||
for (let j = 0; j < 2; j++) {
|
||||
mac += hexDigits.charAt(Math.floor(Math.random() * 16));
|
||||
}
|
||||
}
|
||||
return mac;
|
||||
}
|
||||
|
||||
// 加载配置
|
||||
export function loadConfig() {
|
||||
const deviceMacInput = document.getElementById('deviceMac');
|
||||
const deviceNameInput = document.getElementById('deviceName');
|
||||
const clientIdInput = document.getElementById('clientId');
|
||||
const tokenInput = document.getElementById('token');
|
||||
const otaUrlInput = document.getElementById('otaUrl');
|
||||
|
||||
// 从localStorage加载MAC地址，如果没有则生成新的
|
||||
let savedMac = localStorage.getItem('xz_tester_deviceMac');
|
||||
if (!savedMac) {
|
||||
savedMac = generateRandomMac();
|
||||
localStorage.setItem('xz_tester_deviceMac', savedMac);
|
||||
}
|
||||
deviceMacInput.value = savedMac;
|
||||
|
||||
// 从localStorage加载其他配置
|
||||
const savedDeviceName = localStorage.getItem('xz_tester_deviceName');
|
||||
if (savedDeviceName) {
|
||||
deviceNameInput.value = savedDeviceName;
|
||||
}
|
||||
|
||||
const savedClientId = localStorage.getItem('xz_tester_clientId');
|
||||
if (savedClientId) {
|
||||
clientIdInput.value = savedClientId;
|
||||
}
|
||||
|
||||
const savedToken = localStorage.getItem('xz_tester_token');
|
||||
if (savedToken) {
|
||||
tokenInput.value = savedToken;
|
||||
}
|
||||
|
||||
const savedOtaUrl = localStorage.getItem('xz_tester_otaUrl');
|
||||
if (savedOtaUrl) {
|
||||
otaUrlInput.value = savedOtaUrl;
|
||||
}
|
||||
}
|
||||
|
||||
// 保存配置
|
||||
export function saveConfig() {
|
||||
const deviceMacInput = document.getElementById('deviceMac');
|
||||
const deviceNameInput = document.getElementById('deviceName');
|
||||
const clientIdInput = document.getElementById('clientId');
|
||||
const tokenInput = document.getElementById('token');
|
||||
|
||||
localStorage.setItem('xz_tester_deviceMac', deviceMacInput.value);
|
||||
localStorage.setItem('xz_tester_deviceName', deviceNameInput.value);
|
||||
localStorage.setItem('xz_tester_clientId', clientIdInput.value);
|
||||
localStorage.setItem('xz_tester_token', tokenInput.value);
|
||||
}
|
||||
|
||||
// 获取配置值
|
||||
export function getConfig() {
|
||||
const deviceMac = document.getElementById('deviceMac').value.trim();
|
||||
return {
|
||||
deviceId: deviceMac, // 使用MAC地址作为deviceId
|
||||
deviceName: document.getElementById('deviceName').value.trim(),
|
||||
deviceMac: deviceMac,
|
||||
clientId: document.getElementById('clientId').value.trim(),
|
||||
token: document.getElementById('token').value.trim()
|
||||
};
|
||||
}
|
||||
|
||||
// 保存连接URL
|
||||
export function saveConnectionUrls() {
|
||||
const otaUrl = document.getElementById('otaUrl').value.trim();
|
||||
const wsUrl = document.getElementById('serverUrl').value.trim();
|
||||
localStorage.setItem('xz_tester_otaUrl', otaUrl);
|
||||
localStorage.setItem('xz_tester_wsUrl', wsUrl);
|
||||
}
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
import { log } from './utils/logger.js';
|
||||
import { updateScriptStatus } from './document.js'
|
||||
import { log } from '../../utils/logger.js';
|
||||
import { updateScriptStatus } from '../../ui/dom-helper.js'
|
||||
|
||||
|
||||
// 检查Opus库是否已加载
|
||||
@@ -0,0 +1,297 @@
|
||||
// 音频播放模块
|
||||
import { log } from '../../utils/logger.js';
|
||||
import BlockingQueue from '../../utils/blocking-queue.js';
|
||||
import { createStreamingContext } from './stream-context.js';
|
||||
|
||||
// 音频播放器类
|
||||
export class AudioPlayer {
|
||||
constructor() {
|
||||
// 音频参数
|
||||
this.SAMPLE_RATE = 16000;
|
||||
this.CHANNELS = 1;
|
||||
this.FRAME_SIZE = 960;
|
||||
this.MIN_AUDIO_DURATION = 0.12;
|
||||
|
||||
// 状态
|
||||
this.audioContext = null;
|
||||
this.opusDecoder = null;
|
||||
this.streamingContext = null;
|
||||
this.queue = new BlockingQueue();
|
||||
this.isPlaying = false;
|
||||
}
|
||||
|
||||
// 获取或创建AudioContext
|
||||
getAudioContext() {
|
||||
if (!this.audioContext) {
|
||||
this.audioContext = new (window.AudioContext || window.webkitAudioContext)({
|
||||
sampleRate: this.SAMPLE_RATE,
|
||||
latencyHint: 'interactive'
|
||||
});
|
||||
log('创建音频上下文，采样率: ' + this.SAMPLE_RATE + 'Hz', 'debug');
|
||||
}
|
||||
return this.audioContext;
|
||||
}
|
||||
|
||||
// 初始化Opus解码器
|
||||
async initOpusDecoder() {
|
||||
if (this.opusDecoder) return this.opusDecoder;
|
||||
|
||||
try {
|
||||
if (typeof window.ModuleInstance === 'undefined') {
|
||||
if (typeof Module !== 'undefined') {
|
||||
window.ModuleInstance = Module;
|
||||
log('使用全局Module作为ModuleInstance', 'info');
|
||||
} else {
|
||||
throw new Error('Opus库未加载，ModuleInstance和Module对象都不存在');
|
||||
}
|
||||
}
|
||||
|
||||
const mod = window.ModuleInstance;
|
||||
|
||||
this.opusDecoder = {
|
||||
channels: this.CHANNELS,
|
||||
rate: this.SAMPLE_RATE,
|
||||
frameSize: this.FRAME_SIZE,
|
||||
module: mod,
|
||||
decoderPtr: null,
|
||||
|
||||
init: function () {
|
||||
if (this.decoderPtr) return true;
|
||||
|
||||
const decoderSize = mod._opus_decoder_get_size(this.channels);
|
||||
log(`Opus解码器大小: ${decoderSize}字节`, 'debug');
|
||||
|
||||
this.decoderPtr = mod._malloc(decoderSize);
|
||||
if (!this.decoderPtr) {
|
||||
throw new Error("无法分配解码器内存");
|
||||
}
|
||||
|
||||
const err = mod._opus_decoder_init(
|
||||
this.decoderPtr,
|
||||
this.rate,
|
||||
this.channels
|
||||
);
|
||||
|
||||
if (err < 0) {
|
||||
this.destroy();
|
||||
throw new Error(`Opus解码器初始化失败: ${err}`);
|
||||
}
|
||||
|
||||
log("Opus解码器初始化成功", 'success');
|
||||
return true;
|
||||
},
|
||||
|
||||
decode: function (opusData) {
|
||||
if (!this.decoderPtr) {
|
||||
if (!this.init()) {
|
||||
throw new Error("解码器未初始化且无法初始化");
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const mod = this.module;
|
||||
|
||||
const opusPtr = mod._malloc(opusData.length);
|
||||
mod.HEAPU8.set(opusData, opusPtr);
|
||||
|
||||
const pcmPtr = mod._malloc(this.frameSize * 2);
|
||||
|
||||
const decodedSamples = mod._opus_decode(
|
||||
this.decoderPtr,
|
||||
opusPtr,
|
||||
opusData.length,
|
||||
pcmPtr,
|
||||
this.frameSize,
|
||||
0
|
||||
);
|
||||
|
||||
if (decodedSamples < 0) {
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
throw new Error(`Opus解码失败: ${decodedSamples}`);
|
||||
}
|
||||
|
||||
const decodedData = new Int16Array(decodedSamples);
|
||||
for (let i = 0; i < decodedSamples; i++) {
|
||||
decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i];
|
||||
}
|
||||
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
|
||||
return decodedData;
|
||||
} catch (error) {
|
||||
log(`Opus解码错误: ${error.message}`, 'error');
|
||||
return new Int16Array(0);
|
||||
}
|
||||
},
|
||||
|
||||
destroy: function () {
|
||||
if (this.decoderPtr) {
|
||||
this.module._free(this.decoderPtr);
|
||||
this.decoderPtr = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (!this.opusDecoder.init()) {
|
||||
throw new Error("Opus解码器初始化失败");
|
||||
}
|
||||
|
||||
return this.opusDecoder;
|
||||
|
||||
} catch (error) {
|
||||
log(`Opus解码器初始化失败: ${error.message}`, 'error');
|
||||
this.opusDecoder = null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 启动音频缓冲
|
||||
async startAudioBuffering() {
|
||||
log("开始音频缓冲...", 'info');
|
||||
|
||||
this.initOpusDecoder().catch(error => {
|
||||
log(`预初始化Opus解码器失败: ${error.message}`, 'warning');
|
||||
});
|
||||
|
||||
const timeout = 400;
|
||||
while (true) {
|
||||
const packets = await this.queue.dequeue(
|
||||
6,
|
||||
timeout,
|
||||
(count) => {
|
||||
log(`缓冲超时，当前缓冲包数: ${count}，开始播放`, 'info');
|
||||
}
|
||||
);
|
||||
if (packets.length) {
|
||||
log(`已缓冲 ${packets.length} 个音频包，开始播放`, 'info');
|
||||
this.streamingContext.pushAudioBuffer(packets);
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const data = await this.queue.dequeue(99, 30);
|
||||
if (data.length) {
|
||||
this.streamingContext.pushAudioBuffer(data);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 播放已缓冲的音频
|
||||
async playBufferedAudio() {
|
||||
try {
|
||||
this.audioContext = this.getAudioContext();
|
||||
|
||||
if (!this.opusDecoder) {
|
||||
log('初始化Opus解码器...', 'info');
|
||||
try {
|
||||
this.opusDecoder = await this.initOpusDecoder();
|
||||
if (!this.opusDecoder) {
|
||||
throw new Error('解码器初始化失败');
|
||||
}
|
||||
log('Opus解码器初始化成功', 'success');
|
||||
} catch (error) {
|
||||
log('Opus解码器初始化失败: ' + error.message, 'error');
|
||||
this.isPlaying = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.streamingContext) {
|
||||
this.streamingContext = createStreamingContext(
|
||||
this.opusDecoder,
|
||||
this.audioContext,
|
||||
this.SAMPLE_RATE,
|
||||
this.CHANNELS,
|
||||
this.MIN_AUDIO_DURATION
|
||||
);
|
||||
}
|
||||
|
||||
this.streamingContext.decodeOpusFrames();
|
||||
this.streamingContext.startPlaying();
|
||||
|
||||
} catch (error) {
|
||||
log(`播放已缓冲的音频出错: ${error.message}`, 'error');
|
||||
this.isPlaying = false;
|
||||
this.streamingContext = null;
|
||||
}
|
||||
}
|
||||
|
||||
// 添加音频数据到队列
|
||||
enqueueAudioData(opusData) {
|
||||
if (opusData.length > 0) {
|
||||
this.queue.enqueue(opusData);
|
||||
} else {
|
||||
log('收到空音频数据帧，可能是结束标志', 'warning');
|
||||
if (this.isPlaying && this.streamingContext) {
|
||||
this.streamingContext.endOfStream = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 预加载解码器
|
||||
async preload() {
|
||||
log('预加载Opus解码器...', 'info');
|
||||
try {
|
||||
await this.initOpusDecoder();
|
||||
log('Opus解码器预加载成功', 'success');
|
||||
} catch (error) {
|
||||
log(`Opus解码器预加载失败: ${error.message}，将在需要时重试`, 'warning');
|
||||
}
|
||||
}
|
||||
|
||||
// 启动播放系统
|
||||
async start() {
|
||||
await this.preload();
|
||||
this.playBufferedAudio();
|
||||
this.startAudioBuffering();
|
||||
}
|
||||
|
||||
// 获取音频包统计信息
|
||||
getAudioStats() {
|
||||
if (!this.streamingContext) {
|
||||
return {
|
||||
pendingDecode: 0,
|
||||
pendingPlay: 0,
|
||||
totalPending: 0
|
||||
};
|
||||
}
|
||||
|
||||
const pendingDecode = this.streamingContext.getPendingDecodeCount();
|
||||
const pendingPlay = this.streamingContext.getPendingPlayCount();
|
||||
|
||||
return {
|
||||
pendingDecode, // 待解码包数
|
||||
pendingPlay, // 待播放包数
|
||||
totalPending: pendingDecode + pendingPlay // 总待处理包数
|
||||
};
|
||||
}
|
||||
|
||||
// 清空所有音频缓冲并停止播放
|
||||
clearAllAudio() {
|
||||
log('AudioPlayer: 清空所有音频', 'info');
|
||||
|
||||
// 清空接收队列（使用clear方法保持对象引用）
|
||||
this.queue.clear();
|
||||
|
||||
// 清空流上下文的所有缓冲
|
||||
if (this.streamingContext) {
|
||||
this.streamingContext.clearAllBuffers();
|
||||
}
|
||||
|
||||
log('AudioPlayer: 音频已清空', 'success');
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let audioPlayerInstance = null;
|
||||
|
||||
export function getAudioPlayer() {
|
||||
if (!audioPlayerInstance) {
|
||||
audioPlayerInstance = new AudioPlayer();
|
||||
}
|
||||
return audioPlayerInstance;
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
// 音频录制模块
|
||||
import { log } from '../../utils/logger.js';
|
||||
import { initOpusEncoder } from './opus-codec.js';
|
||||
import { getAudioPlayer } from './player.js';
|
||||
|
||||
// 音频录制器类
|
||||
export class AudioRecorder {
|
||||
constructor() {
|
||||
this.isRecording = false;
|
||||
this.audioContext = null;
|
||||
this.analyser = null;
|
||||
this.audioProcessor = null;
|
||||
this.audioProcessorType = null;
|
||||
this.audioSource = null;
|
||||
this.opusEncoder = null;
|
||||
this.pcmDataBuffer = new Int16Array();
|
||||
this.audioBuffers = [];
|
||||
this.totalAudioSize = 0;
|
||||
this.visualizationRequest = null;
|
||||
this.recordingTimer = null;
|
||||
this.websocket = null;
|
||||
|
||||
// 回调函数
|
||||
this.onRecordingStart = null;
|
||||
this.onRecordingStop = null;
|
||||
this.onVisualizerUpdate = null;
|
||||
}
|
||||
|
||||
// 设置WebSocket实例
|
||||
setWebSocket(ws) {
|
||||
this.websocket = ws;
|
||||
}
|
||||
|
||||
// 获取AudioContext实例
|
||||
getAudioContext() {
|
||||
const audioPlayer = getAudioPlayer();
|
||||
return audioPlayer.getAudioContext();
|
||||
}
|
||||
|
||||
// 初始化编码器
|
||||
initEncoder() {
|
||||
if (!this.opusEncoder) {
|
||||
this.opusEncoder = initOpusEncoder();
|
||||
}
|
||||
return this.opusEncoder;
|
||||
}
|
||||
|
||||
// PCM处理器代码
|
||||
getAudioProcessorCode() {
|
||||
return `
|
||||
class AudioRecorderProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.buffers = [];
|
||||
this.frameSize = 960;
|
||||
this.buffer = new Int16Array(this.frameSize);
|
||||
this.bufferIndex = 0;
|
||||
this.isRecording = false;
|
||||
|
||||
this.port.onmessage = (event) => {
|
||||
if (event.data.command === 'start') {
|
||||
this.isRecording = true;
|
||||
this.port.postMessage({ type: 'status', status: 'started' });
|
||||
} else if (event.data.command === 'stop') {
|
||||
this.isRecording = false;
|
||||
|
||||
if (this.bufferIndex > 0) {
|
||||
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
|
||||
this.port.postMessage({
|
||||
type: 'buffer',
|
||||
buffer: finalBuffer
|
||||
});
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
|
||||
this.port.postMessage({ type: 'status', status: 'stopped' });
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
process(inputs, outputs, parameters) {
|
||||
if (!this.isRecording) return true;
|
||||
|
||||
const input = inputs[0][0];
|
||||
if (!input) return true;
|
||||
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
if (this.bufferIndex >= this.frameSize) {
|
||||
this.port.postMessage({
|
||||
type: 'buffer',
|
||||
buffer: this.buffer.slice(0)
|
||||
});
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
|
||||
this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
|
||||
`;
|
||||
}
|
||||
|
||||
// 创建音频处理器
|
||||
async createAudioProcessor() {
|
||||
this.audioContext = this.getAudioContext();
|
||||
|
||||
try {
|
||||
if (this.audioContext.audioWorklet) {
|
||||
const blob = new Blob([this.getAudioProcessorCode()], { type: 'application/javascript' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
await this.audioContext.audioWorklet.addModule(url);
|
||||
URL.revokeObjectURL(url);
|
||||
|
||||
const audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-recorder-processor');
|
||||
|
||||
audioProcessor.port.onmessage = (event) => {
|
||||
if (event.data.type === 'buffer') {
|
||||
this.processPCMBuffer(event.data.buffer);
|
||||
}
|
||||
};
|
||||
|
||||
log('使用AudioWorklet处理音频', 'success');
|
||||
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
audioProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
return { node: audioProcessor, type: 'worklet' };
|
||||
} else {
|
||||
log('AudioWorklet不可用，使用ScriptProcessorNode作为回退方案', 'warning');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
} catch (error) {
|
||||
log(`创建音频处理器失败: ${error.message}，尝试回退方案`, 'error');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
}
|
||||
|
||||
// 创建ScriptProcessor作为回退
|
||||
createScriptProcessor() {
|
||||
try {
|
||||
const frameSize = 4096;
|
||||
const scriptProcessor = this.audioContext.createScriptProcessor(frameSize, 1, 1);
|
||||
|
||||
scriptProcessor.onaudioprocess = (event) => {
|
||||
if (!this.isRecording) return;
|
||||
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
const buffer = new Int16Array(input.length);
|
||||
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
|
||||
this.processPCMBuffer(buffer);
|
||||
};
|
||||
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
scriptProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
|
||||
log('使用ScriptProcessorNode作为回退方案成功', 'warning');
|
||||
return { node: scriptProcessor, type: 'processor' };
|
||||
} catch (fallbackError) {
|
||||
log(`回退方案也失败: ${fallbackError.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// 处理PCM缓冲数据
|
||||
processPCMBuffer(buffer) {
|
||||
if (!this.isRecording) return;
|
||||
|
||||
const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length);
|
||||
newBuffer.set(this.pcmDataBuffer);
|
||||
newBuffer.set(buffer, this.pcmDataBuffer.length);
|
||||
this.pcmDataBuffer = newBuffer;
|
||||
|
||||
const samplesPerFrame = 960;
|
||||
|
||||
while (this.pcmDataBuffer.length >= samplesPerFrame) {
|
||||
const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame);
|
||||
this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame);
|
||||
|
||||
this.encodeAndSendOpus(frameData);
|
||||
}
|
||||
}
|
||||
|
||||
// 编码并发送Opus数据
|
||||
encodeAndSendOpus(pcmData = null) {
|
||||
if (!this.opusEncoder) {
|
||||
log('Opus编码器未初始化', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (pcmData) {
|
||||
const opusData = this.opusEncoder.encode(pcmData);
|
||||
|
||||
if (opusData && opusData.length > 0) {
|
||||
this.audioBuffers.push(opusData.buffer);
|
||||
this.totalAudioSize += opusData.length;
|
||||
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
this.websocket.send(opusData.buffer);
|
||||
log(`发送Opus帧，大小：${opusData.length}字节`, 'debug');
|
||||
} catch (error) {
|
||||
log(`WebSocket发送错误: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log('Opus编码失败，无有效数据返回', 'error');
|
||||
}
|
||||
} else {
|
||||
if (this.pcmDataBuffer.length > 0) {
|
||||
const samplesPerFrame = 960;
|
||||
if (this.pcmDataBuffer.length < samplesPerFrame) {
|
||||
const paddedBuffer = new Int16Array(samplesPerFrame);
|
||||
paddedBuffer.set(this.pcmDataBuffer);
|
||||
this.encodeAndSendOpus(paddedBuffer);
|
||||
} else {
|
||||
this.encodeAndSendOpus(this.pcmDataBuffer.slice(0, samplesPerFrame));
|
||||
}
|
||||
this.pcmDataBuffer = new Int16Array(0);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log(`Opus编码错误: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 开始录音
|
||||
async start() {
|
||||
if (this.isRecording) return false;
|
||||
|
||||
try {
|
||||
// 检查是否有WebSocketHandler实例
|
||||
const { getWebSocketHandler } = await import('../network/websocket.js');
|
||||
const wsHandler = getWebSocketHandler();
|
||||
|
||||
// 如果机器正在说话，发送打断消息
|
||||
if (wsHandler && wsHandler.isRemoteSpeaking && wsHandler.currentSessionId) {
|
||||
const abortMessage = {
|
||||
session_id: wsHandler.currentSessionId,
|
||||
type: 'abort',
|
||||
reason: 'wake_word_detected'
|
||||
};
|
||||
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
this.websocket.send(JSON.stringify(abortMessage));
|
||||
log('发送打断消息', 'info');
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.initEncoder()) {
|
||||
log('无法启动录音: Opus编码器初始化失败', 'error');
|
||||
return false;
|
||||
}
|
||||
|
||||
log('请至少录制1-2秒钟的音频，确保采集到足够数据', 'info');
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
sampleRate: 16000,
|
||||
channelCount: 1
|
||||
}
|
||||
});
|
||||
|
||||
this.audioContext = this.getAudioContext();
|
||||
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume();
|
||||
}
|
||||
|
||||
const processorResult = await this.createAudioProcessor();
|
||||
if (!processorResult) {
|
||||
log('无法创建音频处理器', 'error');
|
||||
return false;
|
||||
}
|
||||
|
||||
this.audioProcessor = processorResult.node;
|
||||
this.audioProcessorType = processorResult.type;
|
||||
|
||||
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 2048;
|
||||
|
||||
this.audioSource.connect(this.analyser);
|
||||
this.audioSource.connect(this.audioProcessor);
|
||||
|
||||
this.pcmDataBuffer = new Int16Array();
|
||||
this.audioBuffers = [];
|
||||
this.totalAudioSize = 0;
|
||||
this.isRecording = true;
|
||||
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'start' });
|
||||
}
|
||||
|
||||
// 发送监听开始消息
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
const listenMessage = {
|
||||
type: 'listen',
|
||||
mode: 'manual',
|
||||
state: 'start'
|
||||
};
|
||||
|
||||
log(`发送录音开始消息: ${JSON.stringify(listenMessage)}`, 'info');
|
||||
this.websocket.send(JSON.stringify(listenMessage));
|
||||
} else {
|
||||
log('WebSocket未连接，无法发送开始消息', 'error');
|
||||
return false;
|
||||
}
|
||||
|
||||
// 开始可视化
|
||||
if (this.onVisualizerUpdate) {
|
||||
const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
|
||||
this.startVisualization(dataArray);
|
||||
}
|
||||
|
||||
// 启动录音计时器
|
||||
let recordingSeconds = 0;
|
||||
this.recordingTimer = setInterval(() => {
|
||||
recordingSeconds += 0.1;
|
||||
if (this.onRecordingStart) {
|
||||
this.onRecordingStart(recordingSeconds);
|
||||
}
|
||||
}, 100);
|
||||
|
||||
log('开始PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音启动错误: ${error.message}`, 'error');
|
||||
this.isRecording = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 开始可视化
|
||||
startVisualization(dataArray) {
|
||||
const draw = () => {
|
||||
this.visualizationRequest = requestAnimationFrame(() => draw());
|
||||
|
||||
if (!this.isRecording) return;
|
||||
|
||||
this.analyser.getByteFrequencyData(dataArray);
|
||||
|
||||
if (this.onVisualizerUpdate) {
|
||||
this.onVisualizerUpdate(dataArray);
|
||||
}
|
||||
};
|
||||
draw();
|
||||
}
|
||||
|
||||
// 停止录音
|
||||
stop() {
|
||||
if (!this.isRecording) return false;
|
||||
|
||||
try {
|
||||
this.isRecording = false;
|
||||
|
||||
if (this.audioProcessor) {
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'stop' });
|
||||
}
|
||||
|
||||
this.audioProcessor.disconnect();
|
||||
this.audioProcessor = null;
|
||||
}
|
||||
|
||||
if (this.audioSource) {
|
||||
this.audioSource.disconnect();
|
||||
this.audioSource = null;
|
||||
}
|
||||
|
||||
if (this.visualizationRequest) {
|
||||
cancelAnimationFrame(this.visualizationRequest);
|
||||
this.visualizationRequest = null;
|
||||
}
|
||||
|
||||
if (this.recordingTimer) {
|
||||
clearInterval(this.recordingTimer);
|
||||
this.recordingTimer = null;
|
||||
}
|
||||
|
||||
// 编码并发送剩余的数据
|
||||
this.encodeAndSendOpus();
|
||||
|
||||
// 发送结束信号
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
const emptyOpusFrame = new Uint8Array(0);
|
||||
this.websocket.send(emptyOpusFrame);
|
||||
|
||||
const stopMessage = {
|
||||
type: 'listen',
|
||||
mode: 'manual',
|
||||
state: 'stop'
|
||||
};
|
||||
|
||||
this.websocket.send(JSON.stringify(stopMessage));
|
||||
log('已发送录音停止信号', 'info');
|
||||
}
|
||||
|
||||
if (this.onRecordingStop) {
|
||||
this.onRecordingStop();
|
||||
}
|
||||
|
||||
log('停止PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音停止错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 获取分析器
|
||||
getAnalyser() {
|
||||
return this.analyser;
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let audioRecorderInstance = null;
|
||||
|
||||
export function getAudioRecorder() {
|
||||
if (!audioRecorderInstance) {
|
||||
audioRecorderInstance = new AudioRecorder();
|
||||
}
|
||||
return audioRecorderInstance;
|
||||
}
|
||||
+95
-33
@@ -1,5 +1,5 @@
|
||||
import BlockingQueue from './utils/BlockingQueue.js';
|
||||
import { log } from './utils/logger.js';
|
||||
import BlockingQueue from '../../utils/blocking-queue.js';
|
||||
import { log } from '../../utils/logger.js';
|
||||
|
||||
// 音频流播放上下文类
|
||||
export class StreamingContext {
|
||||
@@ -22,6 +22,7 @@ export class StreamingContext {
|
||||
this.source = null; // 当前音频源
|
||||
this.totalSamples = 0; // 累积的总样本数
|
||||
this.lastPlayTime = 0; // 上次播放的时间戳
|
||||
this.scheduledEndTime = 0; // 已调度音频的结束时间
|
||||
}
|
||||
|
||||
// 缓存音频数组
|
||||
@@ -31,29 +32,82 @@ export class StreamingContext {
|
||||
|
||||
// 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题
|
||||
async getPendingAudioBufferQueue() {
|
||||
// 原子交换 + 清空
|
||||
[this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()];
|
||||
// 等待数据到达并获取
|
||||
const data = await this.audioBufferQueue.dequeue();
|
||||
// 赋值给待处理队列
|
||||
this.pendingAudioBufferQueue = data;
|
||||
}
|
||||
|
||||
// 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题
|
||||
async getQueue(minSamples) {
|
||||
let TepArray = [];
|
||||
const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1;
|
||||
// 原子交换 + 清空
|
||||
[TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()];
|
||||
this.queue.push(...TepArray);
|
||||
|
||||
// 等待数据并获取
|
||||
const tempArray = await this.activeQueue.dequeue(num);
|
||||
this.queue.push(...tempArray);
|
||||
}
|
||||
|
||||
// 将Int16音频数据转换为Float32音频数据
|
||||
convertInt16ToFloat32(int16Data) {
|
||||
const float32Data = new Float32Array(int16Data.length);
|
||||
for (let i = 0; i < int16Data.length; i++) {
|
||||
// 将[-32768,32767]范围转换为[-1,1]
|
||||
float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF);
|
||||
// 将[-32768,32767]范围转换为[-1,1]，统一使用32768.0避免不对称失真
|
||||
float32Data[i] = int16Data[i] / 32768.0;
|
||||
}
|
||||
return float32Data;
|
||||
}
|
||||
|
||||
// 获取待解码包数
|
||||
getPendingDecodeCount() {
|
||||
return this.audioBufferQueue.length + this.pendingAudioBufferQueue.length;
|
||||
}
|
||||
|
||||
// 获取待播放样本数（转换为包数，每包960样本）
|
||||
getPendingPlayCount() {
|
||||
// 计算已在队列中的样本
|
||||
const queuedSamples = this.activeQueue.length + this.queue.length;
|
||||
|
||||
// 计算已调度但未播放的样本（在Web Audio缓冲区中）
|
||||
let scheduledSamples = 0;
|
||||
if (this.playing && this.scheduledEndTime) {
|
||||
const currentTime = this.audioContext.currentTime;
|
||||
const remainingTime = Math.max(0, this.scheduledEndTime - currentTime);
|
||||
scheduledSamples = Math.floor(remainingTime * this.sampleRate);
|
||||
}
|
||||
|
||||
const totalSamples = queuedSamples + scheduledSamples;
|
||||
return Math.ceil(totalSamples / 960);
|
||||
}
|
||||
|
||||
// 清空所有音频缓冲
|
||||
clearAllBuffers() {
|
||||
log('清空所有音频缓冲', 'info');
|
||||
|
||||
// 清空所有队列（使用clear方法保持对象引用）
|
||||
this.audioBufferQueue.clear();
|
||||
this.pendingAudioBufferQueue = [];
|
||||
this.activeQueue.clear();
|
||||
this.queue = [];
|
||||
|
||||
// 停止当前播放的音频源
|
||||
if (this.source) {
|
||||
try {
|
||||
this.source.stop();
|
||||
this.source.disconnect();
|
||||
} catch (e) {
|
||||
// 忽略已经停止的错误
|
||||
}
|
||||
this.source = null;
|
||||
}
|
||||
|
||||
// 重置状态
|
||||
this.playing = false;
|
||||
this.scheduledEndTime = this.audioContext.currentTime;
|
||||
this.totalSamples = 0;
|
||||
|
||||
log('音频缓冲已清空', 'success');
|
||||
}
|
||||
|
||||
// 将Opus数据解码为PCM
|
||||
async decodeOpusFrames() {
|
||||
if (!this.opusDecoder) {
|
||||
@@ -97,18 +151,26 @@ export class StreamingContext {
|
||||
|
||||
// 开始播放音频
|
||||
async startPlaying() {
|
||||
this.scheduledEndTime = this.audioContext.currentTime; // 跟踪已调度音频的结束时间
|
||||
|
||||
while (true) {
|
||||
// 如果累积了至少0.3秒的音频，开始播放
|
||||
const minSamples = this.sampleRate * this.minAudioDuration * 3;
|
||||
// 初始缓冲：等待足够的样本再开始播放
|
||||
const minSamples = this.sampleRate * this.minAudioDuration * 2;
|
||||
if (!this.playing && this.queue.length < minSamples) {
|
||||
await this.getQueue(minSamples);
|
||||
}
|
||||
this.playing = true;
|
||||
while (this.playing && this.queue.length) {
|
||||
// 创建新的音频缓冲区
|
||||
const minPlaySamples = Math.min(this.queue.length, this.sampleRate);
|
||||
const currentSamples = this.queue.splice(0, minPlaySamples);
|
||||
|
||||
// 持续播放队列中的音频，每次播放一个小块
|
||||
while (this.playing && this.queue.length > 0) {
|
||||
// 每次播放120ms的音频（2个Opus包）
|
||||
const playDuration = 0.12;
|
||||
const targetSamples = Math.floor(this.sampleRate * playDuration);
|
||||
const actualSamples = Math.min(this.queue.length, targetSamples);
|
||||
|
||||
if (actualSamples === 0) break;
|
||||
|
||||
const currentSamples = this.queue.splice(0, actualSamples);
|
||||
const audioBuffer = this.audioContext.createBuffer(this.channels, currentSamples.length, this.sampleRate);
|
||||
audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
|
||||
|
||||
@@ -116,28 +178,28 @@ export class StreamingContext {
|
||||
this.source = this.audioContext.createBufferSource();
|
||||
this.source.buffer = audioBuffer;
|
||||
|
||||
// 创建增益节点用于平滑过渡
|
||||
const gainNode = this.audioContext.createGain();
|
||||
// 精确调度播放时间
|
||||
const currentTime = this.audioContext.currentTime;
|
||||
const startTime = Math.max(this.scheduledEndTime, currentTime);
|
||||
|
||||
// 应用淡入淡出效果避免爆音
|
||||
const fadeDuration = 0.02; // 20毫秒
|
||||
gainNode.gain.setValueAtTime(0, this.audioContext.currentTime);
|
||||
gainNode.gain.linearRampToValueAtTime(1, this.audioContext.currentTime + fadeDuration);
|
||||
// 直接连接到输出
|
||||
this.source.connect(this.audioContext.destination);
|
||||
|
||||
log(`调度播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / this.sampleRate).toFixed(2)} 秒`, 'debug');
|
||||
this.source.start(startTime);
|
||||
|
||||
// 更新下一个音频块的调度时间
|
||||
const duration = audioBuffer.duration;
|
||||
if (duration > fadeDuration * 2) {
|
||||
gainNode.gain.setValueAtTime(1, this.audioContext.currentTime + duration - fadeDuration);
|
||||
gainNode.gain.linearRampToValueAtTime(0, this.audioContext.currentTime + duration);
|
||||
this.scheduledEndTime = startTime + duration;
|
||||
this.lastPlayTime = startTime;
|
||||
|
||||
// 如果队列中数据不足，等待新数据
|
||||
if (this.queue.length < targetSamples) {
|
||||
break;
|
||||
}
|
||||
|
||||
// 连接节点并开始播放
|
||||
this.source.connect(gainNode);
|
||||
gainNode.connect(this.audioContext.destination);
|
||||
|
||||
this.lastPlayTime = this.audioContext.currentTime;
|
||||
log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / this.sampleRate).toFixed(2)} 秒`, 'info');
|
||||
this.source.start();
|
||||
}
|
||||
|
||||
// 等待新数据
|
||||
await this.getQueue(minSamples);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,477 @@
|
||||
import { log } from '../../utils/logger.js';
|
||||
|
||||
// ==========================================
|
||||
// MCP 工具管理逻辑
|
||||
// ==========================================
|
||||
|
||||
// 全局变量
|
||||
let mcpTools = [];
|
||||
let mcpEditingIndex = null;
|
||||
let mcpProperties = [];
|
||||
let websocket = null; // 将从外部设置
|
||||
|
||||
/**
|
||||
* 设置 WebSocket 实例
|
||||
* @param {WebSocket} ws - WebSocket 连接实例
|
||||
*/
|
||||
export function setWebSocket(ws) {
|
||||
websocket = ws;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化 MCP 工具
|
||||
*/
|
||||
export async function initMcpTools() {
|
||||
// 加载默认工具数据
|
||||
const defaultMcpTools = await fetch("js/config/default-mcp-tools.json").then(res => res.json());
|
||||
|
||||
const savedTools = localStorage.getItem('mcpTools');
|
||||
if (savedTools) {
|
||||
try {
|
||||
mcpTools = JSON.parse(savedTools);
|
||||
} catch (e) {
|
||||
log('加载MCP工具失败，使用默认工具', 'warning');
|
||||
mcpTools = [...defaultMcpTools];
|
||||
}
|
||||
} else {
|
||||
mcpTools = [...defaultMcpTools];
|
||||
}
|
||||
|
||||
renderMcpTools();
|
||||
setupMcpEventListeners();
|
||||
}
|
||||
|
||||
/**
|
||||
* 渲染工具列表
|
||||
*/
|
||||
function renderMcpTools() {
|
||||
const container = document.getElementById('mcpToolsContainer');
|
||||
const countSpan = document.getElementById('mcpToolsCount');
|
||||
|
||||
countSpan.textContent = `${mcpTools.length} 个工具`;
|
||||
|
||||
if (mcpTools.length === 0) {
|
||||
container.innerHTML = '<div style="text-align: center; padding: 30px; color: #999;">暂无工具，点击下方按钮添加新工具</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = mcpTools.map((tool, index) => {
|
||||
const paramCount = tool.inputSchema.properties ? Object.keys(tool.inputSchema.properties).length : 0;
|
||||
const requiredCount = tool.inputSchema.required ? tool.inputSchema.required.length : 0;
|
||||
const hasMockResponse = tool.mockResponse && Object.keys(tool.mockResponse).length > 0;
|
||||
|
||||
return `
|
||||
<div class="mcp-tool-card">
|
||||
<div class="mcp-tool-header">
|
||||
<div class="mcp-tool-name">${tool.name}</div>
|
||||
<div class="mcp-tool-actions">
|
||||
<button onclick="window.mcpModule.editMcpTool(${index})"
|
||||
style="padding: 4px 10px; border: none; border-radius: 4px; background-color: #2196f3; color: white; cursor: pointer; font-size: 12px;">
|
||||
✏️ 编辑
|
||||
</button>
|
||||
<button onclick="window.mcpModule.deleteMcpTool(${index})"
|
||||
style="padding: 4px 10px; border: none; border-radius: 4px; background-color: #f44336; color: white; cursor: pointer; font-size: 12px;">
|
||||
🗑️ 删除
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mcp-tool-description">${tool.description}</div>
|
||||
<div class="mcp-tool-info">
|
||||
<div class="mcp-tool-info-row">
|
||||
<span class="mcp-tool-info-label">参数数量:</span>
|
||||
<span class="mcp-tool-info-value">${paramCount} 个 ${requiredCount > 0 ? `(${requiredCount} 个必填)` : ''}</span>
|
||||
</div>
|
||||
<div class="mcp-tool-info-row">
|
||||
<span class="mcp-tool-info-label">模拟返回:</span>
|
||||
<span class="mcp-tool-info-value">${hasMockResponse ? '✅ 已配置: ' + JSON.stringify(tool.mockResponse) : '⚪ 使用默认'}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* 渲染参数列表
|
||||
*/
|
||||
function renderMcpProperties() {
|
||||
const container = document.getElementById('mcpPropertiesContainer');
|
||||
|
||||
if (mcpProperties.length === 0) {
|
||||
container.innerHTML = '<div style="text-align: center; padding: 20px; color: #999; font-size: 14px;">暂无参数，点击下方按钮添加参数</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = mcpProperties.map((prop, index) => `
|
||||
<div class="mcp-property-item">
|
||||
<div class="mcp-property-header">
|
||||
<span class="mcp-property-name">${prop.name}</span>
|
||||
<button type="button" onclick="window.mcpModule.deleteMcpProperty(${index})"
|
||||
style="padding: 3px 8px; border: none; border-radius: 3px; background-color: #f44336; color: white; cursor: pointer; font-size: 11px;">
|
||||
删除
|
||||
</button>
|
||||
</div>
|
||||
<div class="mcp-property-row">
|
||||
<div>
|
||||
<label class="mcp-small-label">参数名称 *</label>
|
||||
<input type="text" class="mcp-small-input" value="${prop.name}"
|
||||
onchange="window.mcpModule.updateMcpProperty(${index}, 'name', this.value)" required>
|
||||
</div>
|
||||
<div>
|
||||
<label class="mcp-small-label">数据类型 *</label>
|
||||
<select class="mcp-small-input" onchange="window.mcpModule.updateMcpProperty(${index}, 'type', this.value)">
|
||||
<option value="string" ${prop.type === 'string' ? 'selected' : ''}>字符串</option>
|
||||
<option value="integer" ${prop.type === 'integer' ? 'selected' : ''}>整数</option>
|
||||
<option value="number" ${prop.type === 'number' ? 'selected' : ''}>数字</option>
|
||||
<option value="boolean" ${prop.type === 'boolean' ? 'selected' : ''}>布尔值</option>
|
||||
<option value="array" ${prop.type === 'array' ? 'selected' : ''}>数组</option>
|
||||
<option value="object" ${prop.type === 'object' ? 'selected' : ''}>对象</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
${(prop.type === 'integer' || prop.type === 'number') ? `
|
||||
<div class="mcp-property-row">
|
||||
<div>
|
||||
<label class="mcp-small-label">最小值</label>
|
||||
<input type="number" class="mcp-small-input" value="${prop.minimum !== undefined ? prop.minimum : ''}"
|
||||
placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'minimum', this.value ? parseFloat(this.value) : undefined)">
|
||||
</div>
|
||||
<div>
|
||||
<label class="mcp-small-label">最大值</label>
|
||||
<input type="number" class="mcp-small-input" value="${prop.maximum !== undefined ? prop.maximum : ''}"
|
||||
placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'maximum', this.value ? parseFloat(this.value) : undefined)">
|
||||
</div>
|
||||
</div>
|
||||
` : ''}
|
||||
<div class="mcp-property-row-full">
|
||||
<label class="mcp-small-label">参数描述</label>
|
||||
<input type="text" class="mcp-small-input" value="${prop.description || ''}"
|
||||
placeholder="可选" onchange="window.mcpModule.updateMcpProperty(${index}, 'description', this.value)">
|
||||
</div>
|
||||
<label class="mcp-checkbox-label">
|
||||
<input type="checkbox" ${prop.required ? 'checked' : ''}
|
||||
onchange="window.mcpModule.updateMcpProperty(${index}, 'required', this.checked)">
|
||||
必填参数
|
||||
</label>
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加参数
|
||||
*/
|
||||
function addMcpProperty() {
|
||||
mcpProperties.push({
|
||||
name: `param_${mcpProperties.length + 1}`,
|
||||
type: 'string',
|
||||
required: false,
|
||||
description: ''
|
||||
});
|
||||
renderMcpProperties();
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新参数
|
||||
*/
|
||||
function updateMcpProperty(index, field, value) {
|
||||
if (field === 'name') {
|
||||
const isDuplicate = mcpProperties.some((p, i) => i !== index && p.name === value);
|
||||
if (isDuplicate) {
|
||||
alert('参数名称已存在，请使用不同的名称');
|
||||
renderMcpProperties();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
mcpProperties[index][field] = value;
|
||||
|
||||
if (field === 'type' && value !== 'integer' && value !== 'number') {
|
||||
delete mcpProperties[index].minimum;
|
||||
delete mcpProperties[index].maximum;
|
||||
renderMcpProperties();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除参数
|
||||
*/
|
||||
function deleteMcpProperty(index) {
|
||||
mcpProperties.splice(index, 1);
|
||||
renderMcpProperties();
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置事件监听
|
||||
*/
|
||||
function setupMcpEventListeners() {
|
||||
const toggleBtn = document.getElementById('toggleMcpTools');
|
||||
const panel = document.getElementById('mcpToolsPanel');
|
||||
const addBtn = document.getElementById('addMcpToolBtn');
|
||||
const modal = document.getElementById('mcpToolModal');
|
||||
const closeBtn = document.getElementById('closeMcpModalBtn');
|
||||
const cancelBtn = document.getElementById('cancelMcpBtn');
|
||||
const form = document.getElementById('mcpToolForm');
|
||||
const addPropertyBtn = document.getElementById('addMcpPropertyBtn');
|
||||
|
||||
toggleBtn.addEventListener('click', () => {
|
||||
const isExpanded = panel.classList.contains('expanded');
|
||||
panel.classList.toggle('expanded');
|
||||
toggleBtn.textContent = isExpanded ? '展开' : '收起';
|
||||
});
|
||||
|
||||
addBtn.addEventListener('click', () => openMcpModal());
|
||||
closeBtn.addEventListener('click', closeMcpModal);
|
||||
cancelBtn.addEventListener('click', closeMcpModal);
|
||||
addPropertyBtn.addEventListener('click', addMcpProperty);
|
||||
|
||||
modal.addEventListener('click', (e) => {
|
||||
if (e.target === modal) closeMcpModal();
|
||||
});
|
||||
|
||||
form.addEventListener('submit', handleMcpSubmit);
|
||||
}
|
||||
|
||||
/**
|
||||
* 打开模态框
|
||||
*/
|
||||
function openMcpModal(index = null) {
|
||||
const isConnected = websocket && websocket.readyState === WebSocket.OPEN;
|
||||
if (isConnected) {
|
||||
alert('WebSocket 已连接，无法编辑工具');
|
||||
return;
|
||||
}
|
||||
|
||||
mcpEditingIndex = index;
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = '';
|
||||
|
||||
if (index !== null) {
|
||||
document.getElementById('mcpModalTitle').textContent = '编辑工具';
|
||||
const tool = mcpTools[index];
|
||||
document.getElementById('mcpToolName').value = tool.name;
|
||||
document.getElementById('mcpToolDescription').value = tool.description;
|
||||
document.getElementById('mcpMockResponse').value = tool.mockResponse ? JSON.stringify(tool.mockResponse, null, 2) : '';
|
||||
|
||||
mcpProperties = [];
|
||||
const schema = tool.inputSchema;
|
||||
if (schema.properties) {
|
||||
Object.keys(schema.properties).forEach(key => {
|
||||
const prop = schema.properties[key];
|
||||
mcpProperties.push({
|
||||
name: key,
|
||||
type: prop.type || 'string',
|
||||
minimum: prop.minimum,
|
||||
maximum: prop.maximum,
|
||||
description: prop.description || '',
|
||||
required: schema.required && schema.required.includes(key)
|
||||
});
|
||||
});
|
||||
}
|
||||
} else {
|
||||
document.getElementById('mcpModalTitle').textContent = '添加工具';
|
||||
document.getElementById('mcpToolForm').reset();
|
||||
mcpProperties = [];
|
||||
}
|
||||
|
||||
renderMcpProperties();
|
||||
document.getElementById('mcpToolModal').style.display = 'block';
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭模态框
|
||||
*/
|
||||
function closeMcpModal() {
|
||||
document.getElementById('mcpToolModal').style.display = 'none';
|
||||
mcpEditingIndex = null;
|
||||
document.getElementById('mcpToolForm').reset();
|
||||
mcpProperties = [];
|
||||
document.getElementById('mcpErrorContainer').innerHTML = '';
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理表单提交
|
||||
*/
|
||||
function handleMcpSubmit(e) {
|
||||
e.preventDefault();
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = '';
|
||||
|
||||
const name = document.getElementById('mcpToolName').value.trim();
|
||||
const description = document.getElementById('mcpToolDescription').value.trim();
|
||||
const mockResponseText = document.getElementById('mcpMockResponse').value.trim();
|
||||
|
||||
// 检查名称重复
|
||||
const isDuplicate = mcpTools.some((tool, index) =>
|
||||
tool.name === name && index !== mcpEditingIndex
|
||||
);
|
||||
|
||||
if (isDuplicate) {
|
||||
showMcpError('工具名称已存在，请使用不同的名称');
|
||||
return;
|
||||
}
|
||||
|
||||
// 解析模拟返回结果
|
||||
let mockResponse = null;
|
||||
if (mockResponseText) {
|
||||
try {
|
||||
mockResponse = JSON.parse(mockResponseText);
|
||||
} catch (e) {
|
||||
showMcpError('模拟返回结果不是有效的 JSON 格式: ' + e.message);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 构建 inputSchema
|
||||
const inputSchema = {
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: []
|
||||
};
|
||||
|
||||
mcpProperties.forEach(prop => {
|
||||
const propSchema = { type: prop.type };
|
||||
|
||||
if (prop.description) {
|
||||
propSchema.description = prop.description;
|
||||
}
|
||||
|
||||
if ((prop.type === 'integer' || prop.type === 'number')) {
|
||||
if (prop.minimum !== undefined && prop.minimum !== '') {
|
||||
propSchema.minimum = prop.minimum;
|
||||
}
|
||||
if (prop.maximum !== undefined && prop.maximum !== '') {
|
||||
propSchema.maximum = prop.maximum;
|
||||
}
|
||||
}
|
||||
|
||||
inputSchema.properties[prop.name] = propSchema;
|
||||
|
||||
if (prop.required) {
|
||||
inputSchema.required.push(prop.name);
|
||||
}
|
||||
});
|
||||
|
||||
if (inputSchema.required.length === 0) {
|
||||
delete inputSchema.required;
|
||||
}
|
||||
|
||||
const tool = { name, description, inputSchema, mockResponse };
|
||||
|
||||
if (mcpEditingIndex !== null) {
|
||||
mcpTools[mcpEditingIndex] = tool;
|
||||
log(`已更新工具: ${name}`, 'success');
|
||||
} else {
|
||||
mcpTools.push(tool);
|
||||
log(`已添加工具: ${name}`, 'success');
|
||||
}
|
||||
|
||||
saveMcpTools();
|
||||
renderMcpTools();
|
||||
closeMcpModal();
|
||||
}
|
||||
|
||||
/**
|
||||
* 显示错误
|
||||
*/
|
||||
function showMcpError(message) {
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = `<div class="mcp-error">${message}</div>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 编辑工具
|
||||
*/
|
||||
function editMcpTool(index) {
|
||||
openMcpModal(index);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除工具
|
||||
*/
|
||||
function deleteMcpTool(index) {
|
||||
const isConnected = websocket && websocket.readyState === WebSocket.OPEN;
|
||||
if (isConnected) {
|
||||
alert('WebSocket 已连接，无法编辑工具');
|
||||
return;
|
||||
}
|
||||
if (confirm(`确定要删除工具 "${mcpTools[index].name}" 吗？`)) {
|
||||
const toolName = mcpTools[index].name;
|
||||
mcpTools.splice(index, 1);
|
||||
saveMcpTools();
|
||||
renderMcpTools();
|
||||
log(`已删除工具: ${toolName}`, 'info');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存工具
|
||||
*/
|
||||
function saveMcpTools() {
|
||||
localStorage.setItem('mcpTools', JSON.stringify(mcpTools));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取工具列表
|
||||
*/
|
||||
export function getMcpTools() {
|
||||
return mcpTools.map(tool => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
inputSchema: tool.inputSchema
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行工具调用
|
||||
*/
|
||||
export function executeMcpTool(toolName, toolArgs) {
|
||||
const tool = mcpTools.find(t => t.name === toolName);
|
||||
|
||||
if (!tool) {
|
||||
log(`未找到工具: ${toolName}`, 'error');
|
||||
return {
|
||||
success: false,
|
||||
error: `未知工具: ${toolName}`
|
||||
};
|
||||
}
|
||||
|
||||
// 如果有模拟返回结果，使用它
|
||||
if (tool.mockResponse) {
|
||||
// 替换模板变量
|
||||
let responseStr = JSON.stringify(tool.mockResponse);
|
||||
|
||||
// 替换 ${paramName} 格式的变量
|
||||
if (toolArgs) {
|
||||
Object.keys(toolArgs).forEach(key => {
|
||||
const regex = new RegExp(`\\$\\{${key}\\}`, 'g');
|
||||
responseStr = responseStr.replace(regex, toolArgs[key]);
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const response = JSON.parse(responseStr);
|
||||
log(`工具 ${toolName} 执行成功，返回模拟结果: ${responseStr}`, 'success');
|
||||
return response;
|
||||
} catch (e) {
|
||||
log(`解析模拟返回结果失败: ${e.message}`, 'error');
|
||||
return tool.mockResponse;
|
||||
}
|
||||
}
|
||||
|
||||
// 没有模拟返回结果，返回默认成功消息
|
||||
log(`工具 ${toolName} 执行成功，返回默认结果`, 'success');
|
||||
return {
|
||||
success: true,
|
||||
message: `工具 ${toolName} 执行成功`,
|
||||
tool: toolName,
|
||||
arguments: toolArgs
|
||||
};
|
||||
}
|
||||
|
||||
// 暴露全局方法供 HTML 内联事件调用
|
||||
window.mcpModule = {
|
||||
updateMcpProperty,
|
||||
deleteMcpProperty,
|
||||
editMcpTool,
|
||||
deleteMcpTool
|
||||
};
|
||||
+43
-22
@@ -1,24 +1,50 @@
|
||||
import { log } from './utils/logger.js';
|
||||
import { otaStatusStyle } from './document.js'
|
||||
import { otaStatusStyle } from '../../ui/dom-helper.js';
|
||||
import { log } from '../../utils/logger.js';
|
||||
|
||||
// WebSocket 连接
|
||||
export async function webSocketConnect(otaUrl,wsUrl,config){
|
||||
if (!validateWsUrl(wsUrl)) {
|
||||
return; // 直接返回，不再往下执行
|
||||
}
|
||||
export async function webSocketConnect(otaUrl, config) {
|
||||
|
||||
if (!validateConfig(config)) {
|
||||
return;
|
||||
}
|
||||
const ok = await sendOTA(otaUrl, config);
|
||||
if (!ok) return;
|
||||
|
||||
// 使用自定义WebSocket实现以添加认证头信息
|
||||
let connUrl = new URL(wsUrl);
|
||||
// 添加认证参数
|
||||
// 发送OTA请求并获取返回的websocket信息
|
||||
const otaResult = await sendOTA(otaUrl, config);
|
||||
if (!otaResult) {
|
||||
log('无法从OTA服务器获取信息', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 从OTA响应中提取websocket信息
|
||||
const { websocket } = otaResult;
|
||||
if (!websocket || !websocket.url) {
|
||||
log('OTA响应中缺少websocket信息', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 使用OTA返回的websocket URL
|
||||
let connUrl = new URL(websocket.url);
|
||||
|
||||
// 添加token参数（从OTA响应中获取）
|
||||
if (websocket.token) {
|
||||
if (websocket.token.startsWith("Bearer ")) {
|
||||
connUrl.searchParams.append('authorization', websocket.token);
|
||||
} else {
|
||||
connUrl.searchParams.append('authorization', 'Bearer ' + websocket.token);
|
||||
}
|
||||
}
|
||||
|
||||
// 添加认证参数（保持原有逻辑）
|
||||
connUrl.searchParams.append('device-id', config.deviceId);
|
||||
connUrl.searchParams.append('client-id', config.clientId);
|
||||
log(`正在连接: ${connUrl.toString()}`, 'info');
|
||||
|
||||
const wsurl = connUrl.toString()
|
||||
|
||||
log(`正在连接: ${wsurl}`, 'info');
|
||||
|
||||
if (wsurl) {
|
||||
document.getElementById('serverUrl').value = wsurl;
|
||||
}
|
||||
|
||||
return new WebSocket(connUrl.toString());
|
||||
}
|
||||
@@ -37,7 +63,7 @@ function validateConfig(config) {
|
||||
}
|
||||
|
||||
// 判断wsUrl路径是否存在错误
|
||||
function validateWsUrl(wsUrl){
|
||||
function validateWsUrl(wsUrl) {
|
||||
if (wsUrl === '') return false;
|
||||
// 检查URL格式
|
||||
if (!wsUrl.startsWith('ws://') && !wsUrl.startsWith('wss://')) {
|
||||
@@ -48,7 +74,7 @@ function validateWsUrl(wsUrl){
|
||||
}
|
||||
|
||||
|
||||
// OTA发送请求，验证状态
|
||||
// OTA发送请求，验证状态，并返回响应数据
|
||||
async function sendOTA(otaUrl, config) {
|
||||
try {
|
||||
const res = await fetch(otaUrl, {
|
||||
@@ -90,14 +116,9 @@ async function sendOTA(otaUrl, config) {
|
||||
|
||||
const result = await res.json();
|
||||
otaStatusStyle(true)
|
||||
return true; // 成功
|
||||
return result; // 返回完整的响应数据
|
||||
} catch (err) {
|
||||
otaStatusStyle(false)
|
||||
return false; // 失败
|
||||
return null; // 失败返回null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
// WebSocket消息处理模块
|
||||
import { log } from '../../utils/logger.js';
|
||||
import { addMessage } from '../../ui/dom-helper.js';
|
||||
import { webSocketConnect } from './ota-connector.js';
|
||||
import { getConfig, saveConnectionUrls } from '../../config/manager.js';
|
||||
import { getAudioPlayer } from '../audio/player.js';
|
||||
import { getAudioRecorder } from '../audio/recorder.js';
|
||||
import { getMcpTools, executeMcpTool, setWebSocket as setMcpWebSocket } from '../mcp/tools.js';
|
||||
|
||||
// WebSocket处理器类
|
||||
export class WebSocketHandler {
|
||||
constructor() {
|
||||
this.websocket = null;
|
||||
this.onConnectionStateChange = null;
|
||||
this.onRecordButtonStateChange = null;
|
||||
this.onSessionStateChange = null;
|
||||
this.onSessionEmotionChange = null;
|
||||
this.currentSessionId = null;
|
||||
this.isRemoteSpeaking = false;
|
||||
}
|
||||
|
||||
// 发送hello握手消息
|
||||
async sendHelloMessage() {
|
||||
if (!this.websocket || this.websocket.readyState !== WebSocket.OPEN) return false;
|
||||
|
||||
try {
|
||||
const config = getConfig();
|
||||
|
||||
const helloMessage = {
|
||||
type: 'hello',
|
||||
device_id: config.deviceId,
|
||||
device_name: config.deviceName,
|
||||
device_mac: config.deviceMac,
|
||||
token: config.token,
|
||||
features: {
|
||||
mcp: true
|
||||
}
|
||||
};
|
||||
|
||||
log('发送hello握手消息', 'info');
|
||||
this.websocket.send(JSON.stringify(helloMessage));
|
||||
|
||||
return new Promise(resolve => {
|
||||
const timeout = setTimeout(() => {
|
||||
log('等待hello响应超时', 'error');
|
||||
log('提示: 请尝试点击"测试认证"按钮进行连接排查', 'info');
|
||||
resolve(false);
|
||||
}, 5000);
|
||||
|
||||
const onMessageHandler = (event) => {
|
||||
try {
|
||||
const response = JSON.parse(event.data);
|
||||
if (response.type === 'hello' && response.session_id) {
|
||||
log(`服务器握手成功，会话ID: ${response.session_id}`, 'success');
|
||||
clearTimeout(timeout);
|
||||
this.websocket.removeEventListener('message', onMessageHandler);
|
||||
resolve(true);
|
||||
}
|
||||
} catch (e) {
|
||||
// 忽略非JSON消息
|
||||
}
|
||||
};
|
||||
|
||||
this.websocket.addEventListener('message', onMessageHandler);
|
||||
});
|
||||
} catch (error) {
|
||||
log(`发送hello消息错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 处理文本消息
|
||||
handleTextMessage(message) {
|
||||
if (message.type === 'hello') {
|
||||
log(`服务器回应：${JSON.stringify(message, null, 2)}`, 'success');
|
||||
} else if (message.type === 'tts') {
|
||||
this.handleTTSMessage(message);
|
||||
} else if (message.type === 'audio') {
|
||||
log(`收到音频控制消息: ${JSON.stringify(message)}`, 'info');
|
||||
} else if (message.type === 'stt') {
|
||||
log(`识别结果: ${message.text}`, 'info');
|
||||
addMessage(`${message.text}`, true);
|
||||
} else if (message.type === 'llm') {
|
||||
log(`大模型回复: ${message.text}`, 'info');
|
||||
|
||||
// 如果包含表情，更新sessionStatus表情
|
||||
if (message.text && /[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u.test(message.text)) {
|
||||
// 提取表情符号
|
||||
const emojiMatch = message.text.match(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/u);
|
||||
if (emojiMatch && this.onSessionEmotionChange) {
|
||||
this.onSessionEmotionChange(emojiMatch[0]);
|
||||
}
|
||||
}
|
||||
|
||||
// 只有当文本不仅仅是表情时，才添加到对话中
|
||||
// 移除文本中的表情后检查是否还有内容
|
||||
const textWithoutEmoji = message.text ? message.text.replace(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/gu, '').trim() : '';
|
||||
if (textWithoutEmoji) {
|
||||
addMessage(message.text);
|
||||
}
|
||||
} else if (message.type === 'mcp') {
|
||||
this.handleMCPMessage(message);
|
||||
} else {
|
||||
log(`未知消息类型: ${message.type}`, 'info');
|
||||
addMessage(JSON.stringify(message, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
// 处理TTS消息
|
||||
handleTTSMessage(message) {
|
||||
if (message.state === 'start') {
|
||||
log('服务器开始发送语音', 'info');
|
||||
this.currentSessionId = message.session_id;
|
||||
this.isRemoteSpeaking = true;
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(true);
|
||||
}
|
||||
} else if (message.state === 'sentence_start') {
|
||||
log(`服务器发送语音段: ${message.text}`, 'info');
|
||||
if (message.text) {
|
||||
addMessage(message.text);
|
||||
}
|
||||
} else if (message.state === 'sentence_end') {
|
||||
log(`语音段结束: ${message.text}`, 'info');
|
||||
} else if (message.state === 'stop') {
|
||||
log('服务器语音传输结束，清空所有音频缓冲', 'info');
|
||||
|
||||
// 清空所有音频缓冲并停止播放
|
||||
const audioPlayer = getAudioPlayer();
|
||||
audioPlayer.clearAllAudio();
|
||||
|
||||
this.isRemoteSpeaking = false;
|
||||
if (this.onRecordButtonStateChange) {
|
||||
this.onRecordButtonStateChange(false);
|
||||
}
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 处理MCP消息
|
||||
handleMCPMessage(message) {
|
||||
const payload = message.payload || {};
|
||||
log(`服务器下发: ${JSON.stringify(message)}`, 'info');
|
||||
|
||||
if (payload.method === 'tools/list') {
|
||||
const tools = getMcpTools();
|
||||
|
||||
const replyMessage = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"tools": tools
|
||||
}
|
||||
}
|
||||
});
|
||||
log(`客户端上报: ${replyMessage}`, 'info');
|
||||
this.websocket.send(replyMessage);
|
||||
log(`回复MCP工具列表: ${tools.length} 个工具`, 'info');
|
||||
|
||||
} else if (payload.method === 'tools/call') {
|
||||
const toolName = payload.params?.name;
|
||||
const toolArgs = payload.params?.arguments;
|
||||
|
||||
log(`调用工具: ${toolName} 参数: ${JSON.stringify(toolArgs)}`, 'info');
|
||||
|
||||
const result = executeMcpTool(toolName, toolArgs);
|
||||
|
||||
const replyMessage = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": JSON.stringify(result)
|
||||
}
|
||||
],
|
||||
"isError": false
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
log(`客户端上报: ${replyMessage}`, 'info');
|
||||
this.websocket.send(replyMessage);
|
||||
} else if (payload.method === 'initialize') {
|
||||
log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info');
|
||||
} else {
|
||||
log(`未知的MCP方法: ${payload.method}`, 'warning');
|
||||
}
|
||||
}
|
||||
|
||||
// 处理二进制消息
|
||||
async handleBinaryMessage(data) {
|
||||
try {
|
||||
let arrayBuffer;
|
||||
if (data instanceof ArrayBuffer) {
|
||||
arrayBuffer = data;
|
||||
log(`收到ArrayBuffer音频数据，大小: ${data.byteLength}字节`, 'debug');
|
||||
} else if (data instanceof Blob) {
|
||||
arrayBuffer = await data.arrayBuffer();
|
||||
log(`收到Blob音频数据，大小: ${arrayBuffer.byteLength}字节`, 'debug');
|
||||
} else {
|
||||
log(`收到未知类型的二进制数据: ${typeof data}`, 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
const opusData = new Uint8Array(arrayBuffer);
|
||||
const audioPlayer = getAudioPlayer();
|
||||
audioPlayer.enqueueAudioData(opusData);
|
||||
} catch (error) {
|
||||
log(`处理二进制消息出错: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 连接WebSocket服务器
|
||||
async connect() {
|
||||
const config = getConfig();
|
||||
log('正在检查OTA状态...', 'info');
|
||||
saveConnectionUrls();
|
||||
|
||||
try {
|
||||
const otaUrl = document.getElementById('otaUrl').value.trim();
|
||||
const ws = await webSocketConnect(otaUrl, config);
|
||||
if (ws === undefined) {
|
||||
return false;
|
||||
}
|
||||
this.websocket = ws;
|
||||
|
||||
// 设置接收二进制数据的类型为ArrayBuffer
|
||||
this.websocket.binaryType = 'arraybuffer';
|
||||
|
||||
// 设置 MCP 模块的 WebSocket 实例
|
||||
setMcpWebSocket(this.websocket);
|
||||
|
||||
// 设置录音器的WebSocket
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.setWebSocket(this.websocket);
|
||||
|
||||
this.setupEventHandlers();
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`连接错误: ${error.message}`, 'error');
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 设置事件处理器
|
||||
setupEventHandlers() {
|
||||
this.websocket.onopen = async () => {
|
||||
const url = document.getElementById('serverUrl').value;
|
||||
log(`已连接到服务器: ${url}`, 'success');
|
||||
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(true);
|
||||
}
|
||||
|
||||
// 连接成功后，默认状态为聆听中
|
||||
this.isRemoteSpeaking = false;
|
||||
if (this.onSessionStateChange) {
|
||||
this.onSessionStateChange(false);
|
||||
}
|
||||
|
||||
await this.sendHelloMessage();
|
||||
};
|
||||
|
||||
this.websocket.onclose = () => {
|
||||
log('已断开连接', 'info');
|
||||
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.stop();
|
||||
};
|
||||
|
||||
this.websocket.onerror = (error) => {
|
||||
log(`WebSocket错误: ${error.message || '未知错误'}`, 'error');
|
||||
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
};
|
||||
|
||||
this.websocket.onmessage = (event) => {
|
||||
try {
|
||||
if (typeof event.data === 'string') {
|
||||
const message = JSON.parse(event.data);
|
||||
this.handleTextMessage(message);
|
||||
} else {
|
||||
this.handleBinaryMessage(event.data);
|
||||
}
|
||||
} catch (error) {
|
||||
log(`WebSocket消息处理错误: ${error.message}`, 'error');
|
||||
if (typeof event.data === 'string') {
|
||||
addMessage(event.data);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// 断开连接
|
||||
disconnect() {
|
||||
if (!this.websocket) return;
|
||||
|
||||
this.websocket.close();
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.stop();
|
||||
}
|
||||
|
||||
// 发送文本消息
|
||||
sendTextMessage(text) {
|
||||
if (text === '' || !this.websocket || this.websocket.readyState !== WebSocket.OPEN) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
// 如果对方正在说话，先发送打断消息
|
||||
if (this.isRemoteSpeaking && this.currentSessionId) {
|
||||
const abortMessage = {
|
||||
session_id: this.currentSessionId,
|
||||
type: 'abort',
|
||||
reason: 'wake_word_detected'
|
||||
};
|
||||
this.websocket.send(JSON.stringify(abortMessage));
|
||||
log('发送打断消息', 'info');
|
||||
}
|
||||
|
||||
const listenMessage = {
|
||||
type: 'listen',
|
||||
mode: 'manual',
|
||||
state: 'detect',
|
||||
text: text
|
||||
};
|
||||
|
||||
this.websocket.send(JSON.stringify(listenMessage));
|
||||
log(`发送文本消息: ${text}`, 'info');
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`发送消息错误: ${error.message}`, 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 获取WebSocket实例
|
||||
getWebSocket() {
|
||||
return this.websocket;
|
||||
}
|
||||
|
||||
// 检查是否已连接
|
||||
isConnected() {
|
||||
return this.websocket && this.websocket.readyState === WebSocket.OPEN;
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let wsHandlerInstance = null;
|
||||
|
||||
export function getWebSocketHandler() {
|
||||
if (!wsHandlerInstance) {
|
||||
wsHandlerInstance = new WebSocketHandler();
|
||||
}
|
||||
return wsHandlerInstance;
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
// UI控制模块
|
||||
import { loadConfig, saveConfig } from '../config/manager.js';
|
||||
import { getAudioRecorder } from '../core/audio/recorder.js';
|
||||
import { getWebSocketHandler } from '../core/network/websocket.js';
|
||||
import { getAudioPlayer } from '../core/audio/player.js';
|
||||
|
||||
// UI控制器类
|
||||
export class UIController {
|
||||
constructor() {
|
||||
this.isEditing = false;
|
||||
this.visualizerCanvas = null;
|
||||
this.visualizerContext = null;
|
||||
this.audioStatsTimer = null;
|
||||
}
|
||||
|
||||
// 初始化
|
||||
init() {
|
||||
this.visualizerCanvas = document.getElementById('audioVisualizer');
|
||||
this.visualizerContext = this.visualizerCanvas.getContext('2d');
|
||||
|
||||
this.initVisualizer();
|
||||
this.initEventListeners();
|
||||
this.startAudioStatsMonitor();
|
||||
loadConfig();
|
||||
}
|
||||
|
||||
// 初始化可视化器
|
||||
initVisualizer() {
|
||||
this.visualizerCanvas.width = this.visualizerCanvas.clientWidth;
|
||||
this.visualizerCanvas.height = this.visualizerCanvas.clientHeight;
|
||||
this.visualizerContext.fillStyle = '#fafafa';
|
||||
this.visualizerContext.fillRect(0, 0, this.visualizerCanvas.width, this.visualizerCanvas.height);
|
||||
}
|
||||
|
||||
// 更新状态显示
|
||||
updateStatusDisplay(element, text) {
|
||||
element.textContent = text;
|
||||
element.removeAttribute('style');
|
||||
element.classList.remove('connected');
|
||||
if (text.includes('已连接')) {
|
||||
element.classList.add('connected');
|
||||
}
|
||||
console.log('更新状态:', text, '类列表:', element.className, '样式属性:', element.getAttribute('style'));
|
||||
}
|
||||
|
||||
// 更新连接状态UI
|
||||
updateConnectionUI(isConnected) {
|
||||
const connectionStatus = document.getElementById('connectionStatus');
|
||||
const otaStatus = document.getElementById('otaStatus');
|
||||
const connectButton = document.getElementById('connectButton');
|
||||
const messageInput = document.getElementById('messageInput');
|
||||
const sendTextButton = document.getElementById('sendTextButton');
|
||||
const recordButton = document.getElementById('recordButton');
|
||||
|
||||
if (isConnected) {
|
||||
this.updateStatusDisplay(connectionStatus, '● WS已连接');
|
||||
this.updateStatusDisplay(otaStatus, '● OTA已连接');
|
||||
connectButton.textContent = '断开';
|
||||
messageInput.disabled = false;
|
||||
sendTextButton.disabled = false;
|
||||
recordButton.disabled = false;
|
||||
} else {
|
||||
this.updateStatusDisplay(connectionStatus, '● WS未连接');
|
||||
this.updateStatusDisplay(otaStatus, '● OTA未连接');
|
||||
connectButton.textContent = '连接';
|
||||
messageInput.disabled = true;
|
||||
sendTextButton.disabled = true;
|
||||
recordButton.disabled = true;
|
||||
// 断开连接时，会话状态变为离线
|
||||
this.updateSessionStatus(null);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新录音按钮状态
|
||||
updateRecordButtonState(isRecording, seconds = 0) {
|
||||
const recordButton = document.getElementById('recordButton');
|
||||
if (isRecording) {
|
||||
recordButton.textContent = `停止录音 ${seconds.toFixed(1)}秒`;
|
||||
recordButton.classList.add('recording');
|
||||
} else {
|
||||
recordButton.textContent = '开始录音';
|
||||
recordButton.classList.remove('recording');
|
||||
}
|
||||
recordButton.disabled = false;
|
||||
}
|
||||
|
||||
// 更新会话状态UI
|
||||
updateSessionStatus(isSpeaking) {
|
||||
const sessionStatus = document.getElementById('sessionStatus');
|
||||
if (!sessionStatus) return;
|
||||
|
||||
// 保留背景元素
|
||||
const bgHtml = '<span id="sessionStatusBg" style="position: absolute; left: 0; top: 0; bottom: 0; width: 0%; background: linear-gradient(90deg, rgba(76, 175, 80, 0.2), rgba(33, 150, 243, 0.2)); transition: width 0.15s ease-out, background 0.3s ease; z-index: 0; border-radius: 20px;"></span>';
|
||||
|
||||
if (isSpeaking === null) {
|
||||
// 离线状态
|
||||
sessionStatus.innerHTML = bgHtml + '<span style="position: relative; z-index: 1;"><span class="emoji-large">😶</span> 小智离线中</span>';
|
||||
sessionStatus.className = 'status offline';
|
||||
} else if (isSpeaking) {
|
||||
// 说话中
|
||||
sessionStatus.innerHTML = bgHtml + '<span style="position: relative; z-index: 1;"><span class="emoji-large">😶</span> 小智说话中</span>';
|
||||
sessionStatus.className = 'status speaking';
|
||||
} else {
|
||||
// 聆听中
|
||||
sessionStatus.innerHTML = bgHtml + '<span style="position: relative; z-index: 1;"><span class="emoji-large">😶</span> 小智聆听中</span>';
|
||||
sessionStatus.className = 'status listening';
|
||||
}
|
||||
}
|
||||
|
||||
// 更新会话表情
|
||||
updateSessionEmotion(emoji) {
|
||||
const sessionStatus = document.getElementById('sessionStatus');
|
||||
if (!sessionStatus) return;
|
||||
|
||||
// 获取当前文本内容，提取非表情部分
|
||||
let currentText = sessionStatus.textContent;
|
||||
// 移除现有的表情符号
|
||||
currentText = currentText.replace(/[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]/gu, '').trim();
|
||||
|
||||
// 保留背景元素
|
||||
const bgHtml = '<span id="sessionStatusBg" style="position: absolute; left: 0; top: 0; bottom: 0; width: 0%; background: linear-gradient(90deg, rgba(76, 175, 80, 0.2), rgba(33, 150, 243, 0.2)); transition: width 0.15s ease-out, background 0.3s ease; z-index: 0; border-radius: 20px;"></span>';
|
||||
|
||||
// 使用 innerHTML 添加带样式的表情
|
||||
sessionStatus.innerHTML = bgHtml + `<span style="position: relative; z-index: 1;"><span class="emoji-large">${emoji}</span> ${currentText}</span>`;
|
||||
}
|
||||
|
||||
// 更新音频统计信息
|
||||
updateAudioStats() {
|
||||
const audioPlayer = getAudioPlayer();
|
||||
const stats = audioPlayer.getAudioStats();
|
||||
|
||||
const sessionStatus = document.getElementById('sessionStatus');
|
||||
const sessionStatusBg = document.getElementById('sessionStatusBg');
|
||||
|
||||
// 只在说话状态下显示背景进度
|
||||
if (sessionStatus && sessionStatus.classList.contains('speaking') && sessionStatusBg) {
|
||||
if (stats.pendingPlay > 0) {
|
||||
// 计算进度：5包=50%，10包及以上=100%
|
||||
let percentage;
|
||||
if (stats.pendingPlay >= 10) {
|
||||
percentage = 100;
|
||||
} else {
|
||||
percentage = (stats.pendingPlay / 10) * 100;
|
||||
}
|
||||
|
||||
sessionStatusBg.style.width = `${percentage}%`;
|
||||
|
||||
// 根据缓冲量改变背景颜色
|
||||
if (stats.pendingPlay < 5) {
|
||||
// 缓冲不足：橙红色半透明
|
||||
sessionStatusBg.style.background = 'linear-gradient(90deg, rgba(255, 152, 0, 0.25), rgba(255, 87, 34, 0.25))';
|
||||
} else if (stats.pendingPlay < 10) {
|
||||
// 一般：黄绿色半透明
|
||||
sessionStatusBg.style.background = 'linear-gradient(90deg, rgba(205, 220, 57, 0.25), rgba(76, 175, 80, 0.25))';
|
||||
} else {
|
||||
// 充足：绿蓝色半透明
|
||||
sessionStatusBg.style.background = 'linear-gradient(90deg, rgba(76, 175, 80, 0.25), rgba(33, 150, 243, 0.25))';
|
||||
}
|
||||
} else {
|
||||
// 没有缓冲，隐藏背景
|
||||
sessionStatusBg.style.width = '0%';
|
||||
}
|
||||
} else {
|
||||
// 非说话状态，隐藏背景
|
||||
if (sessionStatusBg) {
|
||||
sessionStatusBg.style.width = '0%';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 启动音频统计监控
|
||||
startAudioStatsMonitor() {
|
||||
// 每100ms更新一次音频统计
|
||||
this.audioStatsTimer = setInterval(() => {
|
||||
this.updateAudioStats();
|
||||
}, 100);
|
||||
}
|
||||
|
||||
// 停止音频统计监控
|
||||
stopAudioStatsMonitor() {
|
||||
if (this.audioStatsTimer) {
|
||||
clearInterval(this.audioStatsTimer);
|
||||
this.audioStatsTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
// 绘制音频可视化效果
|
||||
drawVisualizer(dataArray) {
|
||||
this.visualizerContext.fillStyle = '#fafafa';
|
||||
this.visualizerContext.fillRect(0, 0, this.visualizerCanvas.width, this.visualizerCanvas.height);
|
||||
|
||||
const barWidth = (this.visualizerCanvas.width / dataArray.length) * 2.5;
|
||||
let barHeight;
|
||||
let x = 0;
|
||||
|
||||
for (let i = 0; i < dataArray.length; i++) {
|
||||
barHeight = dataArray[i] / 2;
|
||||
|
||||
// 创建渐变色：从紫色到蓝色到青色
|
||||
const hue = 200 + (barHeight / this.visualizerCanvas.height) * 60; // 200-260度，从青色到紫色
|
||||
const saturation = 80 + (barHeight / this.visualizerCanvas.height) * 20; // 饱和度 80-100%
|
||||
const lightness = 45 + (barHeight / this.visualizerCanvas.height) * 15; // 亮度 45-60%
|
||||
|
||||
this.visualizerContext.fillStyle = `hsl(${hue}, ${saturation}%, ${lightness}%)`;
|
||||
this.visualizerContext.fillRect(x, this.visualizerCanvas.height - barHeight, barWidth, barHeight);
|
||||
|
||||
x += barWidth + 1;
|
||||
}
|
||||
}
|
||||
|
||||
// 初始化事件监听器
|
||||
initEventListeners() {
|
||||
const wsHandler = getWebSocketHandler();
|
||||
const audioRecorder = getAudioRecorder();
|
||||
|
||||
// 设置WebSocket回调
|
||||
wsHandler.onConnectionStateChange = (isConnected) => {
|
||||
this.updateConnectionUI(isConnected);
|
||||
};
|
||||
|
||||
wsHandler.onRecordButtonStateChange = (isRecording) => {
|
||||
this.updateRecordButtonState(isRecording);
|
||||
};
|
||||
|
||||
wsHandler.onSessionStateChange = (isSpeaking) => {
|
||||
this.updateSessionStatus(isSpeaking);
|
||||
};
|
||||
|
||||
wsHandler.onSessionEmotionChange = (emoji) => {
|
||||
this.updateSessionEmotion(emoji);
|
||||
};
|
||||
|
||||
// 设置录音器回调
|
||||
audioRecorder.onRecordingStart = (seconds) => {
|
||||
this.updateRecordButtonState(true, seconds);
|
||||
};
|
||||
|
||||
audioRecorder.onRecordingStop = () => {
|
||||
this.updateRecordButtonState(false);
|
||||
};
|
||||
|
||||
audioRecorder.onVisualizerUpdate = (dataArray) => {
|
||||
this.drawVisualizer(dataArray);
|
||||
};
|
||||
|
||||
// 连接按钮
|
||||
const connectButton = document.getElementById('connectButton');
|
||||
let isConnecting = false;
|
||||
|
||||
const handleConnect = async () => {
|
||||
if (isConnecting) return;
|
||||
|
||||
if (wsHandler.isConnected()) {
|
||||
wsHandler.disconnect();
|
||||
} else {
|
||||
isConnecting = true;
|
||||
await wsHandler.connect();
|
||||
isConnecting = false;
|
||||
}
|
||||
};
|
||||
|
||||
connectButton.addEventListener('click', handleConnect);
|
||||
|
||||
// 设备配置面板编辑/确定切换
|
||||
const toggleButton = document.getElementById('toggleConfig');
|
||||
const deviceMacInput = document.getElementById('deviceMac');
|
||||
const deviceNameInput = document.getElementById('deviceName');
|
||||
const clientIdInput = document.getElementById('clientId');
|
||||
const tokenInput = document.getElementById('token');
|
||||
|
||||
toggleButton.addEventListener('click', () => {
|
||||
this.isEditing = !this.isEditing;
|
||||
|
||||
deviceMacInput.disabled = !this.isEditing;
|
||||
deviceNameInput.disabled = !this.isEditing;
|
||||
clientIdInput.disabled = !this.isEditing;
|
||||
tokenInput.disabled = !this.isEditing;
|
||||
|
||||
toggleButton.textContent = this.isEditing ? '确定' : '编辑';
|
||||
|
||||
if (!this.isEditing) {
|
||||
saveConfig();
|
||||
}
|
||||
});
|
||||
|
||||
// 标签页切换
|
||||
const tabs = document.querySelectorAll('.tab');
|
||||
tabs.forEach(tab => {
|
||||
tab.addEventListener('click', () => {
|
||||
tabs.forEach(t => t.classList.remove('active'));
|
||||
document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('active'));
|
||||
|
||||
tab.classList.add('active');
|
||||
const tabContent = document.getElementById(`${tab.dataset.tab}Tab`);
|
||||
tabContent.classList.add('active');
|
||||
|
||||
if (tab.dataset.tab === 'voice') {
|
||||
setTimeout(() => {
|
||||
this.initVisualizer();
|
||||
}, 50);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// 发送文本消息
|
||||
const messageInput = document.getElementById('messageInput');
|
||||
const sendTextButton = document.getElementById('sendTextButton');
|
||||
|
||||
const sendMessage = () => {
|
||||
const message = messageInput.value.trim();
|
||||
if (message && wsHandler.sendTextMessage(message)) {
|
||||
messageInput.value = '';
|
||||
}
|
||||
};
|
||||
|
||||
sendTextButton.addEventListener('click', sendMessage);
|
||||
messageInput.addEventListener('keypress', (e) => {
|
||||
if (e.key === 'Enter') sendMessage();
|
||||
});
|
||||
|
||||
// 录音按钮
|
||||
const recordButton = document.getElementById('recordButton');
|
||||
recordButton.addEventListener('click', () => {
|
||||
if (audioRecorder.isRecording) {
|
||||
audioRecorder.stop();
|
||||
} else {
|
||||
audioRecorder.start();
|
||||
}
|
||||
});
|
||||
|
||||
// 窗口大小变化
|
||||
window.addEventListener('resize', () => this.initVisualizer());
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
let uiControllerInstance = null;
|
||||
|
||||
export function getUIController() {
|
||||
if (!uiControllerInstance) {
|
||||
uiControllerInstance = new UIController();
|
||||
}
|
||||
return uiControllerInstance;
|
||||
}
|
||||
+7
-7
@@ -12,19 +12,19 @@ const logContainer = document.getElementById('logContainer');
|
||||
let visualizerCanvas = document.getElementById('audioVisualizer');
|
||||
|
||||
// ota 是否连接成功，修改成对应的样式
|
||||
export function otaStatusStyle (flan) {
|
||||
if(flan){
|
||||
document.getElementById('otaStatus').textContent = 'ota已连接';
|
||||
export function otaStatusStyle(flan) {
|
||||
if (flan) {
|
||||
document.getElementById('otaStatus').textContent = 'OTA已连接';
|
||||
document.getElementById('otaStatus').style.color = 'green';
|
||||
}else{
|
||||
document.getElementById('otaStatus').textContent = 'ota未连接';
|
||||
} else {
|
||||
document.getElementById('otaStatus').textContent = 'OTA未连接';
|
||||
document.getElementById('otaStatus').style.color = 'red';
|
||||
}
|
||||
}
|
||||
|
||||
// ota 是否连接成功，修改成对应的样式
|
||||
export function getLogContainer (flan) {
|
||||
return logContainer;
|
||||
export function getLogContainer(flan) {
|
||||
return logContainer;
|
||||
}
|
||||
|
||||
// 更新Opus库状态显示
|
||||
+5
@@ -95,4 +95,9 @@ export default class BlockingQueue {
|
||||
get length() {
|
||||
return this.#items.length;
|
||||
}
|
||||
|
||||
/* 清空队列（保持对象引用，不影响等待者） */
|
||||
clear() {
|
||||
this.#items.length = 0;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { getLogContainer } from '../document.js'
|
||||
import { getLogContainer } from '../ui/dom-helper.js'
|
||||
|
||||
const logContainer = getLogContainer();
|
||||
// 日志记录函数
|
||||
|
||||
@@ -1,670 +0,0 @@
|
||||
const SAMPLE_RATE = 16000;
|
||||
const CHANNELS = 1;
|
||||
const FRAME_SIZE = 960; // 对应于60ms帧大小 (16000Hz * 0.06s = 960 samples)
|
||||
const OPUS_APPLICATION = 2049; // OPUS_APPLICATION_AUDIO
|
||||
const BUFFER_SIZE = 4096;
|
||||
|
||||
let audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: SAMPLE_RATE });
|
||||
let mediaStream, mediaSource, audioProcessor;
|
||||
let recordedPcmData = []; // 存储原始PCM数据
|
||||
let recordedOpusData = []; // 存储Opus编码后的数据
|
||||
let opusEncoder, opusDecoder;
|
||||
let isRecording = false;
|
||||
|
||||
const startButton = document.getElementById("start");
|
||||
const stopButton = document.getElementById("stop");
|
||||
const playButton = document.getElementById("play");
|
||||
const statusLabel = document.getElementById("status");
|
||||
|
||||
startButton.addEventListener("click", startRecording);
|
||||
stopButton.addEventListener("click", stopRecording);
|
||||
playButton.addEventListener("click", playRecording);
|
||||
|
||||
// 初始化Opus编码器与解码器
|
||||
async function initOpus() {
|
||||
if (typeof window.ModuleInstance === 'undefined') {
|
||||
if (typeof Module !== 'undefined') {
|
||||
// 尝试使用全局Module
|
||||
window.ModuleInstance = Module;
|
||||
console.log('使用全局Module作为ModuleInstance');
|
||||
} else {
|
||||
console.error("Opus库未加载，ModuleInstance和Module对象都不存在");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const mod = window.ModuleInstance;
|
||||
|
||||
// 创建编码器
|
||||
opusEncoder = {
|
||||
channels: CHANNELS,
|
||||
sampleRate: SAMPLE_RATE,
|
||||
frameSize: FRAME_SIZE,
|
||||
maxPacketSize: 4000,
|
||||
module: mod,
|
||||
|
||||
// 初始化编码器
|
||||
init: function() {
|
||||
// 获取编码器大小
|
||||
const encoderSize = mod._opus_encoder_get_size(this.channels);
|
||||
console.log(`Opus编码器大小: ${encoderSize}字节`);
|
||||
|
||||
// 分配内存
|
||||
this.encoderPtr = mod._malloc(encoderSize);
|
||||
if (!this.encoderPtr) {
|
||||
throw new Error("无法分配编码器内存");
|
||||
}
|
||||
|
||||
// 初始化编码器
|
||||
const err = mod._opus_encoder_init(
|
||||
this.encoderPtr,
|
||||
this.sampleRate,
|
||||
this.channels,
|
||||
OPUS_APPLICATION
|
||||
);
|
||||
|
||||
if (err < 0) {
|
||||
throw new Error(`Opus编码器初始化失败: ${err}`);
|
||||
}
|
||||
|
||||
return true;
|
||||
},
|
||||
|
||||
// 编码方法
|
||||
encode: function(pcmData) {
|
||||
const mod = this.module;
|
||||
|
||||
// 为PCM数据分配内存
|
||||
const pcmPtr = mod._malloc(pcmData.length * 2); // Int16 = 2字节
|
||||
|
||||
// 将数据复制到WASM内存
|
||||
for (let i = 0; i < pcmData.length; i++) {
|
||||
mod.HEAP16[(pcmPtr >> 1) + i] = pcmData[i];
|
||||
}
|
||||
|
||||
// 为Opus编码数据分配内存
|
||||
const maxEncodedSize = this.maxPacketSize;
|
||||
const encodedPtr = mod._malloc(maxEncodedSize);
|
||||
|
||||
// 编码
|
||||
const encodedBytes = mod._opus_encode(
|
||||
this.encoderPtr,
|
||||
pcmPtr,
|
||||
this.frameSize,
|
||||
encodedPtr,
|
||||
maxEncodedSize
|
||||
);
|
||||
|
||||
if (encodedBytes < 0) {
|
||||
mod._free(pcmPtr);
|
||||
mod._free(encodedPtr);
|
||||
throw new Error(`Opus编码失败: ${encodedBytes}`);
|
||||
}
|
||||
|
||||
// 复制编码后的数据
|
||||
const encodedData = new Uint8Array(encodedBytes);
|
||||
for (let i = 0; i < encodedBytes; i++) {
|
||||
encodedData[i] = mod.HEAPU8[encodedPtr + i];
|
||||
}
|
||||
|
||||
// 释放内存
|
||||
mod._free(pcmPtr);
|
||||
mod._free(encodedPtr);
|
||||
|
||||
return encodedData;
|
||||
},
|
||||
|
||||
// 销毁方法
|
||||
destroy: function() {
|
||||
if (this.encoderPtr) {
|
||||
this.module._free(this.encoderPtr);
|
||||
this.encoderPtr = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 创建解码器
|
||||
opusDecoder = {
|
||||
channels: CHANNELS,
|
||||
rate: SAMPLE_RATE,
|
||||
frameSize: FRAME_SIZE,
|
||||
module: mod,
|
||||
|
||||
// 初始化解码器
|
||||
init: function() {
|
||||
// 获取解码器大小
|
||||
const decoderSize = mod._opus_decoder_get_size(this.channels);
|
||||
console.log(`Opus解码器大小: ${decoderSize}字节`);
|
||||
|
||||
// 分配内存
|
||||
this.decoderPtr = mod._malloc(decoderSize);
|
||||
if (!this.decoderPtr) {
|
||||
throw new Error("无法分配解码器内存");
|
||||
}
|
||||
|
||||
// 初始化解码器
|
||||
const err = mod._opus_decoder_init(
|
||||
this.decoderPtr,
|
||||
this.rate,
|
||||
this.channels
|
||||
);
|
||||
|
||||
if (err < 0) {
|
||||
throw new Error(`Opus解码器初始化失败: ${err}`);
|
||||
}
|
||||
|
||||
return true;
|
||||
},
|
||||
|
||||
// 解码方法
|
||||
decode: function(opusData) {
|
||||
const mod = this.module;
|
||||
|
||||
// 为Opus数据分配内存
|
||||
const opusPtr = mod._malloc(opusData.length);
|
||||
mod.HEAPU8.set(opusData, opusPtr);
|
||||
|
||||
// 为PCM输出分配内存
|
||||
const pcmPtr = mod._malloc(this.frameSize * 2); // Int16 = 2字节
|
||||
|
||||
// 解码
|
||||
const decodedSamples = mod._opus_decode(
|
||||
this.decoderPtr,
|
||||
opusPtr,
|
||||
opusData.length,
|
||||
pcmPtr,
|
||||
this.frameSize,
|
||||
0 // 不使用FEC
|
||||
);
|
||||
|
||||
if (decodedSamples < 0) {
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
throw new Error(`Opus解码失败: ${decodedSamples}`);
|
||||
}
|
||||
|
||||
// 复制解码后的数据
|
||||
const decodedData = new Int16Array(decodedSamples);
|
||||
for (let i = 0; i < decodedSamples; i++) {
|
||||
decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i];
|
||||
}
|
||||
|
||||
// 释放内存
|
||||
mod._free(opusPtr);
|
||||
mod._free(pcmPtr);
|
||||
|
||||
return decodedData;
|
||||
},
|
||||
|
||||
// 销毁方法
|
||||
destroy: function() {
|
||||
if (this.decoderPtr) {
|
||||
this.module._free(this.decoderPtr);
|
||||
this.decoderPtr = null;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 初始化编码器和解码器
|
||||
if (opusEncoder.init() && opusDecoder.init()) {
|
||||
console.log("Opus 编码器和解码器初始化成功。");
|
||||
return true;
|
||||
} else {
|
||||
console.error("Opus 初始化失败");
|
||||
return false;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Opus 初始化失败:", error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 将Float32音频数据转换为Int16音频数据
|
||||
function convertFloat32ToInt16(float32Data) {
|
||||
const int16Data = new Int16Array(float32Data.length);
|
||||
for (let i = 0; i < float32Data.length; i++) {
|
||||
// 将[-1,1]范围转换为[-32768,32767]
|
||||
const s = Math.max(-1, Math.min(1, float32Data[i]));
|
||||
int16Data[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
|
||||
}
|
||||
return int16Data;
|
||||
}
|
||||
|
||||
// 将Int16音频数据转换为Float32音频数据
|
||||
function convertInt16ToFloat32(int16Data) {
|
||||
const float32Data = new Float32Array(int16Data.length);
|
||||
for (let i = 0; i < int16Data.length; i++) {
|
||||
// 将[-32768,32767]范围转换为[-1,1]
|
||||
float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF);
|
||||
}
|
||||
return float32Data;
|
||||
}
|
||||
|
||||
function startRecording() {
|
||||
if (isRecording) return;
|
||||
|
||||
// 确保有权限并且AudioContext是活跃的
|
||||
if (audioContext.state === 'suspended') {
|
||||
audioContext.resume().then(() => {
|
||||
console.log("AudioContext已恢复");
|
||||
continueStartRecording();
|
||||
}).catch(err => {
|
||||
console.error("恢复AudioContext失败:", err);
|
||||
statusLabel.textContent = "无法激活音频上下文，请再次点击";
|
||||
});
|
||||
} else {
|
||||
continueStartRecording();
|
||||
}
|
||||
}
|
||||
|
||||
// 实际开始录音的逻辑
|
||||
function continueStartRecording() {
|
||||
// 重置录音数据
|
||||
recordedPcmData = [];
|
||||
recordedOpusData = [];
|
||||
window.audioDataBuffer = new Int16Array(0); // 重置缓冲区
|
||||
|
||||
// 初始化Opus
|
||||
initOpus().then(success => {
|
||||
if (!success) {
|
||||
statusLabel.textContent = "Opus初始化失败";
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("开始录音，参数：", {
|
||||
sampleRate: SAMPLE_RATE,
|
||||
channels: CHANNELS,
|
||||
frameSize: FRAME_SIZE,
|
||||
bufferSize: BUFFER_SIZE
|
||||
});
|
||||
|
||||
// 请求麦克风权限
|
||||
navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
sampleRate: SAMPLE_RATE,
|
||||
channelCount: CHANNELS,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true
|
||||
}
|
||||
})
|
||||
.then(stream => {
|
||||
console.log("获取到麦克风流，实际参数：", stream.getAudioTracks()[0].getSettings());
|
||||
|
||||
// 检查流是否有效
|
||||
if (!stream || !stream.getAudioTracks().length || !stream.getAudioTracks()[0].enabled) {
|
||||
throw new Error("获取到的音频流无效");
|
||||
}
|
||||
|
||||
mediaStream = stream;
|
||||
mediaSource = audioContext.createMediaStreamSource(stream);
|
||||
|
||||
// 创建ScriptProcessor(虽然已弃用，但兼容性好)
|
||||
// 在降级到ScriptProcessor之前尝试使用AudioWorklet
|
||||
createAudioProcessor().then(processor => {
|
||||
if (processor) {
|
||||
console.log("使用AudioWorklet处理音频");
|
||||
audioProcessor = processor;
|
||||
// 连接音频处理链
|
||||
mediaSource.connect(audioProcessor);
|
||||
audioProcessor.connect(audioContext.destination);
|
||||
} else {
|
||||
console.log("回退到ScriptProcessor");
|
||||
// 创建ScriptProcessor节点
|
||||
audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, CHANNELS, CHANNELS);
|
||||
|
||||
// 处理音频数据
|
||||
audioProcessor.onaudioprocess = processAudioData;
|
||||
|
||||
// 连接音频处理链
|
||||
mediaSource.connect(audioProcessor);
|
||||
audioProcessor.connect(audioContext.destination);
|
||||
}
|
||||
|
||||
// 更新UI
|
||||
isRecording = true;
|
||||
statusLabel.textContent = "录音中...";
|
||||
startButton.disabled = true;
|
||||
stopButton.disabled = false;
|
||||
playButton.disabled = true;
|
||||
}).catch(error => {
|
||||
console.error("创建音频处理器失败:", error);
|
||||
statusLabel.textContent = "创建音频处理器失败";
|
||||
});
|
||||
})
|
||||
.catch(error => {
|
||||
console.error("获取麦克风失败:", error);
|
||||
statusLabel.textContent = "获取麦克风失败: " + error.message;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// 创建AudioWorklet处理器
|
||||
async function createAudioProcessor() {
|
||||
try {
|
||||
// 尝试使用更现代的AudioWorklet API
|
||||
if ('AudioWorklet' in window && 'AudioWorkletNode' in window) {
|
||||
// 定义AudioWorklet处理器代码
|
||||
const workletCode = `
|
||||
class OpusRecorderProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.buffers = [];
|
||||
this.frameSize = ${FRAME_SIZE};
|
||||
this.buffer = new Float32Array(this.frameSize);
|
||||
this.bufferIndex = 0;
|
||||
this.isRecording = false;
|
||||
|
||||
this.port.onmessage = (event) => {
|
||||
if (event.data.command === 'start') {
|
||||
this.isRecording = true;
|
||||
} else if (event.data.command === 'stop') {
|
||||
this.isRecording = false;
|
||||
// 发送最后的缓冲区
|
||||
if (this.bufferIndex > 0) {
|
||||
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
|
||||
this.port.postMessage({ buffer: finalBuffer });
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
process(inputs, outputs) {
|
||||
if (!this.isRecording) return true;
|
||||
|
||||
// 获取输入数据
|
||||
const input = inputs[0][0]; // mono channel
|
||||
if (!input || input.length === 0) return true;
|
||||
|
||||
// 将输入数据添加到缓冲区
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
this.buffer[this.bufferIndex++] = input[i];
|
||||
|
||||
// 当缓冲区填满时，发送给主线程
|
||||
if (this.bufferIndex >= this.frameSize) {
|
||||
this.port.postMessage({ buffer: this.buffer.slice() });
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('opus-recorder-processor', OpusRecorderProcessor);
|
||||
`;
|
||||
|
||||
// 创建Blob URL
|
||||
const blob = new Blob([workletCode], { type: 'application/javascript' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
|
||||
// 加载AudioWorklet模块
|
||||
await audioContext.audioWorklet.addModule(url);
|
||||
|
||||
// 创建AudioWorkletNode
|
||||
const workletNode = new AudioWorkletNode(audioContext, 'opus-recorder-processor');
|
||||
|
||||
// 处理从AudioWorklet接收的消息
|
||||
workletNode.port.onmessage = (event) => {
|
||||
if (event.data.buffer) {
|
||||
// 使用与ScriptProcessor相同的处理逻辑
|
||||
processAudioData({
|
||||
inputBuffer: {
|
||||
getChannelData: () => event.data.buffer
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// 启动录音
|
||||
workletNode.port.postMessage({ command: 'start' });
|
||||
|
||||
// 保存停止函数
|
||||
workletNode.stopRecording = () => {
|
||||
workletNode.port.postMessage({ command: 'stop' });
|
||||
};
|
||||
|
||||
console.log("AudioWorklet 音频处理器创建成功");
|
||||
return workletNode;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("创建AudioWorklet失败，将使用ScriptProcessor:", error);
|
||||
}
|
||||
|
||||
// 如果AudioWorklet不可用或失败，返回null以便回退到ScriptProcessor
|
||||
return null;
|
||||
}
|
||||
|
||||
// 处理音频数据
|
||||
function processAudioData(e) {
|
||||
// 获取输入缓冲区
|
||||
const inputBuffer = e.inputBuffer;
|
||||
|
||||
// 获取第一个通道的Float32数据
|
||||
const inputData = inputBuffer.getChannelData(0);
|
||||
|
||||
// 添加调试信息
|
||||
const nonZeroCount = Array.from(inputData).filter(x => Math.abs(x) > 0.001).length;
|
||||
console.log(`接收到音频数据: ${inputData.length} 个样本, 非零样本数: ${nonZeroCount}`);
|
||||
|
||||
// 如果全是0，可能是麦克风没有正确获取声音
|
||||
if (nonZeroCount < 5) {
|
||||
console.warn("警告: 检测到大量静音样本，请检查麦克风是否正常工作");
|
||||
// 继续处理，以防有些样本确实是静音
|
||||
}
|
||||
|
||||
// 存储PCM数据用于调试
|
||||
recordedPcmData.push(new Float32Array(inputData));
|
||||
|
||||
// 转换为Int16数据供Opus编码
|
||||
const int16Data = convertFloat32ToInt16(inputData);
|
||||
|
||||
// 如果收集到的数据不是FRAME_SIZE的整数倍，需要进行处理
|
||||
// 创建静态缓冲区来存储不足一帧的数据
|
||||
if (!window.audioDataBuffer) {
|
||||
window.audioDataBuffer = new Int16Array(0);
|
||||
}
|
||||
|
||||
// 合并之前缓存的数据和新数据
|
||||
const combinedData = new Int16Array(window.audioDataBuffer.length + int16Data.length);
|
||||
combinedData.set(window.audioDataBuffer);
|
||||
combinedData.set(int16Data, window.audioDataBuffer.length);
|
||||
|
||||
// 处理完整帧
|
||||
const frameCount = Math.floor(combinedData.length / FRAME_SIZE);
|
||||
console.log(`可编码的完整帧数: ${frameCount}, 缓冲区总大小: ${combinedData.length}`);
|
||||
|
||||
for (let i = 0; i < frameCount; i++) {
|
||||
const frameData = combinedData.subarray(i * FRAME_SIZE, (i + 1) * FRAME_SIZE);
|
||||
|
||||
try {
|
||||
console.log(`编码第 ${i+1}/${frameCount} 帧, 帧大小: ${frameData.length}`);
|
||||
const encodedData = opusEncoder.encode(frameData);
|
||||
if (encodedData) {
|
||||
console.log(`编码成功: ${encodedData.length} 字节`);
|
||||
recordedOpusData.push(encodedData);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Opus编码帧 ${i+1} 失败:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
// 保存剩余不足一帧的数据
|
||||
const remainingSamples = combinedData.length % FRAME_SIZE;
|
||||
if (remainingSamples > 0) {
|
||||
window.audioDataBuffer = combinedData.subarray(frameCount * FRAME_SIZE);
|
||||
console.log(`保留 ${remainingSamples} 个样本到下一次处理`);
|
||||
} else {
|
||||
window.audioDataBuffer = new Int16Array(0);
|
||||
}
|
||||
}
|
||||
|
||||
function stopRecording() {
|
||||
if (!isRecording) return;
|
||||
|
||||
// 处理剩余的缓冲数据
|
||||
if (window.audioDataBuffer && window.audioDataBuffer.length > 0) {
|
||||
console.log(`停止录音，处理剩余的 ${window.audioDataBuffer.length} 个样本`);
|
||||
// 如果剩余数据不足一帧，可以通过补零的方式凑成一帧
|
||||
if (window.audioDataBuffer.length < FRAME_SIZE) {
|
||||
const paddedFrame = new Int16Array(FRAME_SIZE);
|
||||
paddedFrame.set(window.audioDataBuffer);
|
||||
// 剩余部分填充为0
|
||||
for (let i = window.audioDataBuffer.length; i < FRAME_SIZE; i++) {
|
||||
paddedFrame[i] = 0;
|
||||
}
|
||||
try {
|
||||
console.log(`编码最后一帧(补零): ${paddedFrame.length} 样本`);
|
||||
const encodedData = opusEncoder.encode(paddedFrame);
|
||||
if (encodedData) {
|
||||
recordedOpusData.push(encodedData);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("最后一帧Opus编码失败:", error);
|
||||
}
|
||||
} else {
|
||||
// 如果数据超过一帧，按正常流程处理
|
||||
processAudioData({
|
||||
inputBuffer: {
|
||||
getChannelData: () => convertInt16ToFloat32(window.audioDataBuffer)
|
||||
}
|
||||
});
|
||||
}
|
||||
window.audioDataBuffer = null;
|
||||
}
|
||||
|
||||
// 如果使用的是AudioWorklet，调用其特定的停止方法
|
||||
if (audioProcessor && typeof audioProcessor.stopRecording === 'function') {
|
||||
audioProcessor.stopRecording();
|
||||
}
|
||||
|
||||
// 停止麦克风
|
||||
if (mediaStream) {
|
||||
mediaStream.getTracks().forEach(track => track.stop());
|
||||
}
|
||||
|
||||
// 断开音频处理链
|
||||
if (audioProcessor) {
|
||||
try {
|
||||
audioProcessor.disconnect();
|
||||
if (mediaSource) mediaSource.disconnect();
|
||||
} catch (error) {
|
||||
console.warn("断开音频处理链时出错:", error);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新UI
|
||||
isRecording = false;
|
||||
statusLabel.textContent = "已停止录音，收集了 " + recordedOpusData.length + " 帧Opus数据";
|
||||
startButton.disabled = false;
|
||||
stopButton.disabled = true;
|
||||
playButton.disabled = recordedOpusData.length === 0;
|
||||
|
||||
console.log("录制完成:",
|
||||
"PCM帧数:", recordedPcmData.length,
|
||||
"Opus帧数:", recordedOpusData.length);
|
||||
}
|
||||
|
||||
function playRecording() {
|
||||
if (!recordedOpusData.length) {
|
||||
statusLabel.textContent = "没有可播放的录音";
|
||||
return;
|
||||
}
|
||||
|
||||
// 将所有Opus数据解码为PCM
|
||||
let allDecodedData = [];
|
||||
|
||||
for (const opusData of recordedOpusData) {
|
||||
try {
|
||||
// 解码为Int16数据
|
||||
const decodedData = opusDecoder.decode(opusData);
|
||||
|
||||
if (decodedData && decodedData.length > 0) {
|
||||
// 将Int16数据转换为Float32
|
||||
const float32Data = convertInt16ToFloat32(decodedData);
|
||||
|
||||
// 添加到总解码数据中
|
||||
allDecodedData.push(...float32Data);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Opus解码失败:", error);
|
||||
}
|
||||
}
|
||||
|
||||
// 如果没有解码出数据，返回
|
||||
if (allDecodedData.length === 0) {
|
||||
statusLabel.textContent = "解码失败，无法播放";
|
||||
return;
|
||||
}
|
||||
|
||||
// 创建音频缓冲区
|
||||
const audioBuffer = audioContext.createBuffer(CHANNELS, allDecodedData.length, SAMPLE_RATE);
|
||||
audioBuffer.copyToChannel(new Float32Array(allDecodedData), 0);
|
||||
|
||||
// 创建音频源并播放
|
||||
const source = audioContext.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(audioContext.destination);
|
||||
source.start();
|
||||
|
||||
// 更新UI
|
||||
statusLabel.textContent = "正在播放...";
|
||||
playButton.disabled = true;
|
||||
|
||||
// 播放结束后恢复UI
|
||||
source.onended = () => {
|
||||
statusLabel.textContent = "播放完毕";
|
||||
playButton.disabled = false;
|
||||
};
|
||||
}
|
||||
|
||||
// 模拟服务端返回的Opus数据进行解码播放
|
||||
function playOpusFromServer(opusData) {
|
||||
// 这个函数展示如何处理服务端返回的opus数据
|
||||
// opusData应该是一个包含opus帧的数组
|
||||
|
||||
if (!opusDecoder) {
|
||||
initOpus().then(success => {
|
||||
if (success) {
|
||||
decodeAndPlayOpusData(opusData);
|
||||
} else {
|
||||
statusLabel.textContent = "Opus解码器初始化失败";
|
||||
}
|
||||
});
|
||||
} else {
|
||||
decodeAndPlayOpusData(opusData);
|
||||
}
|
||||
}
|
||||
|
||||
function decodeAndPlayOpusData(opusData) {
|
||||
let allDecodedData = [];
|
||||
|
||||
for (const frame of opusData) {
|
||||
try {
|
||||
const decodedData = opusDecoder.decode(frame);
|
||||
if (decodedData && decodedData.length > 0) {
|
||||
const float32Data = convertInt16ToFloat32(decodedData);
|
||||
allDecodedData.push(...float32Data);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("服务端Opus数据解码失败:", error);
|
||||
}
|
||||
}
|
||||
|
||||
if (allDecodedData.length === 0) {
|
||||
statusLabel.textContent = "服务端数据解码失败";
|
||||
return;
|
||||
}
|
||||
|
||||
const audioBuffer = audioContext.createBuffer(CHANNELS, allDecodedData.length, SAMPLE_RATE);
|
||||
audioBuffer.copyToChannel(new Float32Array(allDecodedData), 0);
|
||||
|
||||
const source = audioContext.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(audioContext.destination);
|
||||
source.start();
|
||||
|
||||
statusLabel.textContent = "正在播放服务端数据...";
|
||||
source.onended = () => statusLabel.textContent = "服务端数据播放完毕";
|
||||
}
|
||||
@@ -1,464 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Opus 编解码测试</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
}
|
||||
.container {
|
||||
background-color: #f5f5f5;
|
||||
padding: 20px;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
button {
|
||||
padding: 8px 15px;
|
||||
margin-right: 10px;
|
||||
border: none;
|
||||
border-radius: 5px;
|
||||
background-color: #4285f4;
|
||||
color: white;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.2s;
|
||||
}
|
||||
button:hover {
|
||||
background-color: #3367d6;
|
||||
}
|
||||
button:disabled {
|
||||
background-color: #cccccc;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
#status {
|
||||
font-weight: bold;
|
||||
}
|
||||
#scriptStatus {
|
||||
position: fixed;
|
||||
top: 10px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
padding: 15px;
|
||||
border-radius: 5px;
|
||||
display: block;
|
||||
z-index: 100;
|
||||
transition: all 0.3s ease;
|
||||
}
|
||||
#scriptStatus.info {
|
||||
background-color: #e8f0fe;
|
||||
color: #4285f4;
|
||||
border-left: 4px solid #4285f4;
|
||||
}
|
||||
#scriptStatus.success {
|
||||
background-color: #e6f4ea;
|
||||
color: #0f9d58;
|
||||
border-left: 4px solid #0f9d58;
|
||||
}
|
||||
#scriptStatus.error {
|
||||
background-color: #fce8e6;
|
||||
color: #db4437;
|
||||
border-left: 4px solid #db4437;
|
||||
}
|
||||
#debugInfo {
|
||||
margin-top: 20px;
|
||||
border: 1px solid #ccc;
|
||||
border-radius: 5px;
|
||||
padding: 10px;
|
||||
font-family: monospace;
|
||||
font-size: 12px;
|
||||
max-height: 200px;
|
||||
overflow-y: auto;
|
||||
display: none;
|
||||
}
|
||||
#showDebug {
|
||||
margin-top: 10px;
|
||||
background-color: #f5f5f5;
|
||||
color: #444;
|
||||
font-size: 12px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h2>Opus 编解码录音播放测试</h2>
|
||||
|
||||
<div id="scriptStatus" class="info">正在加载Opus库...</div>
|
||||
|
||||
<div>
|
||||
<button id="initAudio" style="background-color: #34a853;">初始化音频</button>
|
||||
<button id="testMic" style="background-color: #fbbc05;">测试麦克风</button>
|
||||
<p></p>
|
||||
<button id="start" disabled>开始录音</button>
|
||||
<button id="stop" disabled>停止录音</button>
|
||||
<button id="play" disabled>播放录音</button>
|
||||
<p>录音状态: <span id="status">待机，正在初始化...</span></p>
|
||||
<button id="showDebug">显示/隐藏调试信息</button>
|
||||
<div id="debugInfo"></div>
|
||||
<div id="audioMeter" style="margin-top: 10px; height: 20px; background-color: #eee; border-radius: 4px; overflow: hidden; display: none;">
|
||||
<div id="audioLevel" style="height: 100%; width: 0%; background-color: #4285f4; transition: width 0.1s;"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Opus解码库 - 这个库会设置一个全局Module变量 -->
|
||||
<script>
|
||||
// 定义全局变量以跟踪库加载状态
|
||||
window.opusLoaded = false;
|
||||
window.startButton = document.getElementById("start");
|
||||
window.stopButton = document.getElementById("stop");
|
||||
window.playButton = document.getElementById("play");
|
||||
window.statusLabel = document.getElementById("status");
|
||||
window.debugInfo = document.getElementById("debugInfo");
|
||||
window.audioContextReady = false;
|
||||
window.testMicActive = false;
|
||||
|
||||
// 显示/隐藏调试信息
|
||||
document.getElementById("showDebug").addEventListener("click", function() {
|
||||
if (debugInfo.style.display === "none" || !debugInfo.style.display) {
|
||||
debugInfo.style.display = "block";
|
||||
this.textContent = "隐藏调试信息";
|
||||
} else {
|
||||
debugInfo.style.display = "none";
|
||||
this.textContent = "显示调试信息";
|
||||
}
|
||||
});
|
||||
|
||||
// 添加初始化音频按钮事件
|
||||
document.getElementById("initAudio").addEventListener("click", function() {
|
||||
initializeAudioSystem();
|
||||
});
|
||||
|
||||
// 添加测试麦克风按钮事件
|
||||
document.getElementById("testMic").addEventListener("click", function() {
|
||||
if (window.testMicActive) {
|
||||
stopMicTest();
|
||||
} else {
|
||||
startMicTest();
|
||||
}
|
||||
});
|
||||
|
||||
// 初始化音频系统
|
||||
function initializeAudioSystem() {
|
||||
try {
|
||||
log("初始化音频系统...");
|
||||
// 创建临时AudioContext来触发用户授权
|
||||
const tempContext = new (window.AudioContext || window.webkitAudioContext)({
|
||||
sampleRate: 16000,
|
||||
latencyHint: 'interactive'
|
||||
});
|
||||
|
||||
// 创建振荡器并播放短促的声音
|
||||
const oscillator = tempContext.createOscillator();
|
||||
const gain = tempContext.createGain();
|
||||
gain.gain.value = 0.1; // 很小的音量
|
||||
oscillator.connect(gain);
|
||||
gain.connect(tempContext.destination);
|
||||
oscillator.frequency.value = 440; // A4
|
||||
oscillator.start();
|
||||
|
||||
// 0.2秒后停止
|
||||
setTimeout(() => {
|
||||
oscillator.stop();
|
||||
// 关闭上下文
|
||||
tempContext.close().then(() => {
|
||||
log("音频系统初始化成功", "success");
|
||||
updateScriptStatus("音频系统已激活", "success");
|
||||
document.getElementById("initAudio").disabled = true;
|
||||
document.getElementById("initAudio").textContent = "音频已初始化";
|
||||
window.audioContextReady = true;
|
||||
|
||||
// 如果Opus已加载，启用开始录音按钮
|
||||
if (window.opusLoaded) {
|
||||
startButton.disabled = false;
|
||||
}
|
||||
});
|
||||
}, 200);
|
||||
} catch (err) {
|
||||
log("初始化音频系统失败: " + err.message, "error");
|
||||
updateScriptStatus("初始化音频失败: " + err.message, "error");
|
||||
}
|
||||
}
|
||||
|
||||
// 测试麦克风
|
||||
function startMicTest() {
|
||||
const audioMeter = document.getElementById("audioMeter");
|
||||
const audioLevel = document.getElementById("audioLevel");
|
||||
const testMicBtn = document.getElementById("testMic");
|
||||
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
log("浏览器不支持麦克风访问", "error");
|
||||
return;
|
||||
}
|
||||
|
||||
log("开始麦克风测试...");
|
||||
audioMeter.style.display = "block";
|
||||
testMicBtn.textContent = "停止测试";
|
||||
testMicBtn.style.backgroundColor = "#ea4335";
|
||||
window.testMicActive = true;
|
||||
|
||||
// 创建音频上下文
|
||||
const testContext = new (window.AudioContext || window.webkitAudioContext)();
|
||||
window.testContext = testContext;
|
||||
|
||||
// 获取麦克风权限
|
||||
navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true
|
||||
}
|
||||
})
|
||||
.then(stream => {
|
||||
log("已获取麦克风访问权限", "success");
|
||||
|
||||
// 保存流以便稍后关闭
|
||||
window.testStream = stream;
|
||||
|
||||
// 创建音频分析器
|
||||
const source = testContext.createMediaStreamSource(stream);
|
||||
const analyser = testContext.createAnalyser();
|
||||
analyser.fftSize = 256;
|
||||
source.connect(analyser);
|
||||
|
||||
// 创建音量显示更新函数
|
||||
const bufferLength = analyser.frequencyBinCount;
|
||||
const dataArray = new Uint8Array(bufferLength);
|
||||
|
||||
function updateMeter() {
|
||||
if (!window.testMicActive) return;
|
||||
|
||||
analyser.getByteFrequencyData(dataArray);
|
||||
|
||||
// 计算音量级别 (0-100)
|
||||
let sum = 0;
|
||||
for (let i = 0; i < bufferLength; i++) {
|
||||
sum += dataArray[i];
|
||||
}
|
||||
const average = sum / bufferLength;
|
||||
const level = Math.min(100, Math.max(0, average * 2));
|
||||
|
||||
// 更新音量计
|
||||
audioLevel.style.width = level + "%";
|
||||
|
||||
// 如果有声音，记录日志
|
||||
if (level > 10) {
|
||||
log(`检测到声音: ${level.toFixed(1)}%`);
|
||||
}
|
||||
|
||||
// 循环更新
|
||||
window.testMicAnimationFrame = requestAnimationFrame(updateMeter);
|
||||
}
|
||||
|
||||
// 开始更新
|
||||
updateMeter();
|
||||
})
|
||||
.catch(err => {
|
||||
log("麦克风测试失败: " + err.message, "error");
|
||||
window.testMicActive = false;
|
||||
testMicBtn.textContent = "测试麦克风";
|
||||
testMicBtn.style.backgroundColor = "#fbbc05";
|
||||
audioMeter.style.display = "none";
|
||||
});
|
||||
}
|
||||
|
||||
// 停止麦克风测试
|
||||
function stopMicTest() {
|
||||
const audioMeter = document.getElementById("audioMeter");
|
||||
const testMicBtn = document.getElementById("testMic");
|
||||
|
||||
log("停止麦克风测试");
|
||||
window.testMicActive = false;
|
||||
testMicBtn.textContent = "测试麦克风";
|
||||
testMicBtn.style.backgroundColor = "#fbbc05";
|
||||
|
||||
// 停止分析器动画
|
||||
if (window.testMicAnimationFrame) {
|
||||
cancelAnimationFrame(window.testMicAnimationFrame);
|
||||
}
|
||||
|
||||
// 停止麦克风流
|
||||
if (window.testStream) {
|
||||
window.testStream.getTracks().forEach(track => track.stop());
|
||||
}
|
||||
|
||||
// 关闭测试上下文
|
||||
if (window.testContext) {
|
||||
window.testContext.close();
|
||||
}
|
||||
|
||||
// 隐藏音量计
|
||||
audioMeter.style.display = "none";
|
||||
}
|
||||
|
||||
// 初始化AudioContext
|
||||
function initAudioContext() {
|
||||
try {
|
||||
window.AudioContext = window.AudioContext || window.webkitAudioContext;
|
||||
if (!window.AudioContext) {
|
||||
throw new Error("浏览器不支持AudioContext");
|
||||
}
|
||||
|
||||
// 我们在app.js中会创建AudioContext，这里只检查兼容性
|
||||
log("AudioContext API 可用");
|
||||
return true;
|
||||
} catch (err) {
|
||||
log("AudioContext初始化失败: " + err.message, "error");
|
||||
updateScriptStatus("AudioContext初始化失败，录音功能不可用", "error");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 检查麦克风权限
|
||||
function checkMicrophonePermission() {
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
log("浏览器不支持getUserMedia API", "error");
|
||||
updateScriptStatus("浏览器不支持录音功能", "error");
|
||||
return false;
|
||||
}
|
||||
|
||||
log("getUserMedia API 可用");
|
||||
return true;
|
||||
}
|
||||
|
||||
// 添加调试日志
|
||||
function log(message, type = "info") {
|
||||
console.log(message);
|
||||
const time = new Date().toLocaleTimeString();
|
||||
const entry = document.createElement("div");
|
||||
entry.textContent = `[${time}] ${message}`;
|
||||
|
||||
if (type === "error") {
|
||||
entry.style.color = "#db4437";
|
||||
} else if (type === "success") {
|
||||
entry.style.color = "#0f9d58";
|
||||
}
|
||||
|
||||
debugInfo.appendChild(entry);
|
||||
debugInfo.scrollTop = debugInfo.scrollHeight;
|
||||
}
|
||||
|
||||
// 更新脚本状态显示
|
||||
function updateScriptStatus(message, type) {
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) {
|
||||
statusElement.textContent = message;
|
||||
statusElement.className = type;
|
||||
statusElement.style.display = 'block';
|
||||
}
|
||||
log(message, type);
|
||||
}
|
||||
|
||||
// 检查Opus库是否已加载
|
||||
function checkOpusLoaded() {
|
||||
try {
|
||||
// 检查Module是否存在（本地库导出的全局变量）
|
||||
if (typeof Module === 'undefined') {
|
||||
log("Module对象不存在", "error");
|
||||
throw new Error('Opus库未加载，Module对象不存在');
|
||||
}
|
||||
|
||||
// 记录Module对象结构以便调试
|
||||
log("Module对象结构: " + Object.keys(Module).join(", "));
|
||||
|
||||
// 尝试使用全局Module函数
|
||||
if (typeof Module._opus_decoder_get_size === 'function') {
|
||||
window.ModuleInstance = Module;
|
||||
log('Opus库加载成功（使用全局Module）', "success");
|
||||
updateScriptStatus('Opus库加载成功', 'success');
|
||||
|
||||
// 启用开始录音按钮
|
||||
startButton.disabled = false;
|
||||
statusLabel.textContent = "待机";
|
||||
|
||||
// 3秒后隐藏状态
|
||||
setTimeout(() => {
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
}, 3000);
|
||||
|
||||
window.opusLoaded = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
// 尝试使用Module.instance
|
||||
if (typeof Module.instance !== 'undefined' && typeof Module.instance._opus_decoder_get_size === 'function') {
|
||||
window.ModuleInstance = Module.instance;
|
||||
log('Opus库加载成功（使用Module.instance）', "success");
|
||||
updateScriptStatus('Opus库加载成功', 'success');
|
||||
|
||||
// 启用开始录音按钮
|
||||
startButton.disabled = false;
|
||||
statusLabel.textContent = "待机";
|
||||
|
||||
// 3秒后隐藏状态
|
||||
setTimeout(() => {
|
||||
const statusElement = document.getElementById('scriptStatus');
|
||||
if (statusElement) statusElement.style.display = 'none';
|
||||
}, 3000);
|
||||
|
||||
window.opusLoaded = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
// 检查是否有其他导出方式
|
||||
log("Module上可用方法: " + Object.getOwnPropertyNames(Module).filter(prop => typeof Module[prop] === 'function').join(", "));
|
||||
|
||||
if (typeof Module.onRuntimeInitialized === 'function' || typeof Module.onRuntimeInitialized === 'undefined') {
|
||||
log("Module.onRuntimeInitialized 尚未执行，等待模块初始化完成...");
|
||||
// Module可能未完成初始化，注册回调
|
||||
Module.onRuntimeInitialized = function() {
|
||||
log("Opus库运行时初始化完成，重新检查");
|
||||
checkOpusLoaded();
|
||||
};
|
||||
return false;
|
||||
}
|
||||
|
||||
throw new Error('Opus解码函数未找到，可能Module结构不正确');
|
||||
} catch (err) {
|
||||
log(`Opus库加载失败: ${err.message}`, "error");
|
||||
updateScriptStatus(`Opus库加载失败: ${err.message}`, 'error');
|
||||
statusLabel.textContent = "错误：Opus库加载失败";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 页面加载完成后检查浏览器能力和Opus库
|
||||
window.addEventListener('load', function() {
|
||||
log("页面加载完成，开始初始化...");
|
||||
updateScriptStatus('正在初始化录音环境...', 'info');
|
||||
|
||||
// 检查浏览器能力
|
||||
const audioContextSupported = initAudioContext();
|
||||
const microphoneSupported = checkMicrophonePermission();
|
||||
|
||||
if (!audioContextSupported || !microphoneSupported) {
|
||||
log("浏览器不支持必要API，无法进行录音", "error");
|
||||
updateScriptStatus('浏览器不支持录音功能', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
log("检查环境完成，开始加载Opus库");
|
||||
updateScriptStatus('正在加载Opus库...', 'info');
|
||||
|
||||
// 延迟一小段时间再检查，确保库有时间加载
|
||||
setTimeout(function checkAndRetry() {
|
||||
if (!checkOpusLoaded()) {
|
||||
// 如果加载失败，5秒后重试
|
||||
log("Opus库加载失败，5秒后重试...");
|
||||
updateScriptStatus('Opus库加载失败，正在重试...', 'error');
|
||||
setTimeout(checkAndRetry, 5000);
|
||||
}
|
||||
}, 1000);
|
||||
});
|
||||
|
||||
// 防止按钮误操作
|
||||
document.getElementById("stop").disabled = true;
|
||||
document.getElementById("play").disabled = true;
|
||||
</script>
|
||||
<script src="./../libopus.js"></script>
|
||||
<script src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user