mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-22 07:03:53 +08:00
Merge branch 'py_test_error_response' into fix/2075
This commit is contained in:
@@ -37,7 +37,7 @@ jobs:
|
||||
file: Dockerfile-server-base
|
||||
push: true
|
||||
tags: ghcr.io/${{ github.repository }}:server-base
|
||||
platforms: linux/amd64
|
||||
platforms: linux/amd64,linux/arm64
|
||||
cache-from: type=gha,scope=server-base
|
||||
cache-to: type=gha,mode=max,scope=server-base
|
||||
build-args: |
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:server_{1},ghcr.io/{0}:server_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:server_latest', github.repository) }}
|
||||
platforms: linux/amd64
|
||||
platforms: linux/amd64,linux/arm64
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:web_{1},ghcr.io/{0}:web_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:web_latest', github.repository) }}
|
||||
platforms: linux/amd64
|
||||
platforms: linux/amd64,linux/arm64
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
|
||||
+9
-8
@@ -1,5 +1,5 @@
|
||||
# 第一阶段:构建Vue前端
|
||||
FROM node:18 as web-builder
|
||||
FROM node:18 AS web-builder
|
||||
WORKDIR /app
|
||||
COPY main/manager-web/package*.json ./
|
||||
RUN npm install
|
||||
@@ -7,7 +7,7 @@ COPY main/manager-web .
|
||||
RUN npm run build
|
||||
|
||||
# 第二阶段:构建Java后端
|
||||
FROM maven:3.9.4-eclipse-temurin-21 as api-builder
|
||||
FROM maven:3.9.4-eclipse-temurin-21 AS api-builder
|
||||
WORKDIR /app
|
||||
COPY main/manager-api/pom.xml .
|
||||
COPY main/manager-api/src ./src
|
||||
@@ -18,18 +18,19 @@ FROM bellsoft/liberica-runtime-container:jre-21-glibc
|
||||
|
||||
# 安装Nginx和字体库
|
||||
RUN apk update && \
|
||||
apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
|
||||
apk add --no-cache --no-scripts \
|
||||
nginx \
|
||||
bash \
|
||||
fontconfig \
|
||||
ttf-dejavu \
|
||||
msttcorefonts-installer \
|
||||
&& ACCEPT_EULA=Y apk add --no-cache msttcorefonts-installer \
|
||||
&& fc-cache -f -v \
|
||||
&& rm -rf /var/cache/apk/*
|
||||
&& rm -rf /var/cache/apk/* \
|
||||
&& mkdir -p /run/nginx /var/log/nginx /var/tmp/nginx /etc/nginx/conf.d
|
||||
|
||||
# 复制项目自带的中文字体
|
||||
COPY main/manager-web/public/generator/static/fonts/*.ttf /usr/share/fonts/
|
||||
|
||||
# 更新字体缓存
|
||||
RUN (printf 'YES\n' | update-ms-fonts || true) && fc-cache -f -v
|
||||
RUN fc-cache -f -v
|
||||
|
||||
# 配置Nginx
|
||||
COPY docs/docker/nginx.conf /etc/nginx/nginx.conf
|
||||
|
||||
@@ -304,7 +304,7 @@ public interface Constant {
|
||||
/**
|
||||
* 版本号
|
||||
*/
|
||||
public static final String VERSION = "0.8.11";
|
||||
public static final String VERSION = "0.9.1";
|
||||
|
||||
/**
|
||||
* 无效固件URL
|
||||
|
||||
+12
-5
@@ -1,6 +1,12 @@
|
||||
package xiaozhi.modules.agent.service.impl;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@@ -81,9 +87,9 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
|
||||
|
||||
if (agent.getMemModelId() != null && agent.getMemModelId().equals(Constant.MEMORY_NO_MEM)) {
|
||||
agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.IGNORE.getCode());
|
||||
if (agent.getChatHistoryConf() == null) {
|
||||
agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode());
|
||||
}
|
||||
}
|
||||
if (agent.getChatHistoryConf() == null) {
|
||||
agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode());
|
||||
}
|
||||
|
||||
// 查询上下文源配置
|
||||
@@ -132,7 +138,8 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
|
||||
if (StringUtils.isNotBlank(keyword)) {
|
||||
if ("mac".equals(searchType)) {
|
||||
// 按MAC地址搜索:先搜索设备,再获取对应的智能体
|
||||
List<DeviceEntity> devices = Optional.ofNullable(deviceService.searchDevicesByMacAddress(keyword, userId)).orElseGet(ArrayList::new);
|
||||
List<DeviceEntity> devices = Optional
|
||||
.ofNullable(deviceService.searchDevicesByMacAddress(keyword, userId)).orElseGet(ArrayList::new);
|
||||
// 获取设备对应的智能体ID列表
|
||||
List<String> agentIds = devices.stream()
|
||||
.map(DeviceEntity::getAgentId)
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
-- 批量清理 ai_model_provider 中的 sample_rate 字段定义
|
||||
UPDATE `ai_model_provider` ap
|
||||
JOIN (
|
||||
SELECT
|
||||
id,
|
||||
JSON_ARRAYAGG(
|
||||
JSON_OBJECT('key', jt.k, 'label', jt.l, 'type', jt.t)
|
||||
) AS new_fields
|
||||
FROM `ai_model_provider`,
|
||||
JSON_TABLE(`fields`, '$[*]' COLUMNS (
|
||||
k VARCHAR(50) PATH '$.key',
|
||||
l VARCHAR(100) PATH '$.label',
|
||||
t VARCHAR(20) PATH '$.type'
|
||||
)) AS jt
|
||||
WHERE `model_type` = 'TTS'
|
||||
AND jt.k != 'sample_rate'
|
||||
GROUP BY id
|
||||
) filtered ON ap.id = filtered.id
|
||||
SET ap.fields = filtered.new_fields;
|
||||
|
||||
-- 清理 config_json 顶层的 sample_rate
|
||||
UPDATE `ai_model_config`
|
||||
SET `config_json` = JSON_REMOVE(`config_json`, '$.sample_rate')
|
||||
WHERE `model_type` = 'TTS'
|
||||
AND JSON_EXTRACT(`config_json`, '$.sample_rate') IS NOT NULL;
|
||||
|
||||
-- 清理Minimax流式TTS的sample_rate参数(位于audio_setting内部)
|
||||
UPDATE `ai_model_config` SET
|
||||
`config_json` = JSON_SET(`config_json`, '$.audio_setting', JSON_REMOVE(JSON_EXTRACT(`config_json`, '$.audio_setting'), '$.sample_rate'))
|
||||
WHERE `id` = 'TTS_MinimaxStreamTTS'
|
||||
AND JSON_EXTRACT(`config_json`, '$.audio_setting.sample_rate') IS NOT NULL;
|
||||
@@ -0,0 +1,87 @@
|
||||
-- 更新HuoshanDoubleStreamTTS供应器配置,将分散的参数改为JSON字典配置
|
||||
-- 将 speech_rate, loudness_rate, pitch, emotion, emotion_scale 等参数整合为 audio_params, additions, mix_speaker 三个JSON字典
|
||||
|
||||
UPDATE `ai_model_provider`
|
||||
SET `fields` = '[
|
||||
{"key": "ws_url", "type": "string", "label": "WebSocket地址"},
|
||||
{"key": "appid", "type": "string", "label": "应用ID"},
|
||||
{"key": "access_token", "type": "string", "label": "访问令牌"},
|
||||
{"key": "resource_id", "type": "string", "label": "资源ID"},
|
||||
{"key": "speaker", "type": "string", "label": "默认音色"},
|
||||
{"key": "enable_ws_reuse", "type": "boolean", "label": "是否开启链接复用", "default": true},
|
||||
{"key": "audio_params", "type": "dict", "label": "音频输出配置"},
|
||||
{"key": "additions", "type": "dict", "label": "高级文本处理配置"},
|
||||
{"key": "mix_speaker", "type": "dict", "label": "混音控制配置"}
|
||||
]'
|
||||
WHERE `id` = 'SYSTEM_TTS_HSDSTTS';
|
||||
|
||||
-- 更新现有配置,将旧的分散参数迁移到新的JSON字典结构
|
||||
UPDATE `ai_model_config`
|
||||
SET `config_json` = JSON_SET(
|
||||
`config_json`,
|
||||
'$.audio_params', JSON_OBJECT(
|
||||
'speech_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.speech_rate')), ''), '0') AS SIGNED),
|
||||
'loudness_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.loudness_rate')), ''), '0') AS SIGNED)
|
||||
),
|
||||
'$.additions', JSON_OBJECT(
|
||||
'aigc_metadata', JSON_OBJECT(),
|
||||
'cache_config', JSON_OBJECT(),
|
||||
'post_process', JSON_OBJECT(
|
||||
'pitch', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.pitch')), ''), '0') AS SIGNED)
|
||||
)
|
||||
),
|
||||
'$.mix_speaker', JSON_OBJECT()
|
||||
)
|
||||
WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
|
||||
|
||||
-- 删除旧的分散参数字段
|
||||
UPDATE `ai_model_config`
|
||||
SET `config_json` = JSON_REMOVE(
|
||||
`config_json`,
|
||||
'$.speech_rate',
|
||||
'$.loudness_rate',
|
||||
'$.pitch',
|
||||
'$.emotion',
|
||||
'$.emotion_scale'
|
||||
)
|
||||
WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
|
||||
|
||||
-- 更新文档链接和备注说明
|
||||
UPDATE `ai_model_config` SET
|
||||
`doc_link` = 'https://www.volcengine.com/docs/6561/1329505',
|
||||
`remark` = '火山引擎双向流式TTS配置说明:
|
||||
1. 访问 https://www.volcengine.com/ 注册并开通火山引擎账号
|
||||
2. 访问 https://console.volcengine.com/speech/service/10007 开通语音合成大模型,购买音色
|
||||
3. 在页面底部获取appid和access_token
|
||||
4. 资源ID固定为:volc.service_type.10029(大模型语音合成及混音)
|
||||
5. 链接复用:开启WebSocket连接复用,默认true减少链接损耗(注意:复用后设备处于聆听状态时空闲链接会占并发数)
|
||||
|
||||
详细参数文档:https://www.volcengine.com/docs/6561/1329505
|
||||
【audio_params】音频输出配置 - 用户可自定义添加火山引擎支持的任何音频参数
|
||||
- speech_rate: 语速(-50~100),默认0
|
||||
- loudness_rate: 音量(-50~100),默认0
|
||||
- emotion: 情感类型(仅部分音色支持),可选值:neutral、happy、sad、angry、fearful、disgusted、surprised
|
||||
- emotion_scale: 情感强度(1~5),默认4
|
||||
示例:{"speech_rate": 10, "loudness_rate": 5, "emotion": "happy", "emotion_scale": 4}
|
||||
|
||||
【additions】高级文本处理配置 - 用户可自定义添加火山引擎支持的任何高级参数
|
||||
- post_process.pitch: 音高(-12~12),默认0
|
||||
- aigc_metadata: AIGC元数据配置
|
||||
- cache_config: 缓存配置
|
||||
示例:{"post_process": {"pitch": 2}, "aigc_metadata": {}, "cache_config": {}}
|
||||
|
||||
【mix_speaker】混音控制配置 - 多音色混合(仅 TTS 1.0)
|
||||
示例:
|
||||
{"speakers": [
|
||||
{"source_speaker": "zh_male_bvlazysheep","mix_factor": 0.3},
|
||||
{"source_speaker": "BV120_streaming","mix_factor": 0.3},
|
||||
{"source_speaker": "zh_male_ahu_conversation_wvae_bigtts","mix_factor": 0.4}
|
||||
]}
|
||||
|
||||
注意:
|
||||
- 多情感音色参数(emotion、emotion_scale)仅部分音色支持
|
||||
- 相关音色列表:https://www.volcengine.com/docs/6561/1257544
|
||||
- 用户可根据火山引擎API文档自行添加更多参数
|
||||
- 混音功能主要适用于豆包语音合成模型1.0的音色,使用时需要将req_params.speaker设置为custom_mix_bigtts
|
||||
'
|
||||
WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
|
||||
@@ -0,0 +1,14 @@
|
||||
-- 更新小智参数中的默认采样率从 16000 改为 24000
|
||||
UPDATE `sys_params`
|
||||
SET `param_value` = '{
|
||||
"type": "hello",
|
||||
"version": 1,
|
||||
"transport": "websocket",
|
||||
"audio_params": {
|
||||
"format": "opus",
|
||||
"sample_rate": 24000,
|
||||
"channels": 1,
|
||||
"frame_duration": 60
|
||||
}
|
||||
}'
|
||||
WHERE `id` = 309 AND `param_code` = 'xiaozhi';
|
||||
@@ -487,6 +487,27 @@ databaseChangeLog:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202601051433.sql
|
||||
- changeSet:
|
||||
id: 202601141645
|
||||
author: RanChen
|
||||
changes:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202601141645.sql
|
||||
- changeSet:
|
||||
id: 202601231530
|
||||
author: RanChen
|
||||
changes:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202601231530.sql
|
||||
- changeSet:
|
||||
id: 202601261730
|
||||
author: RanChen
|
||||
changes:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202601261730.sql
|
||||
- changeSet:
|
||||
id: 202601291552
|
||||
author: shengzhou1216
|
||||
|
||||
@@ -235,7 +235,7 @@ function showAbout() {
|
||||
title: t('settings.aboutApp', { appName: import.meta.env.VITE_APP_TITLE }),
|
||||
content: t('settings.aboutContent', {
|
||||
appName: import.meta.env.VITE_APP_TITLE,
|
||||
version: '0.8.11'
|
||||
version: '0.9.1'
|
||||
}),
|
||||
showCancel: false,
|
||||
confirmText: t('common.confirm'),
|
||||
|
||||
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
@@ -4,9 +4,10 @@
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<link rel="icon" href="./favicon.ico">
|
||||
<title>Xiaozhi AI Customization</title>
|
||||
<script type="module" crossorigin src="./assets/index-FKVSBRAB.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-NXxBVrod.css">
|
||||
<script type="module" crossorigin src="./assets/index-B8r0c7xg.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-CrIJdTCK.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
@@ -64,7 +64,7 @@
|
||||
<el-button size="mini" type="text" @click="handleUnbind(scope.row.device_id)">
|
||||
{{ $t('device.unbind') }}
|
||||
</el-button>
|
||||
<el-button v-if="isGenerate(scope.row)" size="mini" type="text" @click="handleGenertor">
|
||||
<el-button v-if="isGenerate(scope.row)" size="mini" type="text" @click="handleGenertor(scope.row)">
|
||||
{{ $t('device.deviceThemeGeneration') }}
|
||||
</el-button>
|
||||
</template>
|
||||
@@ -337,10 +337,10 @@ export default {
|
||||
});
|
||||
});
|
||||
},
|
||||
handleGenertor() {
|
||||
handleGenertor(row) {
|
||||
const pathname = window.location.pathname;
|
||||
const basePath = pathname.split('/').slice(0, -1).join('/');
|
||||
const url = `${window.location.origin}${basePath}/generator/`;
|
||||
const url = `${window.location.origin}${basePath}/generator/?deviceId=${row.device_id}`;
|
||||
sessionStorage.setItem('devicePath', window.location.href);
|
||||
window.location.href = url;
|
||||
},
|
||||
|
||||
@@ -89,7 +89,8 @@ xiaozhi:
|
||||
transport: websocket
|
||||
audio_params:
|
||||
format: opus
|
||||
sample_rate: 16000
|
||||
# Opus支持的采样率范围为[8000, 12000, 16000, 24000, 48000]
|
||||
sample_rate: 24000
|
||||
channels: 1
|
||||
frame_duration: 60
|
||||
|
||||
@@ -717,13 +718,31 @@ TTS:
|
||||
speaker: zh_female_wanwanxiaohe_moon_bigtts
|
||||
# 开启WebSocket连接复用,默认复用(注意:复用后设备处于聆听状态时空闲链接会占并发数)
|
||||
enable_ws_reuse: True
|
||||
speech_rate: 0
|
||||
loudness_rate: 0
|
||||
pitch: 0
|
||||
# 多情感音色参数,注意:当前仅部分音色支持设置情感。
|
||||
# 相关音色列表:https://www.volcengine.com/docs/6561/1257544
|
||||
emotion: "neutral" # 情感类型,可选值为:neutral、happy、sad、angry、fearful、disgusted、surprised
|
||||
emotion_scale: 4 # 情感强度,可选值为:1~5,默认值为4
|
||||
# 相关参数文档:https://www.volcengine.com/docs/6561/1329505
|
||||
# 音频输出配置(audio_params)- 用户可自定义添加火山引擎支持的任何音频参数
|
||||
audio_params:
|
||||
speech_rate: 0 # 语速(-50~100)
|
||||
loudness_rate: 0 # 音量(-50~100)
|
||||
# 多情感音色参数,注意:当前仅部分音色支持设置情感。
|
||||
# 相关音色列表:https://www.volcengine.com/docs/6561/1257544
|
||||
# emotion: "neutral" # 情感类型(仅部分音色支持):neutral、happy、sad、angry、fearful、disgusted、surprised
|
||||
# emotion_scale: 4 # 情感强度(1~5)
|
||||
# 高级文本处理配置(additions)- 用户可自定义添加火山引擎支持的任何高级参数
|
||||
additions:
|
||||
post_process:
|
||||
pitch: 0 # 音高(-12~12)
|
||||
# aigc_metadata: {} # AIGC元数据配置
|
||||
# cache_config: {} # 缓存配置
|
||||
# 混音控制配置(mix_speaker)- 多音色混合(仅 TTS 1.0)
|
||||
# 混音功能主要适用于豆包语音合成模型1.0的音色,使用时需要将req_params.speaker设置为custom_mix_bigtts
|
||||
# mix_speaker:
|
||||
# speakers:
|
||||
# - source_speaker: zh_male_bvlazysheep
|
||||
# mix_factor: 0.3
|
||||
# - source_speaker: BV120_streaming
|
||||
# mix_factor: 0.3
|
||||
# - source_speaker: zh_male_ahu_conversation_wvae_bigtts
|
||||
# mix_factor: 0.4
|
||||
CosyVoiceSiliconflow:
|
||||
type: siliconflow
|
||||
# 硅基流动TTS
|
||||
@@ -840,7 +859,6 @@ TTS:
|
||||
# - "处理/(chu3)(li3)"
|
||||
# - "危险/dangerous"
|
||||
# audio_setting:
|
||||
# sample_rate: 24000
|
||||
# bitrate: 128000
|
||||
# format: "mp3"
|
||||
# channel: 1
|
||||
@@ -868,7 +886,6 @@ TTS:
|
||||
|
||||
# 以下可不用设置,使用默认设置
|
||||
# format: wav
|
||||
# sample_rate: 16000
|
||||
# volume: 50
|
||||
# speech_rate: 0
|
||||
# pitch_rate: 0
|
||||
@@ -892,7 +909,6 @@ TTS:
|
||||
host: nls-gateway-cn-beijing.aliyuncs.com
|
||||
# 以下可不用设置,使用默认设置
|
||||
# format: pcm # 音频格式:pcm、wav、mp3
|
||||
# sample_rate: 16000 # 采样率:8000、16000、24000
|
||||
# volume: 50 # 音量:0-100
|
||||
# speech_rate: 0 # 语速:-500到500
|
||||
# pitch_rate: 0 # 语调:-500到500
|
||||
@@ -1007,7 +1023,6 @@ TTS:
|
||||
protocol: websocket # protocol choices = ['websocket', 'http']
|
||||
url: ws://127.0.0.1:8092/paddlespeech/tts/streaming # TTS 服务的 URL 地址,指向本地服务器 [websocket默认ws://127.0.0.1:8092/paddlespeech/tts/streaming,http默认http://127.0.0.1:8090/paddlespeech/tts]
|
||||
spk_id: 0 # 发音人 ID,0 通常表示默认的发音人
|
||||
sample_rate: 24000 # 采样率 [websocket默认24000,http默认0 自动选择]
|
||||
speed: 1.0 # 语速,1.0 表示正常语速,>1 表示加快,<1 表示减慢
|
||||
volume: 1.0 # 音量,1.0 表示正常音量,>1 表示增大,<1 表示减小
|
||||
save_path: # 保存路径
|
||||
@@ -1031,7 +1046,6 @@ TTS:
|
||||
output_dir: tmp/
|
||||
# 以下可不用设置,使用默认设置
|
||||
# format: pcm # 音频格式:pcm、wav、mp3、opus
|
||||
# sample_rate: 24000 # 采样率:16000, 24000, 48000
|
||||
# volume: 50 # 音量:0-100
|
||||
# rate: 1 # 语速:0.5~2
|
||||
# pitch: 1 # 语调:0.5~2
|
||||
@@ -1053,7 +1067,6 @@ TTS:
|
||||
# stop_split: 0 # 关闭服务端拆句 不关闭:0,关闭:1
|
||||
# remain: 0 # 是否保留原书面语的样子 保留:1, 不保留:0
|
||||
# format: raw # 音频格式:raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb
|
||||
# sample_rate: 24000 # 采样率:16000, 8000, 24000
|
||||
# volume: 50 # 音量:0-100
|
||||
# speed: 50 # 语速:0-100
|
||||
# pitch: 50 # 语调:0-100
|
||||
|
||||
@@ -5,7 +5,7 @@ from config.config_loader import load_config
|
||||
from config.settings import check_config_file
|
||||
from datetime import datetime
|
||||
|
||||
SERVER_VERSION = "0.8.11"
|
||||
SERVER_VERSION = "0.9.1"
|
||||
_logger_initialized = False
|
||||
|
||||
|
||||
|
||||
@@ -87,6 +87,7 @@ class ConnectionHandler:
|
||||
self.max_output_size = 0
|
||||
self.chat_history_conf = 0
|
||||
self.audio_format = "opus"
|
||||
self.sample_rate = 24000 # 默认采样率,从客户端 hello 消息中动态更新
|
||||
|
||||
# 客户端状态相关
|
||||
self.client_abort = False
|
||||
@@ -208,6 +209,10 @@ class ConnectionHandler:
|
||||
self.welcome_msg = self.config["xiaozhi"]
|
||||
self.welcome_msg["session_id"] = self.session_id
|
||||
|
||||
# 从配置中读取采样率
|
||||
self.sample_rate = self.welcome_msg["audio_params"]["sample_rate"]
|
||||
self.logger.bind(tag=TAG).info(f"配置输出音频采样率为: {self.sample_rate}")
|
||||
|
||||
# 在后台初始化配置和组件(完全不阻塞主循环)
|
||||
asyncio.create_task(self._background_initialize())
|
||||
|
||||
@@ -1184,6 +1189,8 @@ class ConnectionHandler:
|
||||
|
||||
if self.tts:
|
||||
await self.tts.close()
|
||||
if self.asr:
|
||||
await self.asr.close()
|
||||
|
||||
# 最后关闭线程池(避免阻塞)
|
||||
if self.executor:
|
||||
@@ -1232,11 +1239,21 @@ class ConnectionHandler:
|
||||
f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
|
||||
)
|
||||
|
||||
def reset_vad_states(self):
|
||||
self.client_audio_buffer = bytearray()
|
||||
def reset_audio_states(self):
|
||||
"""
|
||||
重置所有音频相关状态(VAD + ASR)
|
||||
"""
|
||||
# Reset VAD states
|
||||
self.client_audio_buffer.clear()
|
||||
self.client_have_voice = False
|
||||
self.client_voice_stop = False
|
||||
self.logger.bind(tag=TAG).debug("VAD states reset.")
|
||||
self.client_voice_window.clear()
|
||||
self.last_is_voice = False
|
||||
|
||||
# Clear ASR buffers
|
||||
self.asr_audio.clear()
|
||||
|
||||
self.logger.bind(tag=TAG).debug("All audio states reset.")
|
||||
|
||||
def chat_and_close(self, text):
|
||||
"""Chat with the user and then close the connection"""
|
||||
|
||||
@@ -142,7 +142,8 @@ async def wakeupWordsResponse(conn):
|
||||
# 获取当前音色
|
||||
voice = getattr(conn.tts, "voice", "default")
|
||||
|
||||
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
|
||||
# 使用链接的sample_rate
|
||||
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=conn.sample_rate)
|
||||
file_path = wakeup_words_config.generate_file_path(voice)
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(wav_bytes)
|
||||
|
||||
@@ -17,7 +17,6 @@ async def handleAudioMessage(conn, audio):
|
||||
if hasattr(conn, "just_woken_up") and conn.just_woken_up:
|
||||
have_voice = False
|
||||
# 设置一个短暂延迟后恢复VAD检测
|
||||
conn.asr_audio.clear()
|
||||
if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
|
||||
conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
|
||||
return
|
||||
|
||||
@@ -26,10 +26,9 @@ class ListenTextMessageHandler(TextMessageHandler):
|
||||
f"客户端拾音模式:{conn.client_listen_mode}"
|
||||
)
|
||||
if msg_json["state"] == "start":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = False
|
||||
# 设备从播放模式切回录音模式,清除所有音频状态和缓冲区
|
||||
conn.reset_audio_states()
|
||||
elif msg_json["state"] == "stop":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = True
|
||||
if conn.asr.interface_type == InterfaceType.STREAM:
|
||||
# 流式模式下,发送结束请求
|
||||
@@ -38,14 +37,13 @@ class ListenTextMessageHandler(TextMessageHandler):
|
||||
# 非流式模式:直接触发ASR识别
|
||||
if len(conn.asr_audio) > 0:
|
||||
asr_audio_task = conn.asr_audio.copy()
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
conn.reset_audio_states()
|
||||
|
||||
if len(asr_audio_task) > 0:
|
||||
await conn.asr.handle_voice_stop(conn, asr_audio_task)
|
||||
elif msg_json["state"] == "detect":
|
||||
conn.client_have_voice = False
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_audio_states()
|
||||
if "text" in msg_json:
|
||||
conn.last_activity_time = time.time() * 1000
|
||||
original_text = msg_json["text"] # 保留原始文本
|
||||
|
||||
@@ -213,36 +213,24 @@ class ASRProvider(ASRProviderBase):
|
||||
return None
|
||||
|
||||
async def speech_to_text(
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus"
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
if self._is_token_expired():
|
||||
logger.warning("Token已过期,正在自动刷新...")
|
||||
self._refresh_token()
|
||||
|
||||
file_path = None
|
||||
try:
|
||||
# 解码Opus为PCM
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
|
||||
# 判断是否保存为WAV文件
|
||||
if self.delete_audio_file:
|
||||
pass
|
||||
else:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
if artifacts is None:
|
||||
return "", None
|
||||
# 发送请求并获取文本
|
||||
text = await self._send_request(combined_pcm_data)
|
||||
text = await self._send_request(artifacts.pcm_bytes)
|
||||
|
||||
if text:
|
||||
return text, file_path
|
||||
return text, artifacts.file_path
|
||||
|
||||
return "", file_path
|
||||
return "", artifacts.file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", file_path
|
||||
return "", None
|
||||
|
||||
@@ -126,16 +126,8 @@ class ASRProvider(ASRProviderBase):
|
||||
await super().open_audio_channels(conn)
|
||||
|
||||
async def receive_audio(self, conn, audio, audio_have_voice):
|
||||
# 初始化音频缓存
|
||||
if not hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
|
||||
# 存储音频数据
|
||||
if audio:
|
||||
conn.asr_audio_for_voiceprint.append(audio)
|
||||
|
||||
conn.asr_audio.append(audio)
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
# 先调用父类方法处理基础逻辑
|
||||
await super().receive_audio(conn, audio, audio_have_voice)
|
||||
|
||||
# 只在有声音且没有连接时建立连接(排除正在停止的情况)
|
||||
if audio_have_voice and not self.is_processing and not self.asr_ws:
|
||||
@@ -204,6 +196,8 @@ class ASRProvider(ASRProviderBase):
|
||||
"""转发识别结果"""
|
||||
try:
|
||||
while not conn.stop_event.is_set():
|
||||
# 获取当前连接的音频数据
|
||||
audio_data = conn.asr_audio
|
||||
try:
|
||||
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
|
||||
result = json.loads(response)
|
||||
@@ -257,19 +251,12 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
# 手动模式下,只有在收到stop信号后才触发处理(仅处理一次)
|
||||
if conn.client_voice_stop:
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
if len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
else:
|
||||
# 自动模式下直接覆盖
|
||||
self.text = text
|
||||
conn.reset_vad_states()
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
|
||||
@@ -289,11 +276,7 @@ class ASRProvider(ASRProviderBase):
|
||||
finally:
|
||||
# 清理连接的音频缓存
|
||||
await self._cleanup()
|
||||
if conn:
|
||||
if hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, 'asr_audio'):
|
||||
conn.asr_audio = []
|
||||
conn.reset_audio_states()
|
||||
|
||||
async def _send_stop_request(self):
|
||||
"""发送停止识别请求(不关闭连接)"""
|
||||
@@ -341,7 +324,7 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
logger.bind(tag=TAG).debug("ASR会话清理完成")
|
||||
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format):
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
|
||||
"""获取识别结果"""
|
||||
result = self.text
|
||||
self.text = ""
|
||||
|
||||
@@ -52,16 +52,8 @@ class ASRProvider(ASRProviderBase):
|
||||
await super().open_audio_channels(conn)
|
||||
|
||||
async def receive_audio(self, conn, audio, audio_have_voice):
|
||||
# 初始化音频缓存
|
||||
if not hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
|
||||
# 存储音频数据
|
||||
if audio:
|
||||
conn.asr_audio_for_voiceprint.append(audio)
|
||||
|
||||
conn.asr_audio.append(audio)
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
# 先调用父类方法处理基础逻辑
|
||||
await super().receive_audio(conn, audio, audio_have_voice)
|
||||
|
||||
# 只在有声音且没有连接时建立连接
|
||||
if audio_have_voice and not self.is_processing and not self.asr_ws:
|
||||
@@ -166,6 +158,8 @@ class ASRProvider(ASRProviderBase):
|
||||
"""转发识别结果"""
|
||||
try:
|
||||
while not conn.stop_event.is_set():
|
||||
# 获取当前连接的音频数据
|
||||
audio_data = conn.asr_audio
|
||||
try:
|
||||
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
|
||||
result = json.loads(response)
|
||||
@@ -214,19 +208,12 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
# 手动模式下,只有在收到stop信号后才触发处理
|
||||
if conn.client_voice_stop:
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
if len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
else:
|
||||
# 自动模式下直接覆盖
|
||||
self.text = text
|
||||
conn.reset_vad_states()
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
|
||||
@@ -257,11 +244,7 @@ class ASRProvider(ASRProviderBase):
|
||||
finally:
|
||||
# 清理连接的音频缓存
|
||||
await self._cleanup()
|
||||
if conn:
|
||||
if hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, 'asr_audio'):
|
||||
conn.asr_audio = []
|
||||
conn.reset_audio_states()
|
||||
|
||||
async def _send_stop_request(self):
|
||||
"""发送停止请求(用于手动模式停止录音)"""
|
||||
@@ -325,7 +308,7 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
logger.bind(tag=TAG).debug("ASR会话清理完成")
|
||||
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format):
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
|
||||
"""获取识别结果"""
|
||||
result = self.text
|
||||
self.text = ""
|
||||
|
||||
@@ -30,37 +30,26 @@ class ASRProvider(ASRProviderBase):
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
async def speech_to_text(
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus"
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
if not opus_data:
|
||||
logger.bind(tag=TAG).warning("音频数据为空!")
|
||||
return None, None
|
||||
|
||||
file_path = None
|
||||
try:
|
||||
# 检查配置是否已设置
|
||||
if not self.app_id or not self.api_key or not self.secret_key:
|
||||
logger.bind(tag=TAG).error("百度语音识别配置未设置,无法进行识别")
|
||||
return None, file_path
|
||||
return None, None
|
||||
|
||||
# 将Opus音频数据解码为PCM
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
|
||||
# 判断是否保存为WAV文件
|
||||
if self.delete_audio_file:
|
||||
pass
|
||||
else:
|
||||
self.save_audio_to_file(pcm_data, session_id)
|
||||
if artifacts is None:
|
||||
return "", None
|
||||
|
||||
start_time = time.time()
|
||||
# 识别本地文件
|
||||
result = self.client.asr(
|
||||
combined_pcm_data,
|
||||
artifacts.pcm_bytes,
|
||||
"pcm",
|
||||
16000,
|
||||
{
|
||||
@@ -73,13 +62,13 @@ class ASRProvider(ASRProviderBase):
|
||||
f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
|
||||
)
|
||||
result = result["result"][0]
|
||||
return result, file_path
|
||||
return result, artifacts.file_path
|
||||
else:
|
||||
raise Exception(
|
||||
f"百度语音识别失败,错误码: {result['err_no']},错误信息: {result['err_msg']}"
|
||||
)
|
||||
return None, file_path
|
||||
return None, artifacts.file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理音频时发生错误!{e}", exc_info=True)
|
||||
return None, file_path
|
||||
return None, None
|
||||
|
||||
@@ -8,14 +8,18 @@ import queue
|
||||
import asyncio
|
||||
import traceback
|
||||
import threading
|
||||
import shutil
|
||||
import opuslib_next
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from config.logger import setup_logging
|
||||
from typing import Optional, Tuple, List
|
||||
from typing import Optional, Tuple, List, NamedTuple
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
from core.handle.receiveAudioHandle import startToChat
|
||||
from core.handle.reportHandle import enqueue_asr_report
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from core.handle.receiveAudioHandle import handleAudioMessage
|
||||
import tempfile
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
@@ -57,18 +61,17 @@ class ASRProviderBase(ABC):
|
||||
conn.asr_audio.append(audio)
|
||||
else:
|
||||
# 自动/实时模式:使用VAD检测
|
||||
have_voice = audio_have_voice
|
||||
|
||||
conn.asr_audio.append(audio)
|
||||
if not have_voice and not conn.client_have_voice:
|
||||
|
||||
# 如果没有语音,且之前也没有声音,缓存部分音频
|
||||
if not audio_have_voice and not conn.client_have_voice:
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
return
|
||||
|
||||
# 自动模式下通过VAD检测到语音停止时触发识别
|
||||
if conn.client_voice_stop:
|
||||
if conn.asr.interface_type != InterfaceType.STREAM and conn.client_voice_stop:
|
||||
asr_audio_task = conn.asr_audio.copy()
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
conn.reset_audio_states()
|
||||
|
||||
if len(asr_audio_task) > 15:
|
||||
await self.handle_voice_stop(conn, asr_audio_task)
|
||||
@@ -93,10 +96,14 @@ class ASRProviderBase(ABC):
|
||||
wav_data = self._pcm_to_wav(combined_pcm_data)
|
||||
|
||||
# 定义ASR任务
|
||||
asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
|
||||
asr_task = self.speech_to_text_wrapper(
|
||||
asr_audio_task, conn.session_id, conn.audio_format
|
||||
)
|
||||
|
||||
if conn.voiceprint_provider and wav_data:
|
||||
voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
|
||||
voiceprint_task = conn.voiceprint_provider.identify_speaker(
|
||||
wav_data, conn.session_id
|
||||
)
|
||||
# 并发等待两个结果
|
||||
asr_result, voiceprint_result = await asyncio.gather(
|
||||
asr_task, voiceprint_task, return_exceptions=True
|
||||
@@ -159,20 +166,20 @@ class ASRProviderBase(ABC):
|
||||
if text_len > 0:
|
||||
# 使用自定义模块进行上报
|
||||
await startToChat(conn, enhanced_text)
|
||||
enqueue_asr_report(conn, enhanced_text, asr_audio_task)
|
||||
|
||||
audio_snapshot = asr_audio_task.copy()
|
||||
enqueue_asr_report(conn, enhanced_text, audio_snapshot)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
|
||||
import traceback
|
||||
|
||||
logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
|
||||
|
||||
def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
|
||||
"""构建包含说话人信息的文本(仅用于纯文本ASR)"""
|
||||
if speaker_name and speaker_name.strip():
|
||||
return json.dumps({
|
||||
"speaker": speaker_name,
|
||||
"content": text
|
||||
}, ensure_ascii=False)
|
||||
return json.dumps(
|
||||
{"speaker": speaker_name, "content": text}, ensure_ascii=False
|
||||
)
|
||||
else:
|
||||
return text
|
||||
|
||||
@@ -181,23 +188,23 @@ class ASRProviderBase(ABC):
|
||||
if len(pcm_data) == 0:
|
||||
logger.bind(tag=TAG).warning("PCM数据为空,无法转换WAV")
|
||||
return b""
|
||||
|
||||
|
||||
# 确保数据长度是偶数(16位音频)
|
||||
if len(pcm_data) % 2 != 0:
|
||||
pcm_data = pcm_data[:-1]
|
||||
|
||||
|
||||
# 创建WAV文件头
|
||||
wav_buffer = io.BytesIO()
|
||||
try:
|
||||
with wave.open(wav_buffer, 'wb') as wav_file:
|
||||
wav_file.setnchannels(1) # 单声道
|
||||
wav_file.setsampwidth(2) # 16位
|
||||
with wave.open(wav_buffer, "wb") as wav_file:
|
||||
wav_file.setnchannels(1) # 单声道
|
||||
wav_file.setsampwidth(2) # 16位
|
||||
wav_file.setframerate(16000) # 16kHz采样率
|
||||
wav_file.writeframes(pcm_data)
|
||||
|
||||
|
||||
wav_buffer.seek(0)
|
||||
wav_data = wav_buffer.read()
|
||||
|
||||
|
||||
return wav_data
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"WAV转换失败: {e}")
|
||||
@@ -206,6 +213,44 @@ class ASRProviderBase(ABC):
|
||||
def stop_ws_connection(self):
|
||||
pass
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
class AudioArtifacts(NamedTuple):
|
||||
pcm_frames: List[bytes]
|
||||
"""PCM音频帧列表"""
|
||||
pcm_bytes: bytes
|
||||
"""合并后的PCM音频字节数据"""
|
||||
file_path: Optional[str]
|
||||
"""WAV文件路径"""
|
||||
temp_path: Optional[str]
|
||||
"""临时WAV文件路径"""
|
||||
|
||||
def get_current_artifacts(self) -> Optional["ASRProviderBase.AudioArtifacts"]:
|
||||
return self._current_artifacts
|
||||
|
||||
def requires_file(self) -> bool:
|
||||
"""是否需要文件输入"""
|
||||
return False
|
||||
|
||||
def prefers_temp_file(self) -> bool:
|
||||
"""是否优先使用临时文件"""
|
||||
return False
|
||||
|
||||
def build_temp_file(self, pcm_bytes: bytes) -> Optional[str]:
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
|
||||
temp_path = temp_file.name
|
||||
with wave.open(temp_path, "wb") as wav_file:
|
||||
wav_file.setnchannels(1)
|
||||
wav_file.setsampwidth(2)
|
||||
wav_file.setframerate(16000)
|
||||
wav_file.writeframes(pcm_bytes)
|
||||
return temp_path
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"临时音频文件生成失败: {e}")
|
||||
return None
|
||||
|
||||
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
|
||||
"""PCM数据保存为WAV文件"""
|
||||
module_name = __name__.split(".")[-1]
|
||||
@@ -220,11 +265,80 @@ class ASRProviderBase(ABC):
|
||||
|
||||
return file_path
|
||||
|
||||
@abstractmethod
|
||||
async def speech_to_text(
|
||||
async def speech_to_text_wrapper(
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus"
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
file_path = None
|
||||
temp_path = None
|
||||
try:
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
|
||||
free_space = shutil.disk_usage(self.output_dir).free
|
||||
if free_space < len(combined_pcm_data) * 2:
|
||||
raise OSError("磁盘空间不足")
|
||||
|
||||
if self.requires_file() and self.prefers_temp_file():
|
||||
temp_path = self.build_temp_file(combined_pcm_data)
|
||||
|
||||
if (hasattr(self, "delete_audio_file") and not self.delete_audio_file) or (
|
||||
self.requires_file() and not self.prefers_temp_file()
|
||||
):
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
if len(combined_pcm_data) == 0:
|
||||
artifacts = None
|
||||
else:
|
||||
artifacts = ASRProviderBase.AudioArtifacts(
|
||||
pcm_frames=pcm_data,
|
||||
pcm_bytes=combined_pcm_data,
|
||||
file_path=file_path,
|
||||
temp_path=temp_path,
|
||||
)
|
||||
|
||||
text, _ = await self.speech_to_text(
|
||||
opus_data, session_id, audio_format, artifacts
|
||||
)
|
||||
return text, file_path
|
||||
except OSError as e:
|
||||
logger.bind(tag=TAG).error(f"文件操作错误: {e}")
|
||||
return None, None
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}")
|
||||
return None, None
|
||||
finally:
|
||||
try:
|
||||
if temp_path and os.path.exists(temp_path):
|
||||
os.unlink(temp_path)
|
||||
if (
|
||||
hasattr(self, "delete_audio_file")
|
||||
and self.delete_audio_file
|
||||
and file_path
|
||||
and os.path.exists(file_path)
|
||||
):
|
||||
os.remove(file_path)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"文件清理失败: {e}")
|
||||
|
||||
@abstractmethod
|
||||
async def speech_to_text(
|
||||
self,
|
||||
opus_data: List[bytes],
|
||||
session_id: str,
|
||||
audio_format="opus",
|
||||
artifacts: Optional[AudioArtifacts] = None,
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本
|
||||
|
||||
:param opus_data: 输入的Opus音频数据
|
||||
:param session_id: 会话ID
|
||||
:param audio_format: 音频格式,默认"opus"
|
||||
:param artifacts: 音频工件,包含PCM数据、文件路径等
|
||||
:return: 识别结果文本和文件路径(如果有)
|
||||
"""
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
@@ -235,23 +349,23 @@ class ASRProviderBase(ABC):
|
||||
decoder = opuslib_next.Decoder(16000, 1)
|
||||
pcm_data = []
|
||||
buffer_size = 960 # 每次处理960个采样点 (60ms at 16kHz)
|
||||
|
||||
|
||||
for i, opus_packet in enumerate(opus_data):
|
||||
try:
|
||||
if not opus_packet or len(opus_packet) == 0:
|
||||
continue
|
||||
|
||||
|
||||
pcm_frame = decoder.decode(opus_packet, buffer_size)
|
||||
if pcm_frame and len(pcm_frame) > 0:
|
||||
pcm_data.append(pcm_frame)
|
||||
|
||||
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).warning(f"Opus解码错误,跳过数据包 {i}: {e}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"音频处理错误,数据包 {i}: {e}")
|
||||
|
||||
|
||||
return pcm_data
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
|
||||
return []
|
||||
|
||||
@@ -232,24 +232,13 @@ class ASRProvider(ASRProviderBase):
|
||||
yield data[offset:data_len], True
|
||||
|
||||
async def speech_to_text(
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus"
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
|
||||
file_path = None
|
||||
try:
|
||||
# 合并所有opus数据包
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
|
||||
# 判断是否保存为WAV文件
|
||||
if self.delete_audio_file:
|
||||
pass
|
||||
else:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
if artifacts is None:
|
||||
return "", None
|
||||
|
||||
# 直接使用PCM数据
|
||||
# 计算分段大小 (单声道, 16bit, 16kHz采样率)
|
||||
@@ -258,14 +247,14 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
# 语音识别
|
||||
start_time = time.time()
|
||||
text = await self._send_request(combined_pcm_data, segment_size)
|
||||
text = await self._send_request(artifacts.pcm_bytes, segment_size)
|
||||
if text:
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
|
||||
)
|
||||
return text, file_path
|
||||
return "", file_path
|
||||
return text, artifacts.file_path
|
||||
return "", artifacts.file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", file_path
|
||||
return "", None
|
||||
|
||||
@@ -60,17 +60,8 @@ class ASRProvider(ASRProviderBase):
|
||||
await super().open_audio_channels(conn)
|
||||
|
||||
async def receive_audio(self, conn, audio, audio_have_voice):
|
||||
conn.asr_audio.append(audio)
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
# 存储音频数据
|
||||
if not hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
conn.asr_audio_for_voiceprint.append(audio)
|
||||
|
||||
# 当没有音频数据时处理完整语音片段
|
||||
if conn.client_listen_mode != "manual" and not audio and len(conn.asr_audio_for_voiceprint) > 0:
|
||||
await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint)
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
# 先调用父类方法处理基础逻辑
|
||||
await super().receive_audio(conn, audio, audio_have_voice)
|
||||
|
||||
# 如果本次有声音,且之前没有建立连接
|
||||
if audio_have_voice and self.asr_ws is None and not self.is_processing:
|
||||
@@ -164,7 +155,7 @@ class ASRProvider(ASRProviderBase):
|
||||
try:
|
||||
while self.asr_ws and not conn.stop_event.is_set():
|
||||
# 获取当前连接的音频数据
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
audio_data = conn.asr_audio
|
||||
try:
|
||||
response = await self.asr_ws.recv()
|
||||
result = self.parse_response(response)
|
||||
@@ -189,7 +180,6 @@ class ASRProvider(ASRProviderBase):
|
||||
):
|
||||
logger.bind(tag=TAG).error(f"识别文本:空")
|
||||
self.text = ""
|
||||
conn.reset_vad_states()
|
||||
if len(audio_data) > 15: # 确保有足够音频数据
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
@@ -200,12 +190,9 @@ class ASRProvider(ASRProviderBase):
|
||||
if self.enable_multilingual:
|
||||
continue
|
||||
|
||||
if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0:
|
||||
if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 15:
|
||||
logger.bind(tag=TAG).debug("消息结束收到停止信号,触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
break
|
||||
|
||||
for utterance in utterances:
|
||||
@@ -226,14 +213,10 @@ class ASRProvider(ASRProviderBase):
|
||||
if conn.client_voice_stop and len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("消息中途收到停止信号,触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
break
|
||||
else:
|
||||
# 自动模式下直接覆盖
|
||||
self.text = current_text
|
||||
conn.reset_vad_states()
|
||||
if len(audio_data) > 15: # 确保有足够音频数据
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
@@ -262,11 +245,8 @@ class ASRProvider(ASRProviderBase):
|
||||
await self.asr_ws.close()
|
||||
self.asr_ws = None
|
||||
self.is_processing = False
|
||||
if conn:
|
||||
if hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, 'asr_audio'):
|
||||
conn.asr_audio = []
|
||||
# 重置所有音频相关状态
|
||||
conn.reset_audio_states()
|
||||
|
||||
def stop_ws_connection(self):
|
||||
if self.asr_ws:
|
||||
@@ -408,7 +388,7 @@ class ASRProvider(ASRProviderBase):
|
||||
logger.bind(tag=TAG).error(f"原始响应数据: {res.hex()}")
|
||||
raise
|
||||
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format):
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
|
||||
result = self.text
|
||||
self.text = "" # 清空text
|
||||
return result, None
|
||||
@@ -435,11 +415,3 @@ class ASRProvider(ASRProviderBase):
|
||||
logger.bind(tag=TAG).debug("Doubao decoder resources released")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")
|
||||
|
||||
# 清理所有连接的音频缓冲区
|
||||
if hasattr(self, '_connections'):
|
||||
for conn in self._connections.values():
|
||||
if hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, 'asr_audio'):
|
||||
conn.asr_audio = []
|
||||
|
||||
@@ -64,39 +64,21 @@ class ASRProvider(ASRProviderBase):
|
||||
)
|
||||
|
||||
async def speech_to_text(
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus"
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""语音转文本主处理逻辑"""
|
||||
file_path = None
|
||||
retry_count = 0
|
||||
|
||||
|
||||
while retry_count < MAX_RETRIES:
|
||||
try:
|
||||
# 合并所有opus数据包
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
|
||||
# 检查磁盘空间
|
||||
if not self.delete_audio_file:
|
||||
free_space = shutil.disk_usage(self.output_dir).free
|
||||
if free_space < len(combined_pcm_data) * 2: # 预留2倍空间
|
||||
raise OSError("磁盘空间不足")
|
||||
|
||||
# 判断是否保存为WAV文件
|
||||
if self.delete_audio_file:
|
||||
pass
|
||||
else:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
if artifacts is None:
|
||||
return "", None
|
||||
|
||||
# 语音识别 - 使用线程池避免阻塞事件循环
|
||||
start_time = time.time()
|
||||
result = await asyncio.to_thread(
|
||||
self.model.generate,
|
||||
input=combined_pcm_data,
|
||||
input=artifacts.pcm_bytes,
|
||||
cache={},
|
||||
language="auto",
|
||||
use_itn=True,
|
||||
@@ -107,7 +89,7 @@ class ASRProvider(ASRProviderBase):
|
||||
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}"
|
||||
)
|
||||
|
||||
return text, file_path
|
||||
return text, artifacts.file_path
|
||||
|
||||
except OSError as e:
|
||||
retry_count += 1
|
||||
@@ -115,7 +97,7 @@ class ASRProvider(ASRProviderBase):
|
||||
logger.bind(tag=TAG).error(
|
||||
f"语音识别失败(已重试{retry_count}次): {e}", exc_info=True
|
||||
)
|
||||
return "", file_path
|
||||
return "", None
|
||||
logger.bind(tag=TAG).warning(
|
||||
f"语音识别失败,正在重试({retry_count}/{MAX_RETRIES}): {e}"
|
||||
)
|
||||
@@ -123,15 +105,4 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", file_path
|
||||
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"文件删除失败: {file_path} | 错误: {e}"
|
||||
)
|
||||
return "", None
|
||||
|
||||
@@ -101,7 +101,7 @@ class ASRProvider(ASRProviderBase):
|
||||
logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
|
||||
|
||||
async def speech_to_text(
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus"
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Convert speech data to text using FunASR.
|
||||
@@ -109,18 +109,9 @@ class ASRProvider(ASRProviderBase):
|
||||
:param session_id: Unique session identifier.
|
||||
:return: Tuple containing recognized text and optional timestamp.
|
||||
"""
|
||||
file_path = None
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
|
||||
# 判断是否保存为WAV文件
|
||||
if self.delete_audio_file:
|
||||
pass
|
||||
else:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
if artifacts is None:
|
||||
return "", None
|
||||
auth_header = {"Authorization": "Bearer; {}".format(self.api_key)}
|
||||
async with websockets.connect(
|
||||
self.uri,
|
||||
@@ -132,7 +123,7 @@ class ASRProvider(ASRProviderBase):
|
||||
try:
|
||||
# Use asyncio to handle WebSocket communication
|
||||
send_task = asyncio.create_task(
|
||||
self._send_data(ws, combined_pcm_data, session_id)
|
||||
self._send_data(ws, artifacts.pcm_bytes, session_id)
|
||||
)
|
||||
receive_task = asyncio.create_task(self._receive_responses(ws))
|
||||
|
||||
@@ -161,14 +152,14 @@ class ASRProvider(ASRProviderBase):
|
||||
result = lang_tag_filter(result)
|
||||
return (
|
||||
result,
|
||||
file_path,
|
||||
artifacts.file_path,
|
||||
) # Return the recognized text and timestamp (if any)
|
||||
|
||||
except websockets.exceptions.ConnectionClosed as e:
|
||||
logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
|
||||
return "", file_path
|
||||
return "", artifacts.file_path
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"Error during speech-to-text conversion: {e}", exc_info=True
|
||||
)
|
||||
return "", file_path
|
||||
return "", artifacts.file_path
|
||||
|
||||
@@ -21,20 +21,16 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus") -> Tuple[Optional[str], Optional[str]]:
|
||||
def requires_file(self) -> bool:
|
||||
return True
|
||||
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None) -> Tuple[Optional[str], Optional[str]]:
|
||||
file_path = None
|
||||
try:
|
||||
start_time = time.time()
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
|
||||
)
|
||||
|
||||
if artifacts is None:
|
||||
return "", None
|
||||
file_path = artifacts.file_path
|
||||
|
||||
logger.bind(tag=TAG).info(f"file path: {file_path}")
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
@@ -71,12 +67,4 @@ class ASRProvider(ASRProviderBase):
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}")
|
||||
return "", None
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Optional, Tuple, List
|
||||
import dashscope
|
||||
from config.logger import setup_logging
|
||||
@@ -35,56 +34,25 @@ class ASRProvider(ASRProviderBase):
|
||||
# 确保输出目录存在
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
def _prepare_audio_file(self, pcm_data: bytes) -> str:
|
||||
"""将PCM数据转换为WAV文件并返回文件路径"""
|
||||
try:
|
||||
import wave
|
||||
|
||||
# 创建临时WAV文件
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file:
|
||||
temp_path = temp_file.name
|
||||
|
||||
# 写入WAV格式
|
||||
with wave.open(temp_path, 'wb') as wav_file:
|
||||
wav_file.setnchannels(1) # 单声道
|
||||
wav_file.setsampwidth(2) # 16位
|
||||
wav_file.setframerate(16000) # 16kHz采样率
|
||||
wav_file.writeframes(pcm_data)
|
||||
|
||||
return temp_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=tag).error(f"音频文件准备失败: {e}")
|
||||
return None
|
||||
def prefers_temp_file(self) -> bool:
|
||||
return True
|
||||
|
||||
def requires_file(self) -> bool:
|
||||
return True
|
||||
|
||||
async def speech_to_text(
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus"
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
temp_file_path = None
|
||||
file_path = None
|
||||
|
||||
try:
|
||||
# 解码音频数据
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
if len(combined_pcm_data) == 0:
|
||||
logger.bind(tag=tag).warning("音频数据为空")
|
||||
if artifacts is None:
|
||||
return "", None
|
||||
|
||||
# 准备音频文件
|
||||
temp_file_path = self._prepare_audio_file(combined_pcm_data)
|
||||
temp_file_path = artifacts.temp_path
|
||||
file_path = artifacts.file_path
|
||||
if not temp_file_path:
|
||||
return "", None
|
||||
|
||||
# 保存音频文件(如果需要)
|
||||
if not self.delete_audio_file:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
return "", file_path
|
||||
# 构造请求消息
|
||||
messages = [
|
||||
{
|
||||
@@ -141,11 +109,3 @@ class ASRProvider(ASRProviderBase):
|
||||
except Exception as e:
|
||||
logger.bind(tag=tag).error(f"语音识别失败: {e}")
|
||||
return "", file_path
|
||||
|
||||
finally:
|
||||
# 清理临时文件
|
||||
if temp_file_path and os.path.exists(temp_file_path):
|
||||
try:
|
||||
os.unlink(temp_file_path)
|
||||
except Exception as e:
|
||||
logger.bind(tag=tag).warning(f"清理临时文件失败: {e}")
|
||||
@@ -120,24 +120,19 @@ class ASRProvider(ASRProviderBase):
|
||||
samples_float32 = samples_float32 / 32768
|
||||
return samples_float32, f.getframerate()
|
||||
|
||||
def requires_file(self) -> bool:
|
||||
return True
|
||||
|
||||
async def speech_to_text(
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus"
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""语音转文本主处理逻辑"""
|
||||
file_path = None
|
||||
try:
|
||||
# 保存音频文件
|
||||
start_time = time.time()
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
|
||||
)
|
||||
if artifacts is None:
|
||||
return "", None
|
||||
file_path = artifacts.file_path
|
||||
|
||||
# 语音识别
|
||||
start_time = time.time()
|
||||
s = self.model.create_stream()
|
||||
samples, sample_rate = self.read_wave(file_path)
|
||||
@@ -153,11 +148,3 @@ class ASRProvider(ASRProviderBase):
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", file_path
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
|
||||
|
||||
@@ -32,35 +32,24 @@ class ASRProvider(ASRProviderBase):
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
async def speech_to_text(
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus"
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
if not opus_data:
|
||||
logger.bind(tag=TAG).warning("音频数据为空!")
|
||||
return None, None
|
||||
|
||||
file_path = None
|
||||
try:
|
||||
# 检查配置是否已设置
|
||||
if not self.secret_id or not self.secret_key:
|
||||
logger.bind(tag=TAG).error("腾讯云语音识别配置未设置,无法进行识别")
|
||||
return None, file_path
|
||||
return None, None
|
||||
|
||||
# 将Opus音频数据解码为PCM
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
|
||||
# 判断是否保存为WAV文件
|
||||
if self.delete_audio_file:
|
||||
pass
|
||||
else:
|
||||
self.save_audio_to_file(pcm_data, session_id)
|
||||
if artifacts is None:
|
||||
return "", None
|
||||
|
||||
# 将音频数据转换为Base64编码
|
||||
base64_audio = base64.b64encode(combined_pcm_data).decode("utf-8")
|
||||
base64_audio = base64.b64encode(artifacts.pcm_bytes).decode("utf-8")
|
||||
|
||||
# 构建请求体
|
||||
request_body = self._build_request_body(base64_audio)
|
||||
@@ -77,11 +66,11 @@ class ASRProvider(ASRProviderBase):
|
||||
f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
|
||||
)
|
||||
|
||||
return result, file_path
|
||||
return result, artifacts.file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理音频时发生错误!{e}", exc_info=True)
|
||||
return None, file_path
|
||||
return None, None
|
||||
|
||||
def _build_request_body(self, base64_audio: str) -> str:
|
||||
"""构建请求体"""
|
||||
|
||||
@@ -44,36 +44,21 @@ class ASRProvider(ASRProviderBase):
|
||||
raise
|
||||
|
||||
async def speech_to_text(
|
||||
self, audio_data: List[bytes], session_id: str, audio_format: str = "opus"
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
file_path = None
|
||||
try:
|
||||
# 检查模型是否加载成功
|
||||
if not self.model:
|
||||
logger.bind(tag=TAG).error("VOSK模型未加载,无法进行识别")
|
||||
return "", None
|
||||
|
||||
# 解码音频(如果原始格式是Opus)
|
||||
if audio_format == "pcm":
|
||||
pcm_data = audio_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(audio_data)
|
||||
|
||||
if not pcm_data:
|
||||
logger.bind(tag=TAG).warning("解码后的PCM数据为空,无法进行识别")
|
||||
|
||||
if artifacts is None:
|
||||
return "", None
|
||||
|
||||
# 合并PCM数据
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
if len(combined_pcm_data) == 0:
|
||||
if not artifacts.pcm_bytes:
|
||||
logger.bind(tag=TAG).warning("合并后的PCM数据为空")
|
||||
return "", None
|
||||
|
||||
# 判断是否保存为WAV文件
|
||||
if not self.delete_audio_file:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
|
||||
@@ -81,8 +66,8 @@ class ASRProvider(ASRProviderBase):
|
||||
chunk_size = 2000
|
||||
text_result = ""
|
||||
|
||||
for i in range(0, len(combined_pcm_data), chunk_size):
|
||||
chunk = combined_pcm_data[i:i+chunk_size]
|
||||
for i in range(0, len(artifacts.pcm_bytes), chunk_size):
|
||||
chunk = artifacts.pcm_bytes[i:i+chunk_size]
|
||||
if self.recognizer.AcceptWaveform(chunk):
|
||||
result = json.loads(self.recognizer.Result())
|
||||
text = result.get('text', '')
|
||||
@@ -99,16 +84,8 @@ class ASRProvider(ASRProviderBase):
|
||||
f"VOSK语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text_result.strip()}"
|
||||
)
|
||||
|
||||
return text_result.strip(), file_path
|
||||
return text_result.strip(), artifacts.file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"VOSK语音识别失败: {e}")
|
||||
return "", None
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
|
||||
|
||||
@@ -101,11 +101,6 @@ class ASRProvider(ASRProviderBase):
|
||||
# 先调用父类方法处理基础逻辑
|
||||
await super().receive_audio(conn, audio, audio_have_voice)
|
||||
|
||||
# 存储音频数据用于声纹识别
|
||||
if not hasattr(conn, "asr_audio_for_voiceprint"):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
conn.asr_audio_for_voiceprint.append(audio)
|
||||
|
||||
# 如果本次有声音,且之前没有建立连接
|
||||
if audio_have_voice and self.asr_ws is None and not self.is_processing:
|
||||
try:
|
||||
@@ -232,13 +227,8 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
if status == 2:
|
||||
if conn.client_listen_mode == "manual":
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
if len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
|
||||
await self.handle_voice_stop(conn, conn.asr_audio)
|
||||
break
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
@@ -262,13 +252,7 @@ class ASRProvider(ASRProviderBase):
|
||||
finally:
|
||||
# 清理连接资源
|
||||
await self._cleanup()
|
||||
|
||||
# 清理连接的音频缓存
|
||||
if conn:
|
||||
if hasattr(conn, "asr_audio_for_voiceprint"):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, "asr_audio"):
|
||||
conn.asr_audio = []
|
||||
conn.reset_audio_states()
|
||||
|
||||
async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
|
||||
"""处理语音停止,发送最后一帧并处理识别结果"""
|
||||
@@ -334,7 +318,7 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
logger.bind(tag=TAG).debug("ASR会话清理完成")
|
||||
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format):
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
|
||||
"""获取识别结果"""
|
||||
result = self.text
|
||||
self.text = ""
|
||||
@@ -363,10 +347,3 @@ class ASRProvider(ASRProviderBase):
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
|
||||
|
||||
# 清理所有连接的音频缓冲区
|
||||
if hasattr(self, "_connections"):
|
||||
for conn in self._connections.values():
|
||||
if hasattr(conn, "asr_audio_for_voiceprint"):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, "asr_audio"):
|
||||
conn.asr_audio = []
|
||||
|
||||
@@ -41,8 +41,6 @@ class TTSProvider(TTSProviderBase):
|
||||
|
||||
# 音频参数配置
|
||||
self.format = config.get("format", "pcm")
|
||||
sample_rate = config.get("sample_rate", "24000")
|
||||
self.sample_rate = int(sample_rate) if sample_rate else 24000
|
||||
|
||||
volume = config.get("volume", "50")
|
||||
self.volume = int(volume) if volume else 50
|
||||
@@ -60,11 +58,6 @@ class TTSProvider(TTSProviderBase):
|
||||
"X-DashScope-DataInspection": "enable",
|
||||
}
|
||||
|
||||
# 创建Opus编码器
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
async def _ensure_connection(self):
|
||||
"""确保WebSocket连接可用,支持60秒内连接复用"""
|
||||
try:
|
||||
@@ -245,7 +238,7 @@ class TTSProvider(TTSProviderBase):
|
||||
"text_type": "PlainText",
|
||||
"voice": self.voice,
|
||||
"format": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"sample_rate": self.conn.sample_rate,
|
||||
"volume": self.volume,
|
||||
"rate": self.rate,
|
||||
"pitch": self.pitch,
|
||||
@@ -429,7 +422,7 @@ class TTSProvider(TTSProviderBase):
|
||||
"text_type": "PlainText",
|
||||
"voice": self.voice,
|
||||
"format": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"sample_rate": self.conn.sample_rate,
|
||||
"volume": self.volume,
|
||||
"rate": self.rate,
|
||||
"pitch": self.pitch,
|
||||
|
||||
@@ -95,8 +95,6 @@ class TTSProvider(TTSProviderBase):
|
||||
self.appkey = config.get("appkey")
|
||||
self.format = config.get("format", "wav")
|
||||
self.audio_file_type = config.get("format", "wav")
|
||||
sample_rate = config.get("sample_rate", "16000")
|
||||
self.sample_rate = int(sample_rate) if sample_rate else 16000
|
||||
|
||||
if config.get("private_voice"):
|
||||
self.voice = config.get("private_voice")
|
||||
@@ -172,7 +170,7 @@ class TTSProvider(TTSProviderBase):
|
||||
"token": self.token,
|
||||
"text": text,
|
||||
"format": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"sample_rate": self.conn.sample_rate,
|
||||
"voice": self.voice,
|
||||
"volume": self.volume,
|
||||
"speech_rate": self.speech_rate,
|
||||
|
||||
@@ -99,10 +99,6 @@ class TTSProvider(TTSProviderBase):
|
||||
self.format = config.get("format", "pcm")
|
||||
self.audio_file_type = config.get("format", "pcm")
|
||||
|
||||
# 采样率配置
|
||||
sample_rate = config.get("sample_rate", "16000")
|
||||
self.sample_rate = int(sample_rate) if sample_rate else 16000
|
||||
|
||||
# 音色配置 - CosyVoice大模型音色
|
||||
if config.get("private_voice"):
|
||||
self.voice = config.get("private_voice")
|
||||
@@ -134,11 +130,6 @@ class TTSProvider(TTSProviderBase):
|
||||
# 专属tts设置
|
||||
self.task_id = uuid.uuid4().hex
|
||||
|
||||
# 创建Opus编码器
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=16000, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
# Token管理
|
||||
if self.access_key_id and self.access_key_secret:
|
||||
self._refresh_token()
|
||||
@@ -344,7 +335,7 @@ class TTSProvider(TTSProviderBase):
|
||||
"payload": {
|
||||
"voice": self.voice,
|
||||
"format": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"sample_rate": self.conn.sample_rate,
|
||||
"volume": self.volume,
|
||||
"speech_rate": self.speech_rate,
|
||||
"pitch_rate": self.pitch_rate,
|
||||
@@ -508,7 +499,7 @@ class TTSProvider(TTSProviderBase):
|
||||
"payload": {
|
||||
"voice": self.voice,
|
||||
"format": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"sample_rate": self.conn.sample_rate,
|
||||
"volume": self.volume,
|
||||
"speech_rate": self.speech_rate,
|
||||
"pitch_rate": self.pitch_rate,
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
import queue
|
||||
import asyncio
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
from core.utils import p3
|
||||
from datetime import datetime
|
||||
from core.utils import textUtils
|
||||
from typing import Callable, Any
|
||||
from abc import ABC, abstractmethod
|
||||
from config.logger import setup_logging
|
||||
from core.utils import opus_encoder_utils
|
||||
from core.utils.tts import MarkdownCleaner
|
||||
from core.utils.output_counter import add_device_output
|
||||
from core.handle.reportHandle import enqueue_tts_report
|
||||
@@ -97,6 +98,8 @@ class TTSProviderBase(ABC):
|
||||
file_type=self.audio_file_type,
|
||||
is_opus=True,
|
||||
callback=opus_handler,
|
||||
sample_rate=self.conn.sample_rate,
|
||||
opus_encoder=self.opus_encoder,
|
||||
)
|
||||
break
|
||||
else:
|
||||
@@ -138,7 +141,7 @@ class TTSProviderBase(ABC):
|
||||
logger.bind(tag=TAG).error(
|
||||
f"语音生成失败: {text},请检查网络或服务是否正常"
|
||||
)
|
||||
self.tts_audio_queue.put((SentenceType.FIRST, None, text))
|
||||
self.tts_audio_queue.put((SentenceType.FIRST, None, text))
|
||||
self._process_audio_file_stream(tmp_file, callback=opus_handler)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
|
||||
@@ -158,7 +161,8 @@ class TTSProviderBase(ABC):
|
||||
audio_bytes,
|
||||
file_type=self.audio_file_type,
|
||||
is_opus=True,
|
||||
callback=lambda data: audio_datas.append(data)
|
||||
callback=lambda data: audio_datas.append(data),
|
||||
sample_rate=self.conn.sample_rate,
|
||||
)
|
||||
return audio_datas
|
||||
else:
|
||||
@@ -214,13 +218,13 @@ class TTSProviderBase(ABC):
|
||||
self, audio_file_path, callback: Callable[[Any], Any] = None
|
||||
):
|
||||
"""音频文件转换为PCM编码"""
|
||||
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback)
|
||||
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=None)
|
||||
|
||||
def audio_to_opus_data_stream(
|
||||
self, audio_file_path, callback: Callable[[Any], Any] = None
|
||||
):
|
||||
"""音频文件转换为Opus编码"""
|
||||
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback)
|
||||
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=self.opus_encoder)
|
||||
|
||||
def tts_one_sentence(
|
||||
self,
|
||||
@@ -252,6 +256,13 @@ class TTSProviderBase(ABC):
|
||||
|
||||
async def open_audio_channels(self, conn):
|
||||
self.conn = conn
|
||||
|
||||
# 根据conn的sample_rate创建编码器,如果子类已经创建则不覆盖(IndexTTS接口返回为24kHZ-待重采样处理)
|
||||
if not hasattr(self, 'opus_encoder') or self.opus_encoder is None:
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=conn.sample_rate, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
# tts 消化线程
|
||||
self.tts_priority_thread = threading.Thread(
|
||||
target=self.tts_text_priority_thread, daemon=True
|
||||
|
||||
@@ -154,16 +154,29 @@ class TTSProvider(TTSProviderBase):
|
||||
self.voice = config.get("private_voice")
|
||||
else:
|
||||
self.voice = config.get("speaker")
|
||||
speech_rate = config.get("speech_rate", "0")
|
||||
loudness_rate = config.get("loudness_rate", "0")
|
||||
pitch = config.get("pitch", "0")
|
||||
self.speech_rate = int(speech_rate) if speech_rate else 0
|
||||
self.loudness_rate = int(loudness_rate) if loudness_rate else 0
|
||||
self.pitch = int(pitch) if pitch else 0
|
||||
# 多情感音色参数
|
||||
self.emotion = config.get("emotion", "neutral")
|
||||
emotion_scale = config.get("emotion_scale", "4")
|
||||
self.emotion_scale = int(emotion_scale) if emotion_scale else 4
|
||||
|
||||
# 默认 audio_params 配置
|
||||
default_audio_params = {
|
||||
"speech_rate": 0,
|
||||
"loudness_rate": 0
|
||||
}
|
||||
|
||||
# 默认 additions 配置
|
||||
default_additions = {
|
||||
"aigc_metadata": {},
|
||||
"cache_config": {},
|
||||
"post_process": {
|
||||
"pitch": 0
|
||||
}
|
||||
}
|
||||
|
||||
# 默认 mix_speaker 配置
|
||||
default_mix_speaker = {}
|
||||
|
||||
# 合并用户配置
|
||||
self.audio_params = {**default_audio_params, **config.get("audio_params", {})}
|
||||
self.additions = {**default_additions, **config.get("additions", {})}
|
||||
self.mix_speaker = {**default_mix_speaker, **config.get("mix_speaker", {})}
|
||||
|
||||
self.ws_url = config.get("ws_url")
|
||||
self.authorization = config.get("authorization")
|
||||
@@ -171,9 +184,7 @@ class TTSProvider(TTSProviderBase):
|
||||
enable_ws_reuse_value = config.get("enable_ws_reuse", True)
|
||||
self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True
|
||||
self.tts_text = ""
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=16000, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
model_key_msg = check_model_key("TTS", self.access_token)
|
||||
if model_key_msg:
|
||||
logger.bind(tag=TAG).error(model_key_msg)
|
||||
@@ -181,6 +192,8 @@ class TTSProvider(TTSProviderBase):
|
||||
async def open_audio_channels(self, conn):
|
||||
try:
|
||||
await super().open_audio_channels(conn)
|
||||
# 更新 audio_params 中的采样率为实际的 conn.sample_rate
|
||||
self.audio_params["sample_rate"] = conn.sample_rate
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Failed to open audio channels: {str(e)}")
|
||||
self.ws = None
|
||||
@@ -646,20 +659,18 @@ class TTSProvider(TTSProviderBase):
|
||||
text="",
|
||||
speaker="",
|
||||
audio_format="pcm",
|
||||
audio_sample_rate=16000,
|
||||
):
|
||||
audio_params = {
|
||||
"format": audio_format,
|
||||
"sample_rate": audio_sample_rate,
|
||||
"speech_rate": self.speech_rate,
|
||||
"loudness_rate": self.loudness_rate
|
||||
# 构建 req_params
|
||||
req_params = {
|
||||
"text": text,
|
||||
"speaker": speaker,
|
||||
"audio_params": {**self.audio_params, "format": audio_format},
|
||||
"additions": json.dumps(self.additions)
|
||||
}
|
||||
|
||||
# 如果是多情感音色,添加情感参数
|
||||
if '_emo_' in self.voice:
|
||||
if self.emotion:
|
||||
audio_params["emotion"] = self.emotion
|
||||
audio_params["emotion_scale"] = self.emotion_scale
|
||||
|
||||
# 如果有 mix_speaker 配置,添加到 req_params
|
||||
if self.mix_speaker:
|
||||
req_params["mix_speaker"] = self.mix_speaker
|
||||
|
||||
return str.encode(
|
||||
json.dumps(
|
||||
@@ -667,17 +678,7 @@ class TTSProvider(TTSProviderBase):
|
||||
"user": {"uid": uid},
|
||||
"event": event,
|
||||
"namespace": "BidirectionalTTS",
|
||||
"req_params": {
|
||||
"text": text,
|
||||
"speaker": speaker,
|
||||
"audio_params": audio_params,
|
||||
"additions": json.dumps({
|
||||
"post_process": {
|
||||
"pitch": self.pitch
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
"req_params": req_params
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
@@ -174,6 +174,20 @@ class TTSProvider(TTSProviderBase):
|
||||
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
|
||||
self.tts_audio_queue.put((SentenceType.LAST, [], None))
|
||||
|
||||
def audio_to_pcm_data_stream(
|
||||
self, audio_file_path, callback=None
|
||||
):
|
||||
"""音频文件转换为PCM编码,使用24kHz采样率"""
|
||||
from core.utils.util import audio_to_data_stream
|
||||
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=24000, opus_encoder=None)
|
||||
|
||||
def audio_to_opus_data_stream(
|
||||
self, audio_file_path, callback=None
|
||||
):
|
||||
"""音频文件转换为Opus编码,使用24kHz采样率和自己的编码器"""
|
||||
from core.utils.util import audio_to_data_stream
|
||||
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=24000, opus_encoder=self.opus_encoder)
|
||||
|
||||
async def close(self):
|
||||
"""资源清理"""
|
||||
await super().close()
|
||||
|
||||
@@ -25,11 +25,6 @@ class TTSProvider(TTSProviderBase):
|
||||
self.audio_format = "pcm"
|
||||
self.before_stop_play_files = []
|
||||
|
||||
# 创建Opus编码器
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=16000, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
# PCM缓冲区
|
||||
self.pcm_buffer = bytearray()
|
||||
|
||||
@@ -127,7 +122,7 @@ class TTSProvider(TTSProviderBase):
|
||||
"spk_id": self.voice,
|
||||
"frame_durition": 60,
|
||||
"stream": "true",
|
||||
"target_sr": 16000,
|
||||
"target_sr": self.conn.sample_rate,
|
||||
"audio_format": "pcm",
|
||||
"instruct_text": "请生成一段自然流畅的语音",
|
||||
}
|
||||
@@ -136,7 +131,7 @@ class TTSProvider(TTSProviderBase):
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# 一帧 PCM 所需字节数:60 ms × 16 kHz × 1 ch × 2 B = 1 920
|
||||
# 一帧 PCM 所需字节数:60 ms × sample_rate × 1 ch × 2 B
|
||||
frame_bytes = int(
|
||||
self.opus_encoder.sample_rate
|
||||
* self.opus_encoder.channels # 1
|
||||
@@ -213,7 +208,7 @@ class TTSProvider(TTSProviderBase):
|
||||
"spk_id": self.voice,
|
||||
"frame_duration": 60,
|
||||
"stream": False,
|
||||
"target_sr": 16000,
|
||||
"target_sr": self.conn.sample_rate,
|
||||
"audio_format": self.audio_format,
|
||||
"instruct_text": "请生成一段自然流畅的语音",
|
||||
}
|
||||
|
||||
@@ -64,13 +64,17 @@ class TTSProvider(TTSProviderBase):
|
||||
}
|
||||
self.audio_file_type = defult_audio_setting.get("format", "pcm")
|
||||
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=24000, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
# PCM缓冲区
|
||||
self.pcm_buffer = bytearray()
|
||||
|
||||
async def open_audio_channels(self, conn):
|
||||
"""初始化音频通道,并根据conn.sample_rate更新配置"""
|
||||
# 调用父类方法
|
||||
await super().open_audio_channels(conn)
|
||||
|
||||
# 更新audio_setting中的采样率为实际的conn.sample_rate
|
||||
self.audio_setting["sample_rate"] = conn.sample_rate
|
||||
|
||||
def tts_text_priority_thread(self):
|
||||
"""流式文本处理线程"""
|
||||
while not self.conn.stop_event.is_set():
|
||||
@@ -212,6 +216,18 @@ class TTSProvider(TTSProviderBase):
|
||||
|
||||
try:
|
||||
data = json.loads(json_str)
|
||||
|
||||
# 检查业务层错误
|
||||
base_resp = data.get("base_resp", {})
|
||||
status_code = base_resp.get("status_code", 0)
|
||||
if status_code != 0:
|
||||
status_msg = base_resp.get("status_msg", "未知错误")
|
||||
logger.bind(tag=TAG).error(
|
||||
f"TTS请求失败, 错误码:{status_code}, 错误消息:{status_msg}"
|
||||
)
|
||||
self.tts_audio_queue.put((SentenceType.LAST, [], None))
|
||||
return
|
||||
|
||||
status = data.get("data", {}).get("status", 1)
|
||||
audio_hex = data.get("data", {}).get("audio")
|
||||
|
||||
|
||||
@@ -25,10 +25,7 @@ class TTSProvider(TTSProviderBase):
|
||||
self.spk_id = int(config.get("private_voice"))
|
||||
else:
|
||||
self.spk_id = int(config.get("spk_id", "0"))
|
||||
|
||||
sample_rate = config.get("sample_rate", 24000)
|
||||
self.sample_rate = float(sample_rate) if sample_rate else 24000
|
||||
|
||||
|
||||
speed = config.get("speed", 1.0)
|
||||
self.speed = float(speed) if speed else 1.0
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ class TTSProvider(TTSProviderBase):
|
||||
self.voice = config.get("voice")
|
||||
self.response_format = config.get("response_format", "mp3")
|
||||
self.audio_file_type = config.get("response_format", "mp3")
|
||||
self.sample_rate = config.get("sample_rate")
|
||||
self.speed = float(config.get("speed", 1.0))
|
||||
self.gain = config.get("gain")
|
||||
|
||||
|
||||
@@ -91,9 +91,6 @@ class TTSProvider(TTSProviderBase):
|
||||
# 音频编码配置
|
||||
self.format = config.get("format", "raw")
|
||||
|
||||
sample_rate = config.get("sample_rate", "24000")
|
||||
self.sample_rate = int(sample_rate) if sample_rate else 24000
|
||||
|
||||
# 口语化配置
|
||||
self.oral_level = config.get("oral_level", "mid")
|
||||
|
||||
@@ -113,11 +110,6 @@ class TTSProvider(TTSProviderBase):
|
||||
# 序列号管理
|
||||
self.text_seq = 0
|
||||
|
||||
# 创建Opus编码器
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
# 验证必需参数
|
||||
if not all([self.app_id, self.api_key, self.api_secret]):
|
||||
raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
|
||||
@@ -507,7 +499,7 @@ class TTSProvider(TTSProviderBase):
|
||||
"rhy": 0,
|
||||
"audio": {
|
||||
"encoding": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"sample_rate": self.conn.sample_rate,
|
||||
"channels": 1,
|
||||
"bit_depth": 16,
|
||||
"frame_size": 0
|
||||
|
||||
@@ -227,7 +227,7 @@ def extract_json_from_string(input_string):
|
||||
|
||||
|
||||
def audio_to_data_stream(
|
||||
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None
|
||||
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None
|
||||
) -> None:
|
||||
# 获取文件后缀名
|
||||
file_type = os.path.splitext(audio_file_path)[1]
|
||||
@@ -238,12 +238,12 @@ def audio_to_data_stream(
|
||||
audio_file_path, format=file_type, parameters=["-nostdin"]
|
||||
)
|
||||
|
||||
# 转换为单声道/16kHz采样率/16位小端编码(确保与编码器匹配)
|
||||
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
|
||||
# 转换为单声道/指定采样率/16位小端编码(确保与编码器匹配)
|
||||
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
|
||||
|
||||
# 获取原始PCM数据(16位小端)
|
||||
raw_data = audio.raw_data
|
||||
pcm_to_data_stream(raw_data, is_opus, callback)
|
||||
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
|
||||
|
||||
|
||||
async def audio_to_data(
|
||||
@@ -325,7 +325,7 @@ async def audio_to_data(
|
||||
|
||||
|
||||
def audio_bytes_to_data_stream(
|
||||
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]
|
||||
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any], sample_rate=16000, opus_encoder=None
|
||||
) -> None:
|
||||
"""
|
||||
直接用音频二进制数据转为opus/pcm数据,支持wav、mp3、p3
|
||||
@@ -338,18 +338,30 @@ def audio_bytes_to_data_stream(
|
||||
audio = AudioSegment.from_file(
|
||||
BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"]
|
||||
)
|
||||
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
|
||||
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
|
||||
raw_data = audio.raw_data
|
||||
pcm_to_data_stream(raw_data, is_opus, callback)
|
||||
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
|
||||
|
||||
|
||||
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None):
|
||||
# 初始化Opus编码器
|
||||
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
|
||||
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None):
|
||||
"""
|
||||
将PCM数据流式编码为Opus或直接输出PCM
|
||||
|
||||
Args:
|
||||
raw_data: PCM原始数据
|
||||
is_opus: 是否编码为Opus
|
||||
callback: 回调函数
|
||||
sample_rate: 采样率
|
||||
opus_encoder: OpusEncoderUtils对象(推荐提供以保持编码器状态连续)
|
||||
"""
|
||||
using_temp_encoder = False
|
||||
if is_opus and opus_encoder is None:
|
||||
encoder = opuslib_next.Encoder(sample_rate, 1, opuslib_next.APPLICATION_AUDIO)
|
||||
using_temp_encoder = True
|
||||
|
||||
# 编码参数
|
||||
frame_duration = 60 # 60ms per frame
|
||||
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
|
||||
frame_size = int(sample_rate * frame_duration / 1000) # samples/frame
|
||||
|
||||
# 按帧处理所有音频数据(包括最后一帧可能补零)
|
||||
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
|
||||
@@ -361,12 +373,17 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] =
|
||||
chunk += b"\x00" * (frame_size * 2 - len(chunk))
|
||||
|
||||
if is_opus:
|
||||
# 转换为numpy数组处理
|
||||
np_frame = np.frombuffer(chunk, dtype=np.int16)
|
||||
# 编码Opus数据
|
||||
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
|
||||
callback(frame_data)
|
||||
if using_temp_encoder:
|
||||
# 使用临时编码器(仅用于独立音频场景)
|
||||
np_frame = np.frombuffer(chunk, dtype=np.int16)
|
||||
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
|
||||
callback(frame_data)
|
||||
else:
|
||||
# 使用外部编码器(TTS流式场景,保持状态连续)
|
||||
is_last = (i + frame_size * 2 >= len(raw_data))
|
||||
opus_encoder.encode_pcm_to_opus_stream(chunk, end_of_stream=is_last, callback=callback)
|
||||
else:
|
||||
# PCM模式,直接输出
|
||||
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
|
||||
callback(frame_data)
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 142 KiB After Width: | Height: | Size: 136 KiB |
@@ -1,9 +1,10 @@
|
||||
// 主应用入口
|
||||
import { log } from './utils/logger.js';
|
||||
import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js';
|
||||
import { uiController } from './ui/controller.js';
|
||||
import { getAudioPlayer } from './core/audio/player.js';
|
||||
import { initMcpTools } from './core/mcp/tools.js';
|
||||
import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js?v=0127';
|
||||
import { getAudioPlayer } from './core/audio/player.js?v=0127';
|
||||
import { checkMicrophoneAvailability, isHttpNonLocalhost } from './core/audio/recorder.js?v=0127';
|
||||
import { initMcpTools } from './core/mcp/tools.js?v=0127';
|
||||
import { uiController } from './ui/controller.js?v=0127';
|
||||
import { log } from './utils/logger.js?v=0127';
|
||||
|
||||
// 应用类
|
||||
class App {
|
||||
@@ -16,30 +17,24 @@ class App {
|
||||
// 初始化应用
|
||||
async init() {
|
||||
log('正在初始化应用...', 'info');
|
||||
|
||||
// 初始化UI控制器
|
||||
this.uiController = uiController;
|
||||
this.uiController.init();
|
||||
|
||||
// 检查Opus库
|
||||
checkOpusLoaded();
|
||||
|
||||
// 初始化Opus编码器
|
||||
initOpusEncoder();
|
||||
|
||||
// 初始化音频播放器
|
||||
this.audioPlayer = getAudioPlayer();
|
||||
await this.audioPlayer.start();
|
||||
|
||||
// 初始化MCP工具
|
||||
initMcpTools();
|
||||
|
||||
// 检查麦克风可用性
|
||||
await this.checkMicrophoneAvailability();
|
||||
// 初始化Live2D
|
||||
await this.initLive2D();
|
||||
|
||||
// 关闭加载loading
|
||||
this.setModelLoadingStatus(false);
|
||||
|
||||
log('应用初始化完成', 'success');
|
||||
}
|
||||
|
||||
@@ -50,21 +45,17 @@ class App {
|
||||
if (typeof window.Live2DManager === 'undefined') {
|
||||
throw new Error('Live2DManager未加载,请检查脚本引入顺序');
|
||||
}
|
||||
|
||||
this.live2dManager = new window.Live2DManager();
|
||||
await this.live2dManager.initializeLive2D();
|
||||
|
||||
// 更新UI状态
|
||||
const live2dStatus = document.getElementById('live2dStatus');
|
||||
if (live2dStatus) {
|
||||
live2dStatus.textContent = '● 已加载';
|
||||
live2dStatus.className = 'status loaded';
|
||||
}
|
||||
|
||||
log('Live2D初始化完成', 'success');
|
||||
} catch (error) {
|
||||
log(`Live2D初始化失败: ${error.message}`, 'error');
|
||||
|
||||
// 更新UI状态
|
||||
const live2dStatus = document.getElementById('live2dStatus');
|
||||
if (live2dStatus) {
|
||||
@@ -81,18 +72,41 @@ class App {
|
||||
modelLoading.style.display = isLoading ? 'flex' : 'none';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查麦克风可用性
|
||||
* 在应用初始化时调用,检查麦克风是否可用并更新UI状态
|
||||
*/
|
||||
async checkMicrophoneAvailability() {
|
||||
try {
|
||||
const isAvailable = await checkMicrophoneAvailability();
|
||||
const isHttp = isHttpNonLocalhost();
|
||||
// 保存可用性状态到全局变量
|
||||
window.microphoneAvailable = isAvailable;
|
||||
window.isHttpNonLocalhost = isHttp;
|
||||
// 更新UI
|
||||
if (this.uiController) {
|
||||
this.uiController.updateMicrophoneAvailability(isAvailable, isHttp);
|
||||
}
|
||||
log(`麦克风可用性检查完成: ${isAvailable ? '可用' : '不可用'}`, isAvailable ? 'success' : 'warning');
|
||||
} catch (error) {
|
||||
log(`检查麦克风可用性失败: ${error.message}`, 'error');
|
||||
// 默认设置为不可用
|
||||
window.microphoneAvailable = false;
|
||||
window.isHttpNonLocalhost = isHttpNonLocalhost();
|
||||
if (this.uiController) {
|
||||
this.uiController.updateMicrophoneAvailability(false, window.isHttpNonLocalhost);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 创建并启动应用
|
||||
const app = new App();
|
||||
|
||||
// 将应用实例暴露到全局,供其他模块访问
|
||||
window.chatApp = app;
|
||||
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
// 初始化应用
|
||||
app.init();
|
||||
});
|
||||
|
||||
|
||||
export default app;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { log } from '../../utils/logger.js';
|
||||
import { log } from '../../utils/logger.js?v=0127';
|
||||
|
||||
|
||||
// 检查Opus库是否已加载
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// 音频播放模块
|
||||
import { log } from '../../utils/logger.js';
|
||||
import BlockingQueue from '../../utils/blocking-queue.js';
|
||||
import { createStreamingContext } from './stream-context.js';
|
||||
import BlockingQueue from '../../utils/blocking-queue.js?v=0127';
|
||||
import { log } from '../../utils/logger.js?v=0127';
|
||||
import { createStreamingContext } from './stream-context.js?v=0127';
|
||||
|
||||
// 音频播放器类
|
||||
export class AudioPlayer {
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
// 音频录制模块
|
||||
import { log } from '../../utils/logger.js';
|
||||
import { initOpusEncoder } from './opus-codec.js';
|
||||
import { getAudioPlayer } from './player.js';
|
||||
// Audio recording module
|
||||
import { log } from '../../utils/logger.js?v=0127';
|
||||
import { initOpusEncoder } from './opus-codec.js?v=0127';
|
||||
import { getAudioPlayer } from './player.js?v=0127';
|
||||
|
||||
// 音频录制器类
|
||||
// Audio recorder class
|
||||
export class AudioRecorder {
|
||||
constructor() {
|
||||
this.isRecording = false;
|
||||
@@ -19,25 +19,23 @@ export class AudioRecorder {
|
||||
this.visualizationRequest = null;
|
||||
this.recordingTimer = null;
|
||||
this.websocket = null;
|
||||
|
||||
// 回调函数
|
||||
// Callback functions
|
||||
this.onRecordingStart = null;
|
||||
this.onRecordingStop = null;
|
||||
this.onVisualizerUpdate = null;
|
||||
}
|
||||
|
||||
// 设置WebSocket实例
|
||||
// Set WebSocket instance
|
||||
setWebSocket(ws) {
|
||||
this.websocket = ws;
|
||||
}
|
||||
|
||||
// 获取AudioContext实例
|
||||
// Get AudioContext instance
|
||||
getAudioContext() {
|
||||
const audioPlayer = getAudioPlayer();
|
||||
return audioPlayer.getAudioContext();
|
||||
return getAudioPlayer().getAudioContext();
|
||||
}
|
||||
|
||||
// 初始化编码器
|
||||
// Initialize encoder
|
||||
initEncoder() {
|
||||
if (!this.opusEncoder) {
|
||||
this.opusEncoder = initOpusEncoder();
|
||||
@@ -45,7 +43,7 @@ export class AudioRecorder {
|
||||
return this.opusEncoder;
|
||||
}
|
||||
|
||||
// PCM处理器代码
|
||||
// PCM processor code
|
||||
getAudioProcessorCode() {
|
||||
return `
|
||||
class AudioRecorderProcessor extends AudioWorkletProcessor {
|
||||
@@ -56,166 +54,132 @@ export class AudioRecorder {
|
||||
this.buffer = new Int16Array(this.frameSize);
|
||||
this.bufferIndex = 0;
|
||||
this.isRecording = false;
|
||||
|
||||
this.port.onmessage = (event) => {
|
||||
if (event.data.command === 'start') {
|
||||
this.isRecording = true;
|
||||
this.port.postMessage({ type: 'status', status: 'started' });
|
||||
} else if (event.data.command === 'stop') {
|
||||
this.isRecording = false;
|
||||
|
||||
if (this.bufferIndex > 0) {
|
||||
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
|
||||
this.port.postMessage({
|
||||
type: 'buffer',
|
||||
buffer: finalBuffer
|
||||
});
|
||||
this.port.postMessage({ type: 'buffer', buffer: finalBuffer });
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
|
||||
this.port.postMessage({ type: 'status', status: 'stopped' });
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
process(inputs, outputs, parameters) {
|
||||
if (!this.isRecording) return true;
|
||||
|
||||
const input = inputs[0][0];
|
||||
if (!input) return true;
|
||||
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
if (this.bufferIndex >= this.frameSize) {
|
||||
this.port.postMessage({
|
||||
type: 'buffer',
|
||||
buffer: this.buffer.slice(0)
|
||||
});
|
||||
this.port.postMessage({ type: 'buffer', buffer: this.buffer.slice(0) });
|
||||
this.bufferIndex = 0;
|
||||
}
|
||||
|
||||
this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
|
||||
`;
|
||||
}
|
||||
|
||||
// 创建音频处理器
|
||||
// Create audio processor
|
||||
async createAudioProcessor() {
|
||||
this.audioContext = this.getAudioContext();
|
||||
|
||||
try {
|
||||
if (this.audioContext.audioWorklet) {
|
||||
const blob = new Blob([this.getAudioProcessorCode()], { type: 'application/javascript' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
await this.audioContext.audioWorklet.addModule(url);
|
||||
URL.revokeObjectURL(url);
|
||||
|
||||
const audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-recorder-processor');
|
||||
|
||||
audioProcessor.port.onmessage = (event) => {
|
||||
if (event.data.type === 'buffer') {
|
||||
this.processPCMBuffer(event.data.buffer);
|
||||
}
|
||||
};
|
||||
|
||||
log('使用AudioWorklet处理音频', 'success');
|
||||
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
audioProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
return { node: audioProcessor, type: 'worklet' };
|
||||
} else {
|
||||
log('AudioWorklet不可用,使用ScriptProcessorNode作为回退方案', 'warning');
|
||||
log('AudioWorklet不可用,使用ScriptProcessorNode作为后备方案', 'warning');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
} catch (error) {
|
||||
log(`创建音频处理器失败: ${error.message},尝试回退方案`, 'error');
|
||||
log(`创建音频处理器失败: ${error.message},尝试后备方案`, 'error');
|
||||
return this.createScriptProcessor();
|
||||
}
|
||||
}
|
||||
|
||||
// 创建ScriptProcessor作为回退
|
||||
// Create ScriptProcessor as fallback
|
||||
createScriptProcessor() {
|
||||
try {
|
||||
const frameSize = 4096;
|
||||
const scriptProcessor = this.audioContext.createScriptProcessor(frameSize, 1, 1);
|
||||
|
||||
scriptProcessor.onaudioprocess = (event) => {
|
||||
if (!this.isRecording) return;
|
||||
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
const buffer = new Int16Array(input.length);
|
||||
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
|
||||
}
|
||||
|
||||
this.processPCMBuffer(buffer);
|
||||
};
|
||||
|
||||
const silent = this.audioContext.createGain();
|
||||
silent.gain.value = 0;
|
||||
scriptProcessor.connect(silent);
|
||||
silent.connect(this.audioContext.destination);
|
||||
|
||||
log('使用ScriptProcessorNode作为回退方案成功', 'warning');
|
||||
log('使用ScriptProcessorNode作为后备方案成功', 'warning');
|
||||
return { node: scriptProcessor, type: 'processor' };
|
||||
} catch (fallbackError) {
|
||||
log(`回退方案也失败: ${fallbackError.message}`, 'error');
|
||||
log(`后备方案也失败: ${fallbackError.message}`, 'error');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// 处理PCM缓冲数据
|
||||
// Process PCM buffer data
|
||||
processPCMBuffer(buffer) {
|
||||
if (!this.isRecording) return;
|
||||
|
||||
const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length);
|
||||
newBuffer.set(this.pcmDataBuffer);
|
||||
newBuffer.set(buffer, this.pcmDataBuffer.length);
|
||||
this.pcmDataBuffer = newBuffer;
|
||||
|
||||
const samplesPerFrame = 960;
|
||||
|
||||
while (this.pcmDataBuffer.length >= samplesPerFrame) {
|
||||
const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame);
|
||||
this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame);
|
||||
|
||||
this.encodeAndSendOpus(frameData);
|
||||
}
|
||||
}
|
||||
|
||||
// 编码并发送Opus数据
|
||||
// Encode and send Opus data
|
||||
encodeAndSendOpus(pcmData = null) {
|
||||
if (!this.opusEncoder) {
|
||||
log('Opus编码器未初始化', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (pcmData) {
|
||||
const opusData = this.opusEncoder.encode(pcmData);
|
||||
|
||||
if (opusData && opusData.length > 0) {
|
||||
this.audioBuffers.push(opusData.buffer);
|
||||
this.totalAudioSize += opusData.length;
|
||||
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
this.websocket.send(opusData.buffer);
|
||||
log(`发送Opus帧,大小:${opusData.length}字节`, 'debug');
|
||||
} catch (error) {
|
||||
log(`WebSocket发送错误: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log('Opus编码失败,无有效数据返回', 'error');
|
||||
log('Opus编码失败,未返回有效数据', 'error');
|
||||
}
|
||||
} else {
|
||||
if (this.pcmDataBuffer.length > 0) {
|
||||
@@ -235,96 +199,67 @@ export class AudioRecorder {
|
||||
}
|
||||
}
|
||||
|
||||
// 开始录音
|
||||
// Start recording
|
||||
async start() {
|
||||
if (this.isRecording) return false;
|
||||
|
||||
try {
|
||||
// 检查是否有WebSocketHandler实例
|
||||
const { getWebSocketHandler } = await import('../network/websocket.js');
|
||||
// Check if WebSocketHandler instance exists
|
||||
const { getWebSocketHandler } = await import('../network/websocket.js?v=0127');
|
||||
const wsHandler = getWebSocketHandler();
|
||||
|
||||
// 如果机器正在说话,发送打断消息
|
||||
// If machine is speaking, send abort message
|
||||
if (wsHandler && wsHandler.isRemoteSpeaking && wsHandler.currentSessionId) {
|
||||
const abortMessage = {
|
||||
session_id: wsHandler.currentSessionId,
|
||||
type: 'abort',
|
||||
reason: 'wake_word_detected'
|
||||
};
|
||||
|
||||
const abortMessage = { session_id: wsHandler.currentSessionId, type: 'abort', reason: 'wake_word_detected' };
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
this.websocket.send(JSON.stringify(abortMessage));
|
||||
log('发送打断消息', 'info');
|
||||
log('已发送中止消息', 'info');
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.initEncoder()) {
|
||||
log('无法启动录音: Opus编码器初始化失败', 'error');
|
||||
log('无法开始录音: Opus编码器初始化失败', 'error');
|
||||
return false;
|
||||
}
|
||||
|
||||
log('请至少录制1-2秒钟的音频,确保采集到足够数据', 'info');
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
sampleRate: 16000,
|
||||
channelCount: 1
|
||||
}
|
||||
});
|
||||
|
||||
log('请至少录制1-2秒音频以确保收集足够的数据', 'info');
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
|
||||
this.audioContext = this.getAudioContext();
|
||||
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume();
|
||||
}
|
||||
|
||||
const processorResult = await this.createAudioProcessor();
|
||||
if (!processorResult) {
|
||||
log('无法创建音频处理器', 'error');
|
||||
return false;
|
||||
}
|
||||
|
||||
this.audioProcessor = processorResult.node;
|
||||
this.audioProcessorType = processorResult.type;
|
||||
|
||||
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
||||
this.analyser = this.audioContext.createAnalyser();
|
||||
this.analyser.fftSize = 2048;
|
||||
|
||||
this.audioSource.connect(this.analyser);
|
||||
this.audioSource.connect(this.audioProcessor);
|
||||
|
||||
this.pcmDataBuffer = new Int16Array();
|
||||
this.audioBuffers = [];
|
||||
this.totalAudioSize = 0;
|
||||
this.isRecording = true;
|
||||
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'start' });
|
||||
}
|
||||
|
||||
// 发送监听开始消息
|
||||
// Send listening start message
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
log(`发送录音开始消息`, 'info');
|
||||
log(`已发送录音开始消息`, 'info');
|
||||
} else {
|
||||
log('WebSocket未连接,无法发送开始消息', 'error');
|
||||
return false;
|
||||
}
|
||||
|
||||
// 开始可视化
|
||||
// Start visualization
|
||||
if (this.onVisualizerUpdate) {
|
||||
const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
|
||||
this.startVisualization(dataArray);
|
||||
}
|
||||
|
||||
// 立即通知录音开始,更新按钮状态
|
||||
// Immediately notify recording start, update button state
|
||||
if (this.onRecordingStart) {
|
||||
this.onRecordingStart(0);
|
||||
}
|
||||
|
||||
// 启动录音计时器
|
||||
// Start recording timer
|
||||
let recordingSeconds = 0;
|
||||
this.recordingTimer = setInterval(() => {
|
||||
recordingSeconds += 0.1;
|
||||
@@ -332,8 +267,7 @@ export class AudioRecorder {
|
||||
this.onRecordingStart(recordingSeconds);
|
||||
}
|
||||
}, 100);
|
||||
|
||||
log('开始PCM直接录音', 'success');
|
||||
log('已开始PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音启动错误: ${error.message}`, 'error');
|
||||
@@ -342,15 +276,12 @@ export class AudioRecorder {
|
||||
}
|
||||
}
|
||||
|
||||
// 开始可视化
|
||||
// Start visualization
|
||||
startVisualization(dataArray) {
|
||||
const draw = () => {
|
||||
this.visualizationRequest = requestAnimationFrame(() => draw());
|
||||
|
||||
if (!this.isRecording) return;
|
||||
|
||||
this.analyser.getByteFrequencyData(dataArray);
|
||||
|
||||
if (this.onVisualizerUpdate) {
|
||||
this.onVisualizerUpdate(dataArray);
|
||||
}
|
||||
@@ -358,52 +289,42 @@ export class AudioRecorder {
|
||||
draw();
|
||||
}
|
||||
|
||||
// 停止录音
|
||||
// Stop recording
|
||||
stop() {
|
||||
if (!this.isRecording) return false;
|
||||
|
||||
try {
|
||||
this.isRecording = false;
|
||||
|
||||
if (this.audioProcessor) {
|
||||
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
|
||||
this.audioProcessor.port.postMessage({ command: 'stop' });
|
||||
}
|
||||
|
||||
this.audioProcessor.disconnect();
|
||||
this.audioProcessor = null;
|
||||
}
|
||||
|
||||
if (this.audioSource) {
|
||||
this.audioSource.disconnect();
|
||||
this.audioSource = null;
|
||||
}
|
||||
|
||||
if (this.visualizationRequest) {
|
||||
cancelAnimationFrame(this.visualizationRequest);
|
||||
this.visualizationRequest = null;
|
||||
}
|
||||
|
||||
if (this.recordingTimer) {
|
||||
clearInterval(this.recordingTimer);
|
||||
this.recordingTimer = null;
|
||||
}
|
||||
|
||||
// 编码并发送剩余的数据
|
||||
// Encode and send remaining data
|
||||
this.encodeAndSendOpus();
|
||||
|
||||
// 发送结束信号
|
||||
// Send end signal
|
||||
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
|
||||
const emptyOpusFrame = new Uint8Array(0);
|
||||
this.websocket.send(emptyOpusFrame);
|
||||
log('已发送录音停止信号', 'info');
|
||||
}
|
||||
|
||||
if (this.onRecordingStop) {
|
||||
this.onRecordingStop();
|
||||
}
|
||||
|
||||
log('停止PCM直接录音', 'success');
|
||||
log('已停止PCM直接录音', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`直接录音停止错误: ${error.message}`, 'error');
|
||||
@@ -411,13 +332,13 @@ export class AudioRecorder {
|
||||
}
|
||||
}
|
||||
|
||||
// 获取分析器
|
||||
// Get analyser
|
||||
getAnalyser() {
|
||||
return this.analyser;
|
||||
}
|
||||
}
|
||||
|
||||
// 创建单例
|
||||
// Create singleton instance
|
||||
let audioRecorderInstance = null;
|
||||
|
||||
export function getAudioRecorder() {
|
||||
@@ -426,3 +347,49 @@ export function getAudioRecorder() {
|
||||
}
|
||||
return audioRecorderInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if microphone is available
|
||||
* @returns {Promise<boolean>} Returns true if available, false if not available
|
||||
*/
|
||||
export async function checkMicrophoneAvailability() {
|
||||
// Check if browser supports getUserMedia API
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
log('浏览器不支持getUserMedia API', 'warning');
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
// Try to access microphone
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
|
||||
// Immediately stop all tracks to release microphone
|
||||
stream.getTracks().forEach(track => track.stop());
|
||||
log('麦克风可用性检查成功', 'success');
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(`麦克风不可用: ${error.message}`, 'warning');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if it is HTTP non-localhost access
|
||||
* @returns {boolean} Returns true if it is HTTP non-localhost access
|
||||
*/
|
||||
export function isHttpNonLocalhost() {
|
||||
const protocol = window.location.protocol;
|
||||
const hostname = window.location.hostname;
|
||||
// Check if it is HTTP protocol
|
||||
if (protocol !== 'http:') {
|
||||
return false;
|
||||
}
|
||||
// localhost and 127.0.0.1 can use microphone
|
||||
if (hostname === 'localhost' || hostname === '127.0.0.1') {
|
||||
return false;
|
||||
}
|
||||
// Private IP addresses can also use microphone (browser allows)
|
||||
if (hostname.startsWith('192.168.') || hostname.startsWith('10.') || hostname.startsWith('172.')) {
|
||||
return false;
|
||||
}
|
||||
// Other HTTP access is considered non-localhost
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import BlockingQueue from '../../utils/blocking-queue.js';
|
||||
import { log } from '../../utils/logger.js';
|
||||
import BlockingQueue from '../../utils/blocking-queue.js?v=0127';
|
||||
import { log } from '../../utils/logger.js?v=0127';
|
||||
|
||||
// 音频流播放上下文类
|
||||
export class StreamingContext {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { log } from '../../utils/logger.js';
|
||||
import { log } from '../../utils/logger.js?v=0127';
|
||||
|
||||
// ==========================================
|
||||
// MCP 工具管理逻辑
|
||||
@@ -24,7 +24,6 @@ export function setWebSocket(ws) {
|
||||
export async function initMcpTools() {
|
||||
// 加载默认工具数据
|
||||
const defaultMcpTools = await fetch("js/config/default-mcp-tools.json").then(res => res.json());
|
||||
|
||||
const savedTools = localStorage.getItem('mcpTools');
|
||||
if (savedTools) {
|
||||
try {
|
||||
@@ -36,9 +35,11 @@ export async function initMcpTools() {
|
||||
} else {
|
||||
mcpTools = [...defaultMcpTools];
|
||||
}
|
||||
|
||||
renderMcpTools();
|
||||
setupMcpEventListeners();
|
||||
// Only setup event listeners if DOM elements exist
|
||||
if (document.getElementById('toggleMcpTools')) {
|
||||
setupMcpEventListeners();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -47,21 +48,20 @@ export async function initMcpTools() {
|
||||
function renderMcpTools() {
|
||||
const container = document.getElementById('mcpToolsContainer');
|
||||
const countSpan = document.getElementById('mcpToolsCount');
|
||||
|
||||
if (!container) {
|
||||
return; // Container not found, skip rendering
|
||||
}
|
||||
if (countSpan) {
|
||||
countSpan.textContent = `${mcpTools.length} 个工具`;
|
||||
}
|
||||
|
||||
if (mcpTools.length === 0) {
|
||||
container.innerHTML = '<div style="text-align: center; padding: 30px; color: #999;">暂无工具,点击下方按钮添加新工具</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = mcpTools.map((tool, index) => {
|
||||
const paramCount = tool.inputSchema.properties ? Object.keys(tool.inputSchema.properties).length : 0;
|
||||
const requiredCount = tool.inputSchema.required ? tool.inputSchema.required.length : 0;
|
||||
const hasMockResponse = tool.mockResponse && Object.keys(tool.mockResponse).length > 0;
|
||||
|
||||
return `
|
||||
<div class="mcp-tool-card">
|
||||
<div class="mcp-tool-header">
|
||||
@@ -96,12 +96,13 @@ function renderMcpTools() {
|
||||
*/
|
||||
function renderMcpProperties() {
|
||||
const container = document.getElementById('mcpPropertiesContainer');
|
||||
|
||||
if (!container) {
|
||||
return; // Container not found, skip rendering
|
||||
}
|
||||
if (mcpProperties.length === 0) {
|
||||
container.innerHTML = '<div style="text-align: center; padding: 20px; color: #999; font-size: 14px;">暂无参数,点击下方按钮添加参数</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = mcpProperties.map((prop, index) => `
|
||||
<div class="mcp-property-item">
|
||||
<div class="mcp-property-header">
|
||||
@@ -161,12 +162,7 @@ function renderMcpProperties() {
|
||||
* 添加参数
|
||||
*/
|
||||
function addMcpProperty() {
|
||||
mcpProperties.push({
|
||||
name: `param_${mcpProperties.length + 1}`,
|
||||
type: 'string',
|
||||
required: false,
|
||||
description: ''
|
||||
});
|
||||
mcpProperties.push({ name: `param_${mcpProperties.length + 1}`, type: 'string', required: false, description: '' });
|
||||
renderMcpProperties();
|
||||
}
|
||||
|
||||
@@ -182,9 +178,7 @@ function updateMcpProperty(index, field, value) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
mcpProperties[index][field] = value;
|
||||
|
||||
if (field === 'type' && value !== 'integer' && value !== 'number') {
|
||||
delete mcpProperties[index].minimum;
|
||||
delete mcpProperties[index].maximum;
|
||||
@@ -212,25 +206,24 @@ function setupMcpEventListeners() {
|
||||
const cancelBtn = document.getElementById('cancelMcpBtn');
|
||||
const form = document.getElementById('mcpToolForm');
|
||||
const addPropertyBtn = document.getElementById('addMcpPropertyBtn');
|
||||
|
||||
// Return early if required elements don't exist (e.g., in test environment)
|
||||
if (!toggleBtn || !panel || !addBtn || !modal || !closeBtn || !cancelBtn || !form || !addPropertyBtn) {
|
||||
return;
|
||||
}
|
||||
toggleBtn.addEventListener('click', () => {
|
||||
const isExpanded = panel.classList.contains('expanded');
|
||||
panel.classList.toggle('expanded');
|
||||
toggleBtn.textContent = isExpanded ? '展开' : '收起';
|
||||
toggleBtn.textContent = isExpanded ? '收起' : '展开';
|
||||
});
|
||||
|
||||
// 确保面板默认展开
|
||||
panel.classList.add('expanded');
|
||||
|
||||
addBtn.addEventListener('click', () => openMcpModal());
|
||||
closeBtn.addEventListener('click', closeMcpModal);
|
||||
cancelBtn.addEventListener('click', closeMcpModal);
|
||||
addPropertyBtn.addEventListener('click', addMcpProperty);
|
||||
|
||||
modal.addEventListener('click', (e) => {
|
||||
if (e.target === modal) closeMcpModal();
|
||||
});
|
||||
|
||||
form.addEventListener('submit', handleMcpSubmit);
|
||||
}
|
||||
|
||||
@@ -243,18 +236,15 @@ function openMcpModal(index = null) {
|
||||
alert('WebSocket 已连接,无法编辑工具');
|
||||
return;
|
||||
}
|
||||
|
||||
mcpEditingIndex = index;
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = '';
|
||||
|
||||
if (index !== null) {
|
||||
document.getElementById('mcpModalTitle').textContent = '编辑工具';
|
||||
const tool = mcpTools[index];
|
||||
document.getElementById('mcpToolName').value = tool.name;
|
||||
document.getElementById('mcpToolDescription').value = tool.description;
|
||||
document.getElementById('mcpMockResponse').value = tool.mockResponse ? JSON.stringify(tool.mockResponse, null, 2) : '';
|
||||
|
||||
mcpProperties = [];
|
||||
const schema = tool.inputSchema;
|
||||
if (schema.properties) {
|
||||
@@ -275,7 +265,6 @@ function openMcpModal(index = null) {
|
||||
document.getElementById('mcpToolForm').reset();
|
||||
mcpProperties = [];
|
||||
}
|
||||
|
||||
renderMcpProperties();
|
||||
document.getElementById('mcpToolModal').style.display = 'block';
|
||||
}
|
||||
@@ -298,21 +287,15 @@ function handleMcpSubmit(e) {
|
||||
e.preventDefault();
|
||||
const errorContainer = document.getElementById('mcpErrorContainer');
|
||||
errorContainer.innerHTML = '';
|
||||
|
||||
const name = document.getElementById('mcpToolName').value.trim();
|
||||
const description = document.getElementById('mcpToolDescription').value.trim();
|
||||
const mockResponseText = document.getElementById('mcpMockResponse').value.trim();
|
||||
|
||||
// 检查名称重复
|
||||
const isDuplicate = mcpTools.some((tool, index) =>
|
||||
tool.name === name && index !== mcpEditingIndex
|
||||
);
|
||||
|
||||
const isDuplicate = mcpTools.some((tool, index) => tool.name === name && index !== mcpEditingIndex);
|
||||
if (isDuplicate) {
|
||||
showMcpError('工具名称已存在,请使用不同的名称');
|
||||
return;
|
||||
}
|
||||
|
||||
// 解析模拟返回结果
|
||||
let mockResponse = null;
|
||||
if (mockResponseText) {
|
||||
@@ -323,21 +306,13 @@ function handleMcpSubmit(e) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 构建 inputSchema
|
||||
const inputSchema = {
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: []
|
||||
};
|
||||
|
||||
const inputSchema = { type: "object", properties: {}, required: [] };
|
||||
mcpProperties.forEach(prop => {
|
||||
const propSchema = { type: prop.type };
|
||||
|
||||
if (prop.description) {
|
||||
propSchema.description = prop.description;
|
||||
}
|
||||
|
||||
if ((prop.type === 'integer' || prop.type === 'number')) {
|
||||
if (prop.minimum !== undefined && prop.minimum !== '') {
|
||||
propSchema.minimum = prop.minimum;
|
||||
@@ -346,20 +321,15 @@ function handleMcpSubmit(e) {
|
||||
propSchema.maximum = prop.maximum;
|
||||
}
|
||||
}
|
||||
|
||||
inputSchema.properties[prop.name] = propSchema;
|
||||
|
||||
if (prop.required) {
|
||||
inputSchema.required.push(prop.name);
|
||||
}
|
||||
});
|
||||
|
||||
if (inputSchema.required.length === 0) {
|
||||
delete inputSchema.required;
|
||||
}
|
||||
|
||||
const tool = { name, description, inputSchema, mockResponse };
|
||||
|
||||
if (mcpEditingIndex !== null) {
|
||||
mcpTools[mcpEditingIndex] = tool;
|
||||
log(`已更新工具: ${name}`, 'success');
|
||||
@@ -367,7 +337,6 @@ function handleMcpSubmit(e) {
|
||||
mcpTools.push(tool);
|
||||
log(`已添加工具: ${name}`, 'success');
|
||||
}
|
||||
|
||||
saveMcpTools();
|
||||
renderMcpTools();
|
||||
closeMcpModal();
|
||||
@@ -417,11 +386,7 @@ function saveMcpTools() {
|
||||
* 获取工具列表
|
||||
*/
|
||||
export function getMcpTools() {
|
||||
return mcpTools.map(tool => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
inputSchema: tool.inputSchema
|
||||
}));
|
||||
return mcpTools.map(tool => ({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema }));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -429,20 +394,14 @@ export function getMcpTools() {
|
||||
*/
|
||||
export function executeMcpTool(toolName, toolArgs) {
|
||||
const tool = mcpTools.find(t => t.name === toolName);
|
||||
|
||||
if (!tool) {
|
||||
log(`未找到工具: ${toolName}`, 'error');
|
||||
return {
|
||||
success: false,
|
||||
error: `未知工具: ${toolName}`
|
||||
};
|
||||
return { success: false, error: `未知工具: ${toolName}` };
|
||||
}
|
||||
|
||||
// 如果有模拟返回结果,使用它
|
||||
if (tool.mockResponse) {
|
||||
// 替换模板变量
|
||||
let responseStr = JSON.stringify(tool.mockResponse);
|
||||
|
||||
// 替换 ${paramName} 格式的变量
|
||||
if (toolArgs) {
|
||||
Object.keys(toolArgs).forEach(key => {
|
||||
@@ -450,7 +409,6 @@ export function executeMcpTool(toolName, toolArgs) {
|
||||
responseStr = responseStr.replace(regex, toolArgs[key]);
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const response = JSON.parse(responseStr);
|
||||
log(`工具 ${toolName} 执行成功,返回模拟结果: ${responseStr}`, 'success');
|
||||
@@ -460,21 +418,10 @@ export function executeMcpTool(toolName, toolArgs) {
|
||||
return tool.mockResponse;
|
||||
}
|
||||
}
|
||||
|
||||
// 没有模拟返回结果,返回默认成功消息
|
||||
log(`工具 ${toolName} 执行成功,返回默认结果`, 'success');
|
||||
return {
|
||||
success: true,
|
||||
message: `工具 ${toolName} 执行成功`,
|
||||
tool: toolName,
|
||||
arguments: toolArgs
|
||||
};
|
||||
return { success: true, message: `工具 ${toolName} 执行成功`, tool: toolName, arguments: toolArgs };
|
||||
}
|
||||
|
||||
// 暴露全局方法供 HTML 内联事件调用
|
||||
window.mcpModule = {
|
||||
updateMcpProperty,
|
||||
deleteMcpProperty,
|
||||
editMcpTool,
|
||||
deleteMcpTool
|
||||
};
|
||||
window.mcpModule = { updateMcpProperty, deleteMcpProperty, editMcpTool, deleteMcpTool };
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { log } from '../../utils/logger.js';
|
||||
import { log } from '../../utils/logger.js?v=0127';
|
||||
|
||||
// WebSocket 连接
|
||||
export async function webSocketConnect(otaUrl, config) {
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
// WebSocket消息处理模块
|
||||
import { log } from '../../utils/logger.js';
|
||||
import { webSocketConnect } from './ota-connector.js';
|
||||
import { getConfig, saveConnectionUrls } from '../../config/manager.js';
|
||||
import { getAudioPlayer } from '../audio/player.js';
|
||||
import { getAudioRecorder } from '../audio/recorder.js';
|
||||
import { getMcpTools, executeMcpTool, setWebSocket as setMcpWebSocket } from '../mcp/tools.js';
|
||||
import { uiController } from '../../ui/controller.js'
|
||||
import { getConfig, saveConnectionUrls } from '../../config/manager.js?v=0127';
|
||||
import { uiController } from '../../ui/controller.js?v=0127';
|
||||
import { log } from '../../utils/logger.js?v=0127';
|
||||
import { getAudioPlayer } from '../audio/player.js?v=0127';
|
||||
import { getAudioRecorder } from '../audio/recorder.js?v=0127';
|
||||
import { executeMcpTool, getMcpTools, setWebSocket as setMcpWebSocket } from '../mcp/tools.js?v=0127';
|
||||
import { webSocketConnect } from './ota-connector.js?v=0127';
|
||||
|
||||
// WebSocket处理器类
|
||||
export class WebSocketHandler {
|
||||
@@ -101,10 +101,10 @@ export class WebSocketHandler {
|
||||
}
|
||||
|
||||
// 触发Live2D情绪动作
|
||||
if (message.emotion) {
|
||||
console.log(`收到情绪消息: emotion=${message.emotion}, text=${message.text}`);
|
||||
this.triggerLive2DEmotionAction(message.emotion);
|
||||
}
|
||||
if (message.emotion) {
|
||||
console.log(`收到情绪消息: emotion=${message.emotion}, text=${message.text}`);
|
||||
this.triggerLive2DEmotionAction(message.emotion);
|
||||
}
|
||||
}
|
||||
|
||||
// 只有当文本不仅仅是表情时,才添加到对话中
|
||||
@@ -273,6 +273,26 @@ export class WebSocketHandler {
|
||||
this.websocket.send(replyMessage);
|
||||
} else if (payload.method === 'initialize') {
|
||||
log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info');
|
||||
const replyMessage = JSON.stringify({
|
||||
"session_id": message.session_id || "",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {
|
||||
"tools": {}
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": "xiaozhi-web-test",
|
||||
"version": "2.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
log(`回复初始化响应`, 'info');
|
||||
this.websocket.send(replyMessage);
|
||||
} else {
|
||||
log(`未知的MCP方法: ${payload.method}`, 'warning');
|
||||
}
|
||||
@@ -284,7 +304,6 @@ export class WebSocketHandler {
|
||||
let arrayBuffer;
|
||||
if (data instanceof ArrayBuffer) {
|
||||
arrayBuffer = data;
|
||||
log(`收到ArrayBuffer音频数据,大小: ${data.byteLength}字节`, 'debug');
|
||||
} else if (data instanceof Blob) {
|
||||
arrayBuffer = await data.arrayBuffer();
|
||||
log(`收到Blob音频数据,大小: ${arrayBuffer.byteLength}字节`, 'debug');
|
||||
@@ -372,7 +391,7 @@ export class WebSocketHandler {
|
||||
|
||||
this.websocket.onerror = (error) => {
|
||||
log(`WebSocket错误: ${error.message || '未知错误'}`, 'error');
|
||||
|
||||
uiController.addChatMessage(`⚠️ WebSocket错误: ${error.message || '未知错误'}`, false);
|
||||
if (this.onConnectionStateChange) {
|
||||
this.onConnectionStateChange(false);
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// UI控制模块
|
||||
import { loadConfig, saveConfig } from '../config/manager.js';
|
||||
import { getAudioRecorder } from '../core/audio/recorder.js';
|
||||
import { getWebSocketHandler } from '../core/network/websocket.js';
|
||||
import { getAudioPlayer } from '../core/audio/player.js';
|
||||
// UI controller module
|
||||
import { loadConfig, saveConfig } from '../config/manager.js?v=0127';
|
||||
import { getAudioPlayer } from '../core/audio/player.js?v=0127';
|
||||
import { getAudioRecorder } from '../core/audio/recorder.js?v=0127';
|
||||
import { getWebSocketHandler } from '../core/network/websocket.js?v=0127';
|
||||
|
||||
// UI控制器类
|
||||
// UI controller class
|
||||
class UIController {
|
||||
constructor() {
|
||||
this.isEditing = false;
|
||||
@@ -14,7 +14,7 @@ class UIController {
|
||||
this.currentBackgroundIndex = 0;
|
||||
this.backgroundImages = ['1.png', '2.png', '3.png'];
|
||||
|
||||
// 绑定方法
|
||||
// Bind methods
|
||||
this.init = this.init.bind(this);
|
||||
this.initEventListeners = this.initEventListeners.bind(this);
|
||||
this.updateDialButton = this.updateDialButton.bind(this);
|
||||
@@ -25,7 +25,7 @@ class UIController {
|
||||
this.switchTab = this.switchTab.bind(this);
|
||||
}
|
||||
|
||||
// 初始化
|
||||
// Initialize
|
||||
init() {
|
||||
console.log('UIController init started');
|
||||
|
||||
@@ -35,7 +35,7 @@ class UIController {
|
||||
this.initVisualizer();
|
||||
}
|
||||
|
||||
// 检查连接按钮在初始化时是否存在
|
||||
// Check if connect button exists during initialization
|
||||
const connectBtn = document.getElementById('connectBtn');
|
||||
console.log('connectBtn during init:', connectBtn);
|
||||
|
||||
@@ -43,20 +43,20 @@ class UIController {
|
||||
this.startAudioStatsMonitor();
|
||||
loadConfig();
|
||||
|
||||
// 设置录音器回调
|
||||
// Register recording callback
|
||||
const audioRecorder = getAudioRecorder();
|
||||
audioRecorder.onRecordingStart = (seconds) => {
|
||||
this.updateRecordButtonState(true, seconds);
|
||||
};
|
||||
|
||||
// 初始化状态显示
|
||||
// Initialize status display
|
||||
this.updateConnectionUI(false);
|
||||
this.updateDialButton(false);
|
||||
|
||||
console.log('UIController init completed');
|
||||
}
|
||||
|
||||
// 初始化可视化器
|
||||
// Initialize visualizer
|
||||
initVisualizer() {
|
||||
if (this.visualizerCanvas) {
|
||||
this.visualizerCanvas.width = this.visualizerCanvas.clientWidth;
|
||||
@@ -66,9 +66,9 @@ class UIController {
|
||||
}
|
||||
}
|
||||
|
||||
// 初始化事件监听器
|
||||
// Initialize event listeners
|
||||
initEventListeners() {
|
||||
// 设置按钮
|
||||
// Settings button
|
||||
const settingsBtn = document.getElementById('settingsBtn');
|
||||
if (settingsBtn) {
|
||||
settingsBtn.addEventListener('click', () => {
|
||||
@@ -76,13 +76,13 @@ class UIController {
|
||||
});
|
||||
}
|
||||
|
||||
// 背景切换按钮
|
||||
// Background switch button
|
||||
const backgroundBtn = document.getElementById('backgroundBtn');
|
||||
if (backgroundBtn) {
|
||||
backgroundBtn.addEventListener('click', this.switchBackground);
|
||||
}
|
||||
|
||||
// 拨号按钮
|
||||
// Dial button
|
||||
const dialBtn = document.getElementById('dialBtn');
|
||||
if (dialBtn) {
|
||||
dialBtn.addEventListener('click', () => {
|
||||
@@ -92,40 +92,40 @@ class UIController {
|
||||
if (isConnected) {
|
||||
wsHandler.disconnect();
|
||||
this.updateDialButton(false);
|
||||
this.addChatMessage('已断开连接,期待下次再见~😉', false);
|
||||
this.addChatMessage('Disconnected, see you next time~😊', false);
|
||||
} else {
|
||||
// 检查OTA地址是否已填写
|
||||
// Check if OTA URL is filled
|
||||
const otaUrlInput = document.getElementById('otaUrl');
|
||||
if (!otaUrlInput || !otaUrlInput.value.trim()) {
|
||||
// 如果OTA地址未填写,显示设置弹窗并切换到设备配置页
|
||||
// If OTA URL is not filled, show settings modal and switch to device tab
|
||||
this.showModal('settingsModal');
|
||||
this.switchTab('device');
|
||||
this.addChatMessage('请先填写OTA服务器地址', false);
|
||||
this.addChatMessage('Please fill in OTA server URL', false);
|
||||
return;
|
||||
}
|
||||
|
||||
// 执行连接操作
|
||||
// Start connection process
|
||||
this.handleConnect();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 录音按钮
|
||||
// Record button
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (recordBtn) {
|
||||
recordBtn.addEventListener('click', () => {
|
||||
const audioRecorder = getAudioRecorder();
|
||||
if (audioRecorder.isRecording) {
|
||||
audioRecorder.stop();
|
||||
// 停止录音时移除录音样式
|
||||
// Restore record button to normal state
|
||||
recordBtn.classList.remove('recording');
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
} else {
|
||||
// 先更新按钮状态为录音中
|
||||
// Update button state to recording
|
||||
recordBtn.classList.add('recording');
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音中';
|
||||
|
||||
// 延迟开始录音,确保按钮状态已更新
|
||||
// Start recording, update button state after delay
|
||||
setTimeout(() => {
|
||||
audioRecorder.start();
|
||||
}, 100);
|
||||
@@ -133,7 +133,7 @@ class UIController {
|
||||
});
|
||||
}
|
||||
|
||||
// 消息输入框事件
|
||||
// Chat input event listener
|
||||
const chatIpt = document.getElementById('chatIpt');
|
||||
if (chatIpt) {
|
||||
const wsHandler = getWebSocketHandler();
|
||||
@@ -148,7 +148,7 @@ class UIController {
|
||||
});
|
||||
}
|
||||
|
||||
// 关闭按钮
|
||||
// Close button
|
||||
const closeButtons = document.querySelectorAll('.close-btn');
|
||||
closeButtons.forEach(btn => {
|
||||
btn.addEventListener('click', (e) => {
|
||||
@@ -163,7 +163,7 @@ class UIController {
|
||||
});
|
||||
});
|
||||
|
||||
// 设置标签页切换
|
||||
// Settings tab switch
|
||||
const tabBtns = document.querySelectorAll('.tab-btn');
|
||||
tabBtns.forEach(btn => {
|
||||
btn.addEventListener('click', (e) => {
|
||||
@@ -171,7 +171,7 @@ class UIController {
|
||||
});
|
||||
});
|
||||
|
||||
// 点击模态框外部关闭
|
||||
// Click modal background to close
|
||||
const modals = document.querySelectorAll('.modal');
|
||||
modals.forEach(modal => {
|
||||
modal.addEventListener('click', (e) => {
|
||||
@@ -184,7 +184,7 @@ class UIController {
|
||||
});
|
||||
});
|
||||
|
||||
// 添加MCP工具按钮
|
||||
// Add MCP tool button
|
||||
const addMCPToolBtn = document.getElementById('addMCPToolBtn');
|
||||
if (addMCPToolBtn) {
|
||||
addMCPToolBtn.addEventListener('click', (e) => {
|
||||
@@ -193,10 +193,10 @@ class UIController {
|
||||
});
|
||||
}
|
||||
|
||||
// 连接按钮和取消按钮已被移除,功能已集成到拨号按钮中
|
||||
// Connect button and send button are not removed, can be added to dial button later
|
||||
}
|
||||
|
||||
// 更新连接状态UI
|
||||
// Update connection status UI
|
||||
updateConnectionUI(isConnected) {
|
||||
const connectionStatus = document.getElementById('connectionStatus');
|
||||
const statusDot = document.querySelector('.status-dot');
|
||||
@@ -216,7 +216,7 @@ class UIController {
|
||||
}
|
||||
}
|
||||
|
||||
// 更新拨号按钮状态
|
||||
// Update dial button state
|
||||
updateDialButton(isConnected) {
|
||||
const dialBtn = document.getElementById('dialBtn');
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
@@ -225,39 +225,44 @@ class UIController {
|
||||
if (isConnected) {
|
||||
dialBtn.classList.add('dial-active');
|
||||
dialBtn.querySelector('.btn-text').textContent = '挂断';
|
||||
// 更新拨号按钮图标为挂断图标
|
||||
// Update dial button icon to hang up icon
|
||||
dialBtn.querySelector('svg').innerHTML = `
|
||||
<path d="M12,9C10.4,9 9,10.4 9,12C9,13.6 10.4,15 12,15C13.6,15 15,13.6 15,12C15,10.4 13.6,9 12,9M12,17C9.2,17 7,14.8 7,12C7,9.2 9.2,7 12,7C14.8,7 17,9.2 17,12C17,14.8 14.8,17 12,17M12,4.5C7,4.5 2.7,7.6 1,12C2.7,16.4 7,19.5 12,19.5C17,19.5 21.3,16.4 23,12C21.3,7.6 17,4.5 12,4.5Z"/>
|
||||
`;
|
||||
} else {
|
||||
dialBtn.classList.remove('dial-active');
|
||||
dialBtn.querySelector('.btn-text').textContent = '拨号';
|
||||
// 恢复拨号按钮图标
|
||||
// Restore dial button icon
|
||||
dialBtn.querySelector('svg').innerHTML = `
|
||||
<path d="M6.62,10.79C8.06,13.62 10.38,15.94 13.21,17.38L15.41,15.18C15.69,14.9 16.08,14.82 16.43,14.93C17.55,15.3 18.75,15.5 20,15.5A1,1 0 0,1 21,16.5V20A1,1 0 0,1 20,21A17,17 0 0,1 3,4A1,1 0 0,1 4,3H7.5A1,1 0 0,1 8.5,4C8.5,5.25 8.7,6.45 9.07,7.57C9.18,7.92 9.1,8.31 8.82,8.59L6.62,10.79Z"/>
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
// 更新录音按钮状态
|
||||
// Update record button state
|
||||
if (recordBtn) {
|
||||
if (isConnected) {
|
||||
const microphoneAvailable = window.microphoneAvailable !== false;
|
||||
if (isConnected && microphoneAvailable) {
|
||||
recordBtn.disabled = false;
|
||||
recordBtn.title = '开始录音';
|
||||
// 确保录音按钮恢复到正常状态
|
||||
// Restore record button to normal state
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
recordBtn.classList.remove('recording');
|
||||
} else {
|
||||
recordBtn.disabled = true;
|
||||
recordBtn.title = '请先连接服务器';
|
||||
// 确保录音按钮恢复到正常状态
|
||||
if (!microphoneAvailable) {
|
||||
recordBtn.title = window.isHttpNonLocalhost ? '当前由于是http访问,无法录音,只能用文字交互' : '麦克风不可用';
|
||||
} else {
|
||||
recordBtn.title = '请先连接服务器';
|
||||
}
|
||||
// Restore record button to normal state
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
recordBtn.classList.remove('recording');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 更新录音按钮状态
|
||||
// Update record button state
|
||||
updateRecordButtonState(isRecording, seconds = 0) {
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (recordBtn) {
|
||||
@@ -268,11 +273,37 @@ class UIController {
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
recordBtn.classList.remove('recording');
|
||||
}
|
||||
recordBtn.disabled = false;
|
||||
// Only enable button when microphone is available
|
||||
recordBtn.disabled = window.microphoneAvailable === false;
|
||||
}
|
||||
}
|
||||
|
||||
// 添加聊天消息
|
||||
/**
|
||||
* Update microphone availability state
|
||||
* @param {boolean} isAvailable - Whether microphone is available
|
||||
* @param {boolean} isHttpNonLocalhost - Whether it is HTTP non-localhost access
|
||||
*/
|
||||
updateMicrophoneAvailability(isAvailable, isHttpNonLocalhost) {
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (!recordBtn) return;
|
||||
if (!isAvailable) {
|
||||
// Disable record button
|
||||
recordBtn.disabled = true;
|
||||
// Update button text and title
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
recordBtn.title = isHttpNonLocalhost ? '当前由于是http访问,无法录音,只能用文字交互' : '麦克风不可用';
|
||||
|
||||
} else {
|
||||
// If connected, enable record button
|
||||
const wsHandler = getWebSocketHandler();
|
||||
if (wsHandler && wsHandler.isConnected()) {
|
||||
recordBtn.disabled = false;
|
||||
recordBtn.title = '开始录音';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add chat message
|
||||
addChatMessage(content, isUser = false) {
|
||||
const chatStream = document.getElementById('chatStream');
|
||||
if (!chatStream) return;
|
||||
@@ -282,11 +313,11 @@ class UIController {
|
||||
messageDiv.innerHTML = `<div class="message-bubble">${content}</div>`;
|
||||
chatStream.appendChild(messageDiv);
|
||||
|
||||
// 自动滚动到底部
|
||||
// Scroll to bottom
|
||||
chatStream.scrollTop = chatStream.scrollHeight;
|
||||
}
|
||||
|
||||
// 切换背景
|
||||
// Switch background
|
||||
switchBackground() {
|
||||
this.currentBackgroundIndex = (this.currentBackgroundIndex + 1) % this.backgroundImages.length;
|
||||
const backgroundContainer = document.querySelector('.background-container');
|
||||
@@ -295,7 +326,7 @@ class UIController {
|
||||
}
|
||||
}
|
||||
|
||||
// 显示模态框
|
||||
// Show modal
|
||||
showModal(modalId) {
|
||||
const modal = document.getElementById(modalId);
|
||||
if (modal) {
|
||||
@@ -303,7 +334,7 @@ class UIController {
|
||||
}
|
||||
}
|
||||
|
||||
// 隐藏模态框
|
||||
// Hide modal
|
||||
hideModal(modalId) {
|
||||
const modal = document.getElementById(modalId);
|
||||
if (modal) {
|
||||
@@ -311,16 +342,16 @@ class UIController {
|
||||
}
|
||||
}
|
||||
|
||||
// 切换标签页
|
||||
// Switch tab
|
||||
switchTab(tabName) {
|
||||
// 移除所有标签页的active类
|
||||
// Remove active class from all tabs
|
||||
const tabBtns = document.querySelectorAll('.tab-btn');
|
||||
const tabContents = document.querySelectorAll('.tab-content');
|
||||
|
||||
tabBtns.forEach(btn => btn.classList.remove('active'));
|
||||
tabContents.forEach(content => content.classList.remove('active'));
|
||||
|
||||
// 激活选中的标签页
|
||||
// Activate selected tab
|
||||
const activeTabBtn = document.querySelector(`[data-tab="${tabName}"]`);
|
||||
const activeTabContent = document.getElementById(`${tabName}Tab`);
|
||||
|
||||
@@ -330,24 +361,34 @@ class UIController {
|
||||
}
|
||||
}
|
||||
|
||||
// 连接成功后开始对话
|
||||
// Start AI chat session after connection
|
||||
startAIChatSession() {
|
||||
this.addChatMessage('连接成功,开始聊天吧~🙂', false);
|
||||
// 开启录音
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (recordBtn) {
|
||||
recordBtn.click();
|
||||
this.addChatMessage('连接成功,开始聊天吧~😊', false);
|
||||
// Check microphone availability and show error messages if needed
|
||||
if (!window.microphoneAvailable) {
|
||||
if (window.isHttpNonLocalhost) {
|
||||
this.addChatMessage('⚠️ 当前由于是http访问,无法录音,只能用文字交互', false);
|
||||
} else {
|
||||
this.addChatMessage('⚠️ 麦克风不可用,请检查权限设置,只能用文字交互', false);
|
||||
}
|
||||
}
|
||||
// Start recording only if microphone is available
|
||||
if (window.microphoneAvailable) {
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (recordBtn) {
|
||||
recordBtn.click();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 处理连接按钮点击
|
||||
// Handle connect button click
|
||||
async handleConnect() {
|
||||
console.log('handleConnect called');
|
||||
|
||||
// 确保切换到设备配置标签页
|
||||
// Switch to device settings tab
|
||||
this.switchTab('device');
|
||||
|
||||
// 等待DOM更新
|
||||
// Wait for DOM update
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
|
||||
const otaUrlInput = document.getElementById('otaUrl');
|
||||
@@ -362,7 +403,7 @@ class UIController {
|
||||
const otaUrl = otaUrlInput.value;
|
||||
console.log('otaUrl value:', otaUrl);
|
||||
|
||||
// 更新拨号按钮状态为连接中
|
||||
// Update dial button state to connecting
|
||||
const dialBtn = document.getElementById('dialBtn');
|
||||
if (dialBtn) {
|
||||
dialBtn.classList.add('dial-active');
|
||||
@@ -370,7 +411,7 @@ class UIController {
|
||||
dialBtn.disabled = true;
|
||||
}
|
||||
|
||||
// 显示连接中消息
|
||||
// Show connecting message
|
||||
this.addChatMessage('正在连接服务器...', false);
|
||||
|
||||
const chatIpt = document.getElementById('chatIpt');
|
||||
@@ -380,41 +421,51 @@ class UIController {
|
||||
|
||||
try {
|
||||
|
||||
// 获取WebSocket处理器
|
||||
// Get WebSocket handler instance
|
||||
const wsHandler = getWebSocketHandler();
|
||||
|
||||
// Register connection state callback BEFORE connecting
|
||||
wsHandler.onConnectionStateChange = (isConnected) => {
|
||||
this.updateConnectionUI(isConnected);
|
||||
this.updateDialButton(isConnected);
|
||||
};
|
||||
|
||||
// Register chat message callback BEFORE connecting
|
||||
wsHandler.onChatMessage = (text, isUser) => {
|
||||
this.addChatMessage(text, isUser);
|
||||
};
|
||||
|
||||
// Register record button state callback BEFORE connecting
|
||||
wsHandler.onRecordButtonStateChange = (isRecording) => {
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (recordBtn) {
|
||||
if (isRecording) {
|
||||
recordBtn.classList.add('recording');
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音中';
|
||||
} else {
|
||||
recordBtn.classList.remove('recording');
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const isConnected = await wsHandler.connect();
|
||||
|
||||
if (isConnected) {
|
||||
// Check microphone availability (check again after connection)
|
||||
const { checkMicrophoneAvailability } = await import('../core/audio/recorder.js?v=0127');
|
||||
const micAvailable = await checkMicrophoneAvailability();
|
||||
|
||||
// 设置连接状态回调
|
||||
wsHandler.onConnectionStateChange = (isConnected) => {
|
||||
this.updateConnectionUI(isConnected);
|
||||
this.updateDialButton(isConnected);
|
||||
};
|
||||
|
||||
// 设置聊天消息回调
|
||||
wsHandler.onChatMessage = (text, isUser) => {
|
||||
this.addChatMessage(text, isUser);
|
||||
};
|
||||
|
||||
// 设置录音按钮状态回调
|
||||
wsHandler.onRecordButtonStateChange = (isRecording) => {
|
||||
const recordBtn = document.getElementById('recordBtn');
|
||||
if (recordBtn) {
|
||||
if (isRecording) {
|
||||
recordBtn.classList.add('recording');
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音中';
|
||||
} else {
|
||||
recordBtn.classList.remove('recording');
|
||||
recordBtn.querySelector('.btn-text').textContent = '录音';
|
||||
}
|
||||
if (!micAvailable) {
|
||||
const isHttp = window.isHttpNonLocalhost;
|
||||
if (isHttp) {
|
||||
this.addChatMessage('⚠️ 当前由于是http访问,无法录音,只能用文字交互', false);
|
||||
}
|
||||
};
|
||||
// Update global state
|
||||
window.microphoneAvailable = false;
|
||||
}
|
||||
|
||||
// 连接成功
|
||||
this.addChatMessage('OTA连接成功,正在建立WebSocket连接...', false);
|
||||
|
||||
// 更新拨号按钮状态
|
||||
// Update dial button state
|
||||
const dialBtn = document.getElementById('dialBtn');
|
||||
if (dialBtn) {
|
||||
dialBtn.disabled = false;
|
||||
@@ -434,14 +485,14 @@ class UIController {
|
||||
name: error.name
|
||||
});
|
||||
|
||||
// 显示错误消息
|
||||
// Show error message
|
||||
const errorMessage = error.message.includes('Cannot set properties of null')
|
||||
? '连接失败:请刷新页面重试'
|
||||
? '连接失败:请检查设备连接'
|
||||
: `连接失败: ${error.message}`;
|
||||
|
||||
this.addChatMessage(errorMessage, false);
|
||||
|
||||
// 恢复拨号按钮状态
|
||||
// Restore dial button state
|
||||
const dialBtn = document.getElementById('dialBtn');
|
||||
if (dialBtn) {
|
||||
dialBtn.disabled = false;
|
||||
@@ -452,7 +503,7 @@ class UIController {
|
||||
}
|
||||
}
|
||||
|
||||
// 添加MCP工具
|
||||
// Add MCP tool
|
||||
addMCPTool() {
|
||||
const mcpToolsList = document.getElementById('mcpToolsList');
|
||||
if (!mcpToolsList) return;
|
||||
@@ -471,7 +522,7 @@ class UIController {
|
||||
mcpToolsList.appendChild(toolDiv);
|
||||
}
|
||||
|
||||
// 移除MCP工具
|
||||
// Remove MCP tool
|
||||
removeMCPTool(toolId) {
|
||||
const toolElement = document.getElementById(toolId);
|
||||
if (toolElement) {
|
||||
@@ -479,24 +530,24 @@ class UIController {
|
||||
}
|
||||
}
|
||||
|
||||
// 更新音频统计信息
|
||||
// Update audio statistics display
|
||||
updateAudioStats() {
|
||||
const audioPlayer = getAudioPlayer();
|
||||
if (!audioPlayer) return;
|
||||
|
||||
const stats = audioPlayer.getAudioStats();
|
||||
// 这里可以添加音频统计的UI更新逻辑
|
||||
// Here can add audio statistics UI update logic
|
||||
}
|
||||
|
||||
// 启动音频统计监控
|
||||
// Start audio statistics monitor
|
||||
startAudioStatsMonitor() {
|
||||
// 每100ms更新一次音频统计
|
||||
// Update audio statistics every 100ms
|
||||
this.audioStatsTimer = setInterval(() => {
|
||||
this.updateAudioStats();
|
||||
}, 100);
|
||||
}
|
||||
|
||||
// 停止音频统计监控
|
||||
// Stop audio statistics monitor
|
||||
stopAudioStatsMonitor() {
|
||||
if (this.audioStatsTimer) {
|
||||
clearInterval(this.audioStatsTimer);
|
||||
@@ -504,7 +555,7 @@ class UIController {
|
||||
}
|
||||
}
|
||||
|
||||
// 绘制音频可视化效果
|
||||
// Draw audio visualizer waveform
|
||||
drawVisualizer(dataArray) {
|
||||
if (!this.visualizerContext || !this.visualizerCanvas) return;
|
||||
|
||||
@@ -518,7 +569,7 @@ class UIController {
|
||||
for (let i = 0; i < dataArray.length; i++) {
|
||||
barHeight = dataArray[i] / 2;
|
||||
|
||||
// 创建渐变色:从紫色到蓝色到青色
|
||||
// Create gradient color: from purple to blue to green
|
||||
const gradient = this.visualizerContext.createLinearGradient(0, 0, 0, this.visualizerCanvas.height);
|
||||
gradient.addColorStop(0, '#8e44ad');
|
||||
gradient.addColorStop(0.5, '#3498db');
|
||||
@@ -530,21 +581,21 @@ class UIController {
|
||||
}
|
||||
}
|
||||
|
||||
// 更新会话状态UI
|
||||
// Update session status UI
|
||||
updateSessionStatus(isSpeaking) {
|
||||
// 这里可以添加会话状态的UI更新逻辑
|
||||
// 例如:更新Live2D角色的表情或状态指示器
|
||||
// Here can add session status UI update logic
|
||||
// For example: update Live2D model's mouth movement status
|
||||
}
|
||||
|
||||
// 更新会话表情
|
||||
// Update session emotion
|
||||
updateSessionEmotion(emoji) {
|
||||
// 这里可以添加表情更新的逻辑
|
||||
// 例如:在状态指示器中显示表情
|
||||
// Here can add emotion update logic
|
||||
// For example: display emoji in status indicator
|
||||
}
|
||||
}
|
||||
|
||||
// 创建全局实例
|
||||
// Create singleton instance
|
||||
export const uiController = new UIController();
|
||||
|
||||
// 导出类供其他模块使用
|
||||
export { UIController };
|
||||
// Export class for module usage
|
||||
export { UIController };
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>小智服务器测试页面</title>
|
||||
<link rel="stylesheet" href="css/test_page.css">
|
||||
<link rel="stylesheet" href="css/test_page.css?v=0127">
|
||||
<script>
|
||||
// 检测是否使用file://协议打开
|
||||
if (window.location.protocol === 'file:') {
|
||||
@@ -143,7 +143,8 @@
|
||||
</div>
|
||||
<div class="config-item">
|
||||
<label for="deviceName">设备名称:</label>
|
||||
<input type="text" id="deviceName" value="Web测试设备" maxlength="50" placeholder="deviceName">
|
||||
<input type="text" id="deviceName" value="Web测试设备" maxlength="50"
|
||||
placeholder="deviceName">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -237,23 +238,23 @@
|
||||
</div>
|
||||
|
||||
<!-- 背景加载 -->
|
||||
<script src="js/ui/background-load.js"></script>
|
||||
<script src="js/ui/background-load.js?v=0127"></script>
|
||||
|
||||
<!-- PIXI.js 2D渲染引擎 -->
|
||||
<script src="js/live2d/pixi.js"></script>
|
||||
<script src="js/live2d/pixi.js?v=0127"></script>
|
||||
|
||||
<!-- Live2D Cubism 4.0 SDK -->
|
||||
<script src="js/live2d/live2dcubismcore.min.js"></script>
|
||||
<script src="js/live2d/cubism4.min.js"></script>
|
||||
<script src="js/live2d/live2dcubismcore.min.js?v=0127"></script>
|
||||
<script src="js/live2d/cubism4.min.js?v=0127"></script>
|
||||
|
||||
<!-- Live2D 管理器 -->
|
||||
<script src="js/live2d/live2d.js"></script>
|
||||
<script src="js/live2d/live2d.js?v=0127"></script>
|
||||
|
||||
<!-- Opus解码库 -->
|
||||
<script src="js/utils/libopus.js"></script>
|
||||
<script src="js/utils/libopus.js?v=0127"></script>
|
||||
|
||||
<!-- 主应用入口 -->
|
||||
<script type="module" src="js/app.js"></script>
|
||||
<script type="module" src="js/app.js?v=0127"></script>
|
||||
|
||||
<!-- 全局错误处理 -->
|
||||
<script>
|
||||
|
||||
Reference in New Issue
Block a user