Merge branch 'py_test_error_response' into fix/2075

This commit is contained in:
Sakura-RanChen
2026-02-02 14:49:26 +08:00
committed by GitHub
61 changed files with 1057 additions and 986 deletions
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
file: Dockerfile-server-base
push: true
tags: ghcr.io/${{ github.repository }}:server-base
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
cache-from: type=gha,scope=server-base
cache-to: type=gha,mode=max,scope=server-base
build-args: |
+2 -2
View File
@@ -66,7 +66,7 @@ jobs:
push: true
tags: |
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:server_{1},ghcr.io/{0}:server_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:server_latest', github.repository) }}
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
@@ -81,7 +81,7 @@ jobs:
push: true
tags: |
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:web_{1},ghcr.io/{0}:web_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:web_latest', github.repository) }}
platforms: linux/amd64
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
+9 -8
View File
@@ -1,5 +1,5 @@
# 第一阶段:构建Vue前端
FROM node:18 as web-builder
FROM node:18 AS web-builder
WORKDIR /app
COPY main/manager-web/package*.json ./
RUN npm install
@@ -7,7 +7,7 @@ COPY main/manager-web .
RUN npm run build
# 第二阶段:构建Java后端
FROM maven:3.9.4-eclipse-temurin-21 as api-builder
FROM maven:3.9.4-eclipse-temurin-21 AS api-builder
WORKDIR /app
COPY main/manager-api/pom.xml .
COPY main/manager-api/src ./src
@@ -18,18 +18,19 @@ FROM bellsoft/liberica-runtime-container:jre-21-glibc
# 安装Nginx和字体库
RUN apk update && \
apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ \
apk add --no-cache --no-scripts \
nginx \
bash \
fontconfig \
ttf-dejavu \
msttcorefonts-installer \
&& ACCEPT_EULA=Y apk add --no-cache msttcorefonts-installer \
&& fc-cache -f -v \
&& rm -rf /var/cache/apk/*
&& rm -rf /var/cache/apk/* \
&& mkdir -p /run/nginx /var/log/nginx /var/tmp/nginx /etc/nginx/conf.d
# 复制项目自带的中文字体
COPY main/manager-web/public/generator/static/fonts/*.ttf /usr/share/fonts/
# 更新字体缓存
RUN (printf 'YES\n' | update-ms-fonts || true) && fc-cache -f -v
RUN fc-cache -f -v
# 配置Nginx
COPY docs/docker/nginx.conf /etc/nginx/nginx.conf
@@ -304,7 +304,7 @@ public interface Constant {
/**
* 版本号
*/
public static final String VERSION = "0.8.11";
public static final String VERSION = "0.9.1";
/**
* 无效固件URL
@@ -1,6 +1,12 @@
package xiaozhi.modules.agent.service.impl;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -81,9 +87,9 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
if (agent.getMemModelId() != null && agent.getMemModelId().equals(Constant.MEMORY_NO_MEM)) {
agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.IGNORE.getCode());
if (agent.getChatHistoryConf() == null) {
agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode());
}
}
if (agent.getChatHistoryConf() == null) {
agent.setChatHistoryConf(Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode());
}
// 查询上下文源配置
@@ -132,7 +138,8 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
if (StringUtils.isNotBlank(keyword)) {
if ("mac".equals(searchType)) {
// 按MAC地址搜索:先搜索设备,再获取对应的智能体
List<DeviceEntity> devices = Optional.ofNullable(deviceService.searchDevicesByMacAddress(keyword, userId)).orElseGet(ArrayList::new);
List<DeviceEntity> devices = Optional
.ofNullable(deviceService.searchDevicesByMacAddress(keyword, userId)).orElseGet(ArrayList::new);
// 获取设备对应的智能体ID列表
List<String> agentIds = devices.stream()
.map(DeviceEntity::getAgentId)
@@ -0,0 +1,31 @@
-- 批量清理 ai_model_provider 中的 sample_rate 字段定义
UPDATE `ai_model_provider` ap
JOIN (
SELECT
id,
JSON_ARRAYAGG(
JSON_OBJECT('key', jt.k, 'label', jt.l, 'type', jt.t)
) AS new_fields
FROM `ai_model_provider`,
JSON_TABLE(`fields`, '$[*]' COLUMNS (
k VARCHAR(50) PATH '$.key',
l VARCHAR(100) PATH '$.label',
t VARCHAR(20) PATH '$.type'
)) AS jt
WHERE `model_type` = 'TTS'
AND jt.k != 'sample_rate'
GROUP BY id
) filtered ON ap.id = filtered.id
SET ap.fields = filtered.new_fields;
-- 清理 config_json 顶层的 sample_rate
UPDATE `ai_model_config`
SET `config_json` = JSON_REMOVE(`config_json`, '$.sample_rate')
WHERE `model_type` = 'TTS'
AND JSON_EXTRACT(`config_json`, '$.sample_rate') IS NOT NULL;
-- 清理Minimax流式TTS的sample_rate参数(位于audio_setting内部)
UPDATE `ai_model_config` SET
`config_json` = JSON_SET(`config_json`, '$.audio_setting', JSON_REMOVE(JSON_EXTRACT(`config_json`, '$.audio_setting'), '$.sample_rate'))
WHERE `id` = 'TTS_MinimaxStreamTTS'
AND JSON_EXTRACT(`config_json`, '$.audio_setting.sample_rate') IS NOT NULL;
@@ -0,0 +1,87 @@
-- 更新HuoshanDoubleStreamTTS供应器配置,将分散的参数改为JSON字典配置
-- 将 speech_rate, loudness_rate, pitch, emotion, emotion_scale 等参数整合为 audio_params, additions, mix_speaker 三个JSON字典
UPDATE `ai_model_provider`
SET `fields` = '[
{"key": "ws_url", "type": "string", "label": "WebSocket地址"},
{"key": "appid", "type": "string", "label": "应用ID"},
{"key": "access_token", "type": "string", "label": "访问令牌"},
{"key": "resource_id", "type": "string", "label": "资源ID"},
{"key": "speaker", "type": "string", "label": "默认音色"},
{"key": "enable_ws_reuse", "type": "boolean", "label": "是否开启链接复用", "default": true},
{"key": "audio_params", "type": "dict", "label": "音频输出配置"},
{"key": "additions", "type": "dict", "label": "高级文本处理配置"},
{"key": "mix_speaker", "type": "dict", "label": "混音控制配置"}
]'
WHERE `id` = 'SYSTEM_TTS_HSDSTTS';
-- 更新现有配置,将旧的分散参数迁移到新的JSON字典结构
UPDATE `ai_model_config`
SET `config_json` = JSON_SET(
`config_json`,
'$.audio_params', JSON_OBJECT(
'speech_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.speech_rate')), ''), '0') AS SIGNED),
'loudness_rate', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.loudness_rate')), ''), '0') AS SIGNED)
),
'$.additions', JSON_OBJECT(
'aigc_metadata', JSON_OBJECT(),
'cache_config', JSON_OBJECT(),
'post_process', JSON_OBJECT(
'pitch', CAST(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(`config_json`, '$.pitch')), ''), '0') AS SIGNED)
)
),
'$.mix_speaker', JSON_OBJECT()
)
WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
-- 删除旧的分散参数字段
UPDATE `ai_model_config`
SET `config_json` = JSON_REMOVE(
`config_json`,
'$.speech_rate',
'$.loudness_rate',
'$.pitch',
'$.emotion',
'$.emotion_scale'
)
WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
-- 更新文档链接和备注说明
UPDATE `ai_model_config` SET
`doc_link` = 'https://www.volcengine.com/docs/6561/1329505',
`remark` = '火山引擎双向流式TTS配置说明:
1. 访问 https://www.volcengine.com/ 注册并开通火山引擎账号
2. 访问 https://console.volcengine.com/speech/service/10007 开通语音合成大模型,购买音色
3. 在页面底部获取appid和access_token
4. 资源ID固定为:volc.service_type.10029(大模型语音合成及混音)
5. 链接复用:开启WebSocket连接复用,默认true减少链接损耗(注意:复用后设备处于聆听状态时空闲链接会占并发数)
详细参数文档:https://www.volcengine.com/docs/6561/1329505
【audio_params】音频输出配置 - 用户可自定义添加火山引擎支持的任何音频参数
- speech_rate: 语速(-50~100),默认0
- loudness_rate: 音量(-50~100),默认0
- emotion: 情感类型(仅部分音色支持),可选值:neutral、happy、sad、angry、fearful、disgusted、surprised
- emotion_scale: 情感强度(1~5),默认4
示例:{"speech_rate": 10, "loudness_rate": 5, "emotion": "happy", "emotion_scale": 4}
【additions】高级文本处理配置 - 用户可自定义添加火山引擎支持的任何高级参数
- post_process.pitch: 音高(-12~12),默认0
- aigc_metadata: AIGC元数据配置
- cache_config: 缓存配置
示例:{"post_process": {"pitch": 2}, "aigc_metadata": {}, "cache_config": {}}
【mix_speaker】混音控制配置 - 多音色混合(仅 TTS 1.0)
示例:
{"speakers": [
{"source_speaker": "zh_male_bvlazysheep","mix_factor": 0.3},
{"source_speaker": "BV120_streaming","mix_factor": 0.3},
{"source_speaker": "zh_male_ahu_conversation_wvae_bigtts","mix_factor": 0.4}
]}
注意:
- 多情感音色参数(emotion、emotion_scale)仅部分音色支持
- 相关音色列表:https://www.volcengine.com/docs/6561/1257544
- 用户可根据火山引擎API文档自行添加更多参数
- 混音功能主要适用于豆包语音合成模型1.0的音色,使用时需要将req_params.speaker设置为custom_mix_bigtts
'
WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';
@@ -0,0 +1,14 @@
-- 更新小智参数中的默认采样率从 16000 改为 24000
UPDATE `sys_params`
SET `param_value` = '{
"type": "hello",
"version": 1,
"transport": "websocket",
"audio_params": {
"format": "opus",
"sample_rate": 24000,
"channels": 1,
"frame_duration": 60
}
}'
WHERE `id` = 309 AND `param_code` = 'xiaozhi';
@@ -487,6 +487,27 @@ databaseChangeLog:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202601051433.sql
- changeSet:
id: 202601141645
author: RanChen
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202601141645.sql
- changeSet:
id: 202601231530
author: RanChen
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202601231530.sql
- changeSet:
id: 202601261730
author: RanChen
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202601261730.sql
- changeSet:
id: 202601291552
author: shengzhou1216
@@ -235,7 +235,7 @@ function showAbout() {
title: t('settings.aboutApp', { appName: import.meta.env.VITE_APP_TITLE }),
content: t('settings.aboutContent', {
appName: import.meta.env.VITE_APP_TITLE,
version: '0.8.11'
version: '0.9.1'
}),
showCancel: false,
confirmText: t('common.confirm'),
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

+3 -2
View File
@@ -4,9 +4,10 @@
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" href="./favicon.ico">
<title>Xiaozhi AI Customization</title>
<script type="module" crossorigin src="./assets/index-FKVSBRAB.js"></script>
<link rel="stylesheet" crossorigin href="./assets/index-NXxBVrod.css">
<script type="module" crossorigin src="./assets/index-B8r0c7xg.js"></script>
<link rel="stylesheet" crossorigin href="./assets/index-CrIJdTCK.css">
</head>
<body>
<div id="app"></div>
@@ -64,7 +64,7 @@
<el-button size="mini" type="text" @click="handleUnbind(scope.row.device_id)">
{{ $t('device.unbind') }}
</el-button>
<el-button v-if="isGenerate(scope.row)" size="mini" type="text" @click="handleGenertor">
<el-button v-if="isGenerate(scope.row)" size="mini" type="text" @click="handleGenertor(scope.row)">
{{ $t('device.deviceThemeGeneration') }}
</el-button>
</template>
@@ -337,10 +337,10 @@ export default {
});
});
},
handleGenertor() {
handleGenertor(row) {
const pathname = window.location.pathname;
const basePath = pathname.split('/').slice(0, -1).join('/');
const url = `${window.location.origin}${basePath}/generator/`;
const url = `${window.location.origin}${basePath}/generator/?deviceId=${row.device_id}`;
sessionStorage.setItem('devicePath', window.location.href);
window.location.href = url;
},
+27 -14
View File
@@ -89,7 +89,8 @@ xiaozhi:
transport: websocket
audio_params:
format: opus
sample_rate: 16000
# Opus支持的采样率范围为[8000, 12000, 16000, 24000, 48000]
sample_rate: 24000
channels: 1
frame_duration: 60
@@ -717,13 +718,31 @@ TTS:
speaker: zh_female_wanwanxiaohe_moon_bigtts
# 开启WebSocket连接复用,默认复用(注意:复用后设备处于聆听状态时空闲链接会占并发数)
enable_ws_reuse: True
speech_rate: 0
loudness_rate: 0
pitch: 0
# 多情感音色参数,注意:当前仅部分音色支持设置情感。
# 相关音色列表:https://www.volcengine.com/docs/6561/1257544
emotion: "neutral" # 情感类型,可选值为:neutral、happy、sad、angry、fearful、disgusted、surprised
emotion_scale: 4 # 情感强度,可选值为:1~5,默认值为4
# 相关参数文档:https://www.volcengine.com/docs/6561/1329505
# 音频输出配置(audio_params)- 用户可自定义添加火山引擎支持的任何音频参数
audio_params:
speech_rate: 0 # 语速(-50~100)
loudness_rate: 0 # 音量(-50~100)
# 情感音色参数,注意:当前仅部分音色支持设置情感。
# 相关音色列表:https://www.volcengine.com/docs/6561/1257544
# emotion: "neutral" # 情感类型(仅部分音色支持):neutral、happy、sad、angry、fearful、disgusted、surprised
# emotion_scale: 4 # 情感强度(1~5)
# 高级文本处理配置(additions)- 用户可自定义添加火山引擎支持的任何高级参数
additions:
post_process:
pitch: 0 # 音高(-12~12)
# aigc_metadata: {} # AIGC元数据配置
# cache_config: {} # 缓存配置
# 混音控制配置(mix_speaker- 多音色混合(仅 TTS 1.0)
# 混音功能主要适用于豆包语音合成模型1.0的音色,使用时需要将req_params.speaker设置为custom_mix_bigtts
# mix_speaker:
# speakers:
# - source_speaker: zh_male_bvlazysheep
# mix_factor: 0.3
# - source_speaker: BV120_streaming
# mix_factor: 0.3
# - source_speaker: zh_male_ahu_conversation_wvae_bigtts
# mix_factor: 0.4
CosyVoiceSiliconflow:
type: siliconflow
# 硅基流动TTS
@@ -840,7 +859,6 @@ TTS:
# - "处理/(chu3)(li3)"
# - "危险/dangerous"
# audio_setting:
# sample_rate: 24000
# bitrate: 128000
# format: "mp3"
# channel: 1
@@ -868,7 +886,6 @@ TTS:
# 以下可不用设置,使用默认设置
# format: wav
# sample_rate: 16000
# volume: 50
# speech_rate: 0
# pitch_rate: 0
@@ -892,7 +909,6 @@ TTS:
host: nls-gateway-cn-beijing.aliyuncs.com
# 以下可不用设置,使用默认设置
# format: pcm # 音频格式:pcm、wav、mp3
# sample_rate: 16000 # 采样率:8000、16000、24000
# volume: 50 # 音量:0-100
# speech_rate: 0 # 语速:-500到500
# pitch_rate: 0 # 语调:-500到500
@@ -1007,7 +1023,6 @@ TTS:
protocol: websocket # protocol choices = ['websocket', 'http']
url: ws://127.0.0.1:8092/paddlespeech/tts/streaming # TTS 服务的 URL 地址,指向本地服务器 [websocket默认ws://127.0.0.1:8092/paddlespeech/tts/streaminghttp默认http://127.0.0.1:8090/paddlespeech/tts]
spk_id: 0 # 发音人 ID,0 通常表示默认的发音人
sample_rate: 24000 # 采样率 [websocket默认24000http默认0 自动选择]
speed: 1.0 # 语速,1.0 表示正常语速,>1 表示加快,<1 表示减慢
volume: 1.0 # 音量,1.0 表示正常音量,>1 表示增大,<1 表示减小
save_path: # 保存路径
@@ -1031,7 +1046,6 @@ TTS:
output_dir: tmp/
# 以下可不用设置,使用默认设置
# format: pcm # 音频格式:pcm、wav、mp3、opus
# sample_rate: 24000 # 采样率:16000, 24000, 48000
# volume: 50 # 音量:0-100
# rate: 1 # 语速:0.5~2
# pitch: 1 # 语调:0.5~2
@@ -1053,7 +1067,6 @@ TTS:
# stop_split: 0 # 关闭服务端拆句 不关闭:0,关闭:1
# remain: 0 # 是否保留原书面语的样子 保留:1, 不保留:0
# format: raw # 音频格式:raw(PCM), lame(MP3), speex, opus, opus-wb, opus-swb, speex-wb
# sample_rate: 24000 # 采样率:16000, 8000, 24000
# volume: 50 # 音量:0-100
# speed: 50 # 语速:0-100
# pitch: 50 # 语调:0-100
+1 -1
View File
@@ -5,7 +5,7 @@ from config.config_loader import load_config
from config.settings import check_config_file
from datetime import datetime
SERVER_VERSION = "0.8.11"
SERVER_VERSION = "0.9.1"
_logger_initialized = False
+20 -3
View File
@@ -87,6 +87,7 @@ class ConnectionHandler:
self.max_output_size = 0
self.chat_history_conf = 0
self.audio_format = "opus"
self.sample_rate = 24000 # 默认采样率,从客户端 hello 消息中动态更新
# 客户端状态相关
self.client_abort = False
@@ -208,6 +209,10 @@ class ConnectionHandler:
self.welcome_msg = self.config["xiaozhi"]
self.welcome_msg["session_id"] = self.session_id
# 从配置中读取采样率
self.sample_rate = self.welcome_msg["audio_params"]["sample_rate"]
self.logger.bind(tag=TAG).info(f"配置输出音频采样率为: {self.sample_rate}")
# 在后台初始化配置和组件(完全不阻塞主循环)
asyncio.create_task(self._background_initialize())
@@ -1184,6 +1189,8 @@ class ConnectionHandler:
if self.tts:
await self.tts.close()
if self.asr:
await self.asr.close()
# 最后关闭线程池(避免阻塞)
if self.executor:
@@ -1232,11 +1239,21 @@ class ConnectionHandler:
f"清理结束: TTS队列大小={self.tts.tts_text_queue.qsize()}, 音频队列大小={self.tts.tts_audio_queue.qsize()}"
)
def reset_vad_states(self):
self.client_audio_buffer = bytearray()
def reset_audio_states(self):
"""
重置所有音频相关状态(VAD + ASR)
"""
# Reset VAD states
self.client_audio_buffer.clear()
self.client_have_voice = False
self.client_voice_stop = False
self.logger.bind(tag=TAG).debug("VAD states reset.")
self.client_voice_window.clear()
self.last_is_voice = False
# Clear ASR buffers
self.asr_audio.clear()
self.logger.bind(tag=TAG).debug("All audio states reset.")
def chat_and_close(self, text):
"""Chat with the user and then close the connection"""
@@ -142,7 +142,8 @@ async def wakeupWordsResponse(conn):
# 获取当前音色
voice = getattr(conn.tts, "voice", "default")
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
# 使用链接的sample_rate
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=conn.sample_rate)
file_path = wakeup_words_config.generate_file_path(voice)
with open(file_path, "wb") as f:
f.write(wav_bytes)
@@ -17,7 +17,6 @@ async def handleAudioMessage(conn, audio):
if hasattr(conn, "just_woken_up") and conn.just_woken_up:
have_voice = False
# 设置一个短暂延迟后恢复VAD检测
conn.asr_audio.clear()
if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
return
@@ -26,10 +26,9 @@ class ListenTextMessageHandler(TextMessageHandler):
f"客户端拾音模式:{conn.client_listen_mode}"
)
if msg_json["state"] == "start":
conn.client_have_voice = True
conn.client_voice_stop = False
# 设备从播放模式切回录音模式,清除所有音频状态和缓冲区
conn.reset_audio_states()
elif msg_json["state"] == "stop":
conn.client_have_voice = True
conn.client_voice_stop = True
if conn.asr.interface_type == InterfaceType.STREAM:
# 流式模式下,发送结束请求
@@ -38,14 +37,13 @@ class ListenTextMessageHandler(TextMessageHandler):
# 非流式模式:直接触发ASR识别
if len(conn.asr_audio) > 0:
asr_audio_task = conn.asr_audio.copy()
conn.asr_audio.clear()
conn.reset_vad_states()
conn.reset_audio_states()
if len(asr_audio_task) > 0:
await conn.asr.handle_voice_stop(conn, asr_audio_task)
elif msg_json["state"] == "detect":
conn.client_have_voice = False
conn.asr_audio.clear()
conn.reset_audio_states()
if "text" in msg_json:
conn.last_activity_time = time.time() * 1000
original_text = msg_json["text"] # 保留原始文本
@@ -213,36 +213,24 @@ class ASRProvider(ASRProviderBase):
return None
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
if self._is_token_expired():
logger.warning("Token已过期,正在自动刷新...")
self._refresh_token()
file_path = None
try:
# 解码Opus为PCM
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
# 发送请求并获取文本
text = await self._send_request(combined_pcm_data)
text = await self._send_request(artifacts.pcm_bytes)
if text:
return text, file_path
return text, artifacts.file_path
return "", file_path
return "", artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", file_path
return "", None
@@ -126,16 +126,8 @@ class ASRProvider(ASRProviderBase):
await super().open_audio_channels(conn)
async def receive_audio(self, conn, audio, audio_have_voice):
# 初始化音频缓存
if not hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
# 存储音频数据
if audio:
conn.asr_audio_for_voiceprint.append(audio)
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-10:]
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 只在有声音且没有连接时建立连接(排除正在停止的情况)
if audio_have_voice and not self.is_processing and not self.asr_ws:
@@ -204,6 +196,8 @@ class ASRProvider(ASRProviderBase):
"""转发识别结果"""
try:
while not conn.stop_event.is_set():
# 获取当前连接的音频数据
audio_data = conn.asr_audio
try:
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
result = json.loads(response)
@@ -257,19 +251,12 @@ class ASRProvider(ASRProviderBase):
# 手动模式下,只有在收到stop信号后才触发处理(仅处理一次)
if conn.client_voice_stop:
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
if len(audio_data) > 0:
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
await self.handle_voice_stop(conn, audio_data)
break
else:
# 自动模式下直接覆盖
self.text = text
conn.reset_vad_states()
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
await self.handle_voice_stop(conn, audio_data)
break
@@ -289,11 +276,7 @@ class ASRProvider(ASRProviderBase):
finally:
# 清理连接的音频缓存
await self._cleanup()
if conn:
if hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
conn.reset_audio_states()
async def _send_stop_request(self):
"""发送停止识别请求(不关闭连接)"""
@@ -341,7 +324,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug("ASR会话清理完成")
async def speech_to_text(self, opus_data, session_id, audio_format):
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
"""获取识别结果"""
result = self.text
self.text = ""
@@ -52,16 +52,8 @@ class ASRProvider(ASRProviderBase):
await super().open_audio_channels(conn)
async def receive_audio(self, conn, audio, audio_have_voice):
# 初始化音频缓存
if not hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
# 存储音频数据
if audio:
conn.asr_audio_for_voiceprint.append(audio)
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-10:]
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 只在有声音且没有连接时建立连接
if audio_have_voice and not self.is_processing and not self.asr_ws:
@@ -166,6 +158,8 @@ class ASRProvider(ASRProviderBase):
"""转发识别结果"""
try:
while not conn.stop_event.is_set():
# 获取当前连接的音频数据
audio_data = conn.asr_audio
try:
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
result = json.loads(response)
@@ -214,19 +208,12 @@ class ASRProvider(ASRProviderBase):
# 手动模式下,只有在收到stop信号后才触发处理
if conn.client_voice_stop:
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
if len(audio_data) > 0:
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
await self.handle_voice_stop(conn, audio_data)
break
else:
# 自动模式下直接覆盖
self.text = text
conn.reset_vad_states()
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
await self.handle_voice_stop(conn, audio_data)
break
@@ -257,11 +244,7 @@ class ASRProvider(ASRProviderBase):
finally:
# 清理连接的音频缓存
await self._cleanup()
if conn:
if hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
conn.reset_audio_states()
async def _send_stop_request(self):
"""发送停止请求(用于手动模式停止录音)"""
@@ -325,7 +308,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug("ASR会话清理完成")
async def speech_to_text(self, opus_data, session_id, audio_format):
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
"""获取识别结果"""
result = self.text
self.text = ""
@@ -30,37 +30,26 @@ class ASRProvider(ASRProviderBase):
os.makedirs(self.output_dir, exist_ok=True)
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
if not opus_data:
logger.bind(tag=TAG).warning("音频数据为空!")
return None, None
file_path = None
try:
# 检查配置是否已设置
if not self.app_id or not self.api_key or not self.secret_key:
logger.bind(tag=TAG).error("百度语音识别配置未设置,无法进行识别")
return None, file_path
return None, None
# 将Opus音频数据解码为PCM
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
start_time = time.time()
# 识别本地文件
result = self.client.asr(
combined_pcm_data,
artifacts.pcm_bytes,
"pcm",
16000,
{
@@ -73,13 +62,13 @@ class ASRProvider(ASRProviderBase):
f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
)
result = result["result"][0]
return result, file_path
return result, artifacts.file_path
else:
raise Exception(
f"百度语音识别失败,错误码: {result['err_no']},错误信息: {result['err_msg']}"
)
return None, file_path
return None, artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(f"处理音频时发生错误!{e}", exc_info=True)
return None, file_path
return None, None
+144 -30
View File
@@ -8,14 +8,18 @@ import queue
import asyncio
import traceback
import threading
import shutil
import opuslib_next
from abc import ABC, abstractmethod
from config.logger import setup_logging
from typing import Optional, Tuple, List
from typing import Optional, Tuple, List, NamedTuple
from core.providers.asr.dto.dto import InterfaceType
from core.handle.receiveAudioHandle import startToChat
from core.handle.reportHandle import enqueue_asr_report
from core.utils.util import remove_punctuation_and_length
from core.handle.receiveAudioHandle import handleAudioMessage
import tempfile
TAG = __name__
logger = setup_logging()
@@ -57,18 +61,17 @@ class ASRProviderBase(ABC):
conn.asr_audio.append(audio)
else:
# 自动/实时模式:使用VAD检测
have_voice = audio_have_voice
conn.asr_audio.append(audio)
if not have_voice and not conn.client_have_voice:
# 如果没有语音,且之前也没有声音,缓存部分音频
if not audio_have_voice and not conn.client_have_voice:
conn.asr_audio = conn.asr_audio[-10:]
return
# 自动模式下通过VAD检测到语音停止时触发识别
if conn.client_voice_stop:
if conn.asr.interface_type != InterfaceType.STREAM and conn.client_voice_stop:
asr_audio_task = conn.asr_audio.copy()
conn.asr_audio.clear()
conn.reset_vad_states()
conn.reset_audio_states()
if len(asr_audio_task) > 15:
await self.handle_voice_stop(conn, asr_audio_task)
@@ -93,10 +96,14 @@ class ASRProviderBase(ABC):
wav_data = self._pcm_to_wav(combined_pcm_data)
# 定义ASR任务
asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
asr_task = self.speech_to_text_wrapper(
asr_audio_task, conn.session_id, conn.audio_format
)
if conn.voiceprint_provider and wav_data:
voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
voiceprint_task = conn.voiceprint_provider.identify_speaker(
wav_data, conn.session_id
)
# 并发等待两个结果
asr_result, voiceprint_result = await asyncio.gather(
asr_task, voiceprint_task, return_exceptions=True
@@ -159,20 +166,20 @@ class ASRProviderBase(ABC):
if text_len > 0:
# 使用自定义模块进行上报
await startToChat(conn, enhanced_text)
enqueue_asr_report(conn, enhanced_text, asr_audio_task)
audio_snapshot = asr_audio_task.copy()
enqueue_asr_report(conn, enhanced_text, audio_snapshot)
except Exception as e:
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
import traceback
logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
"""构建包含说话人信息的文本(仅用于纯文本ASR)"""
if speaker_name and speaker_name.strip():
return json.dumps({
"speaker": speaker_name,
"content": text
}, ensure_ascii=False)
return json.dumps(
{"speaker": speaker_name, "content": text}, ensure_ascii=False
)
else:
return text
@@ -181,23 +188,23 @@ class ASRProviderBase(ABC):
if len(pcm_data) == 0:
logger.bind(tag=TAG).warning("PCM数据为空,无法转换WAV")
return b""
# 确保数据长度是偶数(16位音频)
if len(pcm_data) % 2 != 0:
pcm_data = pcm_data[:-1]
# 创建WAV文件头
wav_buffer = io.BytesIO()
try:
with wave.open(wav_buffer, 'wb') as wav_file:
wav_file.setnchannels(1) # 单声道
wav_file.setsampwidth(2) # 16位
with wave.open(wav_buffer, "wb") as wav_file:
wav_file.setnchannels(1) # 单声道
wav_file.setsampwidth(2) # 16位
wav_file.setframerate(16000) # 16kHz采样率
wav_file.writeframes(pcm_data)
wav_buffer.seek(0)
wav_data = wav_buffer.read()
return wav_data
except Exception as e:
logger.bind(tag=TAG).error(f"WAV转换失败: {e}")
@@ -206,6 +213,44 @@ class ASRProviderBase(ABC):
def stop_ws_connection(self):
pass
async def close(self):
pass
class AudioArtifacts(NamedTuple):
pcm_frames: List[bytes]
"""PCM音频帧列表"""
pcm_bytes: bytes
"""合并后的PCM音频字节数据"""
file_path: Optional[str]
"""WAV文件路径"""
temp_path: Optional[str]
"""临时WAV文件路径"""
def get_current_artifacts(self) -> Optional["ASRProviderBase.AudioArtifacts"]:
return self._current_artifacts
def requires_file(self) -> bool:
"""是否需要文件输入"""
return False
def prefers_temp_file(self) -> bool:
"""是否优先使用临时文件"""
return False
def build_temp_file(self, pcm_bytes: bytes) -> Optional[str]:
try:
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
temp_path = temp_file.name
with wave.open(temp_path, "wb") as wav_file:
wav_file.setnchannels(1)
wav_file.setsampwidth(2)
wav_file.setframerate(16000)
wav_file.writeframes(pcm_bytes)
return temp_path
except Exception as e:
logger.bind(tag=TAG).error(f"临时音频文件生成失败: {e}")
return None
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
@@ -220,11 +265,80 @@ class ASRProviderBase(ABC):
return file_path
@abstractmethod
async def speech_to_text(
async def speech_to_text_wrapper(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
file_path = None
temp_path = None
try:
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
free_space = shutil.disk_usage(self.output_dir).free
if free_space < len(combined_pcm_data) * 2:
raise OSError("磁盘空间不足")
if self.requires_file() and self.prefers_temp_file():
temp_path = self.build_temp_file(combined_pcm_data)
if (hasattr(self, "delete_audio_file") and not self.delete_audio_file) or (
self.requires_file() and not self.prefers_temp_file()
):
file_path = self.save_audio_to_file(pcm_data, session_id)
if len(combined_pcm_data) == 0:
artifacts = None
else:
artifacts = ASRProviderBase.AudioArtifacts(
pcm_frames=pcm_data,
pcm_bytes=combined_pcm_data,
file_path=file_path,
temp_path=temp_path,
)
text, _ = await self.speech_to_text(
opus_data, session_id, audio_format, artifacts
)
return text, file_path
except OSError as e:
logger.bind(tag=TAG).error(f"文件操作错误: {e}")
return None, None
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}")
return None, None
finally:
try:
if temp_path and os.path.exists(temp_path):
os.unlink(temp_path)
if (
hasattr(self, "delete_audio_file")
and self.delete_audio_file
and file_path
and os.path.exists(file_path)
):
os.remove(file_path)
except Exception as e:
logger.bind(tag=TAG).error(f"文件清理失败: {e}")
@abstractmethod
async def speech_to_text(
self,
opus_data: List[bytes],
session_id: str,
audio_format="opus",
artifacts: Optional[AudioArtifacts] = None,
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本
:param opus_data: 输入的Opus音频数据
:param session_id: 会话ID
:param audio_format: 音频格式,默认"opus"
:param artifacts: 音频工件,包含PCM数据、文件路径等
:return: 识别结果文本和文件路径(如果有)
"""
pass
@staticmethod
@@ -235,23 +349,23 @@ class ASRProviderBase(ABC):
decoder = opuslib_next.Decoder(16000, 1)
pcm_data = []
buffer_size = 960 # 每次处理960个采样点 (60ms at 16kHz)
for i, opus_packet in enumerate(opus_data):
try:
if not opus_packet or len(opus_packet) == 0:
continue
pcm_frame = decoder.decode(opus_packet, buffer_size)
if pcm_frame and len(pcm_frame) > 0:
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).warning(f"Opus解码错误,跳过数据包 {i}: {e}")
except Exception as e:
logger.bind(tag=TAG).error(f"音频处理错误,数据包 {i}: {e}")
return pcm_data
except Exception as e:
logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
return []
@@ -232,24 +232,13 @@ class ASRProvider(ASRProviderBase):
yield data[offset:data_len], True
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
file_path = None
try:
# 合并所有opus数据包
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
# 直接使用PCM数据
# 计算分段大小 (单声道, 16bit, 16kHz采样率)
@@ -258,14 +247,14 @@ class ASRProvider(ASRProviderBase):
# 语音识别
start_time = time.time()
text = await self._send_request(combined_pcm_data, segment_size)
text = await self._send_request(artifacts.pcm_bytes, segment_size)
if text:
logger.bind(tag=TAG).debug(
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
)
return text, file_path
return "", file_path
return text, artifacts.file_path
return "", artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", file_path
return "", None
@@ -60,17 +60,8 @@ class ASRProvider(ASRProviderBase):
await super().open_audio_channels(conn)
async def receive_audio(self, conn, audio, audio_have_voice):
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-10:]
# 存储音频数据
if not hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
conn.asr_audio_for_voiceprint.append(audio)
# 当没有音频数据时处理完整语音片段
if conn.client_listen_mode != "manual" and not audio and len(conn.asr_audio_for_voiceprint) > 0:
await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint)
conn.asr_audio_for_voiceprint = []
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 如果本次有声音,且之前没有建立连接
if audio_have_voice and self.asr_ws is None and not self.is_processing:
@@ -164,7 +155,7 @@ class ASRProvider(ASRProviderBase):
try:
while self.asr_ws and not conn.stop_event.is_set():
# 获取当前连接的音频数据
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
audio_data = conn.asr_audio
try:
response = await self.asr_ws.recv()
result = self.parse_response(response)
@@ -189,7 +180,6 @@ class ASRProvider(ASRProviderBase):
):
logger.bind(tag=TAG).error(f"识别文本:空")
self.text = ""
conn.reset_vad_states()
if len(audio_data) > 15: # 确保有足够音频数据
await self.handle_voice_stop(conn, audio_data)
break
@@ -200,12 +190,9 @@ class ASRProvider(ASRProviderBase):
if self.enable_multilingual:
continue
if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0:
if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 15:
logger.bind(tag=TAG).debug("消息结束收到停止信号,触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
break
for utterance in utterances:
@@ -226,14 +213,10 @@ class ASRProvider(ASRProviderBase):
if conn.client_voice_stop and len(audio_data) > 0:
logger.bind(tag=TAG).debug("消息中途收到停止信号,触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
break
else:
# 自动模式下直接覆盖
self.text = current_text
conn.reset_vad_states()
if len(audio_data) > 15: # 确保有足够音频数据
await self.handle_voice_stop(conn, audio_data)
break
@@ -262,11 +245,8 @@ class ASRProvider(ASRProviderBase):
await self.asr_ws.close()
self.asr_ws = None
self.is_processing = False
if conn:
if hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
# 重置所有音频相关状态
conn.reset_audio_states()
def stop_ws_connection(self):
if self.asr_ws:
@@ -408,7 +388,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).error(f"原始响应数据: {res.hex()}")
raise
async def speech_to_text(self, opus_data, session_id, audio_format):
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
result = self.text
self.text = "" # 清空text
return result, None
@@ -435,11 +415,3 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug("Doubao decoder resources released")
except Exception as e:
logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")
# 清理所有连接的音频缓冲区
if hasattr(self, '_connections'):
for conn in self._connections.values():
if hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
@@ -64,39 +64,21 @@ class ASRProvider(ASRProviderBase):
)
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""语音转文本主处理逻辑"""
file_path = None
retry_count = 0
while retry_count < MAX_RETRIES:
try:
# 合并所有opus数据包
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 检查磁盘空间
if not self.delete_audio_file:
free_space = shutil.disk_usage(self.output_dir).free
if free_space < len(combined_pcm_data) * 2: # 预留2倍空间
raise OSError("磁盘空间不足")
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
# 语音识别 - 使用线程池避免阻塞事件循环
start_time = time.time()
result = await asyncio.to_thread(
self.model.generate,
input=combined_pcm_data,
input=artifacts.pcm_bytes,
cache={},
language="auto",
use_itn=True,
@@ -107,7 +89,7 @@ class ASRProvider(ASRProviderBase):
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}"
)
return text, file_path
return text, artifacts.file_path
except OSError as e:
retry_count += 1
@@ -115,7 +97,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).error(
f"语音识别失败(已重试{retry_count}次): {e}", exc_info=True
)
return "", file_path
return "", None
logger.bind(tag=TAG).warning(
f"语音识别失败,正在重试({retry_count}/{MAX_RETRIES}: {e}"
)
@@ -123,15 +105,4 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", file_path
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
try:
os.remove(file_path)
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
except Exception as e:
logger.bind(tag=TAG).error(
f"文件删除失败: {file_path} | 错误: {e}"
)
return "", None
@@ -101,7 +101,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""
Convert speech data to text using FunASR.
@@ -109,18 +109,9 @@ class ASRProvider(ASRProviderBase):
:param session_id: Unique session identifier.
:return: Tuple containing recognized text and optional timestamp.
"""
file_path = None
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
auth_header = {"Authorization": "Bearer; {}".format(self.api_key)}
async with websockets.connect(
self.uri,
@@ -132,7 +123,7 @@ class ASRProvider(ASRProviderBase):
try:
# Use asyncio to handle WebSocket communication
send_task = asyncio.create_task(
self._send_data(ws, combined_pcm_data, session_id)
self._send_data(ws, artifacts.pcm_bytes, session_id)
)
receive_task = asyncio.create_task(self._receive_responses(ws))
@@ -161,14 +152,14 @@ class ASRProvider(ASRProviderBase):
result = lang_tag_filter(result)
return (
result,
file_path,
artifacts.file_path,
) # Return the recognized text and timestamp (if any)
except websockets.exceptions.ConnectionClosed as e:
logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
return "", file_path
return "", artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(
f"Error during speech-to-text conversion: {e}", exc_info=True
)
return "", file_path
return "", artifacts.file_path
@@ -21,20 +21,16 @@ class ASRProvider(ASRProviderBase):
os.makedirs(self.output_dir, exist_ok=True)
async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus") -> Tuple[Optional[str], Optional[str]]:
def requires_file(self) -> bool:
return True
async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None) -> Tuple[Optional[str], Optional[str]]:
file_path = None
try:
start_time = time.time()
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
file_path = self.save_audio_to_file(pcm_data, session_id)
logger.bind(tag=TAG).debug(
f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
)
if artifacts is None:
return "", None
file_path = artifacts.file_path
logger.bind(tag=TAG).info(f"file path: {file_path}")
headers = {
"Authorization": f"Bearer {self.api_key}",
@@ -71,12 +67,4 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}")
return "", None
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
try:
os.remove(file_path)
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
except Exception as e:
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
@@ -1,5 +1,4 @@
import os
import tempfile
from typing import Optional, Tuple, List
import dashscope
from config.logger import setup_logging
@@ -35,56 +34,25 @@ class ASRProvider(ASRProviderBase):
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
def _prepare_audio_file(self, pcm_data: bytes) -> str:
"""将PCM数据转换为WAV文件并返回文件路径"""
try:
import wave
# 创建临时WAV文件
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file:
temp_path = temp_file.name
# 写入WAV格式
with wave.open(temp_path, 'wb') as wav_file:
wav_file.setnchannels(1) # 单声道
wav_file.setsampwidth(2) # 16位
wav_file.setframerate(16000) # 16kHz采样率
wav_file.writeframes(pcm_data)
return temp_path
except Exception as e:
logger.bind(tag=tag).error(f"音频文件准备失败: {e}")
return None
def prefers_temp_file(self) -> bool:
return True
def requires_file(self) -> bool:
return True
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
temp_file_path = None
file_path = None
try:
# 解码音频数据
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
if len(combined_pcm_data) == 0:
logger.bind(tag=tag).warning("音频数据为空")
if artifacts is None:
return "", None
# 准备音频文件
temp_file_path = self._prepare_audio_file(combined_pcm_data)
temp_file_path = artifacts.temp_path
file_path = artifacts.file_path
if not temp_file_path:
return "", None
# 保存音频文件(如果需要)
if not self.delete_audio_file:
file_path = self.save_audio_to_file(pcm_data, session_id)
return "", file_path
# 构造请求消息
messages = [
{
@@ -141,11 +109,3 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=tag).error(f"语音识别失败: {e}")
return "", file_path
finally:
# 清理临时文件
if temp_file_path and os.path.exists(temp_file_path):
try:
os.unlink(temp_file_path)
except Exception as e:
logger.bind(tag=tag).warning(f"清理临时文件失败: {e}")
@@ -120,24 +120,19 @@ class ASRProvider(ASRProviderBase):
samples_float32 = samples_float32 / 32768
return samples_float32, f.getframerate()
def requires_file(self) -> bool:
return True
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""语音转文本主处理逻辑"""
file_path = None
try:
# 保存音频文件
start_time = time.time()
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
file_path = self.save_audio_to_file(pcm_data, session_id)
logger.bind(tag=TAG).debug(
f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
)
if artifacts is None:
return "", None
file_path = artifacts.file_path
# 语音识别
start_time = time.time()
s = self.model.create_stream()
samples, sample_rate = self.read_wave(file_path)
@@ -153,11 +148,3 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", file_path
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
try:
os.remove(file_path)
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
except Exception as e:
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
@@ -32,35 +32,24 @@ class ASRProvider(ASRProviderBase):
os.makedirs(self.output_dir, exist_ok=True)
async def speech_to_text(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
if not opus_data:
logger.bind(tag=TAG).warning("音频数据为空!")
return None, None
file_path = None
try:
# 检查配置是否已设置
if not self.secret_id or not self.secret_key:
logger.bind(tag=TAG).error("腾讯云语音识别配置未设置,无法进行识别")
return None, file_path
return None, None
# 将Opus音频数据解码为PCM
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
if artifacts is None:
return "", None
# 将音频数据转换为Base64编码
base64_audio = base64.b64encode(combined_pcm_data).decode("utf-8")
base64_audio = base64.b64encode(artifacts.pcm_bytes).decode("utf-8")
# 构建请求体
request_body = self._build_request_body(base64_audio)
@@ -77,11 +66,11 @@ class ASRProvider(ASRProviderBase):
f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
)
return result, file_path
return result, artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(f"处理音频时发生错误!{e}", exc_info=True)
return None, file_path
return None, None
def _build_request_body(self, base64_audio: str) -> str:
"""构建请求体"""
+7 -30
View File
@@ -44,36 +44,21 @@ class ASRProvider(ASRProviderBase):
raise
async def speech_to_text(
self, audio_data: List[bytes], session_id: str, audio_format: str = "opus"
self, opus_data: List[bytes], session_id: str, audio_format="opus", artifacts=None
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
file_path = None
try:
# 检查模型是否加载成功
if not self.model:
logger.bind(tag=TAG).error("VOSK模型未加载,无法进行识别")
return "", None
# 解码音频(如果原始格式是Opus
if audio_format == "pcm":
pcm_data = audio_data
else:
pcm_data = self.decode_opus(audio_data)
if not pcm_data:
logger.bind(tag=TAG).warning("解码后的PCM数据为空,无法进行识别")
if artifacts is None:
return "", None
# 合并PCM数据
combined_pcm_data = b"".join(pcm_data)
if len(combined_pcm_data) == 0:
if not artifacts.pcm_bytes:
logger.bind(tag=TAG).warning("合并后的PCM数据为空")
return "", None
# 判断是否保存为WAV文件
if not self.delete_audio_file:
file_path = self.save_audio_to_file(pcm_data, session_id)
start_time = time.time()
@@ -81,8 +66,8 @@ class ASRProvider(ASRProviderBase):
chunk_size = 2000
text_result = ""
for i in range(0, len(combined_pcm_data), chunk_size):
chunk = combined_pcm_data[i:i+chunk_size]
for i in range(0, len(artifacts.pcm_bytes), chunk_size):
chunk = artifacts.pcm_bytes[i:i+chunk_size]
if self.recognizer.AcceptWaveform(chunk):
result = json.loads(self.recognizer.Result())
text = result.get('text', '')
@@ -99,16 +84,8 @@ class ASRProvider(ASRProviderBase):
f"VOSK语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text_result.strip()}"
)
return text_result.strip(), file_path
return text_result.strip(), artifacts.file_path
except Exception as e:
logger.bind(tag=TAG).error(f"VOSK语音识别失败: {e}")
return "", None
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
try:
os.remove(file_path)
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
except Exception as e:
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
@@ -101,11 +101,6 @@ class ASRProvider(ASRProviderBase):
# 先调用父类方法处理基础逻辑
await super().receive_audio(conn, audio, audio_have_voice)
# 存储音频数据用于声纹识别
if not hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
conn.asr_audio_for_voiceprint.append(audio)
# 如果本次有声音,且之前没有建立连接
if audio_have_voice and self.asr_ws is None and not self.is_processing:
try:
@@ -232,13 +227,8 @@ class ASRProvider(ASRProviderBase):
if status == 2:
if conn.client_listen_mode == "manual":
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
if len(audio_data) > 0:
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
logger.bind(tag=TAG).debug("收到最终识别结果,触发处理")
await self.handle_voice_stop(conn, conn.asr_audio)
break
except asyncio.TimeoutError:
@@ -262,13 +252,7 @@ class ASRProvider(ASRProviderBase):
finally:
# 清理连接资源
await self._cleanup()
# 清理连接的音频缓存
if conn:
if hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, "asr_audio"):
conn.asr_audio = []
conn.reset_audio_states()
async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
"""处理语音停止,发送最后一帧并处理识别结果"""
@@ -334,7 +318,7 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug("ASR会话清理完成")
async def speech_to_text(self, opus_data, session_id, audio_format):
async def speech_to_text(self, opus_data, session_id, audio_format, artifacts=None):
"""获取识别结果"""
result = self.text
self.text = ""
@@ -363,10 +347,3 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
# 清理所有连接的音频缓冲区
if hasattr(self, "_connections"):
for conn in self._connections.values():
if hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, "asr_audio"):
conn.asr_audio = []
@@ -41,8 +41,6 @@ class TTSProvider(TTSProviderBase):
# 音频参数配置
self.format = config.get("format", "pcm")
sample_rate = config.get("sample_rate", "24000")
self.sample_rate = int(sample_rate) if sample_rate else 24000
volume = config.get("volume", "50")
self.volume = int(volume) if volume else 50
@@ -60,11 +58,6 @@ class TTSProvider(TTSProviderBase):
"X-DashScope-DataInspection": "enable",
}
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
)
async def _ensure_connection(self):
"""确保WebSocket连接可用,支持60秒内连接复用"""
try:
@@ -245,7 +238,7 @@ class TTSProvider(TTSProviderBase):
"text_type": "PlainText",
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"rate": self.rate,
"pitch": self.pitch,
@@ -429,7 +422,7 @@ class TTSProvider(TTSProviderBase):
"text_type": "PlainText",
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"rate": self.rate,
"pitch": self.pitch,
@@ -95,8 +95,6 @@ class TTSProvider(TTSProviderBase):
self.appkey = config.get("appkey")
self.format = config.get("format", "wav")
self.audio_file_type = config.get("format", "wav")
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
if config.get("private_voice"):
self.voice = config.get("private_voice")
@@ -172,7 +170,7 @@ class TTSProvider(TTSProviderBase):
"token": self.token,
"text": text,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"voice": self.voice,
"volume": self.volume,
"speech_rate": self.speech_rate,
@@ -99,10 +99,6 @@ class TTSProvider(TTSProviderBase):
self.format = config.get("format", "pcm")
self.audio_file_type = config.get("format", "pcm")
# 采样率配置
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
# 音色配置 - CosyVoice大模型音色
if config.get("private_voice"):
self.voice = config.get("private_voice")
@@ -134,11 +130,6 @@ class TTSProvider(TTSProviderBase):
# 专属tts设置
self.task_id = uuid.uuid4().hex
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# Token管理
if self.access_key_id and self.access_key_secret:
self._refresh_token()
@@ -344,7 +335,7 @@ class TTSProvider(TTSProviderBase):
"payload": {
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"speech_rate": self.speech_rate,
"pitch_rate": self.pitch_rate,
@@ -508,7 +499,7 @@ class TTSProvider(TTSProviderBase):
"payload": {
"voice": self.voice,
"format": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"volume": self.volume,
"speech_rate": self.speech_rate,
"pitch_rate": self.pitch_rate,
+16 -5
View File
@@ -1,17 +1,18 @@
import os
import re
import time
import uuid
import queue
import asyncio
import threading
import traceback
from core.utils import p3
from datetime import datetime
from core.utils import textUtils
from typing import Callable, Any
from abc import ABC, abstractmethod
from config.logger import setup_logging
from core.utils import opus_encoder_utils
from core.utils.tts import MarkdownCleaner
from core.utils.output_counter import add_device_output
from core.handle.reportHandle import enqueue_tts_report
@@ -97,6 +98,8 @@ class TTSProviderBase(ABC):
file_type=self.audio_file_type,
is_opus=True,
callback=opus_handler,
sample_rate=self.conn.sample_rate,
opus_encoder=self.opus_encoder,
)
break
else:
@@ -138,7 +141,7 @@ class TTSProviderBase(ABC):
logger.bind(tag=TAG).error(
f"语音生成失败: {text},请检查网络或服务是否正常"
)
self.tts_audio_queue.put((SentenceType.FIRST, None, text))
self.tts_audio_queue.put((SentenceType.FIRST, None, text))
self._process_audio_file_stream(tmp_file, callback=opus_handler)
except Exception as e:
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
@@ -158,7 +161,8 @@ class TTSProviderBase(ABC):
audio_bytes,
file_type=self.audio_file_type,
is_opus=True,
callback=lambda data: audio_datas.append(data)
callback=lambda data: audio_datas.append(data),
sample_rate=self.conn.sample_rate,
)
return audio_datas
else:
@@ -214,13 +218,13 @@ class TTSProviderBase(ABC):
self, audio_file_path, callback: Callable[[Any], Any] = None
):
"""音频文件转换为PCM编码"""
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback)
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=None)
def audio_to_opus_data_stream(
self, audio_file_path, callback: Callable[[Any], Any] = None
):
"""音频文件转换为Opus编码"""
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback)
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=self.conn.sample_rate, opus_encoder=self.opus_encoder)
def tts_one_sentence(
self,
@@ -252,6 +256,13 @@ class TTSProviderBase(ABC):
async def open_audio_channels(self, conn):
self.conn = conn
# 根据conn的sample_rate创建编码器,如果子类已经创建则不覆盖(IndexTTS接口返回为24kHZ-待重采样处理)
if not hasattr(self, 'opus_encoder') or self.opus_encoder is None:
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=conn.sample_rate, channels=1, frame_size_ms=60
)
# tts 消化线程
self.tts_priority_thread = threading.Thread(
target=self.tts_text_priority_thread, daemon=True
@@ -154,16 +154,29 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("private_voice")
else:
self.voice = config.get("speaker")
speech_rate = config.get("speech_rate", "0")
loudness_rate = config.get("loudness_rate", "0")
pitch = config.get("pitch", "0")
self.speech_rate = int(speech_rate) if speech_rate else 0
self.loudness_rate = int(loudness_rate) if loudness_rate else 0
self.pitch = int(pitch) if pitch else 0
# 多情感音色参数
self.emotion = config.get("emotion", "neutral")
emotion_scale = config.get("emotion_scale", "4")
self.emotion_scale = int(emotion_scale) if emotion_scale else 4
# 默认 audio_params 配置
default_audio_params = {
"speech_rate": 0,
"loudness_rate": 0
}
# 默认 additions 配置
default_additions = {
"aigc_metadata": {},
"cache_config": {},
"post_process": {
"pitch": 0
}
}
# 默认 mix_speaker 配置
default_mix_speaker = {}
# 合并用户配置
self.audio_params = {**default_audio_params, **config.get("audio_params", {})}
self.additions = {**default_additions, **config.get("additions", {})}
self.mix_speaker = {**default_mix_speaker, **config.get("mix_speaker", {})}
self.ws_url = config.get("ws_url")
self.authorization = config.get("authorization")
@@ -171,9 +184,7 @@ class TTSProvider(TTSProviderBase):
enable_ws_reuse_value = config.get("enable_ws_reuse", True)
self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True
self.tts_text = ""
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
model_key_msg = check_model_key("TTS", self.access_token)
if model_key_msg:
logger.bind(tag=TAG).error(model_key_msg)
@@ -181,6 +192,8 @@ class TTSProvider(TTSProviderBase):
async def open_audio_channels(self, conn):
try:
await super().open_audio_channels(conn)
# 更新 audio_params 中的采样率为实际的 conn.sample_rate
self.audio_params["sample_rate"] = conn.sample_rate
except Exception as e:
logger.bind(tag=TAG).error(f"Failed to open audio channels: {str(e)}")
self.ws = None
@@ -646,20 +659,18 @@ class TTSProvider(TTSProviderBase):
text="",
speaker="",
audio_format="pcm",
audio_sample_rate=16000,
):
audio_params = {
"format": audio_format,
"sample_rate": audio_sample_rate,
"speech_rate": self.speech_rate,
"loudness_rate": self.loudness_rate
# 构建 req_params
req_params = {
"text": text,
"speaker": speaker,
"audio_params": {**self.audio_params, "format": audio_format},
"additions": json.dumps(self.additions)
}
# 如果是多情感音色,添加情感参数
if '_emo_' in self.voice:
if self.emotion:
audio_params["emotion"] = self.emotion
audio_params["emotion_scale"] = self.emotion_scale
# 如果有 mix_speaker 配置,添加到 req_params
if self.mix_speaker:
req_params["mix_speaker"] = self.mix_speaker
return str.encode(
json.dumps(
@@ -667,17 +678,7 @@ class TTSProvider(TTSProviderBase):
"user": {"uid": uid},
"event": event,
"namespace": "BidirectionalTTS",
"req_params": {
"text": text,
"speaker": speaker,
"audio_params": audio_params,
"additions": json.dumps({
"post_process": {
"pitch": self.pitch
}
})
},
"req_params": req_params
}
)
)
@@ -174,6 +174,20 @@ class TTSProvider(TTSProviderBase):
logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
self.tts_audio_queue.put((SentenceType.LAST, [], None))
def audio_to_pcm_data_stream(
self, audio_file_path, callback=None
):
"""音频文件转换为PCM编码,使用24kHz采样率"""
from core.utils.util import audio_to_data_stream
return audio_to_data_stream(audio_file_path, is_opus=False, callback=callback, sample_rate=24000, opus_encoder=None)
def audio_to_opus_data_stream(
self, audio_file_path, callback=None
):
"""音频文件转换为Opus编码,使用24kHz采样率和自己的编码器"""
from core.utils.util import audio_to_data_stream
return audio_to_data_stream(audio_file_path, is_opus=True, callback=callback, sample_rate=24000, opus_encoder=self.opus_encoder)
async def close(self):
"""资源清理"""
await super().close()
@@ -25,11 +25,6 @@ class TTSProvider(TTSProviderBase):
self.audio_format = "pcm"
self.before_stop_play_files = []
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=16000, channels=1, frame_size_ms=60
)
# PCM缓冲区
self.pcm_buffer = bytearray()
@@ -127,7 +122,7 @@ class TTSProvider(TTSProviderBase):
"spk_id": self.voice,
"frame_durition": 60,
"stream": "true",
"target_sr": 16000,
"target_sr": self.conn.sample_rate,
"audio_format": "pcm",
"instruct_text": "请生成一段自然流畅的语音",
}
@@ -136,7 +131,7 @@ class TTSProvider(TTSProviderBase):
"Content-Type": "application/json",
}
# 一帧 PCM 所需字节数:60 ms &times; 16 kHz &times; 1 ch &times; 2 B = 1 920
# 一帧 PCM 所需字节数:60 ms × sample_rate × 1 ch × 2 B
frame_bytes = int(
self.opus_encoder.sample_rate
* self.opus_encoder.channels # 1
@@ -213,7 +208,7 @@ class TTSProvider(TTSProviderBase):
"spk_id": self.voice,
"frame_duration": 60,
"stream": False,
"target_sr": 16000,
"target_sr": self.conn.sample_rate,
"audio_format": self.audio_format,
"instruct_text": "请生成一段自然流畅的语音",
}
@@ -64,13 +64,17 @@ class TTSProvider(TTSProviderBase):
}
self.audio_file_type = defult_audio_setting.get("format", "pcm")
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=24000, channels=1, frame_size_ms=60
)
# PCM缓冲区
self.pcm_buffer = bytearray()
async def open_audio_channels(self, conn):
"""初始化音频通道,并根据conn.sample_rate更新配置"""
# 调用父类方法
await super().open_audio_channels(conn)
# 更新audio_setting中的采样率为实际的conn.sample_rate
self.audio_setting["sample_rate"] = conn.sample_rate
def tts_text_priority_thread(self):
"""流式文本处理线程"""
while not self.conn.stop_event.is_set():
@@ -212,6 +216,18 @@ class TTSProvider(TTSProviderBase):
try:
data = json.loads(json_str)
# 检查业务层错误
base_resp = data.get("base_resp", {})
status_code = base_resp.get("status_code", 0)
if status_code != 0:
status_msg = base_resp.get("status_msg", "未知错误")
logger.bind(tag=TAG).error(
f"TTS请求失败, 错误码:{status_code}, 错误消息:{status_msg}"
)
self.tts_audio_queue.put((SentenceType.LAST, [], None))
return
status = data.get("data", {}).get("status", 1)
audio_hex = data.get("data", {}).get("audio")
@@ -25,10 +25,7 @@ class TTSProvider(TTSProviderBase):
self.spk_id = int(config.get("private_voice"))
else:
self.spk_id = int(config.get("spk_id", "0"))
sample_rate = config.get("sample_rate", 24000)
self.sample_rate = float(sample_rate) if sample_rate else 24000
speed = config.get("speed", 1.0)
self.speed = float(speed) if speed else 1.0
@@ -13,7 +13,6 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("voice")
self.response_format = config.get("response_format", "mp3")
self.audio_file_type = config.get("response_format", "mp3")
self.sample_rate = config.get("sample_rate")
self.speed = float(config.get("speed", 1.0))
self.gain = config.get("gain")
@@ -91,9 +91,6 @@ class TTSProvider(TTSProviderBase):
# 音频编码配置
self.format = config.get("format", "raw")
sample_rate = config.get("sample_rate", "24000")
self.sample_rate = int(sample_rate) if sample_rate else 24000
# 口语化配置
self.oral_level = config.get("oral_level", "mid")
@@ -113,11 +110,6 @@ class TTSProvider(TTSProviderBase):
# 序列号管理
self.text_seq = 0
# 创建Opus编码器
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
)
# 验证必需参数
if not all([self.app_id, self.api_key, self.api_secret]):
raise ValueError("讯飞TTS需要配置app_id、api_key和api_secret")
@@ -507,7 +499,7 @@ class TTSProvider(TTSProviderBase):
"rhy": 0,
"audio": {
"encoding": self.format,
"sample_rate": self.sample_rate,
"sample_rate": self.conn.sample_rate,
"channels": 1,
"bit_depth": 16,
"frame_size": 0
+33 -16
View File
@@ -227,7 +227,7 @@ def extract_json_from_string(input_string):
def audio_to_data_stream(
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None
audio_file_path, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None
) -> None:
# 获取文件后缀名
file_type = os.path.splitext(audio_file_path)[1]
@@ -238,12 +238,12 @@ def audio_to_data_stream(
audio_file_path, format=file_type, parameters=["-nostdin"]
)
# 转换为单声道/16kHz采样率/16位小端编码(确保与编码器匹配)
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
# 转换为单声道/指定采样率/16位小端编码(确保与编码器匹配)
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
# 获取原始PCM数据(16位小端)
raw_data = audio.raw_data
pcm_to_data_stream(raw_data, is_opus, callback)
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
async def audio_to_data(
@@ -325,7 +325,7 @@ async def audio_to_data(
def audio_bytes_to_data_stream(
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any]
audio_bytes, file_type, is_opus, callback: Callable[[Any], Any], sample_rate=16000, opus_encoder=None
) -> None:
"""
直接用音频二进制数据转为opus/pcm数据支持wavmp3p3
@@ -338,18 +338,30 @@ def audio_bytes_to_data_stream(
audio = AudioSegment.from_file(
BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"]
)
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
audio = audio.set_channels(1).set_frame_rate(sample_rate).set_sample_width(2)
raw_data = audio.raw_data
pcm_to_data_stream(raw_data, is_opus, callback)
pcm_to_data_stream(raw_data, is_opus, callback, sample_rate, opus_encoder)
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None):
# 初始化Opus编码器
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] = None, sample_rate=16000, opus_encoder=None):
"""
将PCM数据流式编码为Opus或直接输出PCM
Args:
raw_data: PCM原始数据
is_opus: 是否编码为Opus
callback: 回调函数
sample_rate: 采样率
opus_encoder: OpusEncoderUtils对象(推荐提供以保持编码器状态连续)
"""
using_temp_encoder = False
if is_opus and opus_encoder is None:
encoder = opuslib_next.Encoder(sample_rate, 1, opuslib_next.APPLICATION_AUDIO)
using_temp_encoder = True
# 编码参数
frame_duration = 60 # 60ms per frame
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
frame_size = int(sample_rate * frame_duration / 1000) # samples/frame
# 按帧处理所有音频数据(包括最后一帧可能补零)
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
@@ -361,12 +373,17 @@ def pcm_to_data_stream(raw_data, is_opus=True, callback: Callable[[Any], Any] =
chunk += b"\x00" * (frame_size * 2 - len(chunk))
if is_opus:
# 转换为numpy数组处理
np_frame = np.frombuffer(chunk, dtype=np.int16)
# 编码Opus数据
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
callback(frame_data)
if using_temp_encoder:
# 使用临时编码器(仅用于独立音频场景)
np_frame = np.frombuffer(chunk, dtype=np.int16)
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
callback(frame_data)
else:
# 使用外部编码器(TTS流式场景,保持状态连续)
is_last = (i + frame_size * 2 >= len(raw_data))
opus_encoder.encode_pcm_to_opus_stream(chunk, end_of_stream=is_last, callback=callback)
else:
# PCM模式,直接输出
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
callback(frame_data)
Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 142 KiB

After

Width:  |  Height:  |  Size: 136 KiB

+35 -21
View File
@@ -1,9 +1,10 @@
// 主应用入口
import { log } from './utils/logger.js';
import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js';
import { uiController } from './ui/controller.js';
import { getAudioPlayer } from './core/audio/player.js';
import { initMcpTools } from './core/mcp/tools.js';
import { checkOpusLoaded, initOpusEncoder } from './core/audio/opus-codec.js?v=0127';
import { getAudioPlayer } from './core/audio/player.js?v=0127';
import { checkMicrophoneAvailability, isHttpNonLocalhost } from './core/audio/recorder.js?v=0127';
import { initMcpTools } from './core/mcp/tools.js?v=0127';
import { uiController } from './ui/controller.js?v=0127';
import { log } from './utils/logger.js?v=0127';
// 应用类
class App {
@@ -16,30 +17,24 @@ class App {
// 初始化应用
async init() {
log('正在初始化应用...', 'info');
// 初始化UI控制器
this.uiController = uiController;
this.uiController.init();
// 检查Opus库
checkOpusLoaded();
// 初始化Opus编码器
initOpusEncoder();
// 初始化音频播放器
this.audioPlayer = getAudioPlayer();
await this.audioPlayer.start();
// 初始化MCP工具
initMcpTools();
// 检查麦克风可用性
await this.checkMicrophoneAvailability();
// 初始化Live2D
await this.initLive2D();
// 关闭加载loading
this.setModelLoadingStatus(false);
log('应用初始化完成', 'success');
}
@@ -50,21 +45,17 @@ class App {
if (typeof window.Live2DManager === 'undefined') {
throw new Error('Live2DManager未加载,请检查脚本引入顺序');
}
this.live2dManager = new window.Live2DManager();
await this.live2dManager.initializeLive2D();
// 更新UI状态
const live2dStatus = document.getElementById('live2dStatus');
if (live2dStatus) {
live2dStatus.textContent = '● 已加载';
live2dStatus.className = 'status loaded';
}
log('Live2D初始化完成', 'success');
} catch (error) {
log(`Live2D初始化失败: ${error.message}`, 'error');
// 更新UI状态
const live2dStatus = document.getElementById('live2dStatus');
if (live2dStatus) {
@@ -81,18 +72,41 @@ class App {
modelLoading.style.display = isLoading ? 'flex' : 'none';
}
}
/**
* 检查麦克风可用性
* 在应用初始化时调用检查麦克风是否可用并更新UI状态
*/
async checkMicrophoneAvailability() {
try {
const isAvailable = await checkMicrophoneAvailability();
const isHttp = isHttpNonLocalhost();
// 保存可用性状态到全局变量
window.microphoneAvailable = isAvailable;
window.isHttpNonLocalhost = isHttp;
// 更新UI
if (this.uiController) {
this.uiController.updateMicrophoneAvailability(isAvailable, isHttp);
}
log(`麦克风可用性检查完成: ${isAvailable ? '可用' : '不可用'}`, isAvailable ? 'success' : 'warning');
} catch (error) {
log(`检查麦克风可用性失败: ${error.message}`, 'error');
// 默认设置为不可用
window.microphoneAvailable = false;
window.isHttpNonLocalhost = isHttpNonLocalhost();
if (this.uiController) {
this.uiController.updateMicrophoneAvailability(false, window.isHttpNonLocalhost);
}
}
}
}
// 创建并启动应用
const app = new App();
// 将应用实例暴露到全局,供其他模块访问
window.chatApp = app;
document.addEventListener('DOMContentLoaded', () => {
// 初始化应用
app.init();
});
export default app;
@@ -1,4 +1,4 @@
import { log } from '../../utils/logger.js';
import { log } from '../../utils/logger.js?v=0127';
// 检查Opus库是否已加载
@@ -1,7 +1,7 @@
// 音频播放模块
import { log } from '../../utils/logger.js';
import BlockingQueue from '../../utils/blocking-queue.js';
import { createStreamingContext } from './stream-context.js';
import BlockingQueue from '../../utils/blocking-queue.js?v=0127';
import { log } from '../../utils/logger.js?v=0127';
import { createStreamingContext } from './stream-context.js?v=0127';
// 音频播放器类
export class AudioPlayer {
@@ -1,9 +1,9 @@
// 音频录制模块
import { log } from '../../utils/logger.js';
import { initOpusEncoder } from './opus-codec.js';
import { getAudioPlayer } from './player.js';
// Audio recording module
import { log } from '../../utils/logger.js?v=0127';
import { initOpusEncoder } from './opus-codec.js?v=0127';
import { getAudioPlayer } from './player.js?v=0127';
// 音频录制器类
// Audio recorder class
export class AudioRecorder {
constructor() {
this.isRecording = false;
@@ -19,25 +19,23 @@ export class AudioRecorder {
this.visualizationRequest = null;
this.recordingTimer = null;
this.websocket = null;
// 回调函数
// Callback functions
this.onRecordingStart = null;
this.onRecordingStop = null;
this.onVisualizerUpdate = null;
}
// 设置WebSocket实例
// Set WebSocket instance
setWebSocket(ws) {
this.websocket = ws;
}
// 获取AudioContext实例
// Get AudioContext instance
getAudioContext() {
const audioPlayer = getAudioPlayer();
return audioPlayer.getAudioContext();
return getAudioPlayer().getAudioContext();
}
// 初始化编码器
// Initialize encoder
initEncoder() {
if (!this.opusEncoder) {
this.opusEncoder = initOpusEncoder();
@@ -45,7 +43,7 @@ export class AudioRecorder {
return this.opusEncoder;
}
// PCM处理器代码
// PCM processor code
getAudioProcessorCode() {
return `
class AudioRecorderProcessor extends AudioWorkletProcessor {
@@ -56,166 +54,132 @@ export class AudioRecorder {
this.buffer = new Int16Array(this.frameSize);
this.bufferIndex = 0;
this.isRecording = false;
this.port.onmessage = (event) => {
if (event.data.command === 'start') {
this.isRecording = true;
this.port.postMessage({ type: 'status', status: 'started' });
} else if (event.data.command === 'stop') {
this.isRecording = false;
if (this.bufferIndex > 0) {
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
this.port.postMessage({
type: 'buffer',
buffer: finalBuffer
});
this.port.postMessage({ type: 'buffer', buffer: finalBuffer });
this.bufferIndex = 0;
}
this.port.postMessage({ type: 'status', status: 'stopped' });
}
};
}
process(inputs, outputs, parameters) {
if (!this.isRecording) return true;
const input = inputs[0][0];
if (!input) return true;
for (let i = 0; i < input.length; i++) {
if (this.bufferIndex >= this.frameSize) {
this.port.postMessage({
type: 'buffer',
buffer: this.buffer.slice(0)
});
this.port.postMessage({ type: 'buffer', buffer: this.buffer.slice(0) });
this.bufferIndex = 0;
}
this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
}
return true;
}
}
registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
`;
}
// 创建音频处理器
// Create audio processor
async createAudioProcessor() {
this.audioContext = this.getAudioContext();
try {
if (this.audioContext.audioWorklet) {
const blob = new Blob([this.getAudioProcessorCode()], { type: 'application/javascript' });
const url = URL.createObjectURL(blob);
await this.audioContext.audioWorklet.addModule(url);
URL.revokeObjectURL(url);
const audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-recorder-processor');
audioProcessor.port.onmessage = (event) => {
if (event.data.type === 'buffer') {
this.processPCMBuffer(event.data.buffer);
}
};
log('使用AudioWorklet处理音频', 'success');
const silent = this.audioContext.createGain();
silent.gain.value = 0;
audioProcessor.connect(silent);
silent.connect(this.audioContext.destination);
return { node: audioProcessor, type: 'worklet' };
} else {
log('AudioWorklet不可用,使用ScriptProcessorNode作为回退方案', 'warning');
log('AudioWorklet不可用,使用ScriptProcessorNode作为后备方案', 'warning');
return this.createScriptProcessor();
}
} catch (error) {
log(`创建音频处理器失败: ${error.message},尝试回退方案`, 'error');
log(`创建音频处理器失败: ${error.message},尝试后备方案`, 'error');
return this.createScriptProcessor();
}
}
// 创建ScriptProcessor作为回退
// Create ScriptProcessor as fallback
createScriptProcessor() {
try {
const frameSize = 4096;
const scriptProcessor = this.audioContext.createScriptProcessor(frameSize, 1, 1);
scriptProcessor.onaudioprocess = (event) => {
if (!this.isRecording) return;
const input = event.inputBuffer.getChannelData(0);
const buffer = new Int16Array(input.length);
for (let i = 0; i < input.length; i++) {
buffer[i] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
}
this.processPCMBuffer(buffer);
};
const silent = this.audioContext.createGain();
silent.gain.value = 0;
scriptProcessor.connect(silent);
silent.connect(this.audioContext.destination);
log('使用ScriptProcessorNode作为回退方案成功', 'warning');
log('使用ScriptProcessorNode作为后备方案成功', 'warning');
return { node: scriptProcessor, type: 'processor' };
} catch (fallbackError) {
log(`回退方案也失败: ${fallbackError.message}`, 'error');
log(`后备方案也失败: ${fallbackError.message}`, 'error');
return null;
}
}
// 处理PCM缓冲数据
// Process PCM buffer data
processPCMBuffer(buffer) {
if (!this.isRecording) return;
const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length);
newBuffer.set(this.pcmDataBuffer);
newBuffer.set(buffer, this.pcmDataBuffer.length);
this.pcmDataBuffer = newBuffer;
const samplesPerFrame = 960;
while (this.pcmDataBuffer.length >= samplesPerFrame) {
const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame);
this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame);
this.encodeAndSendOpus(frameData);
}
}
// 编码并发送Opus数据
// Encode and send Opus data
encodeAndSendOpus(pcmData = null) {
if (!this.opusEncoder) {
log('Opus编码器未初始化', 'error');
return;
}
try {
if (pcmData) {
const opusData = this.opusEncoder.encode(pcmData);
if (opusData && opusData.length > 0) {
this.audioBuffers.push(opusData.buffer);
this.totalAudioSize += opusData.length;
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
try {
this.websocket.send(opusData.buffer);
log(`发送Opus帧,大小:${opusData.length}字节`, 'debug');
} catch (error) {
log(`WebSocket发送错误: ${error.message}`, 'error');
}
}
} else {
log('Opus编码失败,有效数据返回', 'error');
log('Opus编码失败,未返回有效数据', 'error');
}
} else {
if (this.pcmDataBuffer.length > 0) {
@@ -235,96 +199,67 @@ export class AudioRecorder {
}
}
// 开始录音
// Start recording
async start() {
if (this.isRecording) return false;
try {
// 检查是否有WebSocketHandler实例
const { getWebSocketHandler } = await import('../network/websocket.js');
// Check if WebSocketHandler instance exists
const { getWebSocketHandler } = await import('../network/websocket.js?v=0127');
const wsHandler = getWebSocketHandler();
// 如果机器正在说话,发送打断消息
// If machine is speaking, send abort message
if (wsHandler && wsHandler.isRemoteSpeaking && wsHandler.currentSessionId) {
const abortMessage = {
session_id: wsHandler.currentSessionId,
type: 'abort',
reason: 'wake_word_detected'
};
const abortMessage = { session_id: wsHandler.currentSessionId, type: 'abort', reason: 'wake_word_detected' };
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
this.websocket.send(JSON.stringify(abortMessage));
log('发送打断消息', 'info');
log('发送中止消息', 'info');
}
}
if (!this.initEncoder()) {
log('无法启动录音: Opus编码器初始化失败', 'error');
log('无法开始录音: Opus编码器初始化失败', 'error');
return false;
}
log('请至少录制1-2秒钟的音频,确保采集到足够数据', 'info');
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
sampleRate: 16000,
channelCount: 1
}
});
log('请至少录制1-2秒音频以确保收集足够的数据', 'info');
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
this.audioContext = this.getAudioContext();
if (this.audioContext.state === 'suspended') {
await this.audioContext.resume();
}
const processorResult = await this.createAudioProcessor();
if (!processorResult) {
log('无法创建音频处理器', 'error');
return false;
}
this.audioProcessor = processorResult.node;
this.audioProcessorType = processorResult.type;
this.audioSource = this.audioContext.createMediaStreamSource(stream);
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 2048;
this.audioSource.connect(this.analyser);
this.audioSource.connect(this.audioProcessor);
this.pcmDataBuffer = new Int16Array();
this.audioBuffers = [];
this.totalAudioSize = 0;
this.isRecording = true;
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
this.audioProcessor.port.postMessage({ command: 'start' });
}
// 发送监听开始消息
// Send listening start message
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
log(`发送录音开始消息`, 'info');
log(`发送录音开始消息`, 'info');
} else {
log('WebSocket未连接,无法发送开始消息', 'error');
return false;
}
// 开始可视化
// Start visualization
if (this.onVisualizerUpdate) {
const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
this.startVisualization(dataArray);
}
// 立即通知录音开始,更新按钮状态
// Immediately notify recording start, update button state
if (this.onRecordingStart) {
this.onRecordingStart(0);
}
// 启动录音计时器
// Start recording timer
let recordingSeconds = 0;
this.recordingTimer = setInterval(() => {
recordingSeconds += 0.1;
@@ -332,8 +267,7 @@ export class AudioRecorder {
this.onRecordingStart(recordingSeconds);
}
}, 100);
log('开始PCM直接录音', 'success');
log('已开始PCM直接录音', 'success');
return true;
} catch (error) {
log(`直接录音启动错误: ${error.message}`, 'error');
@@ -342,15 +276,12 @@ export class AudioRecorder {
}
}
// 开始可视化
// Start visualization
startVisualization(dataArray) {
const draw = () => {
this.visualizationRequest = requestAnimationFrame(() => draw());
if (!this.isRecording) return;
this.analyser.getByteFrequencyData(dataArray);
if (this.onVisualizerUpdate) {
this.onVisualizerUpdate(dataArray);
}
@@ -358,52 +289,42 @@ export class AudioRecorder {
draw();
}
// 停止录音
// Stop recording
stop() {
if (!this.isRecording) return false;
try {
this.isRecording = false;
if (this.audioProcessor) {
if (this.audioProcessorType === 'worklet' && this.audioProcessor.port) {
this.audioProcessor.port.postMessage({ command: 'stop' });
}
this.audioProcessor.disconnect();
this.audioProcessor = null;
}
if (this.audioSource) {
this.audioSource.disconnect();
this.audioSource = null;
}
if (this.visualizationRequest) {
cancelAnimationFrame(this.visualizationRequest);
this.visualizationRequest = null;
}
if (this.recordingTimer) {
clearInterval(this.recordingTimer);
this.recordingTimer = null;
}
// 编码并发送剩余的数据
// Encode and send remaining data
this.encodeAndSendOpus();
// 发送结束信号
// Send end signal
if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
const emptyOpusFrame = new Uint8Array(0);
this.websocket.send(emptyOpusFrame);
log('已发送录音停止信号', 'info');
}
if (this.onRecordingStop) {
this.onRecordingStop();
}
log('停止PCM直接录音', 'success');
log('已停止PCM直接录音', 'success');
return true;
} catch (error) {
log(`直接录音停止错误: ${error.message}`, 'error');
@@ -411,13 +332,13 @@ export class AudioRecorder {
}
}
// 获取分析器
// Get analyser
getAnalyser() {
return this.analyser;
}
}
// 创建单例
// Create singleton instance
let audioRecorderInstance = null;
export function getAudioRecorder() {
@@ -426,3 +347,49 @@ export function getAudioRecorder() {
}
return audioRecorderInstance;
}
/**
* Check if microphone is available
* @returns {Promise<boolean>} Returns true if available, false if not available
*/
export async function checkMicrophoneAvailability() {
// Check if browser supports getUserMedia API
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
log('浏览器不支持getUserMedia API', 'warning');
return false;
}
try {
// Try to access microphone
const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 16000, channelCount: 1 } });
// Immediately stop all tracks to release microphone
stream.getTracks().forEach(track => track.stop());
log('麦克风可用性检查成功', 'success');
return true;
} catch (error) {
log(`麦克风不可用: ${error.message}`, 'warning');
return false;
}
}
/**
* Check if it is HTTP non-localhost access
* @returns {boolean} Returns true if it is HTTP non-localhost access
*/
export function isHttpNonLocalhost() {
const protocol = window.location.protocol;
const hostname = window.location.hostname;
// Check if it is HTTP protocol
if (protocol !== 'http:') {
return false;
}
// localhost and 127.0.0.1 can use microphone
if (hostname === 'localhost' || hostname === '127.0.0.1') {
return false;
}
// Private IP addresses can also use microphone (browser allows)
if (hostname.startsWith('192.168.') || hostname.startsWith('10.') || hostname.startsWith('172.')) {
return false;
}
// Other HTTP access is considered non-localhost
return true;
}
@@ -1,5 +1,5 @@
import BlockingQueue from '../../utils/blocking-queue.js';
import { log } from '../../utils/logger.js';
import BlockingQueue from '../../utils/blocking-queue.js?v=0127';
import { log } from '../../utils/logger.js?v=0127';
// 音频流播放上下文类
export class StreamingContext {
+23 -76
View File
@@ -1,4 +1,4 @@
import { log } from '../../utils/logger.js';
import { log } from '../../utils/logger.js?v=0127';
// ==========================================
// MCP 工具管理逻辑
@@ -24,7 +24,6 @@ export function setWebSocket(ws) {
export async function initMcpTools() {
// 加载默认工具数据
const defaultMcpTools = await fetch("js/config/default-mcp-tools.json").then(res => res.json());
const savedTools = localStorage.getItem('mcpTools');
if (savedTools) {
try {
@@ -36,9 +35,11 @@ export async function initMcpTools() {
} else {
mcpTools = [...defaultMcpTools];
}
renderMcpTools();
setupMcpEventListeners();
// Only setup event listeners if DOM elements exist
if (document.getElementById('toggleMcpTools')) {
setupMcpEventListeners();
}
}
/**
@@ -47,21 +48,20 @@ export async function initMcpTools() {
function renderMcpTools() {
const container = document.getElementById('mcpToolsContainer');
const countSpan = document.getElementById('mcpToolsCount');
if (!container) {
return; // Container not found, skip rendering
}
if (countSpan) {
countSpan.textContent = `${mcpTools.length} 个工具`;
}
if (mcpTools.length === 0) {
container.innerHTML = '<div style="text-align: center; padding: 30px; color: #999;">暂无工具,点击下方按钮添加新工具</div>';
return;
}
container.innerHTML = mcpTools.map((tool, index) => {
const paramCount = tool.inputSchema.properties ? Object.keys(tool.inputSchema.properties).length : 0;
const requiredCount = tool.inputSchema.required ? tool.inputSchema.required.length : 0;
const hasMockResponse = tool.mockResponse && Object.keys(tool.mockResponse).length > 0;
return `
<div class="mcp-tool-card">
<div class="mcp-tool-header">
@@ -96,12 +96,13 @@ function renderMcpTools() {
*/
function renderMcpProperties() {
const container = document.getElementById('mcpPropertiesContainer');
if (!container) {
return; // Container not found, skip rendering
}
if (mcpProperties.length === 0) {
container.innerHTML = '<div style="text-align: center; padding: 20px; color: #999; font-size: 14px;">暂无参数,点击下方按钮添加参数</div>';
return;
}
container.innerHTML = mcpProperties.map((prop, index) => `
<div class="mcp-property-item">
<div class="mcp-property-header">
@@ -161,12 +162,7 @@ function renderMcpProperties() {
* 添加参数
*/
function addMcpProperty() {
mcpProperties.push({
name: `param_${mcpProperties.length + 1}`,
type: 'string',
required: false,
description: ''
});
mcpProperties.push({ name: `param_${mcpProperties.length + 1}`, type: 'string', required: false, description: '' });
renderMcpProperties();
}
@@ -182,9 +178,7 @@ function updateMcpProperty(index, field, value) {
return;
}
}
mcpProperties[index][field] = value;
if (field === 'type' && value !== 'integer' && value !== 'number') {
delete mcpProperties[index].minimum;
delete mcpProperties[index].maximum;
@@ -212,25 +206,24 @@ function setupMcpEventListeners() {
const cancelBtn = document.getElementById('cancelMcpBtn');
const form = document.getElementById('mcpToolForm');
const addPropertyBtn = document.getElementById('addMcpPropertyBtn');
// Return early if required elements don't exist (e.g., in test environment)
if (!toggleBtn || !panel || !addBtn || !modal || !closeBtn || !cancelBtn || !form || !addPropertyBtn) {
return;
}
toggleBtn.addEventListener('click', () => {
const isExpanded = panel.classList.contains('expanded');
panel.classList.toggle('expanded');
toggleBtn.textContent = isExpanded ? '展开' : '收起';
toggleBtn.textContent = isExpanded ? '收起' : '展开';
});
// 确保面板默认展开
panel.classList.add('expanded');
addBtn.addEventListener('click', () => openMcpModal());
closeBtn.addEventListener('click', closeMcpModal);
cancelBtn.addEventListener('click', closeMcpModal);
addPropertyBtn.addEventListener('click', addMcpProperty);
modal.addEventListener('click', (e) => {
if (e.target === modal) closeMcpModal();
});
form.addEventListener('submit', handleMcpSubmit);
}
@@ -243,18 +236,15 @@ function openMcpModal(index = null) {
alert('WebSocket 已连接,无法编辑工具');
return;
}
mcpEditingIndex = index;
const errorContainer = document.getElementById('mcpErrorContainer');
errorContainer.innerHTML = '';
if (index !== null) {
document.getElementById('mcpModalTitle').textContent = '编辑工具';
const tool = mcpTools[index];
document.getElementById('mcpToolName').value = tool.name;
document.getElementById('mcpToolDescription').value = tool.description;
document.getElementById('mcpMockResponse').value = tool.mockResponse ? JSON.stringify(tool.mockResponse, null, 2) : '';
mcpProperties = [];
const schema = tool.inputSchema;
if (schema.properties) {
@@ -275,7 +265,6 @@ function openMcpModal(index = null) {
document.getElementById('mcpToolForm').reset();
mcpProperties = [];
}
renderMcpProperties();
document.getElementById('mcpToolModal').style.display = 'block';
}
@@ -298,21 +287,15 @@ function handleMcpSubmit(e) {
e.preventDefault();
const errorContainer = document.getElementById('mcpErrorContainer');
errorContainer.innerHTML = '';
const name = document.getElementById('mcpToolName').value.trim();
const description = document.getElementById('mcpToolDescription').value.trim();
const mockResponseText = document.getElementById('mcpMockResponse').value.trim();
// 检查名称重复
const isDuplicate = mcpTools.some((tool, index) =>
tool.name === name && index !== mcpEditingIndex
);
const isDuplicate = mcpTools.some((tool, index) => tool.name === name && index !== mcpEditingIndex);
if (isDuplicate) {
showMcpError('工具名称已存在,请使用不同的名称');
return;
}
// 解析模拟返回结果
let mockResponse = null;
if (mockResponseText) {
@@ -323,21 +306,13 @@ function handleMcpSubmit(e) {
return;
}
}
// 构建 inputSchema
const inputSchema = {
type: "object",
properties: {},
required: []
};
const inputSchema = { type: "object", properties: {}, required: [] };
mcpProperties.forEach(prop => {
const propSchema = { type: prop.type };
if (prop.description) {
propSchema.description = prop.description;
}
if ((prop.type === 'integer' || prop.type === 'number')) {
if (prop.minimum !== undefined && prop.minimum !== '') {
propSchema.minimum = prop.minimum;
@@ -346,20 +321,15 @@ function handleMcpSubmit(e) {
propSchema.maximum = prop.maximum;
}
}
inputSchema.properties[prop.name] = propSchema;
if (prop.required) {
inputSchema.required.push(prop.name);
}
});
if (inputSchema.required.length === 0) {
delete inputSchema.required;
}
const tool = { name, description, inputSchema, mockResponse };
if (mcpEditingIndex !== null) {
mcpTools[mcpEditingIndex] = tool;
log(`已更新工具: ${name}`, 'success');
@@ -367,7 +337,6 @@ function handleMcpSubmit(e) {
mcpTools.push(tool);
log(`已添加工具: ${name}`, 'success');
}
saveMcpTools();
renderMcpTools();
closeMcpModal();
@@ -417,11 +386,7 @@ function saveMcpTools() {
* 获取工具列表
*/
export function getMcpTools() {
return mcpTools.map(tool => ({
name: tool.name,
description: tool.description,
inputSchema: tool.inputSchema
}));
return mcpTools.map(tool => ({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema }));
}
/**
@@ -429,20 +394,14 @@ export function getMcpTools() {
*/
export function executeMcpTool(toolName, toolArgs) {
const tool = mcpTools.find(t => t.name === toolName);
if (!tool) {
log(`未找到工具: ${toolName}`, 'error');
return {
success: false,
error: `未知工具: ${toolName}`
};
return { success: false, error: `未知工具: ${toolName}` };
}
// 如果有模拟返回结果,使用它
if (tool.mockResponse) {
// 替换模板变量
let responseStr = JSON.stringify(tool.mockResponse);
// 替换 ${paramName} 格式的变量
if (toolArgs) {
Object.keys(toolArgs).forEach(key => {
@@ -450,7 +409,6 @@ export function executeMcpTool(toolName, toolArgs) {
responseStr = responseStr.replace(regex, toolArgs[key]);
});
}
try {
const response = JSON.parse(responseStr);
log(`工具 ${toolName} 执行成功,返回模拟结果: ${responseStr}`, 'success');
@@ -460,21 +418,10 @@ export function executeMcpTool(toolName, toolArgs) {
return tool.mockResponse;
}
}
// 没有模拟返回结果,返回默认成功消息
log(`工具 ${toolName} 执行成功,返回默认结果`, 'success');
return {
success: true,
message: `工具 ${toolName} 执行成功`,
tool: toolName,
arguments: toolArgs
};
return { success: true, message: `工具 ${toolName} 执行成功`, tool: toolName, arguments: toolArgs };
}
// 暴露全局方法供 HTML 内联事件调用
window.mcpModule = {
updateMcpProperty,
deleteMcpProperty,
editMcpTool,
deleteMcpTool
};
window.mcpModule = { updateMcpProperty, deleteMcpProperty, editMcpTool, deleteMcpTool };
@@ -1,4 +1,4 @@
import { log } from '../../utils/logger.js';
import { log } from '../../utils/logger.js?v=0127';
// WebSocket 连接
export async function webSocketConnect(otaUrl, config) {
@@ -1,11 +1,11 @@
// WebSocket消息处理模块
import { log } from '../../utils/logger.js';
import { webSocketConnect } from './ota-connector.js';
import { getConfig, saveConnectionUrls } from '../../config/manager.js';
import { getAudioPlayer } from '../audio/player.js';
import { getAudioRecorder } from '../audio/recorder.js';
import { getMcpTools, executeMcpTool, setWebSocket as setMcpWebSocket } from '../mcp/tools.js';
import { uiController } from '../../ui/controller.js'
import { getConfig, saveConnectionUrls } from '../../config/manager.js?v=0127';
import { uiController } from '../../ui/controller.js?v=0127';
import { log } from '../../utils/logger.js?v=0127';
import { getAudioPlayer } from '../audio/player.js?v=0127';
import { getAudioRecorder } from '../audio/recorder.js?v=0127';
import { executeMcpTool, getMcpTools, setWebSocket as setMcpWebSocket } from '../mcp/tools.js?v=0127';
import { webSocketConnect } from './ota-connector.js?v=0127';
// WebSocket处理器类
export class WebSocketHandler {
@@ -101,10 +101,10 @@ export class WebSocketHandler {
}
// 触发Live2D情绪动作
if (message.emotion) {
console.log(`收到情绪消息: emotion=${message.emotion}, text=${message.text}`);
this.triggerLive2DEmotionAction(message.emotion);
}
if (message.emotion) {
console.log(`收到情绪消息: emotion=${message.emotion}, text=${message.text}`);
this.triggerLive2DEmotionAction(message.emotion);
}
}
// 只有当文本不仅仅是表情时,才添加到对话中
@@ -273,6 +273,26 @@ export class WebSocketHandler {
this.websocket.send(replyMessage);
} else if (payload.method === 'initialize') {
log(`收到工具初始化请求: ${JSON.stringify(payload.params)}`, 'info');
const replyMessage = JSON.stringify({
"session_id": message.session_id || "",
"type": "mcp",
"payload": {
"jsonrpc": "2.0",
"id": payload.id,
"result": {
"protocolVersion": "2024-11-05",
"capabilities": {
"tools": {}
},
"serverInfo": {
"name": "xiaozhi-web-test",
"version": "2.1.0"
}
}
}
});
log(`回复初始化响应`, 'info');
this.websocket.send(replyMessage);
} else {
log(`未知的MCP方法: ${payload.method}`, 'warning');
}
@@ -284,7 +304,6 @@ export class WebSocketHandler {
let arrayBuffer;
if (data instanceof ArrayBuffer) {
arrayBuffer = data;
log(`收到ArrayBuffer音频数据,大小: ${data.byteLength}字节`, 'debug');
} else if (data instanceof Blob) {
arrayBuffer = await data.arrayBuffer();
log(`收到Blob音频数据,大小: ${arrayBuffer.byteLength}字节`, 'debug');
@@ -372,7 +391,7 @@ export class WebSocketHandler {
this.websocket.onerror = (error) => {
log(`WebSocket错误: ${error.message || '未知错误'}`, 'error');
uiController.addChatMessage(`⚠️ WebSocket错误: ${error.message || '未知错误'}`, false);
if (this.onConnectionStateChange) {
this.onConnectionStateChange(false);
}
+161 -110
View File
@@ -1,10 +1,10 @@
// UI控制模块
import { loadConfig, saveConfig } from '../config/manager.js';
import { getAudioRecorder } from '../core/audio/recorder.js';
import { getWebSocketHandler } from '../core/network/websocket.js';
import { getAudioPlayer } from '../core/audio/player.js';
// UI controller module
import { loadConfig, saveConfig } from '../config/manager.js?v=0127';
import { getAudioPlayer } from '../core/audio/player.js?v=0127';
import { getAudioRecorder } from '../core/audio/recorder.js?v=0127';
import { getWebSocketHandler } from '../core/network/websocket.js?v=0127';
// UI控制器类
// UI controller class
class UIController {
constructor() {
this.isEditing = false;
@@ -14,7 +14,7 @@ class UIController {
this.currentBackgroundIndex = 0;
this.backgroundImages = ['1.png', '2.png', '3.png'];
// 绑定方法
// Bind methods
this.init = this.init.bind(this);
this.initEventListeners = this.initEventListeners.bind(this);
this.updateDialButton = this.updateDialButton.bind(this);
@@ -25,7 +25,7 @@ class UIController {
this.switchTab = this.switchTab.bind(this);
}
// 初始化
// Initialize
init() {
console.log('UIController init started');
@@ -35,7 +35,7 @@ class UIController {
this.initVisualizer();
}
// 检查连接按钮在初始化时是否存在
// Check if connect button exists during initialization
const connectBtn = document.getElementById('connectBtn');
console.log('connectBtn during init:', connectBtn);
@@ -43,20 +43,20 @@ class UIController {
this.startAudioStatsMonitor();
loadConfig();
// 设置录音器回调
// Register recording callback
const audioRecorder = getAudioRecorder();
audioRecorder.onRecordingStart = (seconds) => {
this.updateRecordButtonState(true, seconds);
};
// 初始化状态显示
// Initialize status display
this.updateConnectionUI(false);
this.updateDialButton(false);
console.log('UIController init completed');
}
// 初始化可视化器
// Initialize visualizer
initVisualizer() {
if (this.visualizerCanvas) {
this.visualizerCanvas.width = this.visualizerCanvas.clientWidth;
@@ -66,9 +66,9 @@ class UIController {
}
}
// 初始化事件监听器
// Initialize event listeners
initEventListeners() {
// 设置按钮
// Settings button
const settingsBtn = document.getElementById('settingsBtn');
if (settingsBtn) {
settingsBtn.addEventListener('click', () => {
@@ -76,13 +76,13 @@ class UIController {
});
}
// 背景切换按钮
// Background switch button
const backgroundBtn = document.getElementById('backgroundBtn');
if (backgroundBtn) {
backgroundBtn.addEventListener('click', this.switchBackground);
}
// 拨号按钮
// Dial button
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.addEventListener('click', () => {
@@ -92,40 +92,40 @@ class UIController {
if (isConnected) {
wsHandler.disconnect();
this.updateDialButton(false);
this.addChatMessage('已断开连接,期待下次再见~😉', false);
this.addChatMessage('Disconnected, see you next time~😊', false);
} else {
// 检查OTA地址是否已填写
// Check if OTA URL is filled
const otaUrlInput = document.getElementById('otaUrl');
if (!otaUrlInput || !otaUrlInput.value.trim()) {
// 如果OTA地址未填写,显示设置弹窗并切换到设备配置页
// If OTA URL is not filled, show settings modal and switch to device tab
this.showModal('settingsModal');
this.switchTab('device');
this.addChatMessage('请先填写OTA服务器地址', false);
this.addChatMessage('Please fill in OTA server URL', false);
return;
}
// 执行连接操作
// Start connection process
this.handleConnect();
}
});
}
// 录音按钮
// Record button
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
recordBtn.addEventListener('click', () => {
const audioRecorder = getAudioRecorder();
if (audioRecorder.isRecording) {
audioRecorder.stop();
// 停止录音时移除录音样式
// Restore record button to normal state
recordBtn.classList.remove('recording');
recordBtn.querySelector('.btn-text').textContent = '录音';
} else {
// 先更新按钮状态为录音中
// Update button state to recording
recordBtn.classList.add('recording');
recordBtn.querySelector('.btn-text').textContent = '录音中';
// 延迟开始录音,确保按钮状态已更新
// Start recording, update button state after delay
setTimeout(() => {
audioRecorder.start();
}, 100);
@@ -133,7 +133,7 @@ class UIController {
});
}
// 消息输入框事件
// Chat input event listener
const chatIpt = document.getElementById('chatIpt');
if (chatIpt) {
const wsHandler = getWebSocketHandler();
@@ -148,7 +148,7 @@ class UIController {
});
}
// 关闭按钮
// Close button
const closeButtons = document.querySelectorAll('.close-btn');
closeButtons.forEach(btn => {
btn.addEventListener('click', (e) => {
@@ -163,7 +163,7 @@ class UIController {
});
});
// 设置标签页切换
// Settings tab switch
const tabBtns = document.querySelectorAll('.tab-btn');
tabBtns.forEach(btn => {
btn.addEventListener('click', (e) => {
@@ -171,7 +171,7 @@ class UIController {
});
});
// 点击模态框外部关闭
// Click modal background to close
const modals = document.querySelectorAll('.modal');
modals.forEach(modal => {
modal.addEventListener('click', (e) => {
@@ -184,7 +184,7 @@ class UIController {
});
});
// 添加MCP工具按钮
// Add MCP tool button
const addMCPToolBtn = document.getElementById('addMCPToolBtn');
if (addMCPToolBtn) {
addMCPToolBtn.addEventListener('click', (e) => {
@@ -193,10 +193,10 @@ class UIController {
});
}
// 连接按钮和取消按钮已被移除,功能已集成到拨号按钮中
// Connect button and send button are not removed, can be added to dial button later
}
// 更新连接状态UI
// Update connection status UI
updateConnectionUI(isConnected) {
const connectionStatus = document.getElementById('connectionStatus');
const statusDot = document.querySelector('.status-dot');
@@ -216,7 +216,7 @@ class UIController {
}
}
// 更新拨号按钮状态
// Update dial button state
updateDialButton(isConnected) {
const dialBtn = document.getElementById('dialBtn');
const recordBtn = document.getElementById('recordBtn');
@@ -225,39 +225,44 @@ class UIController {
if (isConnected) {
dialBtn.classList.add('dial-active');
dialBtn.querySelector('.btn-text').textContent = '挂断';
// 更新拨号按钮图标为挂断图标
// Update dial button icon to hang up icon
dialBtn.querySelector('svg').innerHTML = `
<path d="M12,9C10.4,9 9,10.4 9,12C9,13.6 10.4,15 12,15C13.6,15 15,13.6 15,12C15,10.4 13.6,9 12,9M12,17C9.2,17 7,14.8 7,12C7,9.2 9.2,7 12,7C14.8,7 17,9.2 17,12C17,14.8 14.8,17 12,17M12,4.5C7,4.5 2.7,7.6 1,12C2.7,16.4 7,19.5 12,19.5C17,19.5 21.3,16.4 23,12C21.3,7.6 17,4.5 12,4.5Z"/>
`;
} else {
dialBtn.classList.remove('dial-active');
dialBtn.querySelector('.btn-text').textContent = '拨号';
// 恢复拨号按钮图标
// Restore dial button icon
dialBtn.querySelector('svg').innerHTML = `
<path d="M6.62,10.79C8.06,13.62 10.38,15.94 13.21,17.38L15.41,15.18C15.69,14.9 16.08,14.82 16.43,14.93C17.55,15.3 18.75,15.5 20,15.5A1,1 0 0,1 21,16.5V20A1,1 0 0,1 20,21A17,17 0 0,1 3,4A1,1 0 0,1 4,3H7.5A1,1 0 0,1 8.5,4C8.5,5.25 8.7,6.45 9.07,7.57C9.18,7.92 9.1,8.31 8.82,8.59L6.62,10.79Z"/>
`;
}
}
// 更新录音按钮状态
// Update record button state
if (recordBtn) {
if (isConnected) {
const microphoneAvailable = window.microphoneAvailable !== false;
if (isConnected && microphoneAvailable) {
recordBtn.disabled = false;
recordBtn.title = '开始录音';
// 确保录音按钮恢复到正常状态
// Restore record button to normal state
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.classList.remove('recording');
} else {
recordBtn.disabled = true;
recordBtn.title = '请先连接服务器';
// 确保录音按钮恢复到正常状态
if (!microphoneAvailable) {
recordBtn.title = window.isHttpNonLocalhost ? '当前由于是http访问,无法录音,只能用文字交互' : '麦克风不可用';
} else {
recordBtn.title = '请先连接服务器';
}
// Restore record button to normal state
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.classList.remove('recording');
}
}
}
// 更新录音按钮状态
// Update record button state
updateRecordButtonState(isRecording, seconds = 0) {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
@@ -268,11 +273,37 @@ class UIController {
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.classList.remove('recording');
}
recordBtn.disabled = false;
// Only enable button when microphone is available
recordBtn.disabled = window.microphoneAvailable === false;
}
}
// 添加聊天消息
/**
* Update microphone availability state
* @param {boolean} isAvailable - Whether microphone is available
* @param {boolean} isHttpNonLocalhost - Whether it is HTTP non-localhost access
*/
updateMicrophoneAvailability(isAvailable, isHttpNonLocalhost) {
const recordBtn = document.getElementById('recordBtn');
if (!recordBtn) return;
if (!isAvailable) {
// Disable record button
recordBtn.disabled = true;
// Update button text and title
recordBtn.querySelector('.btn-text').textContent = '录音';
recordBtn.title = isHttpNonLocalhost ? '当前由于是http访问,无法录音,只能用文字交互' : '麦克风不可用';
} else {
// If connected, enable record button
const wsHandler = getWebSocketHandler();
if (wsHandler && wsHandler.isConnected()) {
recordBtn.disabled = false;
recordBtn.title = '开始录音';
}
}
}
// Add chat message
addChatMessage(content, isUser = false) {
const chatStream = document.getElementById('chatStream');
if (!chatStream) return;
@@ -282,11 +313,11 @@ class UIController {
messageDiv.innerHTML = `<div class="message-bubble">${content}</div>`;
chatStream.appendChild(messageDiv);
// 自动滚动到底部
// Scroll to bottom
chatStream.scrollTop = chatStream.scrollHeight;
}
// 切换背景
// Switch background
switchBackground() {
this.currentBackgroundIndex = (this.currentBackgroundIndex + 1) % this.backgroundImages.length;
const backgroundContainer = document.querySelector('.background-container');
@@ -295,7 +326,7 @@ class UIController {
}
}
// 显示模态框
// Show modal
showModal(modalId) {
const modal = document.getElementById(modalId);
if (modal) {
@@ -303,7 +334,7 @@ class UIController {
}
}
// 隐藏模态框
// Hide modal
hideModal(modalId) {
const modal = document.getElementById(modalId);
if (modal) {
@@ -311,16 +342,16 @@ class UIController {
}
}
// 切换标签页
// Switch tab
switchTab(tabName) {
// 移除所有标签页的active类
// Remove active class from all tabs
const tabBtns = document.querySelectorAll('.tab-btn');
const tabContents = document.querySelectorAll('.tab-content');
tabBtns.forEach(btn => btn.classList.remove('active'));
tabContents.forEach(content => content.classList.remove('active'));
// 激活选中的标签页
// Activate selected tab
const activeTabBtn = document.querySelector(`[data-tab="${tabName}"]`);
const activeTabContent = document.getElementById(`${tabName}Tab`);
@@ -330,24 +361,34 @@ class UIController {
}
}
// 连接成功后开始对话
// Start AI chat session after connection
startAIChatSession() {
this.addChatMessage('连接成功,开始聊天吧~🙂', false);
// 开启录音
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
recordBtn.click();
this.addChatMessage('连接成功,开始聊天吧~😊', false);
// Check microphone availability and show error messages if needed
if (!window.microphoneAvailable) {
if (window.isHttpNonLocalhost) {
this.addChatMessage('⚠️ 当前由于是http访问,无法录音,只能用文字交互', false);
} else {
this.addChatMessage('⚠️ 麦克风不可用,请检查权限设置,只能用文字交互', false);
}
}
// Start recording only if microphone is available
if (window.microphoneAvailable) {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
recordBtn.click();
}
}
}
// 处理连接按钮点击
// Handle connect button click
async handleConnect() {
console.log('handleConnect called');
// 确保切换到设备配置标签页
// Switch to device settings tab
this.switchTab('device');
// 等待DOM更新
// Wait for DOM update
await new Promise(resolve => setTimeout(resolve, 50));
const otaUrlInput = document.getElementById('otaUrl');
@@ -362,7 +403,7 @@ class UIController {
const otaUrl = otaUrlInput.value;
console.log('otaUrl value:', otaUrl);
// 更新拨号按钮状态为连接中
// Update dial button state to connecting
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.classList.add('dial-active');
@@ -370,7 +411,7 @@ class UIController {
dialBtn.disabled = true;
}
// 显示连接中消息
// Show connecting message
this.addChatMessage('正在连接服务器...', false);
const chatIpt = document.getElementById('chatIpt');
@@ -380,41 +421,51 @@ class UIController {
try {
// 获取WebSocket处理器
// Get WebSocket handler instance
const wsHandler = getWebSocketHandler();
// Register connection state callback BEFORE connecting
wsHandler.onConnectionStateChange = (isConnected) => {
this.updateConnectionUI(isConnected);
this.updateDialButton(isConnected);
};
// Register chat message callback BEFORE connecting
wsHandler.onChatMessage = (text, isUser) => {
this.addChatMessage(text, isUser);
};
// Register record button state callback BEFORE connecting
wsHandler.onRecordButtonStateChange = (isRecording) => {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
if (isRecording) {
recordBtn.classList.add('recording');
recordBtn.querySelector('.btn-text').textContent = '录音中';
} else {
recordBtn.classList.remove('recording');
recordBtn.querySelector('.btn-text').textContent = '录音';
}
}
};
const isConnected = await wsHandler.connect();
if (isConnected) {
// Check microphone availability (check again after connection)
const { checkMicrophoneAvailability } = await import('../core/audio/recorder.js?v=0127');
const micAvailable = await checkMicrophoneAvailability();
// 设置连接状态回调
wsHandler.onConnectionStateChange = (isConnected) => {
this.updateConnectionUI(isConnected);
this.updateDialButton(isConnected);
};
// 设置聊天消息回调
wsHandler.onChatMessage = (text, isUser) => {
this.addChatMessage(text, isUser);
};
// 设置录音按钮状态回调
wsHandler.onRecordButtonStateChange = (isRecording) => {
const recordBtn = document.getElementById('recordBtn');
if (recordBtn) {
if (isRecording) {
recordBtn.classList.add('recording');
recordBtn.querySelector('.btn-text').textContent = '录音中';
} else {
recordBtn.classList.remove('recording');
recordBtn.querySelector('.btn-text').textContent = '录音';
}
if (!micAvailable) {
const isHttp = window.isHttpNonLocalhost;
if (isHttp) {
this.addChatMessage('⚠️ 当前由于是http访问,无法录音,只能用文字交互', false);
}
};
// Update global state
window.microphoneAvailable = false;
}
// 连接成功
this.addChatMessage('OTA连接成功,正在建立WebSocket连接...', false);
// 更新拨号按钮状态
// Update dial button state
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.disabled = false;
@@ -434,14 +485,14 @@ class UIController {
name: error.name
});
// 显示错误消息
// Show error message
const errorMessage = error.message.includes('Cannot set properties of null')
? '连接失败:请刷新页面重试'
? '连接失败:请检查设备连接'
: `连接失败: ${error.message}`;
this.addChatMessage(errorMessage, false);
// 恢复拨号按钮状态
// Restore dial button state
const dialBtn = document.getElementById('dialBtn');
if (dialBtn) {
dialBtn.disabled = false;
@@ -452,7 +503,7 @@ class UIController {
}
}
// 添加MCP工具
// Add MCP tool
addMCPTool() {
const mcpToolsList = document.getElementById('mcpToolsList');
if (!mcpToolsList) return;
@@ -471,7 +522,7 @@ class UIController {
mcpToolsList.appendChild(toolDiv);
}
// 移除MCP工具
// Remove MCP tool
removeMCPTool(toolId) {
const toolElement = document.getElementById(toolId);
if (toolElement) {
@@ -479,24 +530,24 @@ class UIController {
}
}
// 更新音频统计信息
// Update audio statistics display
updateAudioStats() {
const audioPlayer = getAudioPlayer();
if (!audioPlayer) return;
const stats = audioPlayer.getAudioStats();
// 这里可以添加音频统计的UI更新逻辑
// Here can add audio statistics UI update logic
}
// 启动音频统计监控
// Start audio statistics monitor
startAudioStatsMonitor() {
// 每100ms更新一次音频统计
// Update audio statistics every 100ms
this.audioStatsTimer = setInterval(() => {
this.updateAudioStats();
}, 100);
}
// 停止音频统计监控
// Stop audio statistics monitor
stopAudioStatsMonitor() {
if (this.audioStatsTimer) {
clearInterval(this.audioStatsTimer);
@@ -504,7 +555,7 @@ class UIController {
}
}
// 绘制音频可视化效果
// Draw audio visualizer waveform
drawVisualizer(dataArray) {
if (!this.visualizerContext || !this.visualizerCanvas) return;
@@ -518,7 +569,7 @@ class UIController {
for (let i = 0; i < dataArray.length; i++) {
barHeight = dataArray[i] / 2;
// 创建渐变色:从紫色到蓝色到青色
// Create gradient color: from purple to blue to green
const gradient = this.visualizerContext.createLinearGradient(0, 0, 0, this.visualizerCanvas.height);
gradient.addColorStop(0, '#8e44ad');
gradient.addColorStop(0.5, '#3498db');
@@ -530,21 +581,21 @@ class UIController {
}
}
// 更新会话状态UI
// Update session status UI
updateSessionStatus(isSpeaking) {
// 这里可以添加会话状态的UI更新逻辑
// 例如:更新Live2D角色的表情或状态指示器
// Here can add session status UI update logic
// For example: update Live2D model's mouth movement status
}
// 更新会话表情
// Update session emotion
updateSessionEmotion(emoji) {
// 这里可以添加表情更新的逻辑
// 例如:在状态指示器中显示表情
// Here can add emotion update logic
// For example: display emoji in status indicator
}
}
// 创建全局实例
// Create singleton instance
export const uiController = new UIController();
// 导出类供其他模块使用
export { UIController };
// Export class for module usage
export { UIController };
+10 -9
View File
@@ -5,7 +5,7 @@
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>小智服务器测试页面</title>
<link rel="stylesheet" href="css/test_page.css">
<link rel="stylesheet" href="css/test_page.css?v=0127">
<script>
// 检测是否使用file://协议打开
if (window.location.protocol === 'file:') {
@@ -143,7 +143,8 @@
</div>
<div class="config-item">
<label for="deviceName">设备名称:</label>
<input type="text" id="deviceName" value="Web测试设备" maxlength="50" placeholder="deviceName">
<input type="text" id="deviceName" value="Web测试设备" maxlength="50"
placeholder="deviceName">
</div>
</div>
</div>
@@ -237,23 +238,23 @@
</div>
<!-- 背景加载 -->
<script src="js/ui/background-load.js"></script>
<script src="js/ui/background-load.js?v=0127"></script>
<!-- PIXI.js 2D渲染引擎 -->
<script src="js/live2d/pixi.js"></script>
<script src="js/live2d/pixi.js?v=0127"></script>
<!-- Live2D Cubism 4.0 SDK -->
<script src="js/live2d/live2dcubismcore.min.js"></script>
<script src="js/live2d/cubism4.min.js"></script>
<script src="js/live2d/live2dcubismcore.min.js?v=0127"></script>
<script src="js/live2d/cubism4.min.js?v=0127"></script>
<!-- Live2D 管理器 -->
<script src="js/live2d/live2d.js"></script>
<script src="js/live2d/live2d.js?v=0127"></script>
<!-- Opus解码库 -->
<script src="js/utils/libopus.js"></script>
<script src="js/utils/libopus.js?v=0127"></script>
<!-- 主应用入口 -->
<script type="module" src="js/app.js"></script>
<script type="module" src="js/app.js?v=0127"></script>
<!-- 全局错误处理 -->
<script>