diff --git a/main/manager-api/src/main/resources/db/changelog/202512301430.sql b/main/manager-api/src/main/resources/db/changelog/202512301430.sql new file mode 100644 index 00000000..7ebac686 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202512301430.sql @@ -0,0 +1,94 @@ +-- 添加阿里百炼Paraformer实时语音识别服务配置 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_AliyunBLStream'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_AliyunBLStream', 'ASR', 'aliyunbl_stream', '阿里百炼Paraformer实时语音识别', '[{"key":"api_key","label":"API密钥","type":"password"},{"key":"model","label":"模型名称","type":"string"},{"key":"format","label":"音频格式","type":"string"},{"key":"sample_rate","label":"采样率","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 18, 1, NOW(), 1, NOW()); + +delete from `ai_model_config` where id = 'ASR_AliyunBLStream'; +INSERT INTO `ai_model_config` VALUES ('ASR_AliyunBLStream', 'ASR', 'AliyunBLStream', '阿里百炼Paraformer实时语音识别', 0, 1, '{"type": "aliyunbl_stream", "api_key": "", "model": "paraformer-realtime-v2", "format": "pcm", "sample_rate": 16000, "disfluency_removal_enabled": false, "semantic_punctuation_enabled": false, "max_sentence_silence": 200, "multi_threshold_mode_enabled": false, "punctuation_prediction_enabled": true, "inverse_text_normalization_enabled": true, "output_dir": "tmp/"}', 'https://help.aliyun.com/zh/model-studio/websocket-for-paraformer-real-time-service', '支持多语言、热词定制、语义断句等高级功能', 21, NULL, NULL, NULL, NULL); + +-- 更新阿里百炼Paraformer模型配置的说明文档 +UPDATE `ai_model_config` SET +`doc_link` = 'https://help.aliyun.com/zh/model-studio/websocket-for-paraformer-real-time-service', +`remark` = '阿里百炼Paraformer实时语音识别配置说明： +1. 登录阿里云百炼平台 https://bailian.console.aliyun.com/ +2. 创建API-KEY https://bailian.console.aliyun.com/#/api-key +3. 支持模型：paraformer-realtime-v2(推荐)、paraformer-realtime-8k-v2、paraformer-realtime-v1、paraformer-realtime-8k-v1 +4. 功能特性： + - 多语言支持(中文含方言、英文、日语、韩语、德语、法语、俄语) + - 热词定制(vocabulary_id参数)，详细说明请参考：https://help.aliyun.com/zh/model-studio/custom-hot-words? + - 语义断句/VAD断句(semantic_punctuation_enabled参数) + - 自动标点符号、ITN、过滤语气词等 +5. 参数说明： + - model: 模型名称，推荐paraformer-realtime-v2 + - sample_rate: 采样率(Hz)，v2支持任意采样率，v1仅支持16000，8k版本仅支持8000 + - semantic_punctuation_enabled: false为VAD断句(低延迟)，true为语义断句(高准确) + - max_sentence_silence: VAD断句静音时长阈值(200-6000ms) +' WHERE `id` = 'ASR_AliyunBLStream'; + + +-- 更新豆包流式ASR供应器，增加配置 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_DoubaoStreamASR'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_DoubaoStreamASR', 'ASR', 'doubao_stream', '火山引擎语音识别(流式)', '[{"key":"appid","label":"应用ID","type":"string"},{"key":"access_token","label":"访问令牌","type":"string"},{"key":"cluster","label":"集群","type":"string"},{"key":"boosting_table_name","label":"热词文件名称","type":"string"},{"key":"correct_table_name","label":"替换词文件名称","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"},{"key":"end_window_size","label":"静音判定时长(ms)","type":"number"},{"key":"enable_multilingual","label":"是否开启多语种识别模式","type":"boolean"},{"key":"language","label":"指定语言编码","type":"string"}]', 3, 1, NOW(), 1, NOW()); +UPDATE `ai_model_config` SET +`remark` = '豆包ASR配置说明： +1. 豆包ASR和豆包(流式)ASR的区别是：豆包ASR是按次收费，豆包(流式)ASR是按时收费 +2. 一般来说按次收费的更便宜，但是豆包(流式)ASR使用了大模型技术，效果更好 +3. 需要在火山引擎控制台创建应用并获取appid和access_token +4. 支持中文语音识别 +5. 需要网络连接 +6. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://console.volcengine.com/speech/app +2. 创建新应用 +3. 获取appid和access_token +4. 填入配置文件中 +如需设置热词，请参考：https://www.volcengine.com/docs/6561/155738 +如开启多语种识别模式，请设置language当该键为空时，该模型支持中英文、上海话、闽南语，四川、陕西、粤语识别。其他语种请参考：https://www.volcengine.com/docs/6561/1354869 +' WHERE `id` = 'ASR_DoubaoStreamASR'; + +-- 更新豆包流式ASR模型配置，增加enable_multilingual默认值 +UPDATE `ai_model_config` SET +`config_json` = JSON_SET( + `config_json`, + '$.enable_multilingual', false, + '$.language', 'zh-CN' +) +WHERE `id` = 'ASR_DoubaoStreamASR' +AND JSON_EXTRACT(`config_json`, '$.enable_multilingual') IS NULL +AND JSON_EXTRACT(`config_json`, '$.language') IS NULL; + + +-- 更新HuoshanDoubleStreamTTS供应器配置，增加多情感音色参数 +UPDATE `ai_model_provider` +SET `fields` = '[{"key": "ws_url", "type": "string", "label": "WebSocket地址"}, {"key": "appid", "type": "string", "label": "应用ID"}, {"key": "access_token", "type": "string", "label": "访问令牌"}, {"key": "resource_id", "type": "string", "label": "资源ID"}, {"key": "speaker", "type": "string", "label": "默认音色"}, {"key": "enable_ws_reuse", "type": "boolean", "label": "是否开启链接复用", "default": true}, {"key": "speech_rate", "type": "number", "label": "语速(-50~100)"}, {"key": "loudness_rate", "type": "number", "label": "音量(-50~100)"}, {"key": "pitch", "type": "number", "label": "音高(-12~12)"}, {"key": "emotion_scale", "type": "number", "label": "情感强度(1-5)"}, {"key": "emotion", "type": "string", "label": "情感类型"}]' +WHERE `id` = 'SYSTEM_TTS_HSDSTTS'; + +-- 更新默认值 +UPDATE `ai_model_config` SET +`config_json` = JSON_SET( + `config_json`, + '$.emotion', 'neutral', + '$.emotion_scale', 4 +) +WHERE `id` = 'TTS_HuoshanDoubleStreamTTS' +AND JSON_EXTRACT(`config_json`, '$.emotion') IS NULL +AND JSON_EXTRACT(`config_json`, '$.emotion_scale') IS NULL; + +-- 增加文档链接和备注 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.volcengine.com/speech/service/10007', +`remark` = '火山引擎语音合成服务配置说明： +1. 访问 https://www.volcengine.com/ 注册并开通火山引擎账号 +2. 访问 https://console.volcengine.com/speech/service/10007 开通语音合成大模型，购买音色 +3. 在页面底部获取appid和access_token +5. 资源ID固定为：volc.service_type.10029（大模型语音合成及混音） +6. 链接复用：开启WebSocket连接复用，默认true减少链接损耗（注意：复用后设备处于聆听状态时空闲链接会占并发数） +7. 语速：-50~100，可不填，正常默认值0，可填-50~100 +8. 音量：-50~100，可不填，正常默认值0，可填-50~100 +9. 音高：-12~12，可不填，正常默认值0，可填-12~12 +10. 多情感参数（当前仅部分音色支持设置情感）： + 相关音色列表：https://www.volcengine.com/docs/6561/1257544 + - emotion_scale：情感强度，可选值为：1~5，默认值为4 + - emotion：情感类型，可选值为：neutral、happy、sad、angry、fearful、disgusted、surprised +' WHERE `id` = 'TTS_HuoshanDoubleStreamTTS'; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 00db89db..dd2bdb10 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -472,4 +472,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202512221117.sql \ No newline at end of file + path: classpath:db/changelog/202512221117.sql + - changeSet: + id: 202512301430 + author: RanChen + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202512301430.sql diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 1139dbf4..9009eef8 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -346,6 +346,11 @@ ASR: # 热词、替换词使用流程：https://www.volcengine.com/docs/6561/155738 boosting_table_name: （选填）你的热词文件名称 correct_table_name: （选填）你的替换词文件名称 + # 是否开启多语种识别模式 + enable_multilingual: False + # 多语种识别当该键为空时，该模型支持中英文、上海话、闽南语，四川、陕西、粤语识别。当将其设置为特定键时，它可以识别指定语言。 + # 详细语言列表参考 https://www.volcengine.com/docs/6561/1354869 + # language: zh-cn # 静音判定时长(ms)，默认200ms end_window_size: 200 output_dir: tmp/ @@ -478,7 +483,32 @@ ASR: accent: mandarin # 方言，mandarin:普通话 # 调整音频处理参数以提高长语音识别质量 output_dir: tmp/ - + AliyunBLStreamASR: + # 阿里百炼Paraformer实时语音识别服务 + # WebSocket实时流式语音识别，支持多语言、热词定制、语义断句等高级功能 + # 平台地址：https://bailian.console.aliyun.com/ + # API Key地址：https://bailian.console.aliyun.com/#/api-key + # 文档地址：https://help.aliyun.com/zh/model-studio/websocket-for-paraformer-real-time-service + # 支持模型：paraformer-realtime-v2(推荐), paraformer-realtime-8k-v2, paraformer-realtime-v1, paraformer-realtime-8k-v1 + type: aliyunbl_stream + # 必填参数 + api_key: 你的阿里云百炼API密钥 + # 模型选择，推荐使用v2版本 + model: paraformer-realtime-v2 + # 音频格式和采样率 + format: pcm + sample_rate: 16000 # v2支持任意采样率，v1仅支持16000，8k版本仅支持8000 + # 可选参数 + disfluency_removal_enabled: false # 是否过滤语气词(如"嗯"、"啊"等) + semantic_punctuation_enabled: false # 语义断句(true:会议场景,准确;false:VAD断句,交互场景,低延迟) + max_sentence_silence: 200 # VAD断句静音时长阈值(ms)，范围200-6000，仅VAD断句时生效 + multi_threshold_mode_enabled: false # 防止VAD断句切割过长，仅VAD断句时生效 + punctuation_prediction_enabled: true # 是否自动添加标点符号 + inverse_text_normalization_enabled: true # 是否开启ITN(中文数字转阿拉伯数字) + # 热词定制文档地址：https://help.aliyun.com/zh/model-studio/custom-hot-words? + # vocabulary_id: vocab-xxx-24ee19fa8cfb4d52902170a0xxxxxxxx # 热词ID(可选) + # language_hints: ["zh", "en"] # 指定语言(可选)，支持zh、en、ja、yue、ko、de、fr、ru + output_dir: tmp/ VAD: SileroVAD: type: silero @@ -687,6 +717,10 @@ TTS: speech_rate: 0 loudness_rate: 0 pitch: 0 + # 多情感音色参数，注意：当前仅部分音色支持设置情感。 + # 相关音色列表：https://www.volcengine.com/docs/6561/1257544 + emotion: "neutral" # 情感类型，可选值为：neutral、happy、sad、angry、fearful、disgusted、surprised + emotion_scale: 4 # 情感强度，可选值为：1~5，默认值为4 CosyVoiceSiliconflow: type: siliconflow # 硅基流动TTS diff --git a/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py b/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py new file mode 100644 index 00000000..a321ce9b --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py @@ -0,0 +1,343 @@ +import json +import uuid +import asyncio +import websockets +import opuslib_next +from typing import List +from config.logger import setup_logging +from core.providers.asr.base import ASRProviderBase +from core.providers.asr.dto.dto import InterfaceType + +TAG = __name__ +logger = setup_logging() + + +class ASRProvider(ASRProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__() + self.interface_type = InterfaceType.STREAM + self.config = config + self.text = "" + self.decoder = opuslib_next.Decoder(16000, 1) + self.asr_ws = None + self.forward_task = None + self.is_processing = False + self.server_ready = False # 服务器准备状态 + self.task_id = None # 当前任务ID + + # 阿里百炼配置 + self.api_key = config.get("api_key") + self.model = config.get("model", "paraformer-realtime-v2") + self.sample_rate = config.get("sample_rate", 16000) + self.format = config.get("format", "pcm") + + # 可选参数 + self.vocabulary_id = config.get("vocabulary_id") + self.disfluency_removal_enabled = config.get("disfluency_removal_enabled", False) + self.language_hints = config.get("language_hints") + self.semantic_punctuation_enabled = config.get("semantic_punctuation_enabled", False) + max_sentence_silence = config.get("max_sentence_silence") + self.max_sentence_silence = int(max_sentence_silence) if max_sentence_silence else 200 + self.multi_threshold_mode_enabled = config.get("multi_threshold_mode_enabled", False) + self.punctuation_prediction_enabled = config.get("punctuation_prediction_enabled", True) + self.inverse_text_normalization_enabled = config.get("inverse_text_normalization_enabled", True) + + # WebSocket URL + self.ws_url = "wss://dashscope.aliyuncs.com/api-ws/v1/inference" + + self.output_dir = config.get("output_dir", "./audio_output") + self.delete_audio_file = delete_audio_file + + async def open_audio_channels(self, conn): + await super().open_audio_channels(conn) + + async def receive_audio(self, conn, audio, audio_have_voice): + # 初始化音频缓存 + if not hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + + # 存储音频数据 + if audio: + conn.asr_audio_for_voiceprint.append(audio) + + conn.asr_audio.append(audio) + conn.asr_audio = conn.asr_audio[-10:] + + # 只在有声音且没有连接时建立连接 + if audio_have_voice and not self.is_processing and not self.asr_ws: + try: + await self._start_recognition(conn) + except Exception as e: + logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}") + await self._cleanup() + return + + # 发送音频数据 + if self.asr_ws and self.is_processing and self.server_ready: + try: + pcm_frame = self.decoder.decode(audio, 960) + # 直接发送PCM音频数据(二进制) + await self.asr_ws.send(pcm_frame) + except Exception as e: + logger.bind(tag=TAG).warning(f"发送音频失败: {str(e)}") + await self._cleanup() + + async def _start_recognition(self, conn): + """开始识别会话""" + try: + # 如果为手动模式,设置超时时长为最大值 + if conn.client_listen_mode == "manual": + self.max_sentence_silence = 6000 + + self.is_processing = True + self.task_id = uuid.uuid4().hex + + # 建立WebSocket连接 + headers = { + "Authorization": f"Bearer {self.api_key}" + } + + logger.bind(tag=TAG).debug(f"正在连接阿里百炼ASR服务, task_id: {self.task_id}") + + self.asr_ws = await websockets.connect( + self.ws_url, + additional_headers=headers, + max_size=1000000000, + ping_interval=None, + ping_timeout=None, + close_timeout=5, + ) + + logger.bind(tag=TAG).debug("WebSocket连接建立成功") + + self.server_ready = False + self.forward_task = asyncio.create_task(self._forward_results(conn)) + + # 发送run-task指令 + run_task_msg = self._build_run_task_message() + await self.asr_ws.send(json.dumps(run_task_msg, ensure_ascii=False)) + logger.bind(tag=TAG).debug("已发送run-task指令，等待服务器准备...") + + except Exception as e: + logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}") + if self.asr_ws: + await self.asr_ws.close() + self.asr_ws = None + self.is_processing = False + raise + + def _build_run_task_message(self) -> dict: + """构建run-task指令""" + message = { + "header": { + "action": "run-task", + "task_id": self.task_id, + "streaming": "duplex" + }, + "payload": { + "task_group": "audio", + "task": "asr", + "function": "recognition", + "model": self.model, + "parameters": { + "format": self.format, + "sample_rate": self.sample_rate, + "disfluency_removal_enabled": self.disfluency_removal_enabled, + "semantic_punctuation_enabled": self.semantic_punctuation_enabled, + "max_sentence_silence": self.max_sentence_silence, + "multi_threshold_mode_enabled": self.multi_threshold_mode_enabled, + "punctuation_prediction_enabled": self.punctuation_prediction_enabled, + "inverse_text_normalization_enabled": self.inverse_text_normalization_enabled, + }, + "input": {} + } + } + + # 只有当模型名称以v2结尾时才添加vocabulary_id参数 + if self.model.lower().endswith("v2"): + message["payload"]["parameters"]["vocabulary_id"] = self.vocabulary_id + + if self.language_hints: + message["payload"]["parameters"]["language_hints"] = self.language_hints + + return message + + async def _forward_results(self, conn): + """转发识别结果""" + try: + while not conn.stop_event.is_set(): + try: + response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0) + result = json.loads(response) + + header = result.get("header", {}) + payload = result.get("payload", {}) + event = header.get("event", "") + + # 处理task-started事件 + if event == "task-started": + self.server_ready = True + logger.bind(tag=TAG).debug("服务器已准备，开始发送缓存音频...") + + # 发送缓存音频 + if conn.asr_audio: + for cached_audio in conn.asr_audio[-10:]: + try: + pcm_frame = self.decoder.decode(cached_audio, 960) + await self.asr_ws.send(pcm_frame) + except Exception as e: + logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}") + break + continue + + # 处理result-generated事件 + elif event == "result-generated": + output = payload.get("output", {}) + sentence = output.get("sentence", {}) + + text = sentence.get("text", "") + sentence_end = sentence.get("sentence_end", False) + end_time = sentence.get("end_time") + + # 判断是否为最终结果(sentence_end为True且end_time不为null) + is_final = sentence_end and end_time is not None + + if is_final: + logger.bind(tag=TAG).info(f"识别到文本: {text}") + + # 手动模式下累积识别结果 + if conn.client_listen_mode == "manual": + if self.text: + self.text += text + else: + self.text = text + + # 手动模式下,只有在收到stop信号后才触发处理 + if conn.client_voice_stop: + audio_data = getattr(conn, 'asr_audio_for_voiceprint', []) + if len(audio_data) > 0: + logger.bind(tag=TAG).debug("收到最终识别结果，触发处理") + await self.handle_voice_stop(conn, audio_data) + # 清理音频缓存 + conn.asr_audio.clear() + conn.reset_vad_states() + break + else: + # 自动模式下直接覆盖 + self.text = text + conn.reset_vad_states() + audio_data = getattr(conn, 'asr_audio_for_voiceprint', []) + await self.handle_voice_stop(conn, audio_data) + break + + # 处理task-finished事件 + elif event == "task-finished": + logger.bind(tag=TAG).debug("任务已完成") + break + + # 处理task-failed事件 + elif event == "task-failed": + error_code = header.get("error_code", "UNKNOWN") + error_message = header.get("error_message", "未知错误") + logger.bind(tag=TAG).error(f"任务失败: {error_code} - {error_message}") + break + + except asyncio.TimeoutError: + continue + except websockets.ConnectionClosed: + logger.bind(tag=TAG).info("ASR服务连接已关闭") + self.is_processing = False + break + except Exception as e: + logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}") + break + + except Exception as e: + logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}") + finally: + # 清理连接的音频缓存 + await self._cleanup() + if conn: + if hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + if hasattr(conn, 'asr_audio'): + conn.asr_audio = [] + + async def _send_stop_request(self): + """发送停止请求(用于手动模式停止录音)""" + if self.asr_ws: + try: + # 先停止音频发送 + self.is_processing = False + + logger.bind(tag=TAG).debug("收到停止请求，发送finish-task指令") + await self._send_finish_task() + except Exception as e: + logger.bind(tag=TAG).error(f"发送停止请求失败: {e}") + + async def _send_finish_task(self): + """发送finish-task指令""" + if self.asr_ws and self.task_id: + try: + finish_msg = { + "header": { + "action": "finish-task", + "task_id": self.task_id, + "streaming": "duplex" + }, + "payload": { + "input": {} + } + } + await self.asr_ws.send(json.dumps(finish_msg, ensure_ascii=False)) + logger.bind(tag=TAG).debug("已发送finish-task指令") + except Exception as e: + logger.bind(tag=TAG).error(f"发送finish-task指令失败: {e}") + + async def _cleanup(self): + """清理资源""" + logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}") + + # 状态重置 + self.is_processing = False + self.server_ready = False + logger.bind(tag=TAG).debug("ASR状态已重置") + + # 关闭连接 + if self.asr_ws: + try: + # 先发送finish-task指令 + await self._send_finish_task() + # 等待一小段时间让服务器处理 + await asyncio.sleep(0.1) + + logger.bind(tag=TAG).debug("正在关闭WebSocket连接") + await asyncio.wait_for(self.asr_ws.close(), timeout=2.0) + logger.bind(tag=TAG).debug("WebSocket连接已关闭") + except Exception as e: + logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}") + finally: + self.asr_ws = None + + # 清理任务引用 + self.forward_task = None + self.task_id = None + + logger.bind(tag=TAG).debug("ASR会话清理完成") + + async def speech_to_text(self, opus_data, session_id, audio_format): + """获取识别结果""" + result = self.text + self.text = "" + return result, None + + async def close(self): + """关闭资源""" + await self._cleanup() + if hasattr(self, 'decoder') and self.decoder is not None: + try: + del self.decoder + self.decoder = None + logger.bind(tag=TAG).debug("Aliyun BL decoder resources released") + except Exception as e: + logger.bind(tag=TAG).debug(f"释放Aliyun BL decoder资源时出错: {e}") \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/asr/doubao_stream.py b/main/xiaozhi-server/core/providers/asr/doubao_stream.py index 9c667952..3a726b08 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao_stream.py +++ b/main/xiaozhi-server/core/providers/asr/doubao_stream.py @@ -33,7 +33,12 @@ class ASRProvider(ASRProviderBase): self.delete_audio_file = delete_audio_file # 火山引擎ASR配置 - self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel" + enable_multilingual = config.get("enable_multilingual", False) + self.enable_multilingual = False if str(enable_multilingual).lower() == 'false' else True + if self.enable_multilingual: + self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_nostream" + else: + self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel" self.uid = config.get("uid", "streaming_asr_service") self.workflow = config.get( "workflow", "audio_in,resample,partition,vad,fe,decode,itn,nlu_punctuate" @@ -42,7 +47,8 @@ class ASRProvider(ASRProviderBase): self.format = config.get("format", "pcm") self.codec = config.get("codec", "pcm") self.rate = config.get("sample_rate", 16000) - self.language = config.get("language", "zh-CN") + # language参数仅在多语种模式(bigmodel_nostream)下有效 + self.language = config.get("language") if self.enable_multilingual else None self.bits = config.get("bits", 16) self.channel = config.get("channel", 1) self.auth_method = config.get("auth_method", "token") @@ -175,7 +181,8 @@ class ASRProvider(ASRProviderBase): utterances = payload["result"].get("utterances", []) # 检查duration和空文本的情况 if ( - payload.get("audio_info", {}).get("duration", 0) > 2000 + not self.enable_multilingual # 注意：多语种模式不返回中间结果，需要等待最终结果 + and payload.get("audio_info", {}).get("duration", 0) > 2000 and not utterances and not payload["result"].get("text") and conn.client_listen_mode != "manual" @@ -189,6 +196,10 @@ class ASRProvider(ASRProviderBase): # 专门处理没有文本的识别结果（手动模式下可能已经识别完成但是没松按键） elif not payload["result"].get("text") and not utterances: + # 多语种模式会持续返回空文本，直到最后返回完整结果，所以需要排除 + if self.enable_multilingual: + continue + if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0: logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理") await self.handle_voice_stop(conn, audio_data) @@ -299,12 +310,16 @@ class ASRProvider(ASRProviderBase): "format": self.format, "codec": self.codec, "rate": self.rate, - "language": self.language, "bits": self.bits, "channel": self.channel, "sample_rate": self.rate, }, } + + # language参数仅在多语种模式下添加 + if self.enable_multilingual and self.language: + req["audio"]["language"] = self.language + logger.bind(tag=TAG).debug( f"构造请求参数: {json.dumps(req, ensure_ascii=False)}" ) diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 6135fa17..38be1560 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -160,11 +160,16 @@ class TTSProvider(TTSProviderBase): self.speech_rate = int(speech_rate) if speech_rate else 0 self.loudness_rate = int(loudness_rate) if loudness_rate else 0 self.pitch = int(pitch) if pitch else 0 + # 多情感音色参数 + self.emotion = config.get("emotion", "neutral") + emotion_scale = config.get("emotion_scale", "4") + self.emotion_scale = int(emotion_scale) if emotion_scale else 4 + self.ws_url = config.get("ws_url") self.authorization = config.get("authorization") self.header = {"Authorization": f"{self.authorization}{self.access_token}"} enable_ws_reuse_value = config.get("enable_ws_reuse", True) - self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() in ('false', 'False') else True + self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True self.tts_text = "" self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( sample_rate=16000, channels=1, frame_size_ms=60 @@ -642,6 +647,19 @@ class TTSProvider(TTSProviderBase): audio_format="pcm", audio_sample_rate=16000, ): + audio_params = { + "format": audio_format, + "sample_rate": audio_sample_rate, + "speech_rate": self.speech_rate, + "loudness_rate": self.loudness_rate + } + + # 如果是多情感音色,添加情感参数 + if '_emo_' in self.voice: + if self.emotion: + audio_params["emotion"] = self.emotion + audio_params["emotion_scale"] = self.emotion_scale + return str.encode( json.dumps( { @@ -651,12 +669,7 @@ class TTSProvider(TTSProviderBase): "req_params": { "text": text, "speaker": speaker, - "audio_params": { - "format": audio_format, - "sample_rate": audio_sample_rate, - "speech_rate": self.speech_rate, - "loudness_rate": self.loudness_rate - }, + "audio_params": audio_params, "additions": json.dumps({ "post_process": { "pitch": self.pitch