diff --git a/main/manager-api/src/main/resources/db/changelog/202512261524.sql b/main/manager-api/src/main/resources/db/changelog/202512261524.sql deleted file mode 100644 index 649fb240..00000000 --- a/main/manager-api/src/main/resources/db/changelog/202512261524.sql +++ /dev/null @@ -1,26 +0,0 @@ --- 添加阿里百炼Paraformer实时语音识别服务配置 -delete from `ai_model_provider` where id = 'SYSTEM_ASR_AliyunBLStream'; -INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES -('SYSTEM_ASR_AliyunBLStream', 'ASR', 'aliyunbl_stream', '阿里百炼Paraformer实时语音识别', '[{"key":"api_key","label":"API密钥","type":"password"},{"key":"model","label":"模型名称","type":"string"},{"key":"format","label":"音频格式","type":"string"},{"key":"sample_rate","label":"采样率","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 18, 1, NOW(), 1, NOW()); - -delete from `ai_model_config` where id = 'ASR_AliyunBLStream'; -INSERT INTO `ai_model_config` VALUES ('ASR_AliyunBLStream', 'ASR', 'AliyunBLStream', '阿里百炼Paraformer实时语音识别', 0, 1, '{"type": "aliyunbl_stream", "api_key": "", "model": "paraformer-realtime-v2", "format": "pcm", "sample_rate": 16000, "disfluency_removal_enabled": false, "semantic_punctuation_enabled": false, "max_sentence_silence": 800, "multi_threshold_mode_enabled": false, "punctuation_prediction_enabled": true, "heartbeat": false, "inverse_text_normalization_enabled": true, "output_dir": "tmp/"}', 'https://help.aliyun.com/zh/model-studio/websocket-for-paraformer-real-time-service', '支持多语言、热词定制、语义断句等高级功能', 21, NULL, NULL, NULL, NULL); - --- 更新阿里百炼Paraformer模型配置的说明文档 -UPDATE `ai_model_config` SET -`doc_link` = 'https://help.aliyun.com/zh/model-studio/websocket-for-paraformer-real-time-service', -`remark` = '阿里百炼Paraformer实时语音识别配置说明： -1. 登录阿里云百炼平台 https://bailian.console.aliyun.com/ -2. 创建API-KEY https://bailian.console.aliyun.com/#/api-key -3. 支持模型：paraformer-realtime-v2(推荐)、paraformer-realtime-8k-v2、paraformer-realtime-v1、paraformer-realtime-8k-v1 -4. 功能特性： - - 多语言支持(中文含方言、英文、日语、韩语、德语、法语、俄语) - - 热词定制(vocabulary_id参数) - - 语义断句/VAD断句(semantic_punctuation_enabled参数) - - 自动标点符号、ITN、过滤语气词等 -5. 参数说明： - - model: 模型名称，推荐paraformer-realtime-v2 - - sample_rate: 采样率(Hz)，v2支持任意采样率，v1仅支持16000，8k版本仅支持8000 - - semantic_punctuation_enabled: false为VAD断句(低延迟)，true为语义断句(高准确) - - max_sentence_silence: VAD断句静音时长阈值(200-6000ms) -' WHERE `id` = 'ASR_AliyunBLStream'; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/202512291452.sql b/main/manager-api/src/main/resources/db/changelog/202512291452.sql new file mode 100644 index 00000000..fc206a00 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202512291452.sql @@ -0,0 +1,59 @@ +-- 添加阿里百炼Paraformer实时语音识别服务配置 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_AliyunBLStream'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_AliyunBLStream', 'ASR', 'aliyunbl_stream', '阿里百炼Paraformer实时语音识别', '[{"key":"api_key","label":"API密钥","type":"password"},{"key":"model","label":"模型名称","type":"string"},{"key":"format","label":"音频格式","type":"string"},{"key":"sample_rate","label":"采样率","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 18, 1, NOW(), 1, NOW()); + +delete from `ai_model_config` where id = 'ASR_AliyunBLStream'; +INSERT INTO `ai_model_config` VALUES ('ASR_AliyunBLStream', 'ASR', 'AliyunBLStream', '阿里百炼Paraformer实时语音识别', 0, 1, '{"type": "aliyunbl_stream", "api_key": "", "model": "paraformer-realtime-v2", "format": "pcm", "sample_rate": 16000, "disfluency_removal_enabled": false, "semantic_punctuation_enabled": false, "max_sentence_silence": 200, "multi_threshold_mode_enabled": false, "punctuation_prediction_enabled": true, "inverse_text_normalization_enabled": true, "output_dir": "tmp/"}', 'https://help.aliyun.com/zh/model-studio/websocket-for-paraformer-real-time-service', '支持多语言、热词定制、语义断句等高级功能', 21, NULL, NULL, NULL, NULL); + +-- 更新阿里百炼Paraformer模型配置的说明文档 +UPDATE `ai_model_config` SET +`doc_link` = 'https://help.aliyun.com/zh/model-studio/websocket-for-paraformer-real-time-service', +`remark` = '阿里百炼Paraformer实时语音识别配置说明： +1. 登录阿里云百炼平台 https://bailian.console.aliyun.com/ +2. 创建API-KEY https://bailian.console.aliyun.com/#/api-key +3. 支持模型：paraformer-realtime-v2(推荐)、paraformer-realtime-8k-v2、paraformer-realtime-v1、paraformer-realtime-8k-v1 +4. 功能特性： + - 多语言支持(中文含方言、英文、日语、韩语、德语、法语、俄语) + - 热词定制(vocabulary_id参数)，详细说明请参考：https://help.aliyun.com/zh/model-studio/custom-hot-words? + - 语义断句/VAD断句(semantic_punctuation_enabled参数) + - 自动标点符号、ITN、过滤语气词等 +5. 参数说明： + - model: 模型名称，推荐paraformer-realtime-v2 + - sample_rate: 采样率(Hz)，v2支持任意采样率，v1仅支持16000，8k版本仅支持8000 + - semantic_punctuation_enabled: false为VAD断句(低延迟)，true为语义断句(高准确) + - max_sentence_silence: VAD断句静音时长阈值(200-6000ms) +' WHERE `id` = 'ASR_AliyunBLStream'; + +-- 更新豆包流式ASR供应器，增加配置 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_DoubaoStreamASR'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_DoubaoStreamASR', 'ASR', 'doubao_stream', '火山引擎语音识别(流式)', '[{"key":"appid","label":"应用ID","type":"string"},{"key":"access_token","label":"访问令牌","type":"string"},{"key":"cluster","label":"集群","type":"string"},{"key":"boosting_table_name","label":"热词文件名称","type":"string"},{"key":"correct_table_name","label":"替换词文件名称","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"},{"key":"end_window_size","label":"静音判定时长(ms)","type":"number"},{"key":"enable_multilingual","label":"是否开启多语种识别模式","type":"boolean"},{"key":"language","label":"指定语言编码","type":"string"}]', 3, 1, NOW(), 1, NOW()); +UPDATE `ai_model_config` SET +`remark` = '豆包ASR配置说明： +1. 豆包ASR和豆包(流式)ASR的区别是：豆包ASR是按次收费，豆包(流式)ASR是按时收费 +2. 一般来说按次收费的更便宜，但是豆包(流式)ASR使用了大模型技术，效果更好 +3. 需要在火山引擎控制台创建应用并获取appid和access_token +4. 支持中文语音识别 +5. 需要网络连接 +6. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://console.volcengine.com/speech/app +2. 创建新应用 +3. 获取appid和access_token +4. 填入配置文件中 +如需设置热词，请参考：https://www.volcengine.com/docs/6561/155738 +如开启多语种识别模式，请设置language当该键为空时，该模型支持中英文、上海话、闽南语，四川、陕西、粤语识别。其他语种请参考：https://www.volcengine.com/docs/6561/1354869 +' WHERE `id` = 'ASR_DoubaoStreamASR'; + + +-- 更新豆包流式ASR模型配置，增加enable_multilingual默认值 +UPDATE `ai_model_config` SET +`config_json` = JSON_SET( + `config_json`, + '$.enable_multilingual', false, + '$.language', 'zh-CN' +) +WHERE `id` = 'ASR_DoubaoStreamASR' +AND JSON_EXTRACT(`config_json`, '$.enable_multilingual') IS NULL +AND JSON_EXTRACT(`config_json`, '$.language') IS NULL; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 00db89db..157a2abd 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -472,4 +472,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202512221117.sql \ No newline at end of file + path: classpath:db/changelog/202512221117.sql + - changeSet: + id: 202512291452 + author: RanChen + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202512291452.sql diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 11bdd87a..c1d96e18 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -346,6 +346,11 @@ ASR: # 热词、替换词使用流程：https://www.volcengine.com/docs/6561/155738 boosting_table_name: （选填）你的热词文件名称 correct_table_name: （选填）你的替换词文件名称 + # 是否开启多语种识别模式 + enable_multilingual: False + # 多语种识别当该键为空时，该模型支持中英文、上海话、闽南语，四川、陕西、粤语识别。当将其设置为特定键时，它可以识别指定语言。 + # 详细语言列表参考 https://www.volcengine.com/docs/6561/1354869 + # language: zh-cn # 静音判定时长(ms)，默认200ms end_window_size: 200 output_dir: tmp/ @@ -496,11 +501,11 @@ ASR: # 可选参数 disfluency_removal_enabled: false # 是否过滤语气词(如"嗯"、"啊"等) semantic_punctuation_enabled: false # 语义断句(true:会议场景,准确;false:VAD断句,交互场景,低延迟) - max_sentence_silence: 800 # VAD断句静音时长阈值(ms)，范围200-6000，仅VAD断句时生效 + max_sentence_silence: 200 # VAD断句静音时长阈值(ms)，范围200-6000，仅VAD断句时生效 multi_threshold_mode_enabled: false # 防止VAD断句切割过长，仅VAD断句时生效 punctuation_prediction_enabled: true # 是否自动添加标点符号 - heartbeat: false # 是否开启长连接心跳(持续静音下保持连接) inverse_text_normalization_enabled: true # 是否开启ITN(中文数字转阿拉伯数字) + # 热词定制文档地址：https://help.aliyun.com/zh/model-studio/custom-hot-words? # vocabulary_id: vocab-xxx-24ee19fa8cfb4d52902170a0xxxxxxxx # 热词ID(可选) # language_hints: ["zh", "en"] # 指定语言(可选)，支持zh、en、ja、yue、ko、de、fr、ru output_dir: tmp/ diff --git a/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py b/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py index 43241926..a321ce9b 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py +++ b/main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py @@ -32,15 +32,15 @@ class ASRProvider(ASRProviderBase): self.format = config.get("format", "pcm") # 可选参数 - self.vocabulary_id = config.get("vocabulary_id") # 热词ID - self.disfluency_removal_enabled = config.get("disfluency_removal_enabled", False) # 过滤语气词 - self.language_hints = config.get("language_hints") # 语言提示,如 ["zh", "en"] - self.semantic_punctuation_enabled = config.get("semantic_punctuation_enabled", False) # 语义断句 - self.max_sentence_silence = config.get("max_sentence_silence", 800) # VAD断句静音时长(ms) - self.multi_threshold_mode_enabled = config.get("multi_threshold_mode_enabled", False) # 防止VAD断句切割过长 - self.punctuation_prediction_enabled = config.get("punctuation_prediction_enabled", True) # 标点符号预测 - self.heartbeat = config.get("heartbeat", False) # 长连接心跳 - self.inverse_text_normalization_enabled = config.get("inverse_text_normalization_enabled", True) # ITN + self.vocabulary_id = config.get("vocabulary_id") + self.disfluency_removal_enabled = config.get("disfluency_removal_enabled", False) + self.language_hints = config.get("language_hints") + self.semantic_punctuation_enabled = config.get("semantic_punctuation_enabled", False) + max_sentence_silence = config.get("max_sentence_silence") + self.max_sentence_silence = int(max_sentence_silence) if max_sentence_silence else 200 + self.multi_threshold_mode_enabled = config.get("multi_threshold_mode_enabled", False) + self.punctuation_prediction_enabled = config.get("punctuation_prediction_enabled", True) + self.inverse_text_normalization_enabled = config.get("inverse_text_normalization_enabled", True) # WebSocket URL self.ws_url = "wss://dashscope.aliyuncs.com/api-ws/v1/inference" @@ -85,6 +85,10 @@ class ASRProvider(ASRProviderBase): async def _start_recognition(self, conn): """开始识别会话""" try: + # 如果为手动模式,设置超时时长为最大值 + if conn.client_listen_mode == "manual": + self.max_sentence_silence = 6000 + self.is_processing = True self.task_id = uuid.uuid4().hex @@ -143,21 +147,21 @@ class ASRProvider(ASRProviderBase): "max_sentence_silence": self.max_sentence_silence, "multi_threshold_mode_enabled": self.multi_threshold_mode_enabled, "punctuation_prediction_enabled": self.punctuation_prediction_enabled, - "heartbeat": self.heartbeat, "inverse_text_normalization_enabled": self.inverse_text_normalization_enabled, }, "input": {} } } - # 添加可选参数 - if self.vocabulary_id: + # 只有当模型名称以v2结尾时才添加vocabulary_id参数 + if self.model.lower().endswith("v2"): message["payload"]["parameters"]["vocabulary_id"] = self.vocabulary_id if self.language_hints: message["payload"]["parameters"]["language_hints"] = self.language_hints return message + async def _forward_results(self, conn): """转发识别结果""" try: @@ -191,21 +195,10 @@ class ASRProvider(ASRProviderBase): output = payload.get("output", {}) sentence = output.get("sentence", {}) - if not sentence: - continue - - # 跳过心跳消息 - if sentence.get("heartbeat", False): - continue - text = sentence.get("text", "") sentence_end = sentence.get("sentence_end", False) end_time = sentence.get("end_time") - # 只处理有文本的结果 - if not text: - continue - # 判断是否为最终结果(sentence_end为True且end_time不为null) is_final = sentence_end and end_time is not None @@ -272,8 +265,11 @@ class ASRProvider(ASRProviderBase): async def _send_stop_request(self): """发送停止请求(用于手动模式停止录音)""" - if self.asr_ws and self.task_id and self.is_processing: + if self.asr_ws: try: + # 先停止音频发送 + self.is_processing = False + logger.bind(tag=TAG).debug("收到停止请求，发送finish-task指令") await self._send_finish_task() except Exception as e: diff --git a/main/xiaozhi-server/core/providers/asr/doubao_stream.py b/main/xiaozhi-server/core/providers/asr/doubao_stream.py index 9c667952..3a726b08 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao_stream.py +++ b/main/xiaozhi-server/core/providers/asr/doubao_stream.py @@ -33,7 +33,12 @@ class ASRProvider(ASRProviderBase): self.delete_audio_file = delete_audio_file # 火山引擎ASR配置 - self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel" + enable_multilingual = config.get("enable_multilingual", False) + self.enable_multilingual = False if str(enable_multilingual).lower() == 'false' else True + if self.enable_multilingual: + self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_nostream" + else: + self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel" self.uid = config.get("uid", "streaming_asr_service") self.workflow = config.get( "workflow", "audio_in,resample,partition,vad,fe,decode,itn,nlu_punctuate" @@ -42,7 +47,8 @@ class ASRProvider(ASRProviderBase): self.format = config.get("format", "pcm") self.codec = config.get("codec", "pcm") self.rate = config.get("sample_rate", 16000) - self.language = config.get("language", "zh-CN") + # language参数仅在多语种模式(bigmodel_nostream)下有效 + self.language = config.get("language") if self.enable_multilingual else None self.bits = config.get("bits", 16) self.channel = config.get("channel", 1) self.auth_method = config.get("auth_method", "token") @@ -175,7 +181,8 @@ class ASRProvider(ASRProviderBase): utterances = payload["result"].get("utterances", []) # 检查duration和空文本的情况 if ( - payload.get("audio_info", {}).get("duration", 0) > 2000 + not self.enable_multilingual # 注意：多语种模式不返回中间结果，需要等待最终结果 + and payload.get("audio_info", {}).get("duration", 0) > 2000 and not utterances and not payload["result"].get("text") and conn.client_listen_mode != "manual" @@ -189,6 +196,10 @@ class ASRProvider(ASRProviderBase): # 专门处理没有文本的识别结果（手动模式下可能已经识别完成但是没松按键） elif not payload["result"].get("text") and not utterances: + # 多语种模式会持续返回空文本，直到最后返回完整结果，所以需要排除 + if self.enable_multilingual: + continue + if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0: logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理") await self.handle_voice_stop(conn, audio_data) @@ -299,12 +310,16 @@ class ASRProvider(ASRProviderBase): "format": self.format, "codec": self.codec, "rate": self.rate, - "language": self.language, "bits": self.bits, "channel": self.channel, "sample_rate": self.rate, }, } + + # language参数仅在多语种模式下添加 + if self.enable_multilingual and self.language: + req["audio"]["language"] = self.language + logger.bind(tag=TAG).debug( f"构造请求参数: {json.dumps(req, ensure_ascii=False)}" ) diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 6135fa17..e30878f6 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -164,7 +164,7 @@ class TTSProvider(TTSProviderBase): self.authorization = config.get("authorization") self.header = {"Authorization": f"{self.authorization}{self.access_token}"} enable_ws_reuse_value = config.get("enable_ws_reuse", True) - self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() in ('false', 'False') else True + self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True self.tts_text = "" self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( sample_rate=16000, channels=1, frame_size_ms=60