From 77f0bdc4caab84ea90b016ae5cfb6eac736020e5 Mon Sep 17 00:00:00 2001 From: FAN-yeB <1442100690@qq.com> Date: Tue, 9 Sep 2025 17:56:05 +0800 Subject: [PATCH 1/2] =?UTF-8?q?update:=E6=9B=B4=E6=96=B0Qwen-asr-flash?= =?UTF-8?q?=E8=AF=AD=E9=9F=B3=E8=AF=86=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202509091633.sql | 7 + .../db/changelog/db.changelog-master.yaml | 10 +- main/xiaozhi-server/config.yaml | 17 +- .../core/providers/asr/qwen3_asr_flash.py | 169 ++++++++++++++++++ 4 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202509091633.sql create mode 100644 main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py diff --git a/main/manager-api/src/main/resources/db/changelog/202509091633.sql b/main/manager-api/src/main/resources/db/changelog/202509091633.sql new file mode 100644 index 00000000..8566677a --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202509091633.sql @@ -0,0 +1,7 @@ +-- 添加通义千问Qwen3-ASR-Flash语音识别服务配置 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_Qwen3Flash'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_Qwen3Flash', 'ASR', 'qwen3_asr_flash', 'Qwen3-ASR-Flash语音识别', '[{"key":"api_key","label":"API密钥","type":"password"},{"key":"base_url","label":"服务地址","type":"string"},{"key":"model_name","label":"模型名称","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 17, 1, NOW(), 1, NOW()); + +delete from `ai_model_config` where id = 'ASR_Qwen3Flash'; +INSERT INTO `ai_model_config` VALUES ('ASR_Qwen3Flash', 'ASR', 'Qwen3-ASR-Flash', '通义千问语音识别服务', 0, 1, '{"type": "qwen3_asr_flash", "api_key": "", "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", "model_name": "qwen3-asr-flash", "output_dir": "tmp/", "enable_lid": true, "enable_itn": true}', 'https://help.aliyun.com/zh/bailian/', '支持多语言识别、歌唱识别、噪声拒识功能', 20, NULL, NULL, NULL, NULL); \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 817281be..fb8df775 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -316,4 +316,12 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202508271113.sql \ No newline at end of file + path: classpath:db/changelog/202508271113.sql + + - changeSet: + id: 202509091633 + author: cgd + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202509091633.sql \ No newline at end of file diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index e84cd905..8d3d9575 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -411,7 +411,22 @@ ASR: type: vosk model_path: 你的模型路径，如：models/vosk/vosk-model-small-cn-0.22 output_dir: tmp/ - + Qwen3ASRFlash: + # 通义千问Qwen3-ASR-Flash语音识别服务，需要先在阿里云百炼平台创建API密钥 + # 申请步骤： + # 1.登录阿里云百炼平台。https://bailian.console.aliyun.com/ + # 2.创建API-KEY https://bailian.console.aliyun.com/#/api-key + # 3.Qwen3-ASR-Flash基于通义千问多模态基座，支持多语言识别、歌唱识别、噪声拒识等功能 + type: qwen3_asr_flash + api_key: 你的阿里云百炼API密钥 + base_url: https://dashscope.aliyuncs.com/compatible-mode/v1 + model_name: qwen3-asr-flash + output_dir: tmp/ + # ASR选项配置 + enable_lid: true # 自动语种检测 + enable_itn: true # 逆文本归一化 + #language: "zh" # 语种，支持zh、en、ja、ko等 + context: "" # 上下文信息，用于提高识别准确率，不超过10000 Token VAD: diff --git a/main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py b/main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py new file mode 100644 index 00000000..84fe1979 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py @@ -0,0 +1,169 @@ +import os +import json +import asyncio +import tempfile +import difflib +from typing import Optional, Tuple, List +import dashscope +from config.logger import setup_logging +from core.providers.asr.base import ASRProviderBase +from core.providers.asr.dto.dto import InterfaceType + +tag = __name__ +logger = setup_logging() + + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool): + super().__init__() + self.interface_type = InterfaceType.STREAM + """Qwen3-ASR-Flash ASR初始化""" + + # 配置参数 + self.api_key = config.get("api_key") + if not self.api_key: + raise ValueError("Qwen3-ASR-Flash 需要配置 api_key") + + self.model_name = config.get("model_name", "qwen3-asr-flash") + self.output_dir = config.get("output_dir", "./audio_output") + self.delete_audio_file = delete_audio_file + + # ASR选项配置 + self.enable_lid = config.get("enable_lid", True) # 自动语种检测 + self.enable_itn = config.get("enable_itn", True) # 逆文本归一化 + self.language = config.get("language", None) # 指定语种，默认自动检测 + self.context = config.get("context", "") # 上下文信息，用于提高识别准确率 + + # 确保输出目录存在 + os.makedirs(self.output_dir, exist_ok=True) + + def _prepare_audio_file(self, pcm_data: bytes) -> str: + """将PCM数据转换为WAV文件并返回文件路径""" + try: + import wave + + # 创建临时WAV文件 + with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file: + temp_path = temp_file.name + + # 写入WAV格式 + with wave.open(temp_path, 'wb') as wav_file: + wav_file.setnchannels(1) # 单声道 + wav_file.setsampwidth(2) # 16位 + wav_file.setframerate(16000) # 16kHz采样率 + wav_file.writeframes(pcm_data) + + return temp_path + + except Exception as e: + logger.bind(tag=tag).error(f"音频文件准备失败: {e}") + return None + + async def speech_to_text( + self, opus_data: List[bytes], session_id: str, audio_format="opus" + ) -> Tuple[Optional[str], Optional[str]]: + """将语音数据转换为文本""" + temp_file_path = None + file_path = None + + try: + # 解码音频数据 + if audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + + combined_pcm_data = b"".join(pcm_data) + if len(combined_pcm_data) == 0: + logger.bind(tag=tag).warning("音频数据为空") + return "", None + + # 准备音频文件 + temp_file_path = self._prepare_audio_file(combined_pcm_data) + if not temp_file_path: + return "", None + + # 保存音频文件（如果需要） + if not self.delete_audio_file: + file_path = self.save_audio_to_file(pcm_data, session_id) + + # 构造请求消息 + messages = [ + { + "role": "user", + "content": [ + {"audio": temp_file_path} + ] + } + ] + + # 如果有上下文信息，添加system消息 + if self.context: + messages.insert(0, { + "role": "system", + "content": [ + {"text": self.context} + ] + }) + + # 准备ASR选项 + asr_options = { + "enable_lid": self.enable_lid, + "enable_itn": self.enable_itn + } + + # 如果指定了语种，添加到选项中 + if self.language: + asr_options["language"] = self.language + + # 设置API密钥 + dashscope.api_key = self.api_key + + # 发送流式请求 + response = dashscope.MultiModalConversation.call( + model=self.model_name, + messages=messages, + result_format="message", + asr_options=asr_options, + stream=True + ) + + # 处理流式响应 + full_text = "" + last_text = "" # 用于存储上一个文本片段 + for chunk in response: + try: + text = chunk["output"]["choices"][0]["message"].content[0]["text"] + # 标准化文本片段（去除首尾空格） + normalized_text = text.strip() + # 只有当新文本片段与上一个不同时才处理 + if normalized_text != last_text: + # 提取新增的文本部分 + # 通过比较当前文本和上一个文本，找到新增的部分 + if normalized_text.startswith(last_text): + # 如果当前文本以最后一个文本开头，则新增部分是两者的差集 + new_part = normalized_text[len(last_text):] + else: + # 如果不以最后一个文本开头，说明识别结果发生了较大变化，直接使用当前文本 + new_part = normalized_text + + # 将新增部分添加到完整文本中 + full_text += new_part + last_text = normalized_text + # 这里可以实时处理文本片段，例如通过回调函数 + except: + pass + + return full_text, file_path + + except Exception as e: + logger.bind(tag=tag).error(f"语音识别失败: {e}") + return "", file_path + + finally: + # 清理临时文件 + if temp_file_path and os.path.exists(temp_file_path): + try: + os.unlink(temp_file_path) + except Exception as e: + logger.bind(tag=tag).warning(f"清理临时文件失败: {e}") \ No newline at end of file From 906a322675bbed69575656828211188c1c3b4e97 Mon Sep 17 00:00:00 2001 From: FAN-yeB <1442100690@qq.com> Date: Wed, 10 Sep 2025 11:10:34 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E6=9B=B4=E6=96=B0Qwen3-ASR-Flash=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E9=85=8D=E7=BD=AE=E7=9A=84=E8=AF=B4=E6=98=8E=E6=96=87?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/db/changelog/202509091633.sql | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/main/manager-api/src/main/resources/db/changelog/202509091633.sql b/main/manager-api/src/main/resources/db/changelog/202509091633.sql index 8566677a..087bab41 100644 --- a/main/manager-api/src/main/resources/db/changelog/202509091633.sql +++ b/main/manager-api/src/main/resources/db/changelog/202509091633.sql @@ -4,4 +4,13 @@ INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `f ('SYSTEM_ASR_Qwen3Flash', 'ASR', 'qwen3_asr_flash', 'Qwen3-ASR-Flash语音识别', '[{"key":"api_key","label":"API密钥","type":"password"},{"key":"base_url","label":"服务地址","type":"string"},{"key":"model_name","label":"模型名称","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 17, 1, NOW(), 1, NOW()); delete from `ai_model_config` where id = 'ASR_Qwen3Flash'; -INSERT INTO `ai_model_config` VALUES ('ASR_Qwen3Flash', 'ASR', 'Qwen3-ASR-Flash', '通义千问语音识别服务', 0, 1, '{"type": "qwen3_asr_flash", "api_key": "", "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", "model_name": "qwen3-asr-flash", "output_dir": "tmp/", "enable_lid": true, "enable_itn": true}', 'https://help.aliyun.com/zh/bailian/', '支持多语言识别、歌唱识别、噪声拒识功能', 20, NULL, NULL, NULL, NULL); \ No newline at end of file +INSERT INTO `ai_model_config` VALUES ('ASR_Qwen3Flash', 'ASR', 'Qwen3-ASR-Flash', '通义千问语音识别服务', 0, 1, '{"type": "qwen3_asr_flash", "api_key": "", "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", "model_name": "qwen3-asr-flash", "output_dir": "tmp/", "enable_lid": true, "enable_itn": true}', 'https://help.aliyun.com/zh/bailian/', '支持多语言识别、歌唱识别、噪声拒识功能', 20, NULL, NULL, NULL, NULL); + +-- 更新Qwen3-ASR-Flash模型配置的说明文档 +UPDATE `ai_model_config` SET +`doc_link` = 'https://bailian.console.aliyun.com/?apiKey=1&tab=doc#/doc/?type=model&url=2979031', +`remark` = '通义千问Qwen3-ASR-Flash配置说明： +1. 登录阿里云百炼平台https://bailian.console.aliyun.com/ +2. 创建API-KEY https://bailian.console.aliyun.com/#/api-key +3.Qwen3-ASR-Flash基于通义千问多模态基座，支持多语言识别、歌唱识别、噪声拒识等功能 +' WHERE `id` = 'ASR_Qwen3Flash';