Merge pull request #1081 from xinnan-tech/test-aliyun-asr

update:增加阿里云ASR

README.md

@@ -231,7 +231,7 @@ Websocket接口地址: wss://2662r3426b.vicp.fun/xiaozhi/v1/
 | 使用方式 | 支持平台 | 免费平台 |
 |:---:|:---:|:---:|
 | 本地使用 | FunASR、SherpaASR | FunASR、SherpaASR |
-| 接口调用 | DoubaoASR | - |
+| 接口调用 | DoubaoASR、FunASRServer、TencentASR、AliyunASR | FunASRServer |
 
 ---
 

main/manager-api/src/main/resources/db/changelog/202504301339.sql

@@ -1,3 +0,0 @@
-update `ai_model_provider` set `fields` = 
-'[{"key": "api_url","label": "API地址","type": "string"},{"key": "voice","label": "音色","type": "string"},{"key": "output_dir","label": "输出目录","type": "string"},{"key": "authorization","label": "授权","type": "string"},{"key": "appid","label": "应用ID","type": "string"},{"key": "access_token","label": "访问令牌","type": "string"},{"key": "cluster","label": "集群","type": "string"},{"key": "speed_ratio","label": "语速","type": "number"},{"key": "volume_ratio","label": "音量","type": "number"},{"key": "pitch_ratio","label": "音高","type": "number"}]'
-where `id` = 'SYSTEM_TTS_doubao';

main/manager-api/src/main/resources/db/changelog/202504301340.sql

@@ -0,0 +1,20 @@
+update `ai_model_provider` set `fields` = 
+'[{"key": "api_url","label": "API地址","type": "string"},{"key": "voice","label": "音色","type": "string"},{"key": "output_dir","label": "输出目录","type": "string"},{"key": "authorization","label": "授权","type": "string"},{"key": "appid","label": "应用ID","type": "string"},{"key": "access_token","label": "访问令牌","type": "string"},{"key": "cluster","label": "集群","type": "string"},{"key": "speed_ratio","label": "语速","type": "number"},{"key": "volume_ratio","label": "音量","type": "number"},{"key": "pitch_ratio","label": "音高","type": "number"}]'
+where `id` = 'SYSTEM_TTS_doubao';
+
+-- 添加阿里云ASR供应器
+INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES
+('SYSTEM_ASR_AliyunASR', 'ASR', 'aliyun', '阿里云语音识别', '[{"key":"appkey","label":"应用AppKey","type":"string"},{"key":"token","label":"临时Token","type":"string"},{"key":"access_key_id","label":"AccessKey ID","type":"string"},{"key":"access_key_secret","label":"AccessKey Secret","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 5, 1, NOW(), 1, NOW());
+
+-- 添加阿里云ASR模型配置
+INSERT INTO `ai_model_config` VALUES ('ASR_AliyunASR', 'ASR', 'AliyunASR', '阿里云语音识别', 0, 1, '{\"type\": \"aliyun\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"output_dir\": \"tmp/\"}', NULL, NULL, 6, NULL, NULL, NULL, NULL);
+
+-- 更新阿里云ASR模型配置的说明文档
+UPDATE `ai_model_config` SET 
+`doc_link` = 'https://nls-portal.console.aliyun.com/',
+`remark` = '阿里云ASR配置说明：
+1. 访问 https://nls-portal.console.aliyun.com/ 开通服务
+2. 访问 https://nls-portal.console.aliyun.com/applist 获取appkey
+3. 访问 https://nls-portal.console.aliyun.com/overview 获取token
+4. 获取access_key_id和access_key_secret
+5. 填入配置文件中' WHERE `id` = 'ASR_AliyunASR';

main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -87,9 +87,9 @@ databaseChangeLog:
             encoding: utf8
             path: classpath:db/changelog/202504291043.sql
   - changeSet:
-      id: 202504301339
+      id: 202504301340
       author: Goody
       changes:
         - sqlFile:
             encoding: utf8
-            path: classpath:db/changelog/202504301339.sql
+            path: classpath:db/changelog/202504301340.sql

main/xiaozhi-server/config.yaml

@@ -241,6 +241,19 @@ ASR:
     secret_id: 你的腾讯语音合成服务secret_id
     secret_key: 你的腾讯语音合成服务secret_key
     output_dir: tmp/
+  AliyunASR:
+    # 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息
+    # 平台地址：https://nls-portal.console.aliyun.com/
+    # appkey地址：https://nls-portal.console.aliyun.com/applist
+    # token地址：https://nls-portal.console.aliyun.com/overview
+    # 定义ASR API类型
+    type: aliyun
+    appkey: 你的阿里云智能语音交互服务项目Appkey
+    token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret
+    access_key_id: 的阿里云账号access_key_id
+    access_key_secret: 你的阿里云账号access_key_secret
+    output_dir: tmp/
+
 VAD:
   SileroVAD:
     type: silero

main/xiaozhi-server/core/providers/asr/aliyun.py

@@ -0,0 +1,265 @@
+import http.client
+import json
+import asyncio
+from typing import Optional, Tuple, List
+import opuslib_next
+import wave
+import io
+import os
+import uuid
+import hmac
+import hashlib
+import base64
+import requests
+from urllib import parse
+import time
+from datetime import datetime
+from config.logger import setup_logging
+from core.providers.asr.base import ASRProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class AccessToken:
+    @staticmethod
+    def _encode_text(text):
+        encoded_text = parse.quote_plus(text)
+        return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
+    
+    @staticmethod
+    def _encode_dict(dic):
+        keys = dic.keys()
+        dic_sorted = [(key, dic[key]) for key in sorted(keys)]
+        encoded_text = parse.urlencode(dic_sorted)
+        return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
+    
+    @staticmethod
+    def create_token(access_key_id, access_key_secret):
+        parameters = {'AccessKeyId': access_key_id,
+                      'Action': 'CreateToken',
+                      'Format': 'JSON',
+                      'RegionId': 'cn-shanghai',
+                      'SignatureMethod': 'HMAC-SHA1',
+                      'SignatureNonce': str(uuid.uuid1()),
+                      'SignatureVersion': '1.0',
+                      'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                      'Version': '2019-02-28'}
+        # 构造规范化的请求字符串
+        query_string = AccessToken._encode_dict(parameters)
+        # print('规范化的请求字符串: %s' % query_string)
+        # 构造待签名字符串
+        string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string)
+        # print('待签名的字符串: %s' % string_to_sign)
+        # 计算签名
+        secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'),
+                                   bytes(string_to_sign, encoding='utf-8'),
+                                   hashlib.sha1).digest()
+        signature = base64.b64encode(secreted_string)
+        # print('签名: %s' % signature)
+        # 进行URL编码
+        signature = AccessToken._encode_text(signature)
+        # print('URL编码后的签名: %s' % signature)
+        # 调用服务
+        full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string)
+        # print('url: %s' % full_url)
+        # 提交HTTP GET请求
+        response = requests.get(full_url)
+        if response.ok:
+            root_obj = response.json()
+            key = 'Token'
+            if key in root_obj:
+                token = root_obj[key]['Id']
+                expire_time = root_obj[key]['ExpireTime']
+                return token, expire_time
+        # print(response.text)
+        return None, None
+
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        """阿里云ASR初始化"""
+        # 新增空值判断逻辑
+        self.access_key_id = config.get("access_key_id")
+        self.access_key_secret = config.get("access_key_secret")
+
+        self.app_key = config.get("appkey")
+        self.host = "nls-gateway-cn-shanghai.aliyuncs.com"
+        self.base_url = f"https://{self.host}/stream/v1/asr"
+        self.sample_rate = 16000
+        self.format = "wav"
+        self.output_dir = config.get("output_dir", "./audio_output")
+        self.delete_audio_file = delete_audio_file
+
+        if self.access_key_id and self.access_key_secret:
+            # 使用密钥对生成临时token
+            self._refresh_token()
+        else:
+            # 直接使用预生成的长期token
+            self.token = config.get("token")
+            self.expire_time = None
+        
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+
+    def _refresh_token(self):
+        """刷新Token并记录过期时间"""
+        if self.access_key_id and self.access_key_secret:
+            self.token, expire_time_str = AccessToken.create_token(
+                self.access_key_id, 
+                self.access_key_secret
+            )
+            if not expire_time_str:
+                raise ValueError("无法获取有效的Token过期时间")
+
+            try:
+                #统一转换为字符串处理
+                expire_str = str(expire_time_str).strip()
+
+                if expire_str.isdigit():
+                    expire_time = datetime.fromtimestamp(int(expire_str))
+                else:
+                    expire_time = datetime.strptime(
+                        expire_str, 
+                        "%Y-%m-%dT%H:%M:%SZ"
+                    )
+                self.expire_time = expire_time.timestamp() - 60
+            except Exception as e:
+                raise ValueError(f"无效的过期时间格式: {expire_str}") from e
+        
+        else:
+            self.expire_time = None
+            
+        if not self.token:
+            raise ValueError("无法获取有效的访问Token")
+
+    def _is_token_expired(self):
+        """检查Token是否过期"""
+        if not self.expire_time:
+            return False  # 长期Token不过期
+        # 新增调试日志
+        # current_time = time.time()
+        # remaining = self.expire_time - current_time
+        # print(f"Token过期检查: 当前时间 {datetime.fromtimestamp(current_time)} | "
+        #              f"过期时间 {datetime.fromtimestamp(self.expire_time)} | "
+        #              f"剩余 {remaining:.2f}秒")
+        return time.time() > self.expire_time
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+
+    def _construct_request_url(self) -> str:
+        """构造请求URL，包含参数"""
+        request = f"{self.base_url}?appkey={self.app_key}"
+        request += f"&format={self.format}"
+        request += f"&sample_rate={self.sample_rate}"
+        request += "&enable_punctuation_prediction=true"
+        request += "&enable_inverse_text_normalization=true"
+        request += "&enable_voice_detection=false"
+        return request
+
+    def decode_opus(self, opus_data: List[bytes], session_id: str) -> List[bytes]:
+        """将Opus数据解码为PCM"""
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        return pcm_data
+
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """将Opus音频数据解码并保存为WAV文件"""
+        file_name = f"asr_{session_id}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        pcm_data = self.decode_opus(opus_data, session_id)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)  # 单声道
+            wf.setsampwidth(2)  # 16-bit
+            wf.setframerate(self.sample_rate)
+            wf.writeframes(b''.join(pcm_data))
+
+        logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}")
+        return file_path
+
+    async def _send_request(self, pcm_data: bytes) -> Optional[str]:
+        """发送请求到阿里云ASR服务"""
+        try:
+            # 设置HTTP头
+            headers = {
+                'X-NLS-Token': self.token,
+                'Content-type': 'application/octet-stream',
+                'Content-Length': str(len(pcm_data))
+            }
+
+            # 创建连接并发送请求
+            conn = http.client.HTTPSConnection(self.host)
+            request_url = self._construct_request_url()
+            
+            loop = asyncio.get_event_loop()
+            await loop.run_in_executor(None, lambda: conn.request(
+                method='POST',
+                url=request_url,
+                body=pcm_data,
+                headers=headers
+            ))
+
+            # 获取响应
+            response = await loop.run_in_executor(None, conn.getresponse)
+            body = await loop.run_in_executor(None, response.read)
+            conn.close()
+
+            # 解析响应
+            try:
+                body_json = json.loads(body)
+                status = body_json.get('status')
+                
+                if status == 20000000:
+                    result = body_json.get('result', '')
+                    logger.bind(tag=TAG).debug(f"ASR结果: {result}")
+                    return result
+                else:
+                    logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}")
+                    return None
+                    
+            except ValueError:
+                logger.bind(tag=TAG).error("响应不是JSON格式")
+                return None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True)
+            return None
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        if self._is_token_expired():
+            logger.warning("Token已过期，正在自动刷新...")
+            self._refresh_token()
+            
+        try:
+            # 解码Opus为PCM
+            pcm_data_list = self.decode_opus(opus_data, session_id)
+            combined_pcm_data = b''.join(pcm_data_list)
+
+            # 发送请求并获取文本
+            text = await self._send_request(combined_pcm_data)
+            
+            file_path = self.save_audio_to_file(opus_data, session_id)
+            if self.delete_audio_file:
+                os.remove(file_path)
+                logger.bind(tag=TAG).debug(f"音频文件已删除: {file_path}")
+
+            if text:
+                return text, None
+            return "", None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None