diff --git a/README.md b/README.md index a8d54c15..b1bf6dae 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ Websocket接口地址: wss://2662r3426b.vicp.fun/xiaozhi/v1/ | 使用方式 | 支持平台 | 免费平台 | |:---:|:---:|:---:| | 本地使用 | FunASR、SherpaASR | FunASR、SherpaASR | -| 接口调用 | DoubaoASR | - | +| 接口调用 | DoubaoASR、FunASRServer、TencentASR、AliyunASR | FunASRServer | --- diff --git a/main/manager-api/src/main/resources/db/changelog/202504301339.sql b/main/manager-api/src/main/resources/db/changelog/202504301339.sql deleted file mode 100644 index e4393f6a..00000000 --- a/main/manager-api/src/main/resources/db/changelog/202504301339.sql +++ /dev/null @@ -1,3 +0,0 @@ -update `ai_model_provider` set `fields` = -'[{"key": "api_url","label": "API地址","type": "string"},{"key": "voice","label": "音色","type": "string"},{"key": "output_dir","label": "输出目录","type": "string"},{"key": "authorization","label": "授权","type": "string"},{"key": "appid","label": "应用ID","type": "string"},{"key": "access_token","label": "访问令牌","type": "string"},{"key": "cluster","label": "集群","type": "string"},{"key": "speed_ratio","label": "语速","type": "number"},{"key": "volume_ratio","label": "音量","type": "number"},{"key": "pitch_ratio","label": "音高","type": "number"}]' -where `id` = 'SYSTEM_TTS_doubao'; diff --git a/main/manager-api/src/main/resources/db/changelog/202504301340.sql b/main/manager-api/src/main/resources/db/changelog/202504301340.sql new file mode 100644 index 00000000..1f0e2f7c --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202504301340.sql @@ -0,0 +1,20 @@ +update `ai_model_provider` set `fields` = +'[{"key": "api_url","label": "API地址","type": "string"},{"key": "voice","label": "音色","type": "string"},{"key": "output_dir","label": "输出目录","type": "string"},{"key": "authorization","label": "授权","type": "string"},{"key": "appid","label": "应用ID","type": "string"},{"key": "access_token","label": "访问令牌","type": "string"},{"key": "cluster","label": "集群","type": "string"},{"key": "speed_ratio","label": "语速","type": "number"},{"key": "volume_ratio","label": "音量","type": "number"},{"key": "pitch_ratio","label": "音高","type": "number"}]' +where `id` = 'SYSTEM_TTS_doubao'; + +-- 添加阿里云ASR供应器 +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_AliyunASR', 'ASR', 'aliyun', '阿里云语音识别', '[{"key":"appkey","label":"应用AppKey","type":"string"},{"key":"token","label":"临时Token","type":"string"},{"key":"access_key_id","label":"AccessKey ID","type":"string"},{"key":"access_key_secret","label":"AccessKey Secret","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 5, 1, NOW(), 1, NOW()); + +-- 添加阿里云ASR模型配置 +INSERT INTO `ai_model_config` VALUES ('ASR_AliyunASR', 'ASR', 'AliyunASR', '阿里云语音识别', 0, 1, '{\"type\": \"aliyun\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"output_dir\": \"tmp/\"}', NULL, NULL, 6, NULL, NULL, NULL, NULL); + +-- 更新阿里云ASR模型配置的说明文档 +UPDATE `ai_model_config` SET +`doc_link` = 'https://nls-portal.console.aliyun.com/', +`remark` = '阿里云ASR配置说明： +1. 访问 https://nls-portal.console.aliyun.com/ 开通服务 +2. 访问 https://nls-portal.console.aliyun.com/applist 获取appkey +3. 访问 https://nls-portal.console.aliyun.com/overview 获取token +4. 获取access_key_id和access_key_secret +5. 填入配置文件中' WHERE `id` = 'ASR_AliyunASR'; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 90735fe1..1d965a15 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -87,9 +87,9 @@ databaseChangeLog: encoding: utf8 path: classpath:db/changelog/202504291043.sql - changeSet: - id: 202504301339 + id: 202504301340 author: Goody changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202504301339.sql \ No newline at end of file + path: classpath:db/changelog/202504301340.sql \ No newline at end of file diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 3ce485ec..8b5cca86 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -241,6 +241,19 @@ ASR: secret_id: 你的腾讯语音合成服务secret_id secret_key: 你的腾讯语音合成服务secret_key output_dir: tmp/ + AliyunASR: + # 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息 + # 平台地址：https://nls-portal.console.aliyun.com/ + # appkey地址：https://nls-portal.console.aliyun.com/applist + # token地址：https://nls-portal.console.aliyun.com/overview + # 定义ASR API类型 + type: aliyun + appkey: 你的阿里云智能语音交互服务项目Appkey + token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret + access_key_id: 的阿里云账号access_key_id + access_key_secret: 你的阿里云账号access_key_secret + output_dir: tmp/ + VAD: SileroVAD: type: silero @@ -643,4 +656,4 @@ TTS: headers: # 自定义请求头 # Authorization: Bearer xxxx format: wav # 接口返回的音频格式 - output_dir: tmp/ + output_dir: tmp/ \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py new file mode 100644 index 00000000..4205bec2 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -0,0 +1,265 @@ +import http.client +import json +import asyncio +from typing import Optional, Tuple, List +import opuslib_next +import wave +import io +import os +import uuid +import hmac +import hashlib +import base64 +import requests +from urllib import parse +import time +from datetime import datetime +from config.logger import setup_logging +from core.providers.asr.base import ASRProviderBase + +TAG = __name__ +logger = setup_logging() + +class AccessToken: + @staticmethod + def _encode_text(text): + encoded_text = parse.quote_plus(text) + return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') + + @staticmethod + def _encode_dict(dic): + keys = dic.keys() + dic_sorted = [(key, dic[key]) for key in sorted(keys)] + encoded_text = parse.urlencode(dic_sorted) + return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') + + @staticmethod + def create_token(access_key_id, access_key_secret): + parameters = {'AccessKeyId': access_key_id, + 'Action': 'CreateToken', + 'Format': 'JSON', + 'RegionId': 'cn-shanghai', + 'SignatureMethod': 'HMAC-SHA1', + 'SignatureNonce': str(uuid.uuid1()), + 'SignatureVersion': '1.0', + 'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + 'Version': '2019-02-28'} + # 构造规范化的请求字符串 + query_string = AccessToken._encode_dict(parameters) + # print('规范化的请求字符串: %s' % query_string) + # 构造待签名字符串 + string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string) + # print('待签名的字符串: %s' % string_to_sign) + # 计算签名 + secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'), + bytes(string_to_sign, encoding='utf-8'), + hashlib.sha1).digest() + signature = base64.b64encode(secreted_string) + # print('签名: %s' % signature) + # 进行URL编码 + signature = AccessToken._encode_text(signature) + # print('URL编码后的签名: %s' % signature) + # 调用服务 + full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string) + # print('url: %s' % full_url) + # 提交HTTP GET请求 + response = requests.get(full_url) + if response.ok: + root_obj = response.json() + key = 'Token' + if key in root_obj: + token = root_obj[key]['Id'] + expire_time = root_obj[key]['ExpireTime'] + return token, expire_time + # print(response.text) + return None, None + + + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool): + """阿里云ASR初始化""" + # 新增空值判断逻辑 + self.access_key_id = config.get("access_key_id") + self.access_key_secret = config.get("access_key_secret") + + self.app_key = config.get("appkey") + self.host = "nls-gateway-cn-shanghai.aliyuncs.com" + self.base_url = f"https://{self.host}/stream/v1/asr" + self.sample_rate = 16000 + self.format = "wav" + self.output_dir = config.get("output_dir", "./audio_output") + self.delete_audio_file = delete_audio_file + + if self.access_key_id and self.access_key_secret: + # 使用密钥对生成临时token + self._refresh_token() + else: + # 直接使用预生成的长期token + self.token = config.get("token") + self.expire_time = None + + # 确保输出目录存在 + os.makedirs(self.output_dir, exist_ok=True) + + + def _refresh_token(self): + """刷新Token并记录过期时间""" + if self.access_key_id and self.access_key_secret: + self.token, expire_time_str = AccessToken.create_token( + self.access_key_id, + self.access_key_secret + ) + if not expire_time_str: + raise ValueError("无法获取有效的Token过期时间") + + try: + #统一转换为字符串处理 + expire_str = str(expire_time_str).strip() + + if expire_str.isdigit(): + expire_time = datetime.fromtimestamp(int(expire_str)) + else: + expire_time = datetime.strptime( + expire_str, + "%Y-%m-%dT%H:%M:%SZ" + ) + self.expire_time = expire_time.timestamp() - 60 + except Exception as e: + raise ValueError(f"无效的过期时间格式: {expire_str}") from e + + else: + self.expire_time = None + + if not self.token: + raise ValueError("无法获取有效的访问Token") + + def _is_token_expired(self): + """检查Token是否过期""" + if not self.expire_time: + return False # 长期Token不过期 + # 新增调试日志 + # current_time = time.time() + # remaining = self.expire_time - current_time + # print(f"Token过期检查: 当前时间 {datetime.fromtimestamp(current_time)} | " + # f"过期时间 {datetime.fromtimestamp(self.expire_time)} | " + # f"剩余 {remaining:.2f}秒") + return time.time() > self.expire_time + def generate_filename(self, extension=".wav"): + return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + + + def _construct_request_url(self) -> str: + """构造请求URL，包含参数""" + request = f"{self.base_url}?appkey={self.app_key}" + request += f"&format={self.format}" + request += f"&sample_rate={self.sample_rate}" + request += "&enable_punctuation_prediction=true" + request += "&enable_inverse_text_normalization=true" + request += "&enable_voice_detection=false" + return request + + def decode_opus(self, opus_data: List[bytes], session_id: str) -> List[bytes]: + """将Opus数据解码为PCM""" + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + return pcm_data + + def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: + """将Opus音频数据解码并保存为WAV文件""" + file_name = f"asr_{session_id}.wav" + file_path = os.path.join(self.output_dir, file_name) + + pcm_data = self.decode_opus(opus_data, session_id) + + with wave.open(file_path, "wb") as wf: + wf.setnchannels(1) # 单声道 + wf.setsampwidth(2) # 16-bit + wf.setframerate(self.sample_rate) + wf.writeframes(b''.join(pcm_data)) + + logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}") + return file_path + + async def _send_request(self, pcm_data: bytes) -> Optional[str]: + """发送请求到阿里云ASR服务""" + try: + # 设置HTTP头 + headers = { + 'X-NLS-Token': self.token, + 'Content-type': 'application/octet-stream', + 'Content-Length': str(len(pcm_data)) + } + + # 创建连接并发送请求 + conn = http.client.HTTPSConnection(self.host) + request_url = self._construct_request_url() + + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, lambda: conn.request( + method='POST', + url=request_url, + body=pcm_data, + headers=headers + )) + + # 获取响应 + response = await loop.run_in_executor(None, conn.getresponse) + body = await loop.run_in_executor(None, response.read) + conn.close() + + # 解析响应 + try: + body_json = json.loads(body) + status = body_json.get('status') + + if status == 20000000: + result = body_json.get('result', '') + logger.bind(tag=TAG).debug(f"ASR结果: {result}") + return result + else: + logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}") + return None + + except ValueError: + logger.bind(tag=TAG).error("响应不是JSON格式") + return None + + except Exception as e: + logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True) + return None + + async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + """将语音数据转换为文本""" + if self._is_token_expired(): + logger.warning("Token已过期，正在自动刷新...") + self._refresh_token() + + try: + # 解码Opus为PCM + pcm_data_list = self.decode_opus(opus_data, session_id) + combined_pcm_data = b''.join(pcm_data_list) + + # 发送请求并获取文本 + text = await self._send_request(combined_pcm_data) + + file_path = self.save_audio_to_file(opus_data, session_id) + if self.delete_audio_file: + os.remove(file_path) + logger.bind(tag=TAG).debug(f"音频文件已删除: {file_path}") + + if text: + return text, None + return "", None + + except Exception as e: + logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) + return "", None \ No newline at end of file