From 766a7062505a4c1179e8f90865544f16a08fb95d Mon Sep 17 00:00:00 2001 From: LFNL-scholar Date: Tue, 8 Apr 2025 20:37:23 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=98=BF=E9=87=8C?= =?UTF-8?q?=E4=BA=91=E7=9A=84ASR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 78 +++--- .../core/providers/asr/aliyun.py | 265 ++++++++++++++++++ 2 files changed, 311 insertions(+), 32 deletions(-) create mode 100644 main/xiaozhi-server/core/providers/asr/aliyun.py diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index f1ac85c0..ad85c712 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -86,35 +86,6 @@ wakeup_words: - "小滨小滨" - "小冰小冰" -# 插件的基础配置 -plugins: - # 获取天气插件的配置，这里填写你的api_key - # 这个密钥是项目共用的key，用多了可能会被限制 - # 想稳定一点就自行申请替换，每天有1000次免费调用 - # 申请地址：https://console.qweather.com/#/apps/create-key/over - get_weather: { "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" } - # 获取新闻插件的配置，这里根据需要的新闻类型传入对应的url链接，默认支持社会、科技、财经新闻 - # 更多类型的新闻列表查看 https://www.chinanews.com.cn/rss/ - get_news: - default_rss_url: "https://www.chinanews.com.cn/rss/society.xml" - category_urls: - society: "https://www.chinanews.com.cn/rss/society.xml" - world: "https://www.chinanews.com.cn/rss/world.xml" - finance: "https://www.chinanews.com.cn/rss/finance.xml" - home_assistant: - devices: - - 客厅,玩具灯,switch.cuco_cn_460494544_cp1_on_p_2_1 - - 卧室,台灯,switch.iot_cn_831898993_socn1_on_p_2_1 - base_url: http://homeassistant.local:8123 - api_key: 你的home assistant api访问令牌 - play_music: - music_dir: "./music" # 音乐文件存放路径，将从该目录及子目录下搜索音乐文件 - music_ext: # 音乐文件类型，p3格式效率最高 - - ".mp3" - - ".wav" - - ".p3" - refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒 - # ##################################################################################### # ################################以下是角色模型配置###################################### @@ -138,9 +109,9 @@ selected_module: # 语音活动检测模块，默认使用SileroVAD模型 VAD: SileroVAD # 语音识别模块，默认使用FunASR本地模型 - ASR: FunASR + ASR: AliyunASR # 将根据配置名称对应的type调用实际的LLM适配器 - LLM: ChatGLMLLM + LLM: AliLLM # TTS将根据配置名称对应的type调用实际的TTS适配器 TTS: EdgeTTS # 记忆模块，默认不开启记忆；如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short @@ -182,6 +153,36 @@ Intent: #- hass_set_state #- hass_play_music +# 插件的基础配置 +plugins: + # 获取天气插件的配置，这里填写你的api_key + # 这个密钥是项目共用的key，用多了可能会被限制 + # 想稳定一点就自行申请替换，每天有1000次免费调用 + # 申请地址：https://console.qweather.com/#/apps/create-key/over + get_weather: { "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" } + # 获取新闻插件的配置，这里根据需要的新闻类型传入对应的url链接，默认支持社会、科技、财经新闻 + # 更多类型的新闻列表查看 https://www.chinanews.com.cn/rss/ + get_news: + default_rss_url: "https://www.chinanews.com.cn/rss/society.xml" + category_urls: + society: "https://www.chinanews.com.cn/rss/society.xml" + world: "https://www.chinanews.com.cn/rss/world.xml" + finance: "https://www.chinanews.com.cn/rss/finance.xml" + home_assistant: + devices: + - 客厅,玩具灯,switch.cuco_cn_460494544_cp1_on_p_2_1 + - 卧室,台灯,switch.iot_cn_831898993_socn1_on_p_2_1 + base_url: http://homeassistant.local:8123 + api_key: 你的home assistant api访问令牌 + play_music: + music_dir: "./music" # 音乐文件存放路径，将从该目录及子目录下搜索音乐文件 + music_ext: # 音乐文件类型，p3格式效率最高 + - ".mp3" + - ".wav" + - ".p3" + refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒 + + Memory: mem0ai: type: mem0ai @@ -220,6 +221,19 @@ ASR: secret_id: 你的腾讯语音合成服务secret_id secret_key: 你的腾讯语音合成服务secret_key output_dir: tmp/ + AliyunASR: + # 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息 + # 平台地址：https://nls-portal.console.aliyun.com/ + # appkey地址：https://nls-portal.console.aliyun.com/applist + # token地址：https://nls-portal.console.aliyun.com/overview + # 定义ASR API类型 + type: aliyun + appkey: 你的阿里云智能语音交互服务项目Appkey + token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret + access_key_id: 的阿里云账号access_key_id + access_key_secret: 你的阿里云账号access_key_secret + output_dir: tmp/ + VAD: SileroVAD: threshold: 0.5 @@ -235,7 +249,7 @@ LLM: # 可在这里找到你的 api_key https://bailian.console.aliyun.com/?apiKey=1#/api-key base_url: https://dashscope.aliyuncs.com/compatible-mode/v1 model_name: qwen-turbo - api_key: 你的deepseek web key + api_key: sk-a8b1fe19c885431c8a3769e18c02541e temperature: 0.7 # 温度值 max_tokens: 500 # 最大生成token数 top_p: 1 diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py new file mode 100644 index 00000000..4205bec2 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -0,0 +1,265 @@ +import http.client +import json +import asyncio +from typing import Optional, Tuple, List +import opuslib_next +import wave +import io +import os +import uuid +import hmac +import hashlib +import base64 +import requests +from urllib import parse +import time +from datetime import datetime +from config.logger import setup_logging +from core.providers.asr.base import ASRProviderBase + +TAG = __name__ +logger = setup_logging() + +class AccessToken: + @staticmethod + def _encode_text(text): + encoded_text = parse.quote_plus(text) + return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') + + @staticmethod + def _encode_dict(dic): + keys = dic.keys() + dic_sorted = [(key, dic[key]) for key in sorted(keys)] + encoded_text = parse.urlencode(dic_sorted) + return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') + + @staticmethod + def create_token(access_key_id, access_key_secret): + parameters = {'AccessKeyId': access_key_id, + 'Action': 'CreateToken', + 'Format': 'JSON', + 'RegionId': 'cn-shanghai', + 'SignatureMethod': 'HMAC-SHA1', + 'SignatureNonce': str(uuid.uuid1()), + 'SignatureVersion': '1.0', + 'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + 'Version': '2019-02-28'} + # 构造规范化的请求字符串 + query_string = AccessToken._encode_dict(parameters) + # print('规范化的请求字符串: %s' % query_string) + # 构造待签名字符串 + string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string) + # print('待签名的字符串: %s' % string_to_sign) + # 计算签名 + secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'), + bytes(string_to_sign, encoding='utf-8'), + hashlib.sha1).digest() + signature = base64.b64encode(secreted_string) + # print('签名: %s' % signature) + # 进行URL编码 + signature = AccessToken._encode_text(signature) + # print('URL编码后的签名: %s' % signature) + # 调用服务 + full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string) + # print('url: %s' % full_url) + # 提交HTTP GET请求 + response = requests.get(full_url) + if response.ok: + root_obj = response.json() + key = 'Token' + if key in root_obj: + token = root_obj[key]['Id'] + expire_time = root_obj[key]['ExpireTime'] + return token, expire_time + # print(response.text) + return None, None + + + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool): + """阿里云ASR初始化""" + # 新增空值判断逻辑 + self.access_key_id = config.get("access_key_id") + self.access_key_secret = config.get("access_key_secret") + + self.app_key = config.get("appkey") + self.host = "nls-gateway-cn-shanghai.aliyuncs.com" + self.base_url = f"https://{self.host}/stream/v1/asr" + self.sample_rate = 16000 + self.format = "wav" + self.output_dir = config.get("output_dir", "./audio_output") + self.delete_audio_file = delete_audio_file + + if self.access_key_id and self.access_key_secret: + # 使用密钥对生成临时token + self._refresh_token() + else: + # 直接使用预生成的长期token + self.token = config.get("token") + self.expire_time = None + + # 确保输出目录存在 + os.makedirs(self.output_dir, exist_ok=True) + + + def _refresh_token(self): + """刷新Token并记录过期时间""" + if self.access_key_id and self.access_key_secret: + self.token, expire_time_str = AccessToken.create_token( + self.access_key_id, + self.access_key_secret + ) + if not expire_time_str: + raise ValueError("无法获取有效的Token过期时间") + + try: + #统一转换为字符串处理 + expire_str = str(expire_time_str).strip() + + if expire_str.isdigit(): + expire_time = datetime.fromtimestamp(int(expire_str)) + else: + expire_time = datetime.strptime( + expire_str, + "%Y-%m-%dT%H:%M:%SZ" + ) + self.expire_time = expire_time.timestamp() - 60 + except Exception as e: + raise ValueError(f"无效的过期时间格式: {expire_str}") from e + + else: + self.expire_time = None + + if not self.token: + raise ValueError("无法获取有效的访问Token") + + def _is_token_expired(self): + """检查Token是否过期""" + if not self.expire_time: + return False # 长期Token不过期 + # 新增调试日志 + # current_time = time.time() + # remaining = self.expire_time - current_time + # print(f"Token过期检查: 当前时间 {datetime.fromtimestamp(current_time)} | " + # f"过期时间 {datetime.fromtimestamp(self.expire_time)} | " + # f"剩余 {remaining:.2f}秒") + return time.time() > self.expire_time + def generate_filename(self, extension=".wav"): + return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + + + def _construct_request_url(self) -> str: + """构造请求URL，包含参数""" + request = f"{self.base_url}?appkey={self.app_key}" + request += f"&format={self.format}" + request += f"&sample_rate={self.sample_rate}" + request += "&enable_punctuation_prediction=true" + request += "&enable_inverse_text_normalization=true" + request += "&enable_voice_detection=false" + return request + + def decode_opus(self, opus_data: List[bytes], session_id: str) -> List[bytes]: + """将Opus数据解码为PCM""" + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + return pcm_data + + def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: + """将Opus音频数据解码并保存为WAV文件""" + file_name = f"asr_{session_id}.wav" + file_path = os.path.join(self.output_dir, file_name) + + pcm_data = self.decode_opus(opus_data, session_id) + + with wave.open(file_path, "wb") as wf: + wf.setnchannels(1) # 单声道 + wf.setsampwidth(2) # 16-bit + wf.setframerate(self.sample_rate) + wf.writeframes(b''.join(pcm_data)) + + logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}") + return file_path + + async def _send_request(self, pcm_data: bytes) -> Optional[str]: + """发送请求到阿里云ASR服务""" + try: + # 设置HTTP头 + headers = { + 'X-NLS-Token': self.token, + 'Content-type': 'application/octet-stream', + 'Content-Length': str(len(pcm_data)) + } + + # 创建连接并发送请求 + conn = http.client.HTTPSConnection(self.host) + request_url = self._construct_request_url() + + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, lambda: conn.request( + method='POST', + url=request_url, + body=pcm_data, + headers=headers + )) + + # 获取响应 + response = await loop.run_in_executor(None, conn.getresponse) + body = await loop.run_in_executor(None, response.read) + conn.close() + + # 解析响应 + try: + body_json = json.loads(body) + status = body_json.get('status') + + if status == 20000000: + result = body_json.get('result', '') + logger.bind(tag=TAG).debug(f"ASR结果: {result}") + return result + else: + logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}") + return None + + except ValueError: + logger.bind(tag=TAG).error("响应不是JSON格式") + return None + + except Exception as e: + logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True) + return None + + async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + """将语音数据转换为文本""" + if self._is_token_expired(): + logger.warning("Token已过期，正在自动刷新...") + self._refresh_token() + + try: + # 解码Opus为PCM + pcm_data_list = self.decode_opus(opus_data, session_id) + combined_pcm_data = b''.join(pcm_data_list) + + # 发送请求并获取文本 + text = await self._send_request(combined_pcm_data) + + file_path = self.save_audio_to_file(opus_data, session_id) + if self.delete_audio_file: + os.remove(file_path) + logger.bind(tag=TAG).debug(f"音频文件已删除: {file_path}") + + if text: + return text, None + return "", None + + except Exception as e: + logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) + return "", None \ No newline at end of file From c9eb64051007cfacbf5c2122f7c816679e745be2 Mon Sep 17 00:00:00 2001 From: LFNL-scholar Date: Tue, 8 Apr 2025 20:48:11 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=98=BF=E9=87=8C?= =?UTF-8?q?=E4=BA=91ASR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index ad85c712..b47ad6c9 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -109,9 +109,9 @@ selected_module: # 语音活动检测模块，默认使用SileroVAD模型 VAD: SileroVAD # 语音识别模块，默认使用FunASR本地模型 - ASR: AliyunASR + ASR: FunASR # 将根据配置名称对应的type调用实际的LLM适配器 - LLM: AliLLM + LLM: ChatGLMLLM # TTS将根据配置名称对应的type调用实际的TTS适配器 TTS: EdgeTTS # 记忆模块，默认不开启记忆；如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short @@ -249,7 +249,7 @@ LLM: # 可在这里找到你的 api_key https://bailian.console.aliyun.com/?apiKey=1#/api-key base_url: https://dashscope.aliyuncs.com/compatible-mode/v1 model_name: qwen-turbo - api_key: sk-a8b1fe19c885431c8a3769e18c02541e + api_key: 你的阿里云大模型 key temperature: 0.7 # 温度值 max_tokens: 500 # 最大生成token数 top_p: 1