From c2e000f93705e1a72589327d142cf33aaff98673 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 8 May 2025 11:11:28 +0800 Subject: [PATCH] =?UTF-8?q?add:asr=E8=B5=8B=E5=80=BCaudio=5Fformat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 3 +- .../core/providers/asr/aliyun.py | 106 ++++++++++-------- .../core/providers/asr/doubao.py | 2 +- .../core/providers/asr/fun_local.py | 14 ++- .../core/providers/asr/sherpa_onnx_local.py | 2 +- 5 files changed, 72 insertions(+), 55 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 5da1fa53..2aca4a3e 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -28,7 +28,8 @@ async def handleHelloMessage(conn, msg_json): format = audio_params.get("format") logger.bind(tag=TAG).info(f"客户端音频格式: {format}") conn.audio_format = format - conn.welcome_msg['audio_params'] = audio_params + conn.asr.set_audio_format(format) + conn.welcome_msg["audio_params"] = audio_params await conn.websocket.send(json.dumps(conn.welcome_msg)) diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py index be5541ee..14421d7d 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -20,62 +20,75 @@ from core.providers.asr.base import ASRProviderBase TAG = __name__ logger = setup_logging() + class AccessToken: @staticmethod def _encode_text(text): encoded_text = parse.quote_plus(text) - return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") @staticmethod def _encode_dict(dic): keys = dic.keys() dic_sorted = [(key, dic[key]) for key in sorted(keys)] encoded_text = parse.urlencode(dic_sorted) - return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") @staticmethod def create_token(access_key_id, access_key_secret): - parameters = {'AccessKeyId': access_key_id, - 'Action': 'CreateToken', - 'Format': 'JSON', - 'RegionId': 'cn-shanghai', - 'SignatureMethod': 'HMAC-SHA1', - 'SignatureNonce': str(uuid.uuid1()), - 'SignatureVersion': '1.0', - 'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - 'Version': '2019-02-28'} + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } # 构造规范化的请求字符串 query_string = AccessToken._encode_dict(parameters) # print('规范化的请求字符串: %s' % query_string) # 构造待签名字符串 - string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string) + string_to_sign = ( + "GET" + + "&" + + AccessToken._encode_text("/") + + "&" + + AccessToken._encode_text(query_string) + ) # print('待签名的字符串: %s' % string_to_sign) # 计算签名 - secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'), - bytes(string_to_sign, encoding='utf-8'), - hashlib.sha1).digest() + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() signature = base64.b64encode(secreted_string) # print('签名: %s' % signature) # 进行URL编码 signature = AccessToken._encode_text(signature) # print('URL编码后的签名: %s' % signature) # 调用服务 - full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string) + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % ( + signature, + query_string, + ) # print('url: %s' % full_url) # 提交HTTP GET请求 response = requests.get(full_url) if response.ok: root_obj = response.json() - key = 'Token' + key = "Token" if key in root_obj: - token = root_obj[key]['Id'] - expire_time = root_obj[key]['ExpireTime'] + token = root_obj[key]["Id"] + expire_time = root_obj[key]["ExpireTime"] return token, expire_time # print(response.text) return None, None - class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): """阿里云ASR初始化""" @@ -102,28 +115,23 @@ class ASRProvider(ASRProviderBase): # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) - def _refresh_token(self): """刷新Token并记录过期时间""" if self.access_key_id and self.access_key_secret: self.token, expire_time_str = AccessToken.create_token( - self.access_key_id, - self.access_key_secret + self.access_key_id, self.access_key_secret ) if not expire_time_str: raise ValueError("无法获取有效的Token过期时间") try: - #统一转换为字符串处理 + # 统一转换为字符串处理 expire_str = str(expire_time_str).strip() if expire_str.isdigit(): expire_time = datetime.fromtimestamp(int(expire_str)) else: - expire_time = datetime.strptime( - expire_str, - "%Y-%m-%dT%H:%M:%SZ" - ) + expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ") self.expire_time = expire_time.timestamp() - 60 except Exception as e: raise ValueError(f"无效的过期时间格式: {expire_str}") from e @@ -145,9 +153,12 @@ class ASRProvider(ASRProviderBase): # f"过期时间 {datetime.fromtimestamp(self.expire_time)} | " # f"剩余 {remaining:.2f}秒") return time.time() > self.expire_time - def generate_filename(self, extension=".wav"): - return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + def generate_filename(self, extension=".wav"): + return os.path.join( + self.output_file, + f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}", + ) def _construct_request_url(self) -> str: """构造请求URL，包含参数""" @@ -158,7 +169,6 @@ class ASRProvider(ASRProviderBase): request += "&enable_inverse_text_normalization=true" request += "&enable_voice_detection=false" return request - def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: """PCM数据保存为WAV文件""" @@ -170,7 +180,7 @@ class ASRProvider(ASRProviderBase): wf.setnchannels(1) # 单声道 wf.setsampwidth(2) # 16-bit wf.setframerate(self.sample_rate) - wf.writeframes(b''.join(pcm_data)) + wf.writeframes(b"".join(pcm_data)) logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}") return file_path @@ -180,9 +190,9 @@ class ASRProvider(ASRProviderBase): try: # 设置HTTP头 headers = { - 'X-NLS-Token': self.token, - 'Content-type': 'application/octet-stream', - 'Content-Length': str(len(pcm_data)) + "X-NLS-Token": self.token, + "Content-type": "application/octet-stream", + "Content-Length": str(len(pcm_data)), } # 创建连接并发送请求 @@ -190,12 +200,12 @@ class ASRProvider(ASRProviderBase): request_url = self._construct_request_url() loop = asyncio.get_event_loop() - await loop.run_in_executor(None, lambda: conn.request( - method='POST', - url=request_url, - body=pcm_data, - headers=headers - )) + await loop.run_in_executor( + None, + lambda: conn.request( + method="POST", url=request_url, body=pcm_data, headers=headers + ), + ) # 获取响应 response = await loop.run_in_executor(None, conn.getresponse) @@ -205,10 +215,10 @@ class ASRProvider(ASRProviderBase): # 解析响应 try: body_json = json.loads(body) - status = body_json.get('status') + status = body_json.get("status") if status == 20000000: - result = body_json.get('result', '') + result = body_json.get("result", "") logger.bind(tag=TAG).debug(f"ASR结果: {result}") return result else: @@ -223,7 +233,9 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True) return None - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if self._is_token_expired(): logger.warning("Token已过期，正在自动刷新...") @@ -235,8 +247,8 @@ class ASRProvider(ASRProviderBase): if self.audio_format == "pcm": pcm_data = opus_data else: - pcm_data = self.decode_opus(opus_data, session_id) - combined_pcm_data = b''.join(pcm_data) + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 if self.delete_audio_file: @@ -254,4 +266,4 @@ class ASRProvider(ASRProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", file_path \ No newline at end of file + return "", file_path diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index 0ca089c9..0775c2ab 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -253,7 +253,7 @@ class ASRProvider(ASRProviderBase): if self.audio_format == "pcm": pcm_data = opus_data else: - pcm_data = self.decode_opus(opus_data, session_id) + pcm_data = self.decode_opus(opus_data) combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 373d9c6a..107417b9 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -44,7 +44,7 @@ class ASRProvider(ASRProviderBase): model=self.model_dir, vad_kwargs={"max_single_segment_time": 30000}, disable_update=True, - hub="hf" + hub="hf", # device="cuda:0", # 启用GPU加速 ) @@ -62,7 +62,9 @@ class ASRProvider(ASRProviderBase): return file_path - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None try: @@ -70,8 +72,8 @@ class ASRProvider(ASRProviderBase): if self.audio_format == "pcm": pcm_data = opus_data else: - pcm_data = self.decode_opus(opus_data, session_id) - + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 @@ -90,7 +92,9 @@ class ASRProvider(ASRProviderBase): batch_size_s=60, ) text = rich_transcription_postprocess(result[0]["text"]) - logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}") + logger.bind(tag=TAG).debug( + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" + ) return text, file_path diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index 45e22336..2b86f6d7 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -132,7 +132,7 @@ class ASRProvider(ASRProviderBase): if self.audio_format == "pcm": pcm_data = opus_data else: - pcm_data = self.decode_opus(opus_data, session_id) + pcm_data = self.decode_opus(opus_data) file_path = self.save_audio_to_file(pcm_data, session_id) logger.bind(tag=TAG).debug( f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"