diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index da14a6a4..ed8908f1 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -221,6 +221,8 @@ ASR: type: fun_server host: 127.0.0.1 port: 10096 + is_ssl: true + output_dir: tmp/ SherpaASR: type: sherpa_onnx_local model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17 @@ -253,7 +255,7 @@ ASR: type: aliyun appkey: 你的阿里云智能语音交互服务项目Appkey token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret - access_key_id: 的阿里云账号access_key_id + access_key_id: 你的阿里云账号access_key_id access_key_secret: 你的阿里云账号access_key_secret output_dir: tmp/ diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py index 4205bec2..250d3a66 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -25,14 +25,14 @@ class AccessToken: def _encode_text(text): encoded_text = parse.quote_plus(text) return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') - + @staticmethod def _encode_dict(dic): keys = dic.keys() dic_sorted = [(key, dic[key]) for key in sorted(keys)] encoded_text = parse.urlencode(dic_sorted) return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~') - + @staticmethod def create_token(access_key_id, access_key_secret): parameters = {'AccessKeyId': access_key_id, @@ -98,7 +98,7 @@ class ASRProvider(ASRProviderBase): # 直接使用预生成的长期token self.token = config.get("token") self.expire_time = None - + # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) @@ -107,7 +107,7 @@ class ASRProvider(ASRProviderBase): """刷新Token并记录过期时间""" if self.access_key_id and self.access_key_secret: self.token, expire_time_str = AccessToken.create_token( - self.access_key_id, + self.access_key_id, self.access_key_secret ) if not expire_time_str: @@ -121,16 +121,16 @@ class ASRProvider(ASRProviderBase): expire_time = datetime.fromtimestamp(int(expire_str)) else: expire_time = datetime.strptime( - expire_str, + expire_str, "%Y-%m-%dT%H:%M:%SZ" ) self.expire_time = expire_time.timestamp() - 60 except Exception as e: raise ValueError(f"无效的过期时间格式: {expire_str}") from e - + else: self.expire_time = None - + if not self.token: raise ValueError("无法获取有效的访问Token") @@ -173,13 +173,12 @@ class ASRProvider(ASRProviderBase): return pcm_data - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" - file_name = f"asr_{session_id}.wav" + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) - pcm_data = self.decode_opus(opus_data, session_id) - with wave.open(file_path, "wb") as wf: wf.setnchannels(1) # 单声道 wf.setsampwidth(2) # 16-bit @@ -202,7 +201,7 @@ class ASRProvider(ASRProviderBase): # 创建连接并发送请求 conn = http.client.HTTPSConnection(self.host) request_url = self._construct_request_url() - + loop = asyncio.get_event_loop() await loop.run_in_executor(None, lambda: conn.request( method='POST', @@ -220,7 +219,7 @@ class ASRProvider(ASRProviderBase): try: body_json = json.loads(body) status = body_json.get('status') - + if status == 20000000: result = body_json.get('result', '') logger.bind(tag=TAG).debug(f"ASR结果: {result}") @@ -228,7 +227,7 @@ class ASRProvider(ASRProviderBase): else: logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}") return None - + except ValueError: logger.bind(tag=TAG).error("响应不是JSON格式") return None @@ -242,24 +241,27 @@ class ASRProvider(ASRProviderBase): if self._is_token_expired(): logger.warning("Token已过期，正在自动刷新...") self._refresh_token() - + + file_path = None try: # 解码Opus为PCM - pcm_data_list = self.decode_opus(opus_data, session_id) - combined_pcm_data = b''.join(pcm_data_list) + pcm_data = self.decode_opus(opus_data, session_id) + combined_pcm_data = b''.join(pcm_data) + + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + file_path = self.save_audio_to_file(pcm_data, session_id) # 发送请求并获取文本 text = await self._send_request(combined_pcm_data) - - file_path = self.save_audio_to_file(opus_data, session_id) - if self.delete_audio_file: - os.remove(file_path) - logger.bind(tag=TAG).debug(f"音频文件已删除: {file_path}") if text: - return text, None - return "", None + return text, file_path + + return "", file_path except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", None \ No newline at end of file + return "", file_path \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index 755a19b7..ce7e05ee 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -103,7 +103,8 @@ class ASRProvider(ASRProviderBase): def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: """PCM数据保存为WAV文件""" - file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) with wave.open(file_path, "wb") as wf: @@ -139,8 +140,8 @@ class ASRProvider(ASRProviderBase): "uid": str(uuid.uuid4()), }, "request": { - "reqid": reqid, - "show_utterances": False, + "reqid": reqid, + "show_utterances": False, "sequence": 1, "boosting_table_name": self.boosting_table_name, "correct_table_name": self.correct_table_name, @@ -260,6 +261,8 @@ class ASRProvider(ASRProviderBase): self, opus_data: List[bytes], session_id: str ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" + + file_path = None try: # 合并所有opus数据包 pcm_data = self.decode_opus(opus_data, session_id) @@ -269,7 +272,7 @@ class ASRProvider(ASRProviderBase): if self.delete_audio_file: pass else: - self.save_audio_to_file(pcm_data, session_id) + file_path = self.save_audio_to_file(pcm_data, session_id) # 直接使用PCM数据 # 计算分段大小 (单声道, 16bit, 16kHz采样率) @@ -283,9 +286,9 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).debug( f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" ) - return text, None - return "", None + return text, file_path + return "", file_path except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", None + return "", file_path diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index a678780b..3724ec2e 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -50,21 +50,12 @@ class ASRProvider(ASRProviderBase): # device="cuda:0", # 启用GPU加速 ) - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: """PCM数据保存为WAV文件""" - file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - with wave.open(file_path, "wb") as wf: wf.setnchannels(1) wf.setsampwidth(2) # 2 bytes = 16-bit @@ -72,7 +63,7 @@ class ASRProvider(ASRProviderBase): wf.writeframes(b"".join(pcm_data)) return file_path - + @staticmethod def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: @@ -100,7 +91,7 @@ class ASRProvider(ASRProviderBase): if self.delete_audio_file: pass else: - self.save_audio_to_file(pcm_data, session_id) + file_path = self.save_audio_to_file(pcm_data, session_id) # 语音识别 start_time = time.time() @@ -118,13 +109,13 @@ class ASRProvider(ASRProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", None + return "", file_path - finally: - # 文件清理逻辑 - if self.delete_audio_file and file_path and os.path.exists(file_path): - try: - os.remove(file_path) - logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") - except Exception as e: - logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") + # finally: + # # 文件清理逻辑 + # if self.delete_audio_file and file_path and os.path.exists(file_path): + # try: + # os.remove(file_path) + # logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") + # except Exception as e: + # logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py index d0701d93..501d43bd 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_server.py +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -1,8 +1,11 @@ from typing import Optional, Tuple, List import opuslib_next from core.providers.asr.base import ASRProviderBase -import ssl +import os +import ssl import json +import uuid +import wave import websockets from config.logger import setup_logging import asyncio @@ -19,6 +22,7 @@ class ASRProvider(ASRProviderBase): self.host = config.get('host', 'localhost') self.port = config.get('port', 10095) self.is_ssl = config.get('is_ssl', True) + self.output_dir = config.get("output_dir") self.delete_audio_file = delete_audio_file self.uri = f"wss://{self.host}:{self.port}" if self.is_ssl else f"ws://{self.host}:{self.port}" self.ssl_context = ssl.SSLContext() if self.is_ssl else None @@ -26,25 +30,35 @@ class ASRProvider(ASRProviderBase): self.ssl_context.check_hostname = False self.ssl_context.verify_mode = ssl.CERT_NONE - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """解码Opus数据并保存为WAV文件""" - pass + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" + file_path = os.path.join(self.output_dir, file_name) + + with wave.open(file_path, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) # 2 bytes = 16-bit + wf.setframerate(16000) + wf.writeframes(b"".join(pcm_data)) + + return file_path @staticmethod def decode_opus(opus_data: List[bytes]) -> bytes: - """将Opus音频数据解码为PCM数据""" + """将Opus音频数据解码为PCM数据""" decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 pcm_data = [] - + for opus_packet in opus_data: try: pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms pcm_data.append(pcm_frame) except opuslib_next.OpusError as e: logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return b"".join(pcm_data) - + + return pcm_data + async def _receive_responses(self, ws) -> None: @@ -90,15 +104,15 @@ class ASRProvider(ASRProviderBase): }) await ws.send(config_message) logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}") - + # Send PCM data await ws.send(pcm_data) logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes") - + # Indicate end of speech end_message = json.dumps({"is_speaking": False}) await ws.send(end_message) - logger.bind(tag=TAG).debug(f"Sent end message: {end_message}") + logger.bind(tag=TAG).debug(f"Sent end message: {end_message}") async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: @@ -108,13 +122,20 @@ class ASRProvider(ASRProviderBase): :param session_id: Unique session identifier. :return: Tuple containing recognized text and optional timestamp. ''' - + file_path = None pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) + + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + file_path = self.save_audio_to_file(pcm_data, session_id) async with websockets.connect(self.uri, subprotocols=["binary"], ping_interval=None, ssl=self.ssl_context) as ws: try: # Use asyncio to handle WebSocket communication - send_task = asyncio.create_task(self._send_data(ws, pcm_data, session_id)) + send_task = asyncio.create_task(self._send_data(ws, combined_pcm_data, session_id)) receive_task = asyncio.create_task(self._receive_responses(ws)) # Gather tasks with error handling @@ -134,11 +155,11 @@ class ASRProvider(ASRProviderBase): # Get the result from the receive task result = receive_task.result() - return result, None # Return the recognized text and timestamp (if any) + return result, file_path # Return the recognized text and timestamp (if any) except websockets.exceptions.ConnectionClosed as e: logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}") - return "", None + return "", file_path except Exception as e: logger.bind(tag=TAG).error(f"Error during speech-to-text conversion: {e}", exc_info=True) - return "", None \ No newline at end of file + return "", file_path \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index cfcbe83b..3e720210 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -85,7 +85,8 @@ class ASRProvider(ASRProviderBase): def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: """PCM数据保存为WAV文件""" - file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) with wave.open(file_path, "wb") as wf: @@ -164,7 +165,7 @@ class ASRProvider(ASRProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", None + return "", file_path finally: # 文件清理逻辑 if self.delete_audio_file and file_path and os.path.exists(file_path): diff --git a/main/xiaozhi-server/core/providers/asr/tencent.py b/main/xiaozhi-server/core/providers/asr/tencent.py index 0ac297a6..dbdcbe26 100644 --- a/main/xiaozhi-server/core/providers/asr/tencent.py +++ b/main/xiaozhi-server/core/providers/asr/tencent.py @@ -26,40 +26,40 @@ class ASRProvider(ASRProviderBase): self.secret_id = config.get("secret_id") self.secret_key = config.get("secret_key") self.output_dir = config.get("output_dir") - + self.delete_audio_file = delete_audio_file + # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: """PCM数据保存为WAV文件""" - - file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) - + with wave.open(file_path, "wb") as wf: wf.setnchannels(1) wf.setsampwidth(2) # 2 bytes = 16-bit wf.setframerate(16000) wf.writeframes(b"".join(pcm_data)) - + return file_path @staticmethod def decode_opus(opus_data: List[bytes]) -> bytes: """将Opus音频数据解码为PCM数据""" - import opuslib_next - + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 pcm_data = [] - + for opus_packet in opus_data: try: pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms pcm_data.append(pcm_frame) except opuslib_next.OpusError as e: logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return b"".join(pcm_data) + + return pcm_data async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" @@ -67,23 +67,25 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).warn("音频数据为空！") return None, None + file_path = None try: # 检查配置是否已设置 if not self.secret_id or not self.secret_key: logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别") - return None, None + return None, file_path # 将Opus音频数据解码为PCM pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) # 判断是否保存为WAV文件 if self.delete_audio_file: pass else: self.save_audio_to_file(pcm_data, session_id) - + # 将音频数据转换为Base64编码 - base64_audio = base64.b64encode(pcm_data).decode('utf-8') + base64_audio = base64.b64encode(combined_pcm_data).decode('utf-8') # 构建请求体 request_body = self._build_request_body(base64_audio) @@ -94,15 +96,15 @@ class ASRProvider(ASRProviderBase): # 发送请求 start_time = time.time() result = self._send_request(request_body, timestamp, authorization) - + if result: logger.bind(tag=TAG).debug(f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}") - - return result, None + + return result, file_path except Exception as e: logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True) - return None, None + return None, file_path def _build_request_body(self, base64_audio: str) -> str: """构建请求体""" @@ -206,26 +208,26 @@ class ASRProvider(ASRProviderBase): try: response = requests.post(self.API_URL, headers=headers, data=request_body) - + if not response.ok: raise IOError(f"请求失败: {response.status_code} {response.reason}") - + response_json = response.json() - + # 检查是否有错误 if "Response" in response_json and "Error" in response_json["Response"]: error = response_json["Response"]["Error"] error_code = error["Code"] error_message = error["Message"] raise IOError(f"API返回错误: {error_code}: {error_message}") - + # 提取识别结果 if "Response" in response_json and "Result" in response_json["Response"]: return response_json["Response"]["Result"] else: logger.bind(tag=TAG).warn(f"响应中没有识别结果: {response_json}") return "" - + except Exception as e: logger.bind(tag=TAG).error(f"发送请求失败: {e}", exc_info=True) return None @@ -239,7 +241,7 @@ class ASRProvider(ASRProviderBase): """计算HMAC-SHA256""" if isinstance(key, str): key = key.encode('utf-8') - + return hmac.new(key, data.encode('utf-8'), hashlib.sha256).digest() def _bytes_to_hex(self, bytes_data: bytes) -> str: