Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

规范asr部分的代码

Browse Source
This commit is contained in:
王华侨
2025-05-03 22:20:50 +08:00
parent 0f3806d358
commit 727621fdac
7 changed files with 122 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/config.yaml
+3 -1
View File
@@ -221,6 +221,8 @@ ASR:
type: fun_server
host: 127.0.0.1
port: 10096
is_ssl: true
output_dir: tmp/
SherpaASR:
type: sherpa_onnx_local
model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17
@@ -253,7 +255,7 @@ ASR:
type: aliyun
appkey: 你的阿里云智能语音交互服务项目Appkey
token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret
access_key_id: 的阿里云账号access_key_id
access_key_id: 你的阿里云账号access_key_id
access_key_secret: 你的阿里云账号access_key_secret
output_dir: tmp/
main/xiaozhi-server/core/providers/asr/aliyun.py
+28 -26
View File
@@ -25,14 +25,14 @@ class AccessToken:
def _encode_text(text):
encoded_text = parse.quote_plus(text)
return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
@staticmethod
def _encode_dict(dic):
keys = dic.keys()
dic_sorted = [(key, dic[key]) for key in sorted(keys)]
encoded_text = parse.urlencode(dic_sorted)
return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
@staticmethod
def create_token(access_key_id, access_key_secret):
parameters = {'AccessKeyId': access_key_id,
@@ -98,7 +98,7 @@ class ASRProvider(ASRProviderBase):
# 直接使用预生成的长期token
self.token = config.get("token")
self.expire_time = None
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
@@ -107,7 +107,7 @@ class ASRProvider(ASRProviderBase):
"""刷新Token并记录过期时间"""
if self.access_key_id and self.access_key_secret:
self.token, expire_time_str = AccessToken.create_token(
self.access_key_id,
self.access_key_id,
self.access_key_secret
)
if not expire_time_str:
@@ -121,16 +121,16 @@ class ASRProvider(ASRProviderBase):
expire_time = datetime.fromtimestamp(int(expire_str))
else:
expire_time = datetime.strptime(
expire_str,
expire_str,
"%Y-%m-%dT%H:%M:%SZ"
)
self.expire_time = expire_time.timestamp() - 60
except Exception as e:
raise ValueError(f"无效的过期时间格式: {expire_str}") from e
else:
self.expire_time = None
if not self.token:
raise ValueError("无法获取有效的访问Token")
@@ -173,13 +173,12 @@ class ASRProvider(ASRProviderBase):
return pcm_data
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""将Opus音频数据解码并保存为WAV文件"""
file_name = f"asr_{session_id}.wav"
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
pcm_data = self.decode_opus(opus_data, session_id)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1) # 单声道
wf.setsampwidth(2) # 16-bit
@@ -202,7 +201,7 @@ class ASRProvider(ASRProviderBase):
# 创建连接并发送请求
conn = http.client.HTTPSConnection(self.host)
request_url = self._construct_request_url()
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, lambda: conn.request(
method='POST',
@@ -220,7 +219,7 @@ class ASRProvider(ASRProviderBase):
try:
body_json = json.loads(body)
status = body_json.get('status')
if status == 20000000:
result = body_json.get('result', '')
logger.bind(tag=TAG).debug(f"ASR结果: {result}")
@@ -228,7 +227,7 @@ class ASRProvider(ASRProviderBase):
else:
logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}")
return None
except ValueError:
logger.bind(tag=TAG).error("响应不是JSON格式")
return None
@@ -242,24 +241,27 @@ class ASRProvider(ASRProviderBase):
if self._is_token_expired():
logger.warning("Token已过期，正在自动刷新...")
self._refresh_token()
file_path = None
try:
# 解码Opus为PCM
pcm_data_list = self.decode_opus(opus_data, session_id)
combined_pcm_data = b''.join(pcm_data_list)
pcm_data = self.decode_opus(opus_data, session_id)
combined_pcm_data = b''.join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
# 发送请求并获取文本
text = await self._send_request(combined_pcm_data)
file_path = self.save_audio_to_file(opus_data, session_id)
if self.delete_audio_file:
os.remove(file_path)
logger.bind(tag=TAG).debug(f"音频文件已删除: {file_path}")
if text:
return text, None
return "", None
return text, file_path
return "", file_path
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", None
return "", file_path
main/xiaozhi-server/core/providers/asr/doubao.py
+10 -7
View File
@@ -103,7 +103,8 @@ class ASRProvider(ASRProviderBase):
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
with wave.open(file_path, "wb") as wf:
@@ -139,8 +140,8 @@ class ASRProvider(ASRProviderBase):
"uid": str(uuid.uuid4()),
},
"request": {
"reqid": reqid,
"show_utterances": False,
"reqid": reqid,
"show_utterances": False,
"sequence": 1,
"boosting_table_name": self.boosting_table_name,
"correct_table_name": self.correct_table_name,
@@ -260,6 +261,8 @@ class ASRProvider(ASRProviderBase):
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
file_path = None
try:
# 合并所有opus数据包
pcm_data = self.decode_opus(opus_data, session_id)
@@ -269,7 +272,7 @@ class ASRProvider(ASRProviderBase):
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
file_path = self.save_audio_to_file(pcm_data, session_id)
# 直接使用PCM数据
# 计算分段大小 (单声道, 16bit, 16kHz采样率)
@@ -283,9 +286,9 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).debug(
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
)
return text, None
return "", None
return text, file_path
return "", file_path
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", None
return "", file_path
main/xiaozhi-server/core/providers/asr/fun_local.py
+14 -23
View File
@@ -50,21 +50,12 @@ class ASRProvider(ASRProviderBase):
# device="cuda:0", # 启用GPU加速
)
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
@@ -72,7 +63,7 @@ class ASRProvider(ASRProviderBase):
wf.writeframes(b"".join(pcm_data))
return file_path
@staticmethod
def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
@@ -100,7 +91,7 @@ class ASRProvider(ASRProviderBase):
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
file_path = self.save_audio_to_file(pcm_data, session_id)
# 语音识别
start_time = time.time()
@@ -118,13 +109,13 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", None
return "", file_path
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
try:
os.remove(file_path)
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
except Exception as e:
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
# finally:
# # 文件清理逻辑
# if self.delete_audio_file and file_path and os.path.exists(file_path):
# try:
# os.remove(file_path)
# logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
# except Exception as e:
# logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
main/xiaozhi-server/core/providers/asr/fun_server.py
+38 -17
View File
@@ -1,8 +1,11 @@
from typing import Optional, Tuple, List
import opuslib_next
from core.providers.asr.base import ASRProviderBase
import ssl
import os
import ssl
import json
import uuid
import wave
import websockets
from config.logger import setup_logging
import asyncio
@@ -19,6 +22,7 @@ class ASRProvider(ASRProviderBase):
self.host = config.get('host', 'localhost')
self.port = config.get('port', 10095)
self.is_ssl = config.get('is_ssl', True)
self.output_dir = config.get("output_dir")
self.delete_audio_file = delete_audio_file
self.uri = f"wss://{self.host}:{self.port}" if self.is_ssl else f"ws://{self.host}:{self.port}"
self.ssl_context = ssl.SSLContext() if self.is_ssl else None
@@ -26,25 +30,35 @@ class ASRProvider(ASRProviderBase):
self.ssl_context.check_hostname = False
self.ssl_context.verify_mode = ssl.CERT_NONE
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""解码Opus数据并保存为WAV文件"""
pass
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
wf.setframerate(16000)
wf.writeframes(b"".join(pcm_data))
return file_path
@staticmethod
def decode_opus(opus_data: List[bytes]) -> bytes:
"""将Opus音频数据解码为PCM数据"""
"""将Opus音频数据解码为PCM数据"""
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return b"".join(pcm_data)
return pcm_data
async def _receive_responses(self, ws) -> None:
@@ -90,15 +104,15 @@ class ASRProvider(ASRProviderBase):
})
await ws.send(config_message)
logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}")
# Send PCM data
await ws.send(pcm_data)
logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes")
# Indicate end of speech
end_message = json.dumps({"is_speaking": False})
await ws.send(end_message)
logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
@@ -108,13 +122,20 @@ class ASRProvider(ASRProviderBase):
:param session_id: Unique session identifier.
:return: Tuple containing recognized text and optional timestamp.
'''
file_path = None
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
async with websockets.connect(self.uri, subprotocols=["binary"], ping_interval=None, ssl=self.ssl_context) as ws:
try:
# Use asyncio to handle WebSocket communication
send_task = asyncio.create_task(self._send_data(ws, pcm_data, session_id))
send_task = asyncio.create_task(self._send_data(ws, combined_pcm_data, session_id))
receive_task = asyncio.create_task(self._receive_responses(ws))
# Gather tasks with error handling
@@ -134,11 +155,11 @@ class ASRProvider(ASRProviderBase):
# Get the result from the receive task
result = receive_task.result()
return result, None # Return the recognized text and timestamp (if any)
return result, file_path # Return the recognized text and timestamp (if any)
except websockets.exceptions.ConnectionClosed as e:
logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
return "", None
return "", file_path
except Exception as e:
logger.bind(tag=TAG).error(f"Error during speech-to-text conversion: {e}", exc_info=True)
return "", None
return "", file_path
main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py
+3 -2
View File
@@ -85,7 +85,8 @@ class ASRProvider(ASRProviderBase):
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
with wave.open(file_path, "wb") as wf:
@@ -164,7 +165,7 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", None
return "", file_path
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
main/xiaozhi-server/core/providers/asr/tencent.py
+26 -24
View File
@@ -26,40 +26,40 @@ class ASRProvider(ASRProviderBase):
self.secret_id = config.get("secret_id")
self.secret_key = config.get("secret_key")
self.output_dir = config.get("output_dir")
self.delete_audio_file = delete_audio_file
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav"
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
wf.setframerate(16000)
wf.writeframes(b"".join(pcm_data))
return file_path
@staticmethod
def decode_opus(opus_data: List[bytes]) -> bytes:
"""将Opus音频数据解码为PCM数据"""
import opuslib_next
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return b"".join(pcm_data)
return pcm_data
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
@@ -67,23 +67,25 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).warn("音频数据为空！")
return None, None
file_path = None
try:
# 检查配置是否已设置
if not self.secret_id or not self.secret_key:
logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别")
return None, None
return None, file_path
# 将Opus音频数据解码为PCM
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
# 将音频数据转换为Base64编码
base64_audio = base64.b64encode(pcm_data).decode('utf-8')
base64_audio = base64.b64encode(combined_pcm_data).decode('utf-8')
# 构建请求体
request_body = self._build_request_body(base64_audio)
@@ -94,15 +96,15 @@ class ASRProvider(ASRProviderBase):
# 发送请求
start_time = time.time()
result = self._send_request(request_body, timestamp, authorization)
if result:
logger.bind(tag=TAG).debug(f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}")
return result, None
return result, file_path
except Exception as e:
logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
return None, None
return None, file_path
def _build_request_body(self, base64_audio: str) -> str:
"""构建请求体"""
@@ -206,26 +208,26 @@ class ASRProvider(ASRProviderBase):
try:
response = requests.post(self.API_URL, headers=headers, data=request_body)
if not response.ok:
raise IOError(f"请求失败: {response.status_code} {response.reason}")
response_json = response.json()
# 检查是否有错误
if "Response" in response_json and "Error" in response_json["Response"]:
error = response_json["Response"]["Error"]
error_code = error["Code"]
error_message = error["Message"]
raise IOError(f"API返回错误: {error_code}: {error_message}")
# 提取识别结果
if "Response" in response_json and "Result" in response_json["Response"]:
return response_json["Response"]["Result"]
else:
logger.bind(tag=TAG).warn(f"响应中没有识别结果: {response_json}")
return ""
except Exception as e:
logger.bind(tag=TAG).error(f"发送请求失败: {e}", exc_info=True)
return None
@@ -239,7 +241,7 @@ class ASRProvider(ASRProviderBase):
"""计算HMAC-SHA256"""
if isinstance(key, str):
key = key.encode('utf-8')
return hmac.new(key, data.encode('utf-8'), hashlib.sha256).digest()
def _bytes_to_hex(self, bytes_data: bytes) -> str: