mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-31 03:43:57 +08:00
resolve merge conflict
This commit is contained in:
caixypromise
@@ -8,8 +8,6 @@ import asyncio
|
||||
import requests
|
||||
import websockets
|
||||
import opuslib_next
|
||||
import random
|
||||
from typing import Optional, Tuple, List
|
||||
from urllib import parse
|
||||
from datetime import datetime
|
||||
from config.logger import setup_logging
|
||||
@@ -96,6 +94,8 @@ class ASRProvider(ASRProviderBase):
|
||||
self.delete_audio_file = delete_audio_file
|
||||
self.expire_time = None
|
||||
|
||||
self.task_id = uuid.uuid4().hex
|
||||
|
||||
# Token管理
|
||||
if self.access_key_id and self.access_key_secret:
|
||||
self._refresh_token()
|
||||
@@ -137,13 +137,13 @@ class ASRProvider(ASRProviderBase):
|
||||
conn.asr_audio.append(audio)
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
|
||||
# 只在有声音且没有连接时建立连接
|
||||
if audio_have_voice and not self.is_processing:
|
||||
# 只在有声音且没有连接时建立连接（排除正在停止的情况）
|
||||
if audio_have_voice and not self.is_processing and not self.asr_ws:
|
||||
try:
|
||||
await self._start_recognition(conn)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}")
|
||||
await self._cleanup(conn)
|
||||
await self._cleanup()
|
||||
return
|
||||
|
||||
if self.asr_ws and self.is_processing and self.server_ready:
|
||||
@@ -169,20 +169,22 @@ class ASRProvider(ASRProviderBase):
|
||||
ping_timeout=None,
|
||||
close_timeout=5,
|
||||
)
|
||||
|
||||
|
||||
self.task_id = uuid.uuid4().hex
|
||||
|
||||
logger.bind(tag=TAG).debug(f"WebSocket连接建立成功, task_id: {self.task_id}")
|
||||
|
||||
self.is_processing = True
|
||||
self.server_ready = False # 重置服务器准备状态
|
||||
self.forward_task = asyncio.create_task(self._forward_results(conn))
|
||||
|
||||
|
||||
# 发送开始请求
|
||||
start_request = {
|
||||
"header": {
|
||||
"namespace": "SpeechTranscriber",
|
||||
"name": "StartTranscription",
|
||||
"status": 20000000,
|
||||
"message_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"task_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"status_text": "Gateway:SUCCESS:Success.",
|
||||
"message_id": uuid.uuid4().hex,
|
||||
"task_id": self.task_id,
|
||||
"appkey": self.appkey
|
||||
},
|
||||
"payload": {
|
||||
@@ -196,23 +198,26 @@ class ASRProvider(ASRProviderBase):
|
||||
}
|
||||
}
|
||||
await self.asr_ws.send(json.dumps(start_request, ensure_ascii=False))
|
||||
logger.bind(tag=TAG).info("已发送开始请求，等待服务器准备...")
|
||||
logger.bind(tag=TAG).debug("已发送开始请求，等待服务器准备...")
|
||||
|
||||
async def _forward_results(self, conn):
|
||||
"""转发识别结果"""
|
||||
try:
|
||||
while self.asr_ws and not conn.stop_event.is_set():
|
||||
while not conn.stop_event.is_set():
|
||||
try:
|
||||
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
|
||||
result = json.loads(response)
|
||||
|
||||
|
||||
header = result.get("header", {})
|
||||
payload = result.get("payload", {})
|
||||
message_name = header.get("name", "")
|
||||
status = header.get("status", 0)
|
||||
|
||||
|
||||
if status != 20000000:
|
||||
if status in [40000004, 40010004]: # 连接超时或客户端断开
|
||||
if status == 40010004:
|
||||
logger.bind(tag=TAG).warning(f"请在服务端响应完成后再关闭链接，状态码: {status}")
|
||||
break
|
||||
if status in [40000004, 40010003]: # 连接超时或客户端断开
|
||||
logger.bind(tag=TAG).warning(f"连接问题，状态码: {status}")
|
||||
break
|
||||
elif status in [40270002, 40270003]: # 音频问题
|
||||
@@ -221,12 +226,12 @@ class ASRProvider(ASRProviderBase):
|
||||
else:
|
||||
logger.bind(tag=TAG).error(f"识别错误，状态码: {status}, 消息: {header.get('status_text', '')}")
|
||||
continue
|
||||
|
||||
|
||||
# 收到TranscriptionStarted表示服务器准备好接收音频数据
|
||||
if message_name == "TranscriptionStarted":
|
||||
self.server_ready = True
|
||||
logger.bind(tag=TAG).info("服务器已准备，开始发送缓存音频...")
|
||||
|
||||
logger.bind(tag=TAG).debug("服务器已准备，开始发送缓存音频...")
|
||||
|
||||
# 发送缓存音频
|
||||
if conn.asr_audio:
|
||||
for cached_audio in conn.asr_audio[-10:]:
|
||||
@@ -237,88 +242,89 @@ class ASRProvider(ASRProviderBase):
|
||||
logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}")
|
||||
break
|
||||
continue
|
||||
|
||||
if message_name == "TranscriptionResultChanged":
|
||||
# 中间结果
|
||||
text = payload.get("result", "")
|
||||
if text:
|
||||
self.text = text
|
||||
elif message_name == "SentenceEnd":
|
||||
# 最终结果
|
||||
# 句子结束（每个句子都会触发）
|
||||
text = payload.get("result", "")
|
||||
if text:
|
||||
self.text = text
|
||||
conn.reset_vad_states()
|
||||
# 传递缓存的音频数据
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清空缓存
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
break
|
||||
elif message_name == "TranscriptionCompleted":
|
||||
# 识别完成
|
||||
self.is_processing = False
|
||||
break
|
||||
|
||||
logger.bind(tag=TAG).info(f"识别到文本: {text}")
|
||||
|
||||
# 手动模式下累积识别结果
|
||||
if conn.client_listen_mode == "manual":
|
||||
if self.text:
|
||||
self.text += text
|
||||
else:
|
||||
self.text = text
|
||||
|
||||
# 手动模式下，只有在收到stop信号后才触发处理（仅处理一次）
|
||||
if conn.client_voice_stop:
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
if len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
break
|
||||
else:
|
||||
# 自动模式下直接覆盖
|
||||
self.text = text
|
||||
conn.reset_vad_states()
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
except websockets.exceptions.ConnectionClosed:
|
||||
logger.bind(tag=TAG).error("接收结果超时")
|
||||
break
|
||||
except websockets.ConnectionClosed:
|
||||
logger.bind(tag=TAG).info("ASR服务连接已关闭")
|
||||
self.is_processing = False
|
||||
break
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}")
|
||||
break
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}")
|
||||
finally:
|
||||
await self._cleanup(conn)
|
||||
# 清理连接的音频缓存
|
||||
await self._cleanup()
|
||||
if conn:
|
||||
if hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, 'asr_audio'):
|
||||
conn.asr_audio = []
|
||||
|
||||
async def _cleanup(self, conn):
|
||||
"""清理资源"""
|
||||
logger.bind(tag=TAG).info(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
|
||||
|
||||
# 清理连接的音频缓存
|
||||
if conn and hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
|
||||
# 判断是否需要发送终止请求
|
||||
should_stop = self.is_processing or self.server_ready
|
||||
|
||||
# 发送停止识别请求
|
||||
if self.asr_ws and should_stop:
|
||||
async def _send_stop_request(self):
|
||||
"""发送停止识别请求（不关闭连接）"""
|
||||
if self.asr_ws:
|
||||
try:
|
||||
# 先停止音频发送
|
||||
self.is_processing = False
|
||||
|
||||
stop_msg = {
|
||||
"header": {
|
||||
"namespace": "SpeechTranscriber",
|
||||
"name": "StopTranscription",
|
||||
"status": 20000000,
|
||||
"message_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"status_text": "Client:Stop",
|
||||
"message_id": uuid.uuid4().hex,
|
||||
"task_id": self.task_id,
|
||||
"appkey": self.appkey
|
||||
}
|
||||
}
|
||||
logger.bind(tag=TAG).info("正在发送ASR终止请求")
|
||||
logger.bind(tag=TAG).debug("停止识别请求已发送")
|
||||
await self.asr_ws.send(json.dumps(stop_msg, ensure_ascii=False))
|
||||
await asyncio.sleep(0.1)
|
||||
logger.bind(tag=TAG).info("ASR终止请求已发送")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR终止请求发送失败: {e}")
|
||||
|
||||
# 状态重置（在终止请求发送后）
|
||||
logger.bind(tag=TAG).error(f"发送停止识别请求失败: {e}")
|
||||
|
||||
async def _cleanup(self):
|
||||
"""清理资源（关闭连接）"""
|
||||
logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
|
||||
|
||||
# 状态重置
|
||||
self.is_processing = False
|
||||
self.server_ready = False
|
||||
logger.bind(tag=TAG).info("ASR状态已重置")
|
||||
logger.bind(tag=TAG).debug("ASR状态已重置")
|
||||
|
||||
# 清理任务
|
||||
if self.forward_task and not self.forward_task.done():
|
||||
self.forward_task.cancel()
|
||||
try:
|
||||
await asyncio.wait_for(self.forward_task, timeout=1.0)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}")
|
||||
finally:
|
||||
self.forward_task = None
|
||||
|
||||
# 关闭连接
|
||||
if self.asr_ws:
|
||||
try:
|
||||
@@ -329,8 +335,11 @@ class ASRProvider(ASRProviderBase):
|
||||
logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
|
||||
finally:
|
||||
self.asr_ws = None
|
||||
|
||||
logger.bind(tag=TAG).info("ASR会话清理完成")
|
||||
|
||||
# 清理任务引用
|
||||
self.forward_task = None
|
||||
|
||||
logger.bind(tag=TAG).debug("ASR会话清理完成")
|
||||
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format):
|
||||
"""获取识别结果"""
|
||||
@@ -340,4 +349,11 @@ class ASRProvider(ASRProviderBase):
|
||||
|
||||
async def close(self):
|
||||
"""关闭资源"""
|
||||
await self._cleanup()
|
||||
await self._cleanup(None)
|
||||
if hasattr(self, 'decoder') and self.decoder is not None:
|
||||
try:
|
||||
del self.decoder
|
||||
self.decoder = None
|
||||
logger.bind(tag=TAG).debug("Aliyun decoder resources released")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"释放Aliyun decoder资源时出错: {e}")
|
||||
|
||||
@@ -9,7 +9,6 @@ import asyncio
|
||||
import traceback
|
||||
import threading
|
||||
import opuslib_next
|
||||
import concurrent.futures
|
||||
from abc import ABC, abstractmethod
|
||||
from config.logger import setup_logging
|
||||
from typing import Optional, Tuple, List
|
||||
@@ -118,121 +117,89 @@ class ASRProviderBase(ABC):
|
||||
|
||||
# 接收音频
|
||||
async def receive_audio(self, conn, audio, audio_have_voice):
|
||||
if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime":
|
||||
have_voice = audio_have_voice
|
||||
if conn.client_listen_mode == "manual":
|
||||
# 手动模式：缓存音频用于ASR识别
|
||||
conn.asr_audio.append(audio)
|
||||
else:
|
||||
have_voice = conn.client_have_voice
|
||||
|
||||
conn.asr_audio.append(audio)
|
||||
if not have_voice and not conn.client_have_voice:
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
return
|
||||
# 自动/实时模式：使用VAD检测
|
||||
have_voice = audio_have_voice
|
||||
|
||||
if conn.client_voice_stop:
|
||||
asr_audio_task = conn.asr_audio.copy()
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
conn.asr_audio.append(audio)
|
||||
if not have_voice and not conn.client_have_voice:
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
return
|
||||
|
||||
if len(asr_audio_task) > 15:
|
||||
await self.handle_voice_stop(conn, asr_audio_task)
|
||||
# 自动模式下通过VAD检测到语音停止时触发识别
|
||||
if conn.client_voice_stop:
|
||||
asr_audio_task = conn.asr_audio.copy()
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
|
||||
if len(asr_audio_task) > 15:
|
||||
await self.handle_voice_stop(conn, asr_audio_task)
|
||||
|
||||
# 处理语音停止
|
||||
async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
|
||||
"""并行处理ASR和声纹识别"""
|
||||
try:
|
||||
total_start_time = time.monotonic()
|
||||
|
||||
|
||||
# 准备音频数据
|
||||
if conn.audio_format == "pcm":
|
||||
pcm_data = asr_audio_task
|
||||
else:
|
||||
pcm_data = self.decode_opus(asr_audio_task)
|
||||
|
||||
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
|
||||
|
||||
# 预先准备WAV数据
|
||||
wav_data = None
|
||||
if conn.voiceprint_provider and combined_pcm_data:
|
||||
wav_data = self._pcm_to_wav(combined_pcm_data)
|
||||
|
||||
|
||||
# 定义ASR任务
|
||||
def run_asr():
|
||||
start_time = time.monotonic()
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
result = loop.run_until_complete(
|
||||
self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
|
||||
)
|
||||
end_time = time.monotonic()
|
||||
logger.bind(tag=TAG).info(f"ASR耗时: {end_time - start_time:.3f}s")
|
||||
return result
|
||||
finally:
|
||||
loop.close()
|
||||
except Exception as e:
|
||||
end_time = time.monotonic()
|
||||
logger.bind(tag=TAG).error(f"ASR失败: {e}")
|
||||
return ("", None)
|
||||
|
||||
# 定义声纹识别任务
|
||||
def run_voiceprint():
|
||||
if not wav_data:
|
||||
return None
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
# 使用连接的声纹识别提供者
|
||||
result = loop.run_until_complete(
|
||||
conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
|
||||
)
|
||||
return result
|
||||
finally:
|
||||
loop.close()
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"声纹识别失败: {e}")
|
||||
return None
|
||||
|
||||
# 使用线程池执行器并行运行
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor:
|
||||
asr_future = thread_executor.submit(run_asr)
|
||||
|
||||
if conn.voiceprint_provider and wav_data:
|
||||
voiceprint_future = thread_executor.submit(run_voiceprint)
|
||||
|
||||
# 等待两个线程都完成
|
||||
asr_result = asr_future.result(timeout=15)
|
||||
voiceprint_result = voiceprint_future.result(timeout=15)
|
||||
|
||||
results = {"asr": asr_result, "voiceprint": voiceprint_result}
|
||||
else:
|
||||
asr_result = asr_future.result(timeout=15)
|
||||
results = {"asr": asr_result, "voiceprint": None}
|
||||
|
||||
|
||||
# 处理结果
|
||||
raw_text, _ = results.get("asr", ("", None))
|
||||
speaker_name = results.get("voiceprint", None)
|
||||
|
||||
# 记录识别结果
|
||||
asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
|
||||
|
||||
if conn.voiceprint_provider and wav_data:
|
||||
voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
|
||||
# 并发等待两个结果
|
||||
asr_result, voiceprint_result = await asyncio.gather(
|
||||
asr_task, voiceprint_task, return_exceptions=True
|
||||
)
|
||||
else:
|
||||
asr_result = await asr_task
|
||||
voiceprint_result = None
|
||||
|
||||
# 记录识别结果 - 检查是否为异常
|
||||
if isinstance(asr_result, Exception):
|
||||
logger.bind(tag=TAG).error(f"ASR识别失败: {asr_result}")
|
||||
raw_text = ""
|
||||
else:
|
||||
raw_text, _ = asr_result
|
||||
|
||||
if isinstance(voiceprint_result, Exception):
|
||||
logger.bind(tag=TAG).error(f"声纹识别失败: {voiceprint_result}")
|
||||
speaker_name = ""
|
||||
else:
|
||||
speaker_name = voiceprint_result
|
||||
|
||||
if raw_text:
|
||||
logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
|
||||
if speaker_name:
|
||||
logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
|
||||
|
||||
|
||||
# 性能监控
|
||||
total_time = time.monotonic() - total_start_time
|
||||
logger.bind(tag=TAG).info(f"总处理耗时: {total_time:.3f}s")
|
||||
|
||||
logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s")
|
||||
|
||||
# 检查文本长度
|
||||
text_len, _ = remove_punctuation_and_length(raw_text)
|
||||
self.stop_ws_connection()
|
||||
|
||||
|
||||
if text_len > 0:
|
||||
# 构建包含说话人信息的JSON字符串
|
||||
enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
|
||||
|
||||
|
||||
# 使用自定义模块进行上报
|
||||
await startToChat(conn, enhanced_text)
|
||||
enqueue_asr_report(conn, enhanced_text, asr_audio_task)
|
||||
@@ -306,6 +273,7 @@ class ASRProviderBase(ABC):
|
||||
@staticmethod
|
||||
def decode_opus(opus_data: List[bytes]) -> List[bytes]:
|
||||
"""将Opus音频数据解码为PCM数据"""
|
||||
decoder = None
|
||||
try:
|
||||
decoder = opuslib_next.Decoder(16000, 1)
|
||||
pcm_data = []
|
||||
@@ -330,3 +298,9 @@ class ASRProviderBase(ABC):
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
|
||||
return []
|
||||
finally:
|
||||
if decoder is not None:
|
||||
try:
|
||||
del decoder
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"释放decoder资源时出错: {e}")
|
||||
|
||||
@@ -18,8 +18,6 @@ class ASRProvider(ASRProviderBase):
|
||||
self.interface_type = InterfaceType.STREAM
|
||||
self.config = config
|
||||
self.text = ""
|
||||
self.max_retries = 3
|
||||
self.retry_delay = 2
|
||||
self.decoder = opuslib_next.Decoder(16000, 1)
|
||||
self.asr_ws = None
|
||||
self.forward_task = None
|
||||
@@ -49,6 +47,8 @@ class ASRProvider(ASRProviderBase):
|
||||
self.channel = config.get("channel", 1)
|
||||
self.auth_method = config.get("auth_method", "token")
|
||||
self.secret = config.get("secret", "access_secret")
|
||||
end_window_size = config.get("end_window_size")
|
||||
self.end_window_size = int(end_window_size) if end_window_size else 200
|
||||
|
||||
async def open_audio_channels(self, conn):
|
||||
await super().open_audio_channels(conn)
|
||||
@@ -56,14 +56,13 @@ class ASRProvider(ASRProviderBase):
|
||||
async def receive_audio(self, conn, audio, audio_have_voice):
|
||||
conn.asr_audio.append(audio)
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
|
||||
# 存储音频数据
|
||||
if not hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
conn.asr_audio_for_voiceprint.append(audio)
|
||||
|
||||
|
||||
# 当没有音频数据时处理完整语音片段
|
||||
if not audio and len(conn.asr_audio_for_voiceprint) > 0:
|
||||
if conn.client_listen_mode != "manual" and not audio and len(conn.asr_audio_for_voiceprint) > 0:
|
||||
await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint)
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
|
||||
@@ -179,6 +178,7 @@ class ASRProvider(ASRProviderBase):
|
||||
payload.get("audio_info", {}).get("duration", 0) > 2000
|
||||
and not utterances
|
||||
and not payload["result"].get("text")
|
||||
and conn.client_listen_mode != "manual"
|
||||
):
|
||||
logger.bind(tag=TAG).error(f"识别文本：空")
|
||||
self.text = ""
|
||||
@@ -187,15 +187,44 @@ class ASRProvider(ASRProviderBase):
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
|
||||
# 专门处理没有文本的识别结果（手动模式下可能已经识别完成但是没松按键）
|
||||
elif not payload["result"].get("text") and not utterances:
|
||||
if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
break
|
||||
|
||||
for utterance in utterances:
|
||||
if utterance.get("definite", False):
|
||||
self.text = utterance["text"]
|
||||
current_text = utterance["text"]
|
||||
logger.bind(tag=TAG).info(
|
||||
f"识别到文本: {self.text}"
|
||||
f"识别到文本: {current_text}"
|
||||
)
|
||||
conn.reset_vad_states()
|
||||
if len(audio_data) > 15: # 确保有足够音频数据
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
|
||||
# 手动模式下累积识别结果
|
||||
if conn.client_listen_mode == "manual":
|
||||
if self.text:
|
||||
self.text += current_text
|
||||
else:
|
||||
self.text = current_text
|
||||
|
||||
# 在接收消息中途时收到停止信号
|
||||
if conn.client_voice_stop and len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("消息中途收到停止信号，触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
break
|
||||
else:
|
||||
# 自动模式下直接覆盖
|
||||
self.text = current_text
|
||||
conn.reset_vad_states()
|
||||
if len(audio_data) > 15: # 确保有足够音频数据
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
break
|
||||
elif "error" in payload:
|
||||
error_msg = payload.get("error", "未知错误")
|
||||
@@ -227,8 +256,6 @@ class ASRProvider(ASRProviderBase):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, 'asr_audio'):
|
||||
conn.asr_audio = []
|
||||
if hasattr(conn, 'has_valid_voice'):
|
||||
conn.has_valid_voice = False
|
||||
|
||||
def stop_ws_connection(self):
|
||||
if self.asr_ws:
|
||||
@@ -236,6 +263,20 @@ class ASRProvider(ASRProviderBase):
|
||||
self.asr_ws = None
|
||||
self.is_processing = False
|
||||
|
||||
async def _send_stop_request(self):
|
||||
"""发送最后一个音频帧以通知服务器结束"""
|
||||
if self.asr_ws:
|
||||
try:
|
||||
# 发送结束标记的音频帧（gzip压缩的空数据）
|
||||
empty_payload = gzip.compress(b"")
|
||||
last_audio_request = bytearray(self.generate_last_audio_default_header())
|
||||
last_audio_request.extend(len(empty_payload).to_bytes(4, "big"))
|
||||
last_audio_request.extend(empty_payload)
|
||||
await self.asr_ws.send(last_audio_request)
|
||||
logger.bind(tag=TAG).debug("已发送结束音频帧")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"发送结束音频帧时出错: {e}")
|
||||
|
||||
def construct_request(self, reqid):
|
||||
req = {
|
||||
"app": {
|
||||
@@ -252,7 +293,7 @@ class ASRProvider(ASRProviderBase):
|
||||
"sequence": 1,
|
||||
"boosting_table_name": self.boosting_table_name,
|
||||
"correct_table_name": self.correct_table_name,
|
||||
"end_window_size": 200,
|
||||
"end_window_size": self.end_window_size,
|
||||
},
|
||||
"audio": {
|
||||
"format": self.format,
|
||||
@@ -370,6 +411,16 @@ class ASRProvider(ASRProviderBase):
|
||||
pass
|
||||
self.forward_task = None
|
||||
self.is_processing = False
|
||||
|
||||
# 显式释放decoder资源
|
||||
if hasattr(self, 'decoder') and self.decoder is not None:
|
||||
try:
|
||||
del self.decoder
|
||||
self.decoder = None
|
||||
logger.bind(tag=TAG).debug("Doubao decoder resources released")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")
|
||||
|
||||
# 清理所有连接的音频缓冲区
|
||||
if hasattr(self, '_connections'):
|
||||
for conn in self._connections.values():
|
||||
@@ -377,5 +428,3 @@ class ASRProvider(ASRProviderBase):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, 'asr_audio'):
|
||||
conn.asr_audio = []
|
||||
if hasattr(conn, 'has_valid_voice'):
|
||||
conn.has_valid_voice = False
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
import time
|
||||
import os
|
||||
import sys
|
||||
import io
|
||||
import sys
|
||||
import time
|
||||
import shutil
|
||||
import psutil
|
||||
import asyncio
|
||||
|
||||
from config.logger import setup_logging
|
||||
from typing import Optional, Tuple, List
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from funasr import AutoModel
|
||||
from funasr.utils.postprocess_utils import rich_transcription_postprocess
|
||||
import shutil
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
|
||||
TAG = __name__
|
||||
@@ -90,16 +92,17 @@ class ASRProvider(ASRProviderBase):
|
||||
else:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
# 语音识别
|
||||
# 语音识别 - 使用线程池避免阻塞事件循环
|
||||
start_time = time.time()
|
||||
result = self.model.generate(
|
||||
result = await asyncio.to_thread(
|
||||
self.model.generate,
|
||||
input=combined_pcm_data,
|
||||
cache={},
|
||||
language="auto",
|
||||
use_itn=True,
|
||||
batch_size_s=60,
|
||||
)
|
||||
text = rich_transcription_postprocess(result[0]["text"])
|
||||
text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"])
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Optional, Tuple, List
|
||||
import dashscope
|
||||
from config.logger import setup_logging
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
|
||||
tag = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool):
|
||||
super().__init__()
|
||||
# 音频文件上传类型，流式文本识别输出
|
||||
self.interface_type = InterfaceType.NON_STREAM
|
||||
"""Qwen3-ASR-Flash ASR初始化"""
|
||||
|
||||
# 配置参数
|
||||
self.api_key = config.get("api_key")
|
||||
if not self.api_key:
|
||||
raise ValueError("Qwen3-ASR-Flash 需要配置 api_key")
|
||||
|
||||
self.model_name = config.get("model_name", "qwen3-asr-flash")
|
||||
self.output_dir = config.get("output_dir", "./audio_output")
|
||||
self.delete_audio_file = delete_audio_file
|
||||
|
||||
# ASR选项配置
|
||||
self.enable_lid = config.get("enable_lid", True) # 自动语种检测
|
||||
self.enable_itn = config.get("enable_itn", True) # 逆文本归一化
|
||||
self.language = config.get("language", None) # 指定语种，默认自动检测
|
||||
self.context = config.get("context", "") # 上下文信息，用于提高识别准确率
|
||||
|
||||
# 确保输出目录存在
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
def _prepare_audio_file(self, pcm_data: bytes) -> str:
|
||||
"""将PCM数据转换为WAV文件并返回文件路径"""
|
||||
try:
|
||||
import wave
|
||||
|
||||
# 创建临时WAV文件
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file:
|
||||
temp_path = temp_file.name
|
||||
|
||||
# 写入WAV格式
|
||||
with wave.open(temp_path, 'wb') as wav_file:
|
||||
wav_file.setnchannels(1) # 单声道
|
||||
wav_file.setsampwidth(2) # 16位
|
||||
wav_file.setframerate(16000) # 16kHz采样率
|
||||
wav_file.writeframes(pcm_data)
|
||||
|
||||
return temp_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=tag).error(f"音频文件准备失败: {e}")
|
||||
return None
|
||||
|
||||
async def speech_to_text(
|
||||
self, opus_data: List[bytes], session_id: str, audio_format="opus"
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
temp_file_path = None
|
||||
file_path = None
|
||||
|
||||
try:
|
||||
# 解码音频数据
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
if len(combined_pcm_data) == 0:
|
||||
logger.bind(tag=tag).warning("音频数据为空")
|
||||
return "", None
|
||||
|
||||
# 准备音频文件
|
||||
temp_file_path = self._prepare_audio_file(combined_pcm_data)
|
||||
if not temp_file_path:
|
||||
return "", None
|
||||
|
||||
# 保存音频文件（如果需要）
|
||||
if not self.delete_audio_file:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
# 构造请求消息
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"audio": temp_file_path}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
# 如果有上下文信息，添加system消息
|
||||
if self.context:
|
||||
messages.insert(0, {
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"text": self.context}
|
||||
]
|
||||
})
|
||||
|
||||
# 准备ASR选项
|
||||
asr_options = {
|
||||
"enable_lid": self.enable_lid,
|
||||
"enable_itn": self.enable_itn
|
||||
}
|
||||
|
||||
# 如果指定了语种，添加到选项中
|
||||
if self.language:
|
||||
asr_options["language"] = self.language
|
||||
|
||||
# 设置API密钥
|
||||
dashscope.api_key = self.api_key
|
||||
|
||||
# 发送流式请求
|
||||
response = dashscope.MultiModalConversation.call(
|
||||
model=self.model_name,
|
||||
messages=messages,
|
||||
result_format="message",
|
||||
asr_options=asr_options,
|
||||
stream=True
|
||||
)
|
||||
|
||||
# 处理流式响应
|
||||
full_text = ""
|
||||
for chunk in response:
|
||||
try:
|
||||
text = chunk["output"]["choices"][0]["message"].content[0]["text"]
|
||||
# 更新为最新的完整文本
|
||||
full_text = text.strip()
|
||||
except:
|
||||
pass
|
||||
|
||||
return full_text, file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=tag).error(f"语音识别失败: {e}")
|
||||
return "", file_path
|
||||
|
||||
finally:
|
||||
# 清理临时文件
|
||||
if temp_file_path and os.path.exists(temp_file_path):
|
||||
try:
|
||||
os.unlink(temp_file_path)
|
||||
except Exception as e:
|
||||
logger.bind(tag=tag).warning(f"清理临时文件失败: {e}")
|
||||
@@ -0,0 +1,114 @@
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
from typing import Optional, Tuple, List
|
||||
from .base import ASRProviderBase
|
||||
from config.logger import setup_logging
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
import vosk
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool = True):
|
||||
super().__init__()
|
||||
self.interface_type = InterfaceType.LOCAL
|
||||
self.model_path = config.get("model_path")
|
||||
self.output_dir = config.get("output_dir", "tmp/")
|
||||
self.delete_audio_file = delete_audio_file
|
||||
|
||||
# 初始化VOSK模型
|
||||
self.model = None
|
||||
self.recognizer = None
|
||||
self._load_model()
|
||||
|
||||
# 确保输出目录存在
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
def _load_model(self):
|
||||
"""加载VOSK模型"""
|
||||
try:
|
||||
if not os.path.exists(self.model_path):
|
||||
raise FileNotFoundError(f"VOSK模型路径不存在: {self.model_path}")
|
||||
|
||||
logger.bind(tag=TAG).info(f"正在加载VOSK模型: {self.model_path}")
|
||||
self.model = vosk.Model(self.model_path)
|
||||
|
||||
# 初始化VOSK识别器（采样率必须为16kHz）
|
||||
self.recognizer = vosk.KaldiRecognizer(self.model, 16000)
|
||||
|
||||
logger.bind(tag=TAG).info("VOSK模型加载成功")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"加载VOSK模型失败: {e}")
|
||||
raise
|
||||
|
||||
async def speech_to_text(
|
||||
self, audio_data: List[bytes], session_id: str, audio_format: str = "opus"
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
file_path = None
|
||||
try:
|
||||
# 检查模型是否加载成功
|
||||
if not self.model:
|
||||
logger.bind(tag=TAG).error("VOSK模型未加载，无法进行识别")
|
||||
return "", None
|
||||
|
||||
# 解码音频（如果原始格式是Opus）
|
||||
if audio_format == "pcm":
|
||||
pcm_data = audio_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(audio_data)
|
||||
|
||||
if not pcm_data:
|
||||
logger.bind(tag=TAG).warning("解码后的PCM数据为空，无法进行识别")
|
||||
return "", None
|
||||
|
||||
# 合并PCM数据
|
||||
combined_pcm_data = b"".join(pcm_data)
|
||||
if len(combined_pcm_data) == 0:
|
||||
logger.bind(tag=TAG).warning("合并后的PCM数据为空")
|
||||
return "", None
|
||||
|
||||
# 判断是否保存为WAV文件
|
||||
if not self.delete_audio_file:
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
|
||||
# 进行识别（VOSK推荐每次送入2000字节的数据）
|
||||
chunk_size = 2000
|
||||
text_result = ""
|
||||
|
||||
for i in range(0, len(combined_pcm_data), chunk_size):
|
||||
chunk = combined_pcm_data[i:i+chunk_size]
|
||||
if self.recognizer.AcceptWaveform(chunk):
|
||||
result = json.loads(self.recognizer.Result())
|
||||
text = result.get('text', '')
|
||||
if text:
|
||||
text_result += text + " "
|
||||
|
||||
# 获取最终结果
|
||||
final_result = json.loads(self.recognizer.FinalResult())
|
||||
final_text = final_result.get('text', '')
|
||||
if final_text:
|
||||
text_result += final_text
|
||||
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"VOSK语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text_result.strip()}"
|
||||
)
|
||||
|
||||
return text_result.strip(), file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"VOSK语音识别失败: {e}")
|
||||
return "", None
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
|
||||
@@ -0,0 +1,372 @@
|
||||
import json
|
||||
import hmac
|
||||
import base64
|
||||
import hashlib
|
||||
import asyncio
|
||||
import websockets
|
||||
import opuslib_next
|
||||
import gc
|
||||
from time import mktime
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlencode
|
||||
from typing import List
|
||||
from config.logger import setup_logging
|
||||
from wsgiref.handlers import format_date_time
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
# 帧状态常量
|
||||
STATUS_FIRST_FRAME = 0 # 第一帧的标识
|
||||
STATUS_CONTINUE_FRAME = 1 # 中间帧标识
|
||||
STATUS_LAST_FRAME = 2 # 最后一帧的标识
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__()
|
||||
self.interface_type = InterfaceType.STREAM
|
||||
self.config = config
|
||||
self.text = ""
|
||||
self.decoder = opuslib_next.Decoder(16000, 1)
|
||||
self.asr_ws = None
|
||||
self.forward_task = None
|
||||
self.is_processing = False
|
||||
self.server_ready = False
|
||||
|
||||
# 讯飞配置
|
||||
self.app_id = config.get("app_id")
|
||||
self.api_key = config.get("api_key")
|
||||
self.api_secret = config.get("api_secret")
|
||||
|
||||
if not all([self.app_id, self.api_key, self.api_secret]):
|
||||
raise ValueError("必须提供app_id、api_key和api_secret")
|
||||
|
||||
# 识别参数
|
||||
self.iat_params = {
|
||||
"domain": config.get("domain", "slm"),
|
||||
"language": config.get("language", "zh_cn"),
|
||||
"accent": config.get("accent", "mandarin"),
|
||||
"result": {"encoding": "utf8", "compress": "raw", "format": "plain"},
|
||||
}
|
||||
|
||||
self.output_dir = config.get("output_dir", "tmp/")
|
||||
self.delete_audio_file = delete_audio_file
|
||||
|
||||
def create_url(self) -> str:
|
||||
"""生成认证URL"""
|
||||
url = "ws://iat.cn-huabei-1.xf-yun.com/v1"
|
||||
# 生成RFC1123格式的时间戳
|
||||
now = datetime.now()
|
||||
date = format_date_time(mktime(now.timetuple()))
|
||||
|
||||
# 拼接字符串
|
||||
signature_origin = "host: " + "iat.cn-huabei-1.xf-yun.com" + "\n"
|
||||
signature_origin += "date: " + date + "\n"
|
||||
signature_origin += "GET " + "/v1 " + "HTTP/1.1"
|
||||
|
||||
# 进行hmac-sha256进行加密
|
||||
signature_sha = hmac.new(
|
||||
self.api_secret.encode("utf-8"),
|
||||
signature_origin.encode("utf-8"),
|
||||
digestmod=hashlib.sha256,
|
||||
).digest()
|
||||
signature_sha = base64.b64encode(signature_sha).decode(encoding="utf-8")
|
||||
|
||||
authorization_origin = (
|
||||
'api_key="%s", algorithm="%s", headers="%s", signature="%s"'
|
||||
% (self.api_key, "hmac-sha256", "host date request-line", signature_sha)
|
||||
)
|
||||
authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
# 将请求的鉴权参数组合为字典
|
||||
v = {
|
||||
"authorization": authorization,
|
||||
"date": date,
|
||||
"host": "iat.cn-huabei-1.xf-yun.com",
|
||||
}
|
||||
|
||||
# 拼接鉴权参数，生成url
|
||||
url = url + "?" + urlencode(v)
|
||||
return url
|
||||
|
||||
async def open_audio_channels(self, conn):
|
||||
await super().open_audio_channels(conn)
|
||||
|
||||
async def receive_audio(self, conn, audio, audio_have_voice):
|
||||
# 先调用父类方法处理基础逻辑
|
||||
await super().receive_audio(conn, audio, audio_have_voice)
|
||||
|
||||
# 存储音频数据用于声纹识别
|
||||
if not hasattr(conn, "asr_audio_for_voiceprint"):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
conn.asr_audio_for_voiceprint.append(audio)
|
||||
|
||||
# 如果本次有声音，且之前没有建立连接
|
||||
if audio_have_voice and self.asr_ws is None and not self.is_processing:
|
||||
try:
|
||||
await self._start_recognition(conn)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
|
||||
await self._cleanup()
|
||||
return
|
||||
|
||||
# 发送当前音频数据
|
||||
if self.asr_ws and self.is_processing and self.server_ready:
|
||||
try:
|
||||
pcm_frame = self.decoder.decode(audio, 960)
|
||||
await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"发送音频数据时发生错误: {e}")
|
||||
await self._cleanup()
|
||||
|
||||
async def _start_recognition(self, conn):
|
||||
"""开始识别会话"""
|
||||
try:
|
||||
self.is_processing = True
|
||||
# 建立WebSocket连接
|
||||
ws_url = self.create_url()
|
||||
logger.bind(tag=TAG).info(f"正在连接ASR服务: {ws_url[:50]}...")
|
||||
|
||||
# 如果为手动模式,设置超时时长为一分钟
|
||||
if conn.client_listen_mode == "manual":
|
||||
self.iat_params["eos"] = 60000
|
||||
|
||||
self.asr_ws = await websockets.connect(
|
||||
ws_url,
|
||||
max_size=1000000000,
|
||||
ping_interval=None,
|
||||
ping_timeout=None,
|
||||
close_timeout=10,
|
||||
)
|
||||
|
||||
logger.bind(tag=TAG).info("ASR WebSocket连接已建立")
|
||||
self.server_ready = False
|
||||
self.forward_task = asyncio.create_task(self._forward_results(conn))
|
||||
|
||||
# 发送首帧音频
|
||||
if conn.asr_audio and len(conn.asr_audio) > 0:
|
||||
first_audio = conn.asr_audio[-1] if conn.asr_audio else b""
|
||||
pcm_frame = (
|
||||
self.decoder.decode(first_audio, 960) if first_audio else b""
|
||||
)
|
||||
await self._send_audio_frame(pcm_frame, STATUS_FIRST_FRAME)
|
||||
self.server_ready = True
|
||||
logger.bind(tag=TAG).info("已发送首帧，开始识别")
|
||||
|
||||
# 发送缓存的音频数据
|
||||
for cached_audio in conn.asr_audio[-10:]:
|
||||
try:
|
||||
pcm_frame = self.decoder.decode(cached_audio, 960)
|
||||
await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).info(f"发送缓存音频数据时发生错误: {e}")
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
|
||||
if hasattr(e, "__cause__") and e.__cause__:
|
||||
logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
|
||||
if self.asr_ws:
|
||||
await self.asr_ws.close()
|
||||
self.asr_ws = None
|
||||
self.is_processing = False
|
||||
raise
|
||||
|
||||
async def _send_audio_frame(self, audio_data: bytes, status: int):
|
||||
"""发送音频帧"""
|
||||
if not self.asr_ws:
|
||||
return
|
||||
|
||||
audio_b64 = base64.b64encode(audio_data).decode("utf-8")
|
||||
|
||||
frame_data = {
|
||||
"header": {"status": status, "app_id": self.app_id},
|
||||
"parameter": {"iat": self.iat_params},
|
||||
"payload": {
|
||||
"audio": {"audio": audio_b64, "sample_rate": 16000, "encoding": "raw"}
|
||||
},
|
||||
}
|
||||
|
||||
await self.asr_ws.send(json.dumps(frame_data, ensure_ascii=False))
|
||||
|
||||
async def _forward_results(self, conn):
|
||||
"""转发识别结果"""
|
||||
try:
|
||||
while not conn.stop_event.is_set():
|
||||
try:
|
||||
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=60)
|
||||
result = json.loads(response)
|
||||
logger.bind(tag=TAG).debug(f"收到ASR结果: {result}")
|
||||
|
||||
header = result.get("header", {})
|
||||
payload = result.get("payload", {})
|
||||
code = header.get("code", 0)
|
||||
status = header.get("status", 0)
|
||||
|
||||
if code != 0:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"识别错误，错误码: {code}, 消息: {header.get('message', '')}"
|
||||
)
|
||||
if code in [10114, 10160]: # 连接问题
|
||||
break
|
||||
continue
|
||||
|
||||
# 处理识别结果
|
||||
if payload and "result" in payload:
|
||||
text_data = payload["result"]["text"]
|
||||
if text_data:
|
||||
# 解码base64文本
|
||||
decoded_text = base64.b64decode(text_data).decode("utf-8")
|
||||
text_json = json.loads(decoded_text)
|
||||
# 提取文本内容
|
||||
text_ws = text_json.get("ws", [])
|
||||
for i in text_ws:
|
||||
for j in i.get("cw", []):
|
||||
w = j.get("w", "")
|
||||
self.text += w
|
||||
|
||||
if status == 2:
|
||||
if conn.client_listen_mode == "manual":
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
if len(audio_data) > 0:
|
||||
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清理音频缓存
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
break
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
logger.bind(tag=TAG).error("接收结果超时")
|
||||
break
|
||||
except websockets.ConnectionClosed:
|
||||
logger.bind(tag=TAG).info("ASR服务连接已关闭")
|
||||
self.is_processing = False
|
||||
break
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理ASR结果时发生错误: {str(e)}")
|
||||
if hasattr(e, "__cause__") and e.__cause__:
|
||||
logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
|
||||
self.is_processing = False
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR结果转发任务发生错误: {str(e)}")
|
||||
if hasattr(e, "__cause__") and e.__cause__:
|
||||
logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
|
||||
finally:
|
||||
# 清理连接资源
|
||||
await self._cleanup()
|
||||
|
||||
# 清理连接的音频缓存
|
||||
if conn:
|
||||
if hasattr(conn, "asr_audio_for_voiceprint"):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, "asr_audio"):
|
||||
conn.asr_audio = []
|
||||
|
||||
async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
|
||||
"""处理语音停止，发送最后一帧并处理识别结果"""
|
||||
try:
|
||||
# 先发送最后一帧表示音频结束
|
||||
if self.asr_ws and self.is_processing:
|
||||
try:
|
||||
await self._send_audio_frame(b"", STATUS_LAST_FRAME)
|
||||
logger.bind(tag=TAG).debug(f"已发送停止请求")
|
||||
|
||||
await asyncio.sleep(0.25)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
|
||||
|
||||
await super().handle_voice_stop(conn, asr_audio_task)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
|
||||
import traceback
|
||||
|
||||
logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
|
||||
|
||||
def stop_ws_connection(self):
|
||||
if self.asr_ws:
|
||||
asyncio.create_task(self.asr_ws.close())
|
||||
self.asr_ws = None
|
||||
self.is_processing = False
|
||||
|
||||
async def _send_stop_request(self):
|
||||
"""发送停止识别请求（不关闭连接）"""
|
||||
if self.asr_ws:
|
||||
try:
|
||||
# 先停止音频发送
|
||||
self.is_processing = False
|
||||
await self._send_audio_frame(b"", STATUS_LAST_FRAME)
|
||||
logger.bind(tag=TAG).debug("已发送停止请求")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
|
||||
|
||||
async def _cleanup(self):
|
||||
"""清理资源（关闭连接）"""
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}"
|
||||
)
|
||||
|
||||
# 状态重置
|
||||
self.is_processing = False
|
||||
self.server_ready = False
|
||||
logger.bind(tag=TAG).debug("ASR状态已重置")
|
||||
|
||||
# 关闭连接
|
||||
if self.asr_ws:
|
||||
try:
|
||||
logger.bind(tag=TAG).debug("正在关闭WebSocket连接")
|
||||
await asyncio.wait_for(self.asr_ws.close(), timeout=2.0)
|
||||
logger.bind(tag=TAG).debug("WebSocket连接已关闭")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
|
||||
finally:
|
||||
self.asr_ws = None
|
||||
|
||||
# 清理任务引用
|
||||
self.forward_task = None
|
||||
|
||||
logger.bind(tag=TAG).debug("ASR会话清理完成")
|
||||
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format):
|
||||
"""获取识别结果"""
|
||||
result = self.text
|
||||
self.text = ""
|
||||
return result, None
|
||||
|
||||
async def close(self):
|
||||
"""资源清理方法"""
|
||||
if self.asr_ws:
|
||||
await self.asr_ws.close()
|
||||
self.asr_ws = None
|
||||
if self.forward_task:
|
||||
self.forward_task.cancel()
|
||||
try:
|
||||
await self.forward_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self.forward_task = None
|
||||
self.is_processing = False
|
||||
|
||||
# 显式释放decoder资源
|
||||
if hasattr(self, 'decoder') and self.decoder is not None:
|
||||
try:
|
||||
del self.decoder
|
||||
self.decoder = None
|
||||
logger.bind(tag=TAG).debug("Xunfei decoder resources released")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
|
||||
|
||||
# 清理所有连接的音频缓冲区
|
||||
if hasattr(self, "_connections"):
|
||||
for conn in self._connections.values():
|
||||
if hasattr(conn, "asr_audio_for_voiceprint"):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
if hasattr(conn, "asr_audio"):
|
||||
conn.asr_audio = []
|
||||
Reference in New Issue
Block a user