mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-21 22:53:56 +08:00
337 lines
12 KiB
Python
337 lines
12 KiB
Python
import os
|
|
import io
|
|
import wave
|
|
import uuid
|
|
import json
|
|
import time
|
|
import queue
|
|
import shutil
|
|
import asyncio
|
|
import tempfile
|
|
import traceback
|
|
import threading
|
|
|
|
from abc import ABC, abstractmethod
|
|
from config.logger import setup_logging
|
|
from core.providers.asr.dto.dto import InterfaceType
|
|
from core.handle.receiveAudioHandle import startToChat
|
|
from core.handle.reportHandle import enqueue_asr_report
|
|
from core.utils.util import remove_punctuation_and_length
|
|
from core.handle.receiveAudioHandle import handleAudioMessage
|
|
from typing import Optional, Tuple, List, NamedTuple, TYPE_CHECKING
|
|
|
|
|
|
if TYPE_CHECKING:
|
|
from core.connection import ConnectionHandler
|
|
|
|
TAG = __name__
|
|
logger = setup_logging()
|
|
|
|
|
|
class ASRProviderBase(ABC):
|
|
def __init__(self):
|
|
pass
|
|
|
|
# 打开音频通道
|
|
async def open_audio_channels(self, conn: "ConnectionHandler"):
|
|
conn.asr_priority_thread = threading.Thread(
|
|
target=self.asr_text_priority_thread, args=(conn,), daemon=True
|
|
)
|
|
conn.asr_priority_thread.start()
|
|
|
|
# 有序处理ASR音频
|
|
def asr_text_priority_thread(self, conn: "ConnectionHandler"):
|
|
while not conn.stop_event.is_set():
|
|
try:
|
|
message = conn.asr_audio_queue.get(timeout=1)
|
|
future = asyncio.run_coroutine_threadsafe(
|
|
handleAudioMessage(conn, message),
|
|
conn.loop,
|
|
)
|
|
future.result()
|
|
except queue.Empty:
|
|
continue
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(
|
|
f"处理ASR文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
|
|
)
|
|
continue
|
|
|
|
# 接收音频
|
|
async def receive_audio(self, conn: "ConnectionHandler", pcm_frame, audio_have_voice):
|
|
if conn.client_listen_mode == "manual":
|
|
# 手动模式：缓存音频用于ASR识别
|
|
conn.asr_audio.append(pcm_frame)
|
|
else:
|
|
# 自动/实时模式：使用VAD检测
|
|
conn.asr_audio.append(pcm_frame)
|
|
|
|
# 如果没有语音，且之前也没有声音，缓存部分音频
|
|
if not audio_have_voice and not conn.client_have_voice:
|
|
conn.asr_audio = conn.asr_audio[-10:]
|
|
return
|
|
|
|
# 自动模式下通过VAD检测到语音停止时触发识别
|
|
if conn.asr.interface_type != InterfaceType.STREAM and conn.client_voice_stop:
|
|
# 直接使用asr_audio中的PCM数据
|
|
pcm_bytes = b"".join(conn.asr_audio)
|
|
# 检查是否有足够的音频数据（每帧1920字节，15帧约28800字节）
|
|
if len(pcm_bytes) > 1920 * 15:
|
|
await self.handle_voice_stop(conn, [pcm_bytes])
|
|
conn.reset_audio_states()
|
|
|
|
# 处理语音停止
|
|
async def handle_voice_stop(self, conn: "ConnectionHandler", asr_audio_task: List[bytes]):
|
|
"""并行处理ASR和声纹识别"""
|
|
try:
|
|
total_start_time = time.monotonic()
|
|
|
|
# 数据已经是PCM直接使用
|
|
pcm_data = asr_audio_task
|
|
combined_pcm_data = b"".join(pcm_data)
|
|
|
|
# 预先准备WAV数据
|
|
wav_data = None
|
|
if conn.voiceprint_provider and combined_pcm_data:
|
|
wav_data = self._pcm_to_wav(combined_pcm_data)
|
|
|
|
# 定义ASR任务
|
|
asr_task = self.speech_to_text_wrapper(
|
|
asr_audio_task, conn.session_id
|
|
)
|
|
|
|
if conn.voiceprint_provider and wav_data:
|
|
voiceprint_task = conn.voiceprint_provider.identify_speaker(
|
|
wav_data, conn.session_id
|
|
)
|
|
# 并发等待两个结果
|
|
asr_result, voiceprint_result = await asyncio.gather(
|
|
asr_task, voiceprint_task, return_exceptions=True
|
|
)
|
|
else:
|
|
asr_result = await asr_task
|
|
voiceprint_result = None
|
|
|
|
# 记录识别结果 - 检查是否为异常
|
|
if isinstance(asr_result, Exception):
|
|
logger.bind(tag=TAG).error(f"ASR识别失败: {asr_result}")
|
|
raw_text = ""
|
|
else:
|
|
raw_text, _ = asr_result
|
|
|
|
if isinstance(voiceprint_result, Exception):
|
|
logger.bind(tag=TAG).error(f"声纹识别失败: {voiceprint_result}")
|
|
speaker_name = ""
|
|
else:
|
|
speaker_name = voiceprint_result
|
|
|
|
# 判断 ASR 结果类型
|
|
if isinstance(raw_text, dict):
|
|
# FunASR 返回的 dict 格式
|
|
if speaker_name:
|
|
raw_text["speaker"] = speaker_name
|
|
|
|
# 记录识别结果
|
|
if raw_text.get("language"):
|
|
logger.bind(tag=TAG).info(f"识别语言: {raw_text['language']}")
|
|
if raw_text.get("emotion"):
|
|
logger.bind(tag=TAG).info(f"识别情绪: {raw_text['emotion']}")
|
|
if raw_text.get("content"):
|
|
logger.bind(tag=TAG).info(f"识别文本: {raw_text['content']}")
|
|
if speaker_name:
|
|
logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
|
|
|
|
# 转换为 JSON 字符串用于下游
|
|
enhanced_text = json.dumps(raw_text, ensure_ascii=False)
|
|
content_for_length_check = raw_text.get("content", "")
|
|
else:
|
|
# 其他 ASR 返回的纯文本
|
|
if raw_text:
|
|
logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
|
|
if speaker_name:
|
|
logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
|
|
|
|
# 构建包含说话人信息的JSON字符串
|
|
enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
|
|
content_for_length_check = raw_text
|
|
|
|
# 性能监控
|
|
total_time = time.monotonic() - total_start_time
|
|
logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s")
|
|
|
|
# 检查文本长度
|
|
text_len, _ = remove_punctuation_and_length(content_for_length_check)
|
|
self.stop_ws_connection()
|
|
|
|
if text_len > 0:
|
|
audio_snapshot = asr_audio_task.copy()
|
|
enqueue_asr_report(conn, enhanced_text, audio_snapshot)
|
|
# 使用自定义模块进行上报
|
|
await startToChat(conn, enhanced_text)
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
|
|
import traceback
|
|
|
|
logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
|
|
|
|
def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
|
|
"""构建包含说话人信息的文本（仅用于纯文本ASR）"""
|
|
if speaker_name and speaker_name.strip():
|
|
return json.dumps(
|
|
{"speaker": speaker_name, "content": text}, ensure_ascii=False
|
|
)
|
|
else:
|
|
return text
|
|
|
|
def _pcm_to_wav(self, pcm_data: bytes) -> bytes:
|
|
"""将PCM数据转换为WAV格式"""
|
|
if len(pcm_data) == 0:
|
|
logger.bind(tag=TAG).warning("PCM数据为空，无法转换WAV")
|
|
return b""
|
|
|
|
# 确保数据长度是偶数（16位音频）
|
|
if len(pcm_data) % 2 != 0:
|
|
pcm_data = pcm_data[:-1]
|
|
|
|
# 创建WAV文件头
|
|
wav_buffer = io.BytesIO()
|
|
try:
|
|
with wave.open(wav_buffer, "wb") as wav_file:
|
|
wav_file.setnchannels(1) # 单声道
|
|
wav_file.setsampwidth(2) # 16位
|
|
wav_file.setframerate(16000) # 16kHz采样率
|
|
wav_file.writeframes(pcm_data)
|
|
|
|
wav_buffer.seek(0)
|
|
wav_data = wav_buffer.read()
|
|
|
|
return wav_data
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(f"WAV转换失败: {e}")
|
|
return b""
|
|
|
|
def stop_ws_connection(self):
|
|
pass
|
|
|
|
async def close(self):
|
|
pass
|
|
|
|
class AudioArtifacts(NamedTuple):
|
|
pcm_frames: List[bytes]
|
|
"""PCM音频帧列表"""
|
|
pcm_bytes: bytes
|
|
"""合并后的PCM音频字节数据"""
|
|
file_path: Optional[str]
|
|
"""WAV文件路径"""
|
|
temp_path: Optional[str]
|
|
"""临时WAV文件路径"""
|
|
|
|
def get_current_artifacts(self) -> Optional["ASRProviderBase.AudioArtifacts"]:
|
|
return self._current_artifacts
|
|
|
|
def requires_file(self) -> bool:
|
|
"""是否需要文件输入"""
|
|
return False
|
|
|
|
def prefers_temp_file(self) -> bool:
|
|
"""是否优先使用临时文件"""
|
|
return False
|
|
|
|
def build_temp_file(self, pcm_bytes: bytes) -> Optional[str]:
|
|
try:
|
|
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
|
|
temp_path = temp_file.name
|
|
with wave.open(temp_path, "wb") as wav_file:
|
|
wav_file.setnchannels(1)
|
|
wav_file.setsampwidth(2)
|
|
wav_file.setframerate(16000)
|
|
wav_file.writeframes(pcm_bytes)
|
|
return temp_path
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(f"临时音频文件生成失败: {e}")
|
|
return None
|
|
|
|
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
|
|
"""PCM数据保存为WAV文件"""
|
|
module_name = __name__.split(".")[-1]
|
|
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
|
|
file_path = os.path.join(self.output_dir, file_name)
|
|
|
|
with wave.open(file_path, "wb") as wf:
|
|
wf.setnchannels(1)
|
|
wf.setsampwidth(2) # 2 bytes = 16-bit
|
|
wf.setframerate(16000)
|
|
wf.writeframes(b"".join(pcm_data))
|
|
|
|
return file_path
|
|
|
|
async def speech_to_text_wrapper(
|
|
self, pcm_data: List[bytes], session_id: str
|
|
) -> Tuple[Optional[str], Optional[str]]:
|
|
file_path = None
|
|
temp_path = None
|
|
try:
|
|
combined_pcm_data = b"".join(pcm_data)
|
|
|
|
free_space = shutil.disk_usage(self.output_dir).free
|
|
if free_space < len(combined_pcm_data) * 2:
|
|
raise OSError("磁盘空间不足")
|
|
|
|
if self.requires_file() and self.prefers_temp_file():
|
|
temp_path = self.build_temp_file(combined_pcm_data)
|
|
|
|
if (hasattr(self, "delete_audio_file") and not self.delete_audio_file) or (
|
|
self.requires_file() and not self.prefers_temp_file()
|
|
):
|
|
file_path = self.save_audio_to_file(pcm_data, session_id)
|
|
|
|
if len(combined_pcm_data) == 0:
|
|
artifacts = None
|
|
else:
|
|
artifacts = ASRProviderBase.AudioArtifacts(
|
|
pcm_frames=pcm_data,
|
|
pcm_bytes=combined_pcm_data,
|
|
file_path=file_path,
|
|
temp_path=temp_path,
|
|
)
|
|
|
|
text, _ = await self.speech_to_text(
|
|
pcm_data, session_id, artifacts
|
|
)
|
|
return text, file_path
|
|
except OSError as e:
|
|
logger.bind(tag=TAG).error(f"文件操作错误: {e}")
|
|
return None, None
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(f"语音识别失败: {e}")
|
|
return None, None
|
|
finally:
|
|
try:
|
|
if temp_path and os.path.exists(temp_path):
|
|
os.unlink(temp_path)
|
|
if (
|
|
hasattr(self, "delete_audio_file")
|
|
and self.delete_audio_file
|
|
and file_path
|
|
and os.path.exists(file_path)
|
|
):
|
|
os.remove(file_path)
|
|
except Exception as e:
|
|
logger.bind(tag=TAG).error(f"文件清理失败: {e}")
|
|
|
|
@abstractmethod
|
|
async def speech_to_text(
|
|
self,
|
|
opus_data: List[bytes],
|
|
session_id: str,
|
|
artifacts: Optional[AudioArtifacts] = None,
|
|
) -> Tuple[Optional[str], Optional[str]]:
|
|
"""将语音数据转换为文本
|
|
|
|
:param opus_data: 输入的Opus音频数据
|
|
:param session_id: 会话ID
|
|
:param artifacts: 音频工件，包含PCM数据、文件路径等
|
|
:return: 识别结果文本和文件路径（如果有）
|
|
"""
|
|
pass
|