Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-30 13:33:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'py_test_typing' into feature/python-typing

Browse Source
This commit is contained in:
Sakura-RanChen
2026-02-05 17:14:47 +08:00
committed by GitHub
parent 4b573fb4e2 8a84428724
commit 4f61d21d17
133 changed files with 23165 additions and 1723 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/providers/asr/base.py
+148 -33
View File
@@ -5,20 +5,25 @@ import uuid
import json
import time
import queue
import shutil
import asyncio
import tempfile
import traceback
import threading
import opuslib_next
from abc import ABC, abstractmethod
from config.logger import setup_logging
from typing import Optional, Tuple, List, TYPE_CHECKING
if TYPE_CHECKING:
from core.connection import ConnectionHandler
from core.providers.asr.dto.dto import InterfaceType
from core.handle.receiveAudioHandle import startToChat
from core.handle.reportHandle import enqueue_asr_report
from core.utils.util import remove_punctuation_and_length
from core.handle.receiveAudioHandle import handleAudioMessage
from typing import Optional, Tuple, List, NamedTuple, TYPE_CHECKING
if TYPE_CHECKING:
from core.connection import ConnectionHandler
TAG = __name__
logger = setup_logging()
@@ -60,18 +65,17 @@ class ASRProviderBase(ABC):
conn.asr_audio.append(audio)
else:
# 自动/实时模式：使用VAD检测
have_voice = audio_have_voice
conn.asr_audio.append(audio)
if not have_voice and not conn.client_have_voice:
# 如果没有语音，且之前也没有声音，缓存部分音频
if not audio_have_voice and not conn.client_have_voice:
conn.asr_audio = conn.asr_audio[-10:]
return
# 自动模式下通过VAD检测到语音停止时触发识别
if conn.client_voice_stop:
if conn.asr.interface_type != InterfaceType.STREAM and conn.client_voice_stop:
asr_audio_task = conn.asr_audio.copy()
conn.asr_audio.clear()
conn.reset_vad_states()
conn.reset_audio_states()
if len(asr_audio_task) > 15:
await self.handle_voice_stop(conn, asr_audio_task)
@@ -96,10 +100,14 @@ class ASRProviderBase(ABC):
wav_data = self._pcm_to_wav(combined_pcm_data)
# 定义ASR任务
asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
asr_task = self.speech_to_text_wrapper(
asr_audio_task, conn.session_id, conn.audio_format
)
if conn.voiceprint_provider and wav_data:
voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
voiceprint_task = conn.voiceprint_provider.identify_speaker(
wav_data, conn.session_id
)
# 并发等待两个结果
asr_result, voiceprint_result = await asyncio.gather(
asr_task, voiceprint_task, return_exceptions=True
@@ -162,20 +170,20 @@ class ASRProviderBase(ABC):
if text_len > 0:
# 使用自定义模块进行上报
await startToChat(conn, enhanced_text)
enqueue_asr_report(conn, enhanced_text, asr_audio_task)
audio_snapshot = asr_audio_task.copy()
enqueue_asr_report(conn, enhanced_text, audio_snapshot)
except Exception as e:
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
import traceback
logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
"""构建包含说话人信息的文本（仅用于纯文本ASR）"""
if speaker_name and speaker_name.strip():
return json.dumps({
"speaker": speaker_name,
"content": text
}, ensure_ascii=False)
return json.dumps(
{"speaker": speaker_name, "content": text}, ensure_ascii=False
)
else:
return text
@@ -184,23 +192,23 @@ class ASRProviderBase(ABC):
if len(pcm_data) == 0:
logger.bind(tag=TAG).warning("PCM数据为空，无法转换WAV")
return b""
# 确保数据长度是偶数（16位音频）
if len(pcm_data) % 2 != 0:
pcm_data = pcm_data[:-1]
# 创建WAV文件头
wav_buffer = io.BytesIO()
try:
with wave.open(wav_buffer, 'wb') as wav_file:
wav_file.setnchannels(1) # 单声道
wav_file.setsampwidth(2) # 16位
with wave.open(wav_buffer, "wb") as wav_file:
wav_file.setnchannels(1) # 单声道
wav_file.setsampwidth(2) # 16位
wav_file.setframerate(16000) # 16kHz采样率
wav_file.writeframes(pcm_data)
wav_buffer.seek(0)
wav_data = wav_buffer.read()
return wav_data
except Exception as e:
logger.bind(tag=TAG).error(f"WAV转换失败: {e}")
@@ -209,6 +217,44 @@ class ASRProviderBase(ABC):
def stop_ws_connection(self):
pass
async def close(self):
pass
class AudioArtifacts(NamedTuple):
pcm_frames: List[bytes]
"""PCM音频帧列表"""
pcm_bytes: bytes
"""合并后的PCM音频字节数据"""
file_path: Optional[str]
"""WAV文件路径"""
temp_path: Optional[str]
"""临时WAV文件路径"""
def get_current_artifacts(self) -> Optional["ASRProviderBase.AudioArtifacts"]:
return self._current_artifacts
def requires_file(self) -> bool:
"""是否需要文件输入"""
return False
def prefers_temp_file(self) -> bool:
"""是否优先使用临时文件"""
return False
def build_temp_file(self, pcm_bytes: bytes) -> Optional[str]:
try:
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
temp_path = temp_file.name
with wave.open(temp_path, "wb") as wav_file:
wav_file.setnchannels(1)
wav_file.setsampwidth(2)
wav_file.setframerate(16000)
wav_file.writeframes(pcm_bytes)
return temp_path
except Exception as e:
logger.bind(tag=TAG).error(f"临时音频文件生成失败: {e}")
return None
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
@@ -223,11 +269,80 @@ class ASRProviderBase(ABC):
return file_path
@abstractmethod
async def speech_to_text(
async def speech_to_text_wrapper(
self, opus_data: List[bytes], session_id: str, audio_format="opus"
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
file_path = None
temp_path = None
try:
if audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
free_space = shutil.disk_usage(self.output_dir).free
if free_space < len(combined_pcm_data) * 2:
raise OSError("磁盘空间不足")
if self.requires_file() and self.prefers_temp_file():
temp_path = self.build_temp_file(combined_pcm_data)
if (hasattr(self, "delete_audio_file") and not self.delete_audio_file) or (
self.requires_file() and not self.prefers_temp_file()
):
file_path = self.save_audio_to_file(pcm_data, session_id)
if len(combined_pcm_data) == 0:
artifacts = None
else:
artifacts = ASRProviderBase.AudioArtifacts(
pcm_frames=pcm_data,
pcm_bytes=combined_pcm_data,
file_path=file_path,
temp_path=temp_path,
)
text, _ = await self.speech_to_text(
opus_data, session_id, audio_format, artifacts
)
return text, file_path
except OSError as e:
logger.bind(tag=TAG).error(f"文件操作错误: {e}")
return None, None
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}")
return None, None
finally:
try:
if temp_path and os.path.exists(temp_path):
os.unlink(temp_path)
if (
hasattr(self, "delete_audio_file")
and self.delete_audio_file
and file_path
and os.path.exists(file_path)
):
os.remove(file_path)
except Exception as e:
logger.bind(tag=TAG).error(f"文件清理失败: {e}")
@abstractmethod
async def speech_to_text(
self,
opus_data: List[bytes],
session_id: str,
audio_format="opus",
artifacts: Optional[AudioArtifacts] = None,
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本
:param opus_data: 输入的Opus音频数据
:param session_id: 会话ID
:param audio_format: 音频格式，默认"opus"
:param artifacts: 音频工件，包含PCM数据、文件路径等
:return: 识别结果文本和文件路径（如果有）
"""
pass
@staticmethod
@@ -238,23 +353,23 @@ class ASRProviderBase(ABC):
decoder = opuslib_next.Decoder(16000, 1)
pcm_data = []
buffer_size = 960 # 每次处理960个采样点 (60ms at 16kHz)
for i, opus_packet in enumerate(opus_data):
try:
if not opus_packet or len(opus_packet) == 0:
continue
pcm_frame = decoder.decode(opus_packet, buffer_size)
if pcm_frame and len(pcm_frame) > 0:
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).warning(f"Opus解码错误，跳过数据包 {i}: {e}")
except Exception as e:
logger.bind(tag=TAG).error(f"音频处理错误，数据包 {i}: {e}")
return pcm_data
except Exception as e:
logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
return []