feat(xiaozhi): 新增小智 AI 一键运行 Docker 镜像
This commit is contained in:
Del Wang
@@ -1,17 +1,8 @@
|
||||
import logging
|
||||
import queue
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import opuslib
|
||||
import pyaudio
|
||||
import opuslib_next as opuslib
|
||||
|
||||
from xiaozhi.ref import set_audio_codec
|
||||
from xiaozhi.services.audio.stream import MyAudio
|
||||
from xiaozhi.services.protocols.typing import AudioConfig
|
||||
from xiaozhi.xiaoai import XiaoAI
|
||||
|
||||
logger = logging.getLogger("AudioCodec")
|
||||
|
||||
|
||||
class AudioCodec:
|
||||
@@ -24,50 +15,46 @@ class AudioCodec:
|
||||
self.output_stream = None
|
||||
self.opus_encoder = None
|
||||
self.opus_decoder = None
|
||||
self.audio_decode_queue = queue.Queue()
|
||||
self._is_closing = False
|
||||
|
||||
self._initialize_audio()
|
||||
set_audio_codec(self)
|
||||
|
||||
def _initialize_audio(self):
|
||||
"""初始化音频设备和编解码器"""
|
||||
try:
|
||||
self.audio = MyAudio() if XiaoAI.mode == "xiaoai" else pyaudio.PyAudio()
|
||||
self.audio = MyAudio.create()
|
||||
|
||||
# 初始化音频输入流
|
||||
self.input_stream = self.audio.open(
|
||||
format=pyaudio.paInt16,
|
||||
channels=AudioConfig.CHANNELS,
|
||||
rate=AudioConfig.SAMPLE_RATE,
|
||||
input=True,
|
||||
frames_per_buffer=AudioConfig.FRAME_SIZE,
|
||||
)
|
||||
# 初始化音频输入流
|
||||
self.input_stream = self.audio.open(
|
||||
format=AudioConfig.FORMAT,
|
||||
channels=AudioConfig.CHANNELS,
|
||||
rate=AudioConfig.SAMPLE_RATE,
|
||||
input=True,
|
||||
frames_per_buffer=AudioConfig.FRAME_SIZE,
|
||||
input_device_index=MyAudio.get_input_device_index(self.audio),
|
||||
)
|
||||
|
||||
# 初始化音频输出流
|
||||
self.output_stream = self.audio.open(
|
||||
format=pyaudio.paInt16,
|
||||
channels=AudioConfig.CHANNELS,
|
||||
rate=AudioConfig.SAMPLE_RATE,
|
||||
output=True,
|
||||
frames_per_buffer=AudioConfig.FRAME_SIZE,
|
||||
)
|
||||
# 初始化音频输出流
|
||||
self.output_stream = self.audio.open(
|
||||
format=AudioConfig.FORMAT,
|
||||
channels=AudioConfig.CHANNELS,
|
||||
rate=AudioConfig.SAMPLE_RATE,
|
||||
output=True,
|
||||
frames_per_buffer=AudioConfig.FRAME_SIZE,
|
||||
output_device_index=MyAudio.get_output_device_index(self.audio),
|
||||
)
|
||||
|
||||
# 初始化Opus编码器
|
||||
self.opus_encoder = opuslib.Encoder(
|
||||
fs=AudioConfig.SAMPLE_RATE,
|
||||
channels=AudioConfig.CHANNELS,
|
||||
application=opuslib.APPLICATION_AUDIO,
|
||||
)
|
||||
# 初始化Opus编码器
|
||||
self.opus_encoder = opuslib.Encoder(
|
||||
fs=AudioConfig.SAMPLE_RATE,
|
||||
channels=AudioConfig.CHANNELS,
|
||||
application=opuslib.APPLICATION_AUDIO,
|
||||
)
|
||||
|
||||
# 初始化Opus解码器
|
||||
self.opus_decoder = opuslib.Decoder(
|
||||
fs=AudioConfig.SAMPLE_RATE, channels=AudioConfig.CHANNELS
|
||||
)
|
||||
|
||||
logger.info("音频设备和编解码器初始化成功")
|
||||
except Exception as e:
|
||||
logger.error(f"初始化音频设备失败: {e}")
|
||||
raise
|
||||
# 初始化Opus解码器
|
||||
self.opus_decoder = opuslib.Decoder(
|
||||
fs=AudioConfig.SAMPLE_RATE, channels=AudioConfig.CHANNELS
|
||||
)
|
||||
|
||||
def read_audio(self):
|
||||
"""读取音频输入数据并编码"""
|
||||
@@ -78,96 +65,35 @@ class AudioCodec:
|
||||
if not data:
|
||||
return None
|
||||
return self.opus_encoder.encode(data, AudioConfig.FRAME_SIZE)
|
||||
except Exception as e:
|
||||
logger.error(f"读取音频输入时出错: {e}")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def write_audio(self, opus_data):
|
||||
"""将编码的音频数据添加到播放队列"""
|
||||
self.audio_decode_queue.put(opus_data)
|
||||
|
||||
def play_audio(self):
|
||||
"""处理并播放队列中的音频数据"""
|
||||
"""解码并播放"""
|
||||
try:
|
||||
# 批量处理多个音频包以减少处理延迟
|
||||
batch_size = min(10, self.audio_decode_queue.qsize())
|
||||
if batch_size == 0:
|
||||
return False
|
||||
|
||||
# 创建缓冲区存储解码后的数据
|
||||
buffer = bytearray()
|
||||
|
||||
for _ in range(batch_size):
|
||||
if self.audio_decode_queue.empty():
|
||||
break
|
||||
|
||||
opus_data = self.audio_decode_queue.get_nowait()
|
||||
try:
|
||||
pcm_data = self.opus_decoder.decode(
|
||||
opus_data, AudioConfig.FRAME_SIZE, decode_fec=False
|
||||
)
|
||||
buffer.extend(pcm_data)
|
||||
except Exception as e:
|
||||
logger.error(f"解码音频数据时出错: {e}")
|
||||
|
||||
# 只有在有数据时才处理和播放
|
||||
if len(buffer) > 0:
|
||||
# 转换为numpy数组
|
||||
pcm_array = np.frombuffer(buffer, dtype=np.int16)
|
||||
|
||||
# 播放音频
|
||||
try:
|
||||
if self.output_stream and self.output_stream.is_active():
|
||||
self.output_stream.write(pcm_array.tobytes())
|
||||
return True
|
||||
else:
|
||||
# MAC 特定：如果流不活跃，尝试重新初始化
|
||||
self._reinitialize_output_stream()
|
||||
if self.output_stream and self.output_stream.is_active():
|
||||
self.output_stream.write(pcm_array.tobytes())
|
||||
return True
|
||||
except OSError as e:
|
||||
if "Stream closed" in str(e) or "Internal PortAudio error" in str(
|
||||
e
|
||||
):
|
||||
logger.error(f"播放音频时出错: {e}")
|
||||
self._reinitialize_output_stream()
|
||||
else:
|
||||
logger.error(f"播放音频时出错: {e}")
|
||||
except queue.Empty:
|
||||
pcm_data = self.decode_audio(opus_data) # 解码
|
||||
self.output_stream.write(pcm_data) # 播放
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"播放音频时出错: {e}")
|
||||
self._reinitialize_output_stream()
|
||||
|
||||
return False
|
||||
def decode_audio(self, opus_data, frame_size=AudioConfig.FRAME_SIZE):
|
||||
"""解码音频数据"""
|
||||
return self.opus_decoder.decode(opus_data, frame_size, decode_fec=False)
|
||||
|
||||
def has_pending_audio(self):
|
||||
"""检查是否还有待播放的音频数据"""
|
||||
return not self.audio_decode_queue.empty()
|
||||
|
||||
def wait_for_audio_complete(self, timeout=5.0):
|
||||
# 等待音频队列清空
|
||||
attempt = 0
|
||||
max_attempts = 15
|
||||
while not self.audio_decode_queue.empty() and attempt < max_attempts:
|
||||
time.sleep(0.1)
|
||||
attempt += 1
|
||||
|
||||
# 在关闭前清空任何剩余数据
|
||||
while not self.audio_decode_queue.empty():
|
||||
try:
|
||||
self.audio_decode_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
def clear_audio_queue(self):
|
||||
"""清空音频队列"""
|
||||
while not self.audio_decode_queue.empty():
|
||||
try:
|
||||
self.audio_decode_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
def encode_audio(self, buffer, frame_size=AudioConfig.FRAME_SIZE):
|
||||
"""编码音频数据"""
|
||||
try:
|
||||
opus_frames = []
|
||||
for i in range(0, len(buffer), frame_size * 2):
|
||||
chunk = buffer[i : i + frame_size * 2]
|
||||
if len(chunk) < frame_size * 2:
|
||||
# 如果 buffer 长度不是 FRAME_SIZE 的 2 倍，需要补齐
|
||||
chunk += b"\x00" * (frame_size * 2 - len(chunk))
|
||||
opus_frame = self.opus_encoder.encode(chunk, frame_size)
|
||||
opus_frames.append(opus_frame)
|
||||
return opus_frames
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def start_streams(self):
|
||||
"""启动音频流"""
|
||||
@@ -178,95 +104,50 @@ class AudioCodec:
|
||||
|
||||
def stop_streams(self):
|
||||
"""停止音频流"""
|
||||
if self.input_stream and self.input_stream.is_active():
|
||||
if self.input_stream.is_active():
|
||||
self.input_stream.stop_stream()
|
||||
if self.output_stream and self.output_stream.is_active():
|
||||
if self.output_stream.is_active():
|
||||
self.output_stream.stop_stream()
|
||||
|
||||
def _reinitialize_output_stream(self):
|
||||
"""重新初始化音频输出流"""
|
||||
if self._is_closing: # 如果正在关闭，不要重新初始化
|
||||
def close(self):
|
||||
"""关闭音频编解码器，确保资源正确释放"""
|
||||
if self._is_closing: # 防止重复关闭
|
||||
return
|
||||
self._is_closing = True
|
||||
|
||||
try:
|
||||
# 关闭输入流
|
||||
if self.input_stream:
|
||||
try:
|
||||
if self.input_stream.is_active():
|
||||
self.input_stream.stop_stream()
|
||||
self.input_stream.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.input_stream = None
|
||||
|
||||
# 关闭输出流
|
||||
if self.output_stream:
|
||||
try:
|
||||
if self.output_stream.is_active():
|
||||
self.output_stream.stop_stream()
|
||||
self.output_stream.close()
|
||||
except Exception:
|
||||
# logger.warning(f"关闭旧输出流时出错: {e}")
|
||||
pass
|
||||
|
||||
# 在 MAC 上添加短暂延迟
|
||||
if sys.platform in ("darwin", "linux"):
|
||||
time.sleep(0.1)
|
||||
|
||||
self.output_stream = self.audio.open(
|
||||
format=pyaudio.paInt16,
|
||||
channels=AudioConfig.CHANNELS,
|
||||
rate=AudioConfig.SAMPLE_RATE,
|
||||
output=True,
|
||||
frames_per_buffer=AudioConfig.FRAME_SIZE,
|
||||
)
|
||||
logger.info("音频输出流重新初始化成功")
|
||||
except Exception as e:
|
||||
logger.error(f"重新初始化音频输出流失败: {e}")
|
||||
raise
|
||||
|
||||
def close(self):
|
||||
"""关闭音频编解码器，确保资源正确释放"""
|
||||
if self._is_closing: # 防止重复关闭
|
||||
return
|
||||
|
||||
self._is_closing = True
|
||||
logger.info("开始关闭音频编解码器...")
|
||||
|
||||
try:
|
||||
# 等待并清理剩余音频数据
|
||||
self.wait_for_audio_complete()
|
||||
|
||||
# 关闭输入流
|
||||
if self.input_stream:
|
||||
logger.debug("正在关闭输入流...")
|
||||
try:
|
||||
if self.input_stream.is_active():
|
||||
self.input_stream.stop_stream()
|
||||
self.input_stream.close()
|
||||
except Exception as e:
|
||||
logger.error(f"关闭输入流时出错: {e}")
|
||||
self.input_stream = None
|
||||
|
||||
# 关闭输出流
|
||||
if self.output_stream:
|
||||
logger.debug("正在关闭输出流...")
|
||||
try:
|
||||
if self.output_stream.is_active():
|
||||
self.output_stream.stop_stream()
|
||||
self.output_stream.close()
|
||||
except Exception as e:
|
||||
logger.error(f"关闭输出流时出错: {e}")
|
||||
self.output_stream = None
|
||||
|
||||
# 关闭 PyAudio 实例
|
||||
if self.audio:
|
||||
logger.debug("正在终止 PyAudio...")
|
||||
try:
|
||||
self.audio.terminate()
|
||||
except Exception as e:
|
||||
logger.error(f"终止 PyAudio 时出错: {e}")
|
||||
except Exception:
|
||||
pass
|
||||
self.audio = None
|
||||
|
||||
# 清理编解码器
|
||||
self.opus_encoder = None
|
||||
self.opus_decoder = None
|
||||
|
||||
logger.info("音频编解码器关闭完成")
|
||||
except Exception as e:
|
||||
logger.error(f"关闭音频编解码器时发生错误: {e}")
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
self._is_closing = False
|
||||
|
||||
def __del__(self):
|
||||
"""析构函数，确保资源被释放"""
|
||||
self.close()
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import asyncio
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
from config import APP_CONFIG
|
||||
from xiaozhi.event import EventManager
|
||||
from xiaozhi.ref import get_speaker, get_xiaoai, get_xiaozhi, set_kws
|
||||
from xiaozhi.services.audio.kws.sherpa import SherpaOnnx
|
||||
from xiaozhi.services.audio.stream import MyAudio
|
||||
from xiaozhi.services.protocols.typing import AudioConfig, DeviceState
|
||||
from xiaozhi.utils.base import get_env
|
||||
|
||||
|
||||
class _KWS:
|
||||
def __init__(self):
|
||||
set_kws(self)
|
||||
|
||||
def start(self):
|
||||
if not get_env("CLI"):
|
||||
return
|
||||
|
||||
self.audio = MyAudio.create()
|
||||
self.stream = self.audio.open(
|
||||
format=AudioConfig.FORMAT,
|
||||
channels=1,
|
||||
rate=16000,
|
||||
input=True,
|
||||
frames_per_buffer=AudioConfig.FRAME_SIZE,
|
||||
start=True,
|
||||
)
|
||||
|
||||
# 启动 KWS 服务
|
||||
self.thread = threading.Thread(target=self._detection_loop, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def get_file_path(self, file_name: str):
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
return os.path.join(current_dir, "../../../models", file_name)
|
||||
|
||||
def _detection_loop(self):
|
||||
SherpaOnnx.start()
|
||||
self.stream.start_stream()
|
||||
while True:
|
||||
# 读取缓冲区音频数据
|
||||
frames = self.stream.read()
|
||||
|
||||
# 在说话和监听状态时，暂停 KWS
|
||||
if not frames or get_xiaozhi().device_state in [
|
||||
DeviceState.LISTENING,
|
||||
DeviceState.SPEAKING,
|
||||
]:
|
||||
time.sleep(0.01)
|
||||
continue
|
||||
|
||||
result = SherpaOnnx.kws(frames)
|
||||
if result:
|
||||
print(f"🔥 触发唤醒: {result}")
|
||||
self.on_message(result)
|
||||
|
||||
def on_message(self, text: str):
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self._on_message(text), get_xiaoai().async_loop
|
||||
)
|
||||
|
||||
async def _on_message(self, text: str):
|
||||
before_wakeup = APP_CONFIG["wakeup"]["before_wakeup"]
|
||||
wakeup = await before_wakeup(get_speaker(), text, "kws")
|
||||
if wakeup:
|
||||
EventManager.on_wakeup()
|
||||
|
||||
|
||||
KWS = _KWS()
|
||||
@@ -0,0 +1,52 @@
|
||||
import re
|
||||
|
||||
from sherpa_onnx import text2token
|
||||
|
||||
|
||||
def init_project_context():
|
||||
"""动态导入父模块"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
project_root = os.path.abspath(
|
||||
os.path.join(os.path.dirname(__file__), "../../../..")
|
||||
)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
|
||||
init_project_context()
|
||||
|
||||
from config import APP_CONFIG
|
||||
from xiaozhi.utils.file import get_model_file_path
|
||||
|
||||
|
||||
def get_args():
|
||||
tokens_type = "cjkchar+bpe"
|
||||
tokens = get_model_file_path("tokens.txt")
|
||||
bpe_model = get_model_file_path("bpe.model")
|
||||
output = get_model_file_path("keywords.txt")
|
||||
keywords = APP_CONFIG["wakeup"]["keywords"]
|
||||
texts = [f"{keyword.upper()}" for keyword in keywords]
|
||||
return locals()
|
||||
|
||||
|
||||
def main():
|
||||
args = get_args()
|
||||
encoded_texts = text2token(
|
||||
args["texts"],
|
||||
tokens=args["tokens"],
|
||||
tokens_type=args["tokens_type"],
|
||||
bpe_model=args["bpe_model"],
|
||||
)
|
||||
with open(args["output"], "w", encoding="utf8") as f:
|
||||
for _, txt in enumerate(encoded_texts):
|
||||
line = "".join(txt)
|
||||
if re.match(r"^[▁A-Z\s]+$", line):
|
||||
f.write(" ".join(txt) + "\n")
|
||||
else:
|
||||
f.write(" ".join(txt) + f" @{line}" + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,36 @@
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
|
||||
from xiaozhi.utils.file import get_model_file_path
|
||||
|
||||
|
||||
class _SherpaOnnx:
|
||||
def start(self):
|
||||
self.keyword_spotter = sherpa_onnx.KeywordSpotter(
|
||||
provider="cpu",
|
||||
num_threads=1,
|
||||
max_active_paths=4,
|
||||
keywords_score=2.0,
|
||||
keywords_threshold=0.2,
|
||||
num_trailing_blanks=1,
|
||||
keywords_file=get_model_file_path("keywords.txt"),
|
||||
tokens=get_model_file_path("tokens.txt"),
|
||||
encoder=get_model_file_path("encoder.onnx"),
|
||||
decoder=get_model_file_path("decoder.onnx"),
|
||||
joiner=get_model_file_path("joiner.onnx"),
|
||||
)
|
||||
self.stream = self.keyword_spotter.create_stream()
|
||||
|
||||
def kws(self, frames):
|
||||
samples = np.frombuffer(frames, dtype=np.int16)
|
||||
samples = samples.astype(np.float32) / 32768.0
|
||||
self.stream.accept_waveform(16000, samples)
|
||||
while self.keyword_spotter.is_ready(self.stream):
|
||||
self.keyword_spotter.decode_stream(self.stream)
|
||||
result = self.keyword_spotter.get_result(self.stream)
|
||||
if result:
|
||||
self.keyword_spotter.reset_stream(self.stream)
|
||||
return result.lower()
|
||||
|
||||
|
||||
SherpaOnnx = _SherpaOnnx()
|
||||
Binary file not shown.
@@ -1,45 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger("Opus")
|
||||
|
||||
|
||||
def setup_opus():
|
||||
libs_dir = ""
|
||||
lib_path = ""
|
||||
|
||||
logger.info("正在加载 Opus 动态库，请耐心等待...")
|
||||
|
||||
if sys.platform == "win32":
|
||||
libs_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
lib_path = os.path.join(libs_dir, "opus.dll")
|
||||
elif sys.platform == "darwin":
|
||||
result = subprocess.check_output(
|
||||
"brew list opus | grep libopus.dylib", shell=True
|
||||
)
|
||||
lib_path = result.decode("utf-8").strip()
|
||||
if not lib_path.endswith("libopus.dylib"):
|
||||
raise RuntimeError("请先安装 Opus： brew install opus")
|
||||
libs_dir = os.path.dirname(lib_path)
|
||||
else:
|
||||
raise RuntimeError(f"暂不支持 {sys.platform} 平台")
|
||||
|
||||
import ctypes.util
|
||||
|
||||
original_find_library = ctypes.util.find_library
|
||||
|
||||
def patched_find_library(name):
|
||||
if name == "opus":
|
||||
return lib_path
|
||||
return original_find_library(name)
|
||||
|
||||
ctypes.util.find_library = patched_find_library
|
||||
|
||||
if hasattr(os, "add_dll_directory"):
|
||||
os.add_dll_directory(libs_dir)
|
||||
|
||||
os.environ["PATH"] = libs_dir + os.pathsep + os.environ.get("PATH", "")
|
||||
|
||||
ctypes.CDLL(lib_path)
|
||||
@@ -1,159 +1,38 @@
|
||||
from typing import ClassVar, Optional, Callable, Any
|
||||
from threading import Lock
|
||||
from collections import deque
|
||||
import uuid
|
||||
from typing import Any, Callable, ClassVar, Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
from config import APP_CONFIG
|
||||
from xiaozhi.ref import get_xiaoai
|
||||
|
||||
|
||||
class GlobalStream:
|
||||
"""
|
||||
全局音频缓冲区，用于存储和分发音频数据
|
||||
class __GlobalStream:
|
||||
def __init__(self):
|
||||
self.readers = {}
|
||||
self.on_output_data = None
|
||||
|
||||
用来将小爱音箱的音频输入输出流，适配到小智 AI 的音频输入输出流
|
||||
"""
|
||||
def register_reader(self, reader):
|
||||
if reader.id not in self.readers:
|
||||
self.readers[reader.id] = reader
|
||||
|
||||
_instance = None
|
||||
_lock = Lock()
|
||||
def unregister_reader(self, reader) -> None:
|
||||
if reader.id in self.readers:
|
||||
del self.readers[reader.id]
|
||||
|
||||
# 默认缓冲区大小（以字节为单位）
|
||||
DEFAULT_BUFFER_SIZE = 1024 * 1024 # 1MB
|
||||
|
||||
def __new__(cls):
|
||||
with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = super(GlobalStream, cls).__new__(cls)
|
||||
cls._instance._initialize()
|
||||
return cls._instance
|
||||
|
||||
def _initialize(self):
|
||||
"""初始化实例变量"""
|
||||
self._max_buffer_size = self.DEFAULT_BUFFER_SIZE
|
||||
self._input_buffer = deque(maxlen=self._max_buffer_size)
|
||||
self._is_input_active = False
|
||||
self._is_output_active = False
|
||||
self._input_readers = {} # 跟踪每个流的读取位置
|
||||
self._reader_counter = 0 # 为每个读取器分配唯一ID
|
||||
self._buffer_overflow_count = 0 # 记录缓冲区溢出次数
|
||||
self.on_output_data = None # 输入数据回调函数
|
||||
|
||||
def set_buffer_size(self, frames: int) -> None:
|
||||
if frames <= 0:
|
||||
raise ValueError("缓冲区大小必须大于0")
|
||||
|
||||
# 创建新的有限长度缓冲区
|
||||
new_input_buffer = deque(self._input_buffer, maxlen=frames * 2)
|
||||
|
||||
# 替换旧缓冲区
|
||||
self._input_buffer = new_input_buffer
|
||||
self._max_buffer_size = frames * 2
|
||||
|
||||
# 重置读取位置，因为缓冲区可能已经改变
|
||||
for reader_id in self._input_readers:
|
||||
self._input_readers[reader_id] = 0
|
||||
|
||||
def register_reader(self) -> int:
|
||||
"""注册一个新的读取器并返回其ID"""
|
||||
reader_id = self._reader_counter
|
||||
self._input_readers[reader_id] = 0 # 初始位置为0
|
||||
self._reader_counter += 1
|
||||
return reader_id
|
||||
|
||||
def unregister_reader(self, reader_id: int) -> None:
|
||||
"""注销一个读取器"""
|
||||
if reader_id in self._input_readers:
|
||||
del self._input_readers[reader_id]
|
||||
|
||||
def read(self, reader_id: int, num_frames: int) -> bytes:
|
||||
num_frames = num_frames * 2
|
||||
if not self._is_input_active:
|
||||
return bytes(num_frames)
|
||||
|
||||
if reader_id not in self._input_readers:
|
||||
return bytes(num_frames)
|
||||
|
||||
# 将输入缓冲区转换为列表以便随机访问
|
||||
buffer_list = list(self._input_buffer)
|
||||
current_pos = self._input_readers[reader_id]
|
||||
|
||||
# 如果当前位置超出缓冲区大小，返回空字节
|
||||
if current_pos >= len(buffer_list):
|
||||
return bytes(num_frames)
|
||||
|
||||
# 读取数据
|
||||
end_pos = min(current_pos + num_frames, len(buffer_list))
|
||||
data = bytes(buffer_list[current_pos:end_pos])
|
||||
|
||||
# 更新读取位置
|
||||
self._input_readers[reader_id] = end_pos
|
||||
|
||||
# 如果数据不足，用零填充
|
||||
if len(data) < num_frames:
|
||||
data += bytes(num_frames - len(data))
|
||||
|
||||
return data
|
||||
|
||||
# 转发输出音频流
|
||||
def write(self, frames: bytes) -> None:
|
||||
"""写入数据到输出缓冲区"""
|
||||
if not self._is_output_active:
|
||||
return
|
||||
def input(self, data: bytes) -> None:
|
||||
for key in self.readers:
|
||||
self.readers[key].input(data)
|
||||
|
||||
def output(self, frames: bytes) -> None:
|
||||
if self.on_output_data:
|
||||
self.on_output_data(frames)
|
||||
|
||||
# 添加输入音频流
|
||||
def add_input_data(self, data: bytes) -> None:
|
||||
if not self._is_input_active:
|
||||
return
|
||||
|
||||
"""添加输入数据到全局缓冲区"""
|
||||
# 检查是否会溢出
|
||||
if len(self._input_buffer) + len(data) > self._max_buffer_size:
|
||||
self._buffer_overflow_count += 1
|
||||
|
||||
for b in data:
|
||||
self._input_buffer.append(b)
|
||||
|
||||
# 如果有读取器的位置已经超出了缓冲区大小，需要调整
|
||||
if len(self._input_buffer) >= self._max_buffer_size:
|
||||
for reader_id in self._input_readers:
|
||||
if self._input_readers[reader_id] > len(self._input_buffer) // 2:
|
||||
# 将读取位置重置到缓冲区中间，避免读取器永远跟不上
|
||||
self._input_readers[reader_id] = len(self._input_buffer) // 2
|
||||
|
||||
def start_input(self) -> None:
|
||||
"""启动输入流"""
|
||||
self._is_input_active = True
|
||||
|
||||
def stop_input(self) -> None:
|
||||
"""停止输入流"""
|
||||
self._is_input_active = False
|
||||
self.clear()
|
||||
|
||||
def start_output(self) -> None:
|
||||
"""启动输出流"""
|
||||
self._is_output_active = True
|
||||
|
||||
def stop_output(self) -> None:
|
||||
"""停止输出流"""
|
||||
self._is_output_active = False
|
||||
|
||||
def is_input_active(self) -> bool:
|
||||
"""检查输入流是否活跃"""
|
||||
return self._is_input_active
|
||||
|
||||
def is_output_active(self) -> bool:
|
||||
"""检查输出流是否活跃"""
|
||||
return self._is_output_active
|
||||
|
||||
def clear(self) -> None:
|
||||
"""清空缓冲区"""
|
||||
self._input_buffer.clear()
|
||||
for reader_id in self._input_readers:
|
||||
self._input_readers[reader_id] = 0
|
||||
GlobalStream = __GlobalStream()
|
||||
|
||||
|
||||
class MyStream:
|
||||
"""音频流类，用于读写音频数据"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
rate: int,
|
||||
@@ -164,6 +43,7 @@ class MyStream:
|
||||
frames_per_buffer: int = 1024,
|
||||
start: bool = True,
|
||||
) -> None:
|
||||
self.id = uuid.uuid4()
|
||||
self._rate = rate
|
||||
self._channels = channels
|
||||
self._format = format
|
||||
@@ -171,79 +51,112 @@ class MyStream:
|
||||
self._is_input = input
|
||||
self._is_output = output
|
||||
self._is_active = False
|
||||
self._is_closed = False
|
||||
|
||||
# 获取全局音频缓冲区
|
||||
self._global_buffer = GlobalStream()
|
||||
self.input_bytes: list[int] = []
|
||||
|
||||
# 注册读取器ID
|
||||
self._reader_id = self._global_buffer.register_reader() if input else None
|
||||
|
||||
# 如果需要，启动流
|
||||
if start:
|
||||
self.start_stream()
|
||||
|
||||
def close(self) -> None:
|
||||
"""关闭流"""
|
||||
if not self._is_closed:
|
||||
self.stop_stream()
|
||||
if self._is_input and self._reader_id is not None:
|
||||
self._global_buffer.unregister_reader(self._reader_id)
|
||||
self._is_closed = True
|
||||
self.stop_stream()
|
||||
|
||||
def is_active(self) -> bool:
|
||||
"""检查流是否活跃"""
|
||||
return self._is_active
|
||||
|
||||
def start_stream(self) -> None:
|
||||
"""启动流"""
|
||||
if not self._is_active and not self._is_closed:
|
||||
if not self._is_active:
|
||||
self._is_active = True
|
||||
if self._is_input:
|
||||
self._global_buffer.start_input()
|
||||
if self._is_output:
|
||||
self._global_buffer.start_output()
|
||||
GlobalStream.register_reader(self)
|
||||
|
||||
def stop_stream(self) -> None:
|
||||
"""停止流"""
|
||||
if self._is_active:
|
||||
self._is_active = False
|
||||
if self._is_input:
|
||||
self._global_buffer.stop_input()
|
||||
if self._is_output:
|
||||
self._global_buffer.stop_output()
|
||||
|
||||
def read(self, num_frames: int, exception_on_overflow=False) -> bytes:
|
||||
"""从输入流读取数据"""
|
||||
if (
|
||||
not self._is_input
|
||||
or self._is_closed
|
||||
or not self._is_active
|
||||
or self._reader_id is None
|
||||
):
|
||||
return bytes(num_frames)
|
||||
|
||||
return self._global_buffer.read(self._reader_id, num_frames)
|
||||
GlobalStream.unregister_reader(self)
|
||||
self.input_bytes = []
|
||||
|
||||
def write(self, frames: bytes) -> None:
|
||||
"""写入数据到输出流"""
|
||||
if not self._is_output or self._is_closed or not self._is_active:
|
||||
# 发送输出音频流到扬声器
|
||||
if not self._is_output or not self._is_active:
|
||||
return
|
||||
GlobalStream.output(frames)
|
||||
|
||||
def input(self, data: bytes):
|
||||
# 收到麦克风输入音频流
|
||||
if not self._is_input or not self._is_active:
|
||||
return
|
||||
|
||||
self._global_buffer.write(frames)
|
||||
if len(data) > 0:
|
||||
samples = np.frombuffer(data, dtype=np.int16)
|
||||
# 小爱音箱录音音量较小，需要后期放大一下
|
||||
samples = samples * APP_CONFIG["vad"]["boost"]
|
||||
self.input_bytes.extend(samples.tobytes())
|
||||
|
||||
def read(self, num_frames=None, exception_on_overflow=False) -> bytes:
|
||||
if num_frames is None:
|
||||
data = bytes(self.input_bytes)
|
||||
self.input_bytes = []
|
||||
return data
|
||||
|
||||
num_frames = num_frames * 2
|
||||
if (
|
||||
not self._is_input
|
||||
or not self._is_active
|
||||
# 达不到预期长度时，返回空字节，等待下一次读取
|
||||
or len(self.input_bytes) < num_frames
|
||||
):
|
||||
return bytes([])
|
||||
|
||||
data = bytes(self.input_bytes[:num_frames])
|
||||
self.input_bytes = self.input_bytes[num_frames:]
|
||||
|
||||
return data
|
||||
|
||||
|
||||
class MyAudio:
|
||||
"""PyAudio替代品，用于创建和管理音频流"""
|
||||
"""PyAudio 替代品，用于创建和管理音频流"""
|
||||
|
||||
Stream: ClassVar[type] = MyStream
|
||||
|
||||
def __init__(self, buffer_size: int = GlobalStream.DEFAULT_BUFFER_SIZE) -> None:
|
||||
# 初始化全局音频缓冲区
|
||||
self._global_buffer = GlobalStream()
|
||||
# 设置缓冲区大小
|
||||
if buffer_size != GlobalStream.DEFAULT_BUFFER_SIZE:
|
||||
self._global_buffer.set_buffer_size(buffer_size)
|
||||
@classmethod
|
||||
def create(cls):
|
||||
if get_xiaoai().mode != "xiaozhi":
|
||||
return MyAudio()
|
||||
else:
|
||||
from pyaudio import PyAudio
|
||||
|
||||
return PyAudio()
|
||||
|
||||
@classmethod
|
||||
def get_input_device_index(cls, audio):
|
||||
if get_xiaoai().mode != "xiaozhi":
|
||||
return 0
|
||||
try:
|
||||
device = audio.get_default_input_device_info()
|
||||
return device["index"]
|
||||
except Exception:
|
||||
for i in range(audio.get_device_count()):
|
||||
dev = audio.get_device_info_by_index(i)
|
||||
if dev["maxInputChannels"] > 0:
|
||||
return i
|
||||
return 0
|
||||
|
||||
@classmethod
|
||||
def get_output_device_index(cls, audio):
|
||||
if get_xiaoai().mode != "xiaozhi":
|
||||
return 0
|
||||
try:
|
||||
device = audio.get_default_output_device_info()
|
||||
return device["index"]
|
||||
except Exception:
|
||||
for i in range(audio.get_device_count()):
|
||||
dev = audio.get_device_info_by_index(i)
|
||||
if dev["maxOutputChannels"] > 0:
|
||||
return i
|
||||
return 0
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._is_terminated = False
|
||||
|
||||
def open(
|
||||
@@ -261,17 +174,9 @@ class MyAudio:
|
||||
output_host_api_specific_stream_info: Optional[Any] = None,
|
||||
stream_callback: Optional[Callable] = None,
|
||||
) -> MyStream:
|
||||
"""打开一个新的音频流"""
|
||||
if self._is_terminated:
|
||||
raise RuntimeError("MyAudio instance has been terminated")
|
||||
|
||||
# 启动全局输入/输出流（如果需要）
|
||||
if input:
|
||||
self._global_buffer.start_input()
|
||||
if output:
|
||||
self._global_buffer.start_output()
|
||||
|
||||
# 创建并返回一个新的流实例
|
||||
return MyStream(
|
||||
rate=rate,
|
||||
channels=channels,
|
||||
@@ -283,8 +188,5 @@ class MyAudio:
|
||||
)
|
||||
|
||||
def terminate(self) -> None:
|
||||
"""终止MyAudio实例"""
|
||||
if not self._is_terminated:
|
||||
self._global_buffer.stop_input()
|
||||
self._global_buffer.stop_output()
|
||||
self._is_terminated = True
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
import threading
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
from config import APP_CONFIG
|
||||
from xiaozhi.event import EventManager
|
||||
from xiaozhi.ref import set_vad
|
||||
from xiaozhi.services.audio.stream import MyAudio
|
||||
from xiaozhi.services.audio.vad.silero import VAD_MODEL
|
||||
from xiaozhi.services.protocols.typing import AudioConfig
|
||||
from xiaozhi.utils.base import get_env
|
||||
|
||||
|
||||
class _VAD:
|
||||
"""基于WebRTC VAD的语音活动检测器，用于检测用户打断"""
|
||||
|
||||
def __init__(self):
|
||||
set_vad(self)
|
||||
|
||||
config = APP_CONFIG.get("vad", {})
|
||||
|
||||
# 参数设置
|
||||
self.sample_rate = 16000
|
||||
self.frame_size = 512
|
||||
self.threshold = config.get("threshold", 0.01)
|
||||
self.min_speech_duration = config.get("min_speech_duration", 250)
|
||||
self.min_silence_duration = config.get("min_silence_duration", 500)
|
||||
|
||||
# 状态变量
|
||||
self.paused = True
|
||||
self.thread = None
|
||||
self.speech_count = 0
|
||||
self.silence_count = 0
|
||||
|
||||
# 创建独立的PyAudio实例和流，避免与主音频流冲突
|
||||
self.audio = None
|
||||
self.stream = None
|
||||
|
||||
# 暂存的语音片段
|
||||
self.temp_frames = []
|
||||
self.speech_buffer = []
|
||||
self.target = None # 检测目标 speech/silence
|
||||
|
||||
def _reset_state(self):
|
||||
"""重置状态"""
|
||||
self.speech_count = 0
|
||||
self.silence_count = 0
|
||||
self.speech_buffer = []
|
||||
self.temp_frames = []
|
||||
|
||||
def start(self):
|
||||
"""启动VAD检测器"""
|
||||
if not get_env("CLI"):
|
||||
return
|
||||
|
||||
if self.thread and self.thread.is_alive():
|
||||
return
|
||||
|
||||
self.paused = False
|
||||
|
||||
# 初始化PyAudio和流
|
||||
self._initialize_audio_stream()
|
||||
|
||||
# 启动检测线程
|
||||
self.thread = threading.Thread(target=self._detection_loop, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def pause(self):
|
||||
"""暂停VAD检测"""
|
||||
if not get_env("CLI"):
|
||||
return
|
||||
|
||||
self.paused = True
|
||||
self._reset_state()
|
||||
self.stream.stop_stream()
|
||||
|
||||
def resume(self, target: str):
|
||||
"""恢复VAD检测"""
|
||||
if not get_env("CLI"):
|
||||
return
|
||||
|
||||
self.paused = False
|
||||
self.target = target
|
||||
self.stream.start_stream()
|
||||
|
||||
def _handle_speech_frame(self, frames):
|
||||
"""处理语音帧"""
|
||||
self.speech_count += len(frames)
|
||||
self.silence_count = 0
|
||||
self.speech_buffer.extend(frames)
|
||||
|
||||
speech_bytes = bytes(self.speech_buffer)
|
||||
|
||||
if (
|
||||
self.target == "speech"
|
||||
and self.speech_count > self.min_speech_duration * self.sample_rate / 1000
|
||||
):
|
||||
self.pause()
|
||||
EventManager.on_speech(speech_bytes)
|
||||
|
||||
def _handle_silence_frame(self, frames):
|
||||
"""处理静音帧"""
|
||||
self.silence_count += len(frames)
|
||||
self.speech_count = 0
|
||||
self.speech_buffer = []
|
||||
|
||||
if (
|
||||
self.target == "silence"
|
||||
and self.silence_count > self.min_silence_duration * self.sample_rate / 1000
|
||||
):
|
||||
self.pause()
|
||||
EventManager.on_silence()
|
||||
|
||||
def _detect_speech(self, frames):
|
||||
"""检测是否是语音"""
|
||||
try:
|
||||
audio_int16 = np.frombuffer(frames, dtype=np.int16)
|
||||
audio_float32 = audio_int16.astype(np.float32) / 32768.0
|
||||
speech_prob = VAD_MODEL(audio_float32, 16000).item()
|
||||
is_speech = speech_prob >= self.threshold
|
||||
return is_speech
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _initialize_audio_stream(self):
|
||||
"""初始化独立的音频流"""
|
||||
try:
|
||||
# 创建 PyAudio 实例
|
||||
self.audio = MyAudio.create()
|
||||
# 创建输入流
|
||||
self.stream = self.audio.open(
|
||||
format=AudioConfig.FORMAT,
|
||||
channels=1,
|
||||
rate=self.sample_rate,
|
||||
input=True,
|
||||
frames_per_buffer=self.frame_size,
|
||||
start=True,
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _close_audio_stream(self):
|
||||
"""关闭音频流"""
|
||||
try:
|
||||
if self.stream:
|
||||
self.stream.stop_stream()
|
||||
self.stream.close()
|
||||
self.stream = None
|
||||
|
||||
if self.audio:
|
||||
self.audio.terminate()
|
||||
self.audio = None
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _detection_loop(self):
|
||||
"""VAD检测主循环"""
|
||||
while True:
|
||||
# 如果暂停或者音频流未初始化，则跳过
|
||||
if self.paused or not self.stream:
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
|
||||
# 读取缓冲区音频数据
|
||||
frames = self.stream.read(self.frame_size)
|
||||
if len(frames) != self.frame_size * 2:
|
||||
time.sleep(0.01)
|
||||
continue
|
||||
|
||||
# 检测是否是语音
|
||||
is_speech = self._detect_speech(frames)
|
||||
if is_speech:
|
||||
self._handle_speech_frame(frames)
|
||||
else:
|
||||
self._handle_silence_frame(frames)
|
||||
|
||||
time.sleep(0.01)
|
||||
|
||||
|
||||
VAD = _VAD()
|
||||
@@ -0,0 +1,89 @@
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
|
||||
from xiaozhi.utils.file import get_model_file_path
|
||||
|
||||
|
||||
class OnnxWrapper:
|
||||
def __init__(self, path):
|
||||
opts = ort.SessionOptions()
|
||||
opts.inter_op_num_threads = 1
|
||||
opts.intra_op_num_threads = 1
|
||||
self.session = ort.InferenceSession(
|
||||
path, providers=["CPUExecutionProvider"], sess_options=opts
|
||||
)
|
||||
self.reset_states()
|
||||
self.sample_rates = [8000, 16000]
|
||||
|
||||
def _validate_input(self, x, sr: int):
|
||||
if len(x.shape) == 1:
|
||||
x = np.expand_dims(x, 0)
|
||||
if len(x.shape) > 2:
|
||||
raise ValueError(
|
||||
f"Too many dimensions for input audio chunk {len(x.shape)}"
|
||||
)
|
||||
|
||||
if sr != 16000 and (sr % 16000 == 0):
|
||||
step = sr // 16000
|
||||
x = x[:, ::step]
|
||||
sr = 16000
|
||||
|
||||
if sr not in self.sample_rates:
|
||||
raise ValueError(
|
||||
f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)"
|
||||
)
|
||||
if sr / x.shape[1] > 31.25:
|
||||
raise ValueError("Input audio chunk is too short")
|
||||
|
||||
return x, sr
|
||||
|
||||
def reset_states(self, batch_size=1):
|
||||
self._state = np.zeros((2, batch_size, 128), dtype=np.float32)
|
||||
self._context = np.zeros(0, dtype=np.float32)
|
||||
self._last_sr = 0
|
||||
self._last_batch_size = 0
|
||||
|
||||
def __call__(self, x, sr: int):
|
||||
x, sr = self._validate_input(x, sr)
|
||||
num_samples = 512 if sr == 16000 else 256
|
||||
|
||||
if x.shape[-1] != num_samples:
|
||||
raise ValueError(
|
||||
f"Provided number of samples is {x.shape[-1]} (Supported values: 256 for 8000 sample rate, 512 for 16000)"
|
||||
)
|
||||
|
||||
batch_size = x.shape[0]
|
||||
context_size = 64 if sr == 16000 else 32
|
||||
|
||||
if not self._last_batch_size:
|
||||
self.reset_states(batch_size)
|
||||
if (self._last_sr) and (self._last_sr != sr):
|
||||
self.reset_states(batch_size)
|
||||
if (self._last_batch_size) and (self._last_batch_size != batch_size):
|
||||
self.reset_states(batch_size)
|
||||
|
||||
if not len(self._context):
|
||||
self._context = np.zeros((batch_size, context_size), dtype=np.float32)
|
||||
|
||||
x = np.concatenate([self._context, x], axis=1)
|
||||
if sr in [8000, 16000]:
|
||||
ort_inputs = {
|
||||
"input": x,
|
||||
"state": self._state,
|
||||
"sr": np.array(sr, dtype="int64"),
|
||||
}
|
||||
ort_outs = self.session.run(None, ort_inputs)
|
||||
out, state = ort_outs
|
||||
self._state = state
|
||||
else:
|
||||
raise ValueError()
|
||||
|
||||
self._context = x[..., -context_size:]
|
||||
self._last_sr = sr
|
||||
self._last_batch_size = batch_size
|
||||
return out
|
||||
|
||||
|
||||
VAD_MODEL = OnnxWrapper(
|
||||
path=get_model_file_path("silero_vad.onnx"),
|
||||
)
|
||||
Reference in New Issue
Block a user