Hass/open-xiaoai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(xiaozhi): 新增小智 AI 一键运行 Docker 镜像

Browse Source
This commit is contained in:
Del Wang
2025-05-19 00:16:49 +08:00
parent 59c3280938
commit 58acbc29f4
39 changed files with 1729 additions and 1152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/xiaozhi/xiaozhi/event.py
+159
View File
@@ -0,0 +1,159 @@
import asyncio
from config import APP_CONFIG
from xiaozhi.ref import get_audio_codec, get_speaker, get_vad, get_xiaoai, get_xiaozhi
from xiaozhi.services.protocols.typing import AbortReason, DeviceState, ListeningMode
class Step:
idle = "idle"
on_interrupt = "on_interrupt"
on_wakeup = "on_wakeup"
on_tts_start = "on_tts_start"
on_tts_end = "on_tts_end"
on_speech = "on_speech"
on_silence = "on_silence"
class __EventManager:
def __init__(self):
self.session_id = 0
self.current_step = Step.idle
self.next_step_future = None
def update_step(self, step: Step, step_data=None):
if get_xiaoai().mode == "xiaozhi":
return
self.current_step = step
if self.next_step_future:
get_xiaoai().async_loop.call_soon_threadsafe(
self.next_step_future.set_result, (step, step_data)
)
self.next_step_future = None
async def wait_next_step(self, timeout=None) -> Step | None:
current_session = self.session_id
self.next_step_future = get_xiaoai().async_loop.create_future()
async def _timeout(timeout):
idx = 0
while idx < timeout:
idx += 1
await asyncio.sleep(1)
return ("timeout", None)
futures = [self.next_step_future]
if timeout:
futures.append(get_xiaoai().async_loop.create_task(_timeout(timeout)))
done, _ = await asyncio.wait(
futures,
return_when=asyncio.FIRST_COMPLETED,
)
if current_session != self.session_id:
# 当前 session 已经结束
return ("interrupted", None)
return list(done)[0].result()
def on_interrupt(self):
"""用户打断（小爱同学）"""
self.session_id = self.session_id + 1
self.update_step(Step.on_interrupt)
self.start_session()
def on_wakeup(self):
"""用户唤醒（你好小智）"""
self.session_id = self.session_id + 1
self.update_step(Step.on_wakeup)
self.start_session()
def on_tts_end(self):
"""TTS结束"""
if self.current_step == Step.on_interrupt:
# 当前 session 已经被打断了，不再处理
return
self.session_id = self.session_id + 1
self.update_step(Step.on_tts_end)
self.start_session()
def on_tts_start(self):
"""TTS结束"""
self.update_step(Step.on_tts_start)
def on_speech(self, speech_buffer: bytes):
"""检测到声音（开始说话"""
self.update_step(Step.on_speech, speech_buffer)
def on_silence(self):
"""检测到静音（说话结束）"""
self.update_step(Step.on_silence)
def start_session(self):
asyncio.run_coroutine_threadsafe(
self.__start_session(), get_xiaoai().async_loop
)
async def __start_session(self):
if get_xiaoai().mode == "xiaozhi":
return
vad = get_vad()
codec = get_audio_codec()
speaker = get_speaker()
xiaozhi = get_xiaozhi()
# 先取消之前的 VAD 检测和音频输入输出流
xiaozhi.set_device_state(DeviceState.IDLE)
await xiaozhi.protocol.send_abort_speaking(AbortReason.ABORT)
# 小爱同学唤醒时，直接打断
if self.current_step == Step.on_interrupt:
return
# 尝试打开音频通道
if not xiaozhi.protocol.is_audio_channel_opened():
xiaozhi.set_device_state(DeviceState.CONNECTING)
await xiaozhi.protocol.open_audio_channel()
# 等待 TTS 余音结束
if self.current_step in [Step.on_tts_end]:
vad.resume("silence")
step, _ = await self.wait_next_step()
if step != Step.on_silence:
return
# 检查是否有人说话
vad.resume("speech")
step, speech_buffer = await self.wait_next_step(
timeout=APP_CONFIG["wakeup"]["timeout"]
)
if step == "timeout":
# 如果没人说话，则回到 IDLE 状态
xiaozhi.set_device_state(DeviceState.IDLE)
print("👋 已退出唤醒")
after_wakeup = APP_CONFIG["wakeup"]["after_wakeup"]
await after_wakeup(speaker)
return
if step != Step.on_speech:
return
# 开始说话
await xiaozhi.protocol.send_start_listening(ListeningMode.MANUAL)
codec.input_stream.input(speech_buffer) # 追加音频输入片段
xiaozhi.set_device_state(DeviceState.LISTENING)
# 等待说话结束
vad.resume("silence")
step, _ = await self.wait_next_step()
if step != Step.on_silence:
return
# 停止说话
await xiaozhi.protocol.send_stop_listening()
xiaozhi.set_device_state(DeviceState.IDLE)
EventManager = __EventManager()
examples/xiaozhi/xiaozhi/models/keywords.txt
+7
View File
@@ -0,0 +1,7 @@
天 猫 精 灵 @天猫精灵
小 度 小 度 @小度小度
豆 包 豆 包 @豆包豆包
你 好 小 智 @你好小智
你 好 小 爱 @你好小爱
▁HI ▁S I RI
▁HE Y ▁S I RI
examples/xiaozhi/xiaozhi/ref.py
+51
View File
@@ -0,0 +1,51 @@
from typing import Any
GLOBAL_STATES = {}
def set_xiaozhi(xiaozhi: Any):
GLOBAL_STATES["xiaozhi"] = xiaozhi
def get_xiaozhi() -> Any:
return GLOBAL_STATES.get("xiaozhi")
def set_xiaoai(xiaoai: Any):
GLOBAL_STATES["xiaoai"] = xiaoai
def get_xiaoai() -> Any:
return GLOBAL_STATES.get("xiaoai")
def set_vad(vad: Any):
GLOBAL_STATES["vad"] = vad
def get_vad() -> Any:
return GLOBAL_STATES.get("vad")
def set_audio_codec(opus_encoder: Any):
GLOBAL_STATES["opus_encoder"] = opus_encoder
def get_audio_codec() -> Any:
return GLOBAL_STATES.get("opus_encoder")
def get_speaker() -> Any:
return GLOBAL_STATES.get("speaker")
def set_speaker(speaker: Any):
GLOBAL_STATES["speaker"] = speaker
def set_kws(kws: Any):
GLOBAL_STATES["kws"] = kws
def get_kws() -> Any:
return GLOBAL_STATES.get("kws")
examples/xiaozhi/xiaozhi/services/audio/codec.py
+75 -194
View File
@@ -1,17 +1,8 @@
import logging
import queue
import sys
import time
import numpy as np
import opuslib
import pyaudio
import opuslib_next as opuslib
from xiaozhi.ref import set_audio_codec
from xiaozhi.services.audio.stream import MyAudio
from xiaozhi.services.protocols.typing import AudioConfig
from xiaozhi.xiaoai import XiaoAI
logger = logging.getLogger("AudioCodec")
class AudioCodec:
@@ -24,50 +15,46 @@ class AudioCodec:
self.output_stream = None
self.opus_encoder = None
self.opus_decoder = None
self.audio_decode_queue = queue.Queue()
self._is_closing = False
self._initialize_audio()
set_audio_codec(self)
def _initialize_audio(self):
"""初始化音频设备和编解码器"""
try:
self.audio = MyAudio() if XiaoAI.mode == "xiaoai" else pyaudio.PyAudio()
self.audio = MyAudio.create()
# 初始化音频输入流
self.input_stream = self.audio.open(
format=pyaudio.paInt16,
channels=AudioConfig.CHANNELS,
rate=AudioConfig.SAMPLE_RATE,
input=True,
frames_per_buffer=AudioConfig.FRAME_SIZE,
)
# 初始化音频输入流
self.input_stream = self.audio.open(
format=AudioConfig.FORMAT,
channels=AudioConfig.CHANNELS,
rate=AudioConfig.SAMPLE_RATE,
input=True,
frames_per_buffer=AudioConfig.FRAME_SIZE,
input_device_index=MyAudio.get_input_device_index(self.audio),
)
# 初始化音频输出流
self.output_stream = self.audio.open(
format=pyaudio.paInt16,
channels=AudioConfig.CHANNELS,
rate=AudioConfig.SAMPLE_RATE,
output=True,
frames_per_buffer=AudioConfig.FRAME_SIZE,
)
# 初始化音频输出流
self.output_stream = self.audio.open(
format=AudioConfig.FORMAT,
channels=AudioConfig.CHANNELS,
rate=AudioConfig.SAMPLE_RATE,
output=True,
frames_per_buffer=AudioConfig.FRAME_SIZE,
output_device_index=MyAudio.get_output_device_index(self.audio),
)
# 初始化Opus编码器
self.opus_encoder = opuslib.Encoder(
fs=AudioConfig.SAMPLE_RATE,
channels=AudioConfig.CHANNELS,
application=opuslib.APPLICATION_AUDIO,
)
# 初始化Opus编码器
self.opus_encoder = opuslib.Encoder(
fs=AudioConfig.SAMPLE_RATE,
channels=AudioConfig.CHANNELS,
application=opuslib.APPLICATION_AUDIO,
)
# 初始化Opus解码器
self.opus_decoder = opuslib.Decoder(
fs=AudioConfig.SAMPLE_RATE, channels=AudioConfig.CHANNELS
)
logger.info("音频设备和编解码器初始化成功")
except Exception as e:
logger.error(f"初始化音频设备失败: {e}")
raise
# 初始化Opus解码器
self.opus_decoder = opuslib.Decoder(
fs=AudioConfig.SAMPLE_RATE, channels=AudioConfig.CHANNELS
)
def read_audio(self):
"""读取音频输入数据并编码"""
@@ -78,96 +65,35 @@ class AudioCodec:
if not data:
return None
return self.opus_encoder.encode(data, AudioConfig.FRAME_SIZE)
except Exception as e:
logger.error(f"读取音频输入时出错: {e}")
except Exception:
return None
def write_audio(self, opus_data):
"""将编码的音频数据添加到播放队列"""
self.audio_decode_queue.put(opus_data)
def play_audio(self):
"""处理并播放队列中的音频数据"""
"""解码并播放"""
try:
# 批量处理多个音频包以减少处理延迟
batch_size = min(10, self.audio_decode_queue.qsize())
if batch_size == 0:
return False
# 创建缓冲区存储解码后的数据
buffer = bytearray()
for _ in range(batch_size):
if self.audio_decode_queue.empty():
break
opus_data = self.audio_decode_queue.get_nowait()
try:
pcm_data = self.opus_decoder.decode(
opus_data, AudioConfig.FRAME_SIZE, decode_fec=False
)
buffer.extend(pcm_data)
except Exception as e:
logger.error(f"解码音频数据时出错: {e}")
# 只有在有数据时才处理和播放
if len(buffer) > 0:
# 转换为numpy数组
pcm_array = np.frombuffer(buffer, dtype=np.int16)
# 播放音频
try:
if self.output_stream and self.output_stream.is_active():
self.output_stream.write(pcm_array.tobytes())
return True
else:
# MAC 特定：如果流不活跃，尝试重新初始化
self._reinitialize_output_stream()
if self.output_stream and self.output_stream.is_active():
self.output_stream.write(pcm_array.tobytes())
return True
except OSError as e:
if "Stream closed" in str(e) or "Internal PortAudio error" in str(
e
):
logger.error(f"播放音频时出错: {e}")
self._reinitialize_output_stream()
else:
logger.error(f"播放音频时出错: {e}")
except queue.Empty:
pcm_data = self.decode_audio(opus_data) # 解码
self.output_stream.write(pcm_data) # 播放
except Exception:
pass
except Exception as e:
logger.error(f"播放音频时出错: {e}")
self._reinitialize_output_stream()
return False
def decode_audio(self, opus_data, frame_size=AudioConfig.FRAME_SIZE):
"""解码音频数据"""
return self.opus_decoder.decode(opus_data, frame_size, decode_fec=False)
def has_pending_audio(self):
"""检查是否还有待播放的音频数据"""
return not self.audio_decode_queue.empty()
def wait_for_audio_complete(self, timeout=5.0):
# 等待音频队列清空
attempt = 0
max_attempts = 15
while not self.audio_decode_queue.empty() and attempt < max_attempts:
time.sleep(0.1)
attempt += 1
# 在关闭前清空任何剩余数据
while not self.audio_decode_queue.empty():
try:
self.audio_decode_queue.get_nowait()
except queue.Empty:
break
def clear_audio_queue(self):
"""清空音频队列"""
while not self.audio_decode_queue.empty():
try:
self.audio_decode_queue.get_nowait()
except queue.Empty:
break
def encode_audio(self, buffer, frame_size=AudioConfig.FRAME_SIZE):
"""编码音频数据"""
try:
opus_frames = []
for i in range(0, len(buffer), frame_size * 2):
chunk = buffer[i : i + frame_size * 2]
if len(chunk) < frame_size * 2:
# 如果 buffer 长度不是 FRAME_SIZE 的 2 倍，需要补齐
chunk += b"\x00" * (frame_size * 2 - len(chunk))
opus_frame = self.opus_encoder.encode(chunk, frame_size)
opus_frames.append(opus_frame)
return opus_frames
except Exception:
return None
def start_streams(self):
"""启动音频流"""
@@ -178,95 +104,50 @@ class AudioCodec:
def stop_streams(self):
"""停止音频流"""
if self.input_stream and self.input_stream.is_active():
if self.input_stream.is_active():
self.input_stream.stop_stream()
if self.output_stream and self.output_stream.is_active():
if self.output_stream.is_active():
self.output_stream.stop_stream()
def _reinitialize_output_stream(self):
"""重新初始化音频输出流"""
if self._is_closing: # 如果正在关闭，不要重新初始化
def close(self):
"""关闭音频编解码器，确保资源正确释放"""
if self._is_closing: # 防止重复关闭
return
self._is_closing = True
try:
# 关闭输入流
if self.input_stream:
try:
if self.input_stream.is_active():
self.input_stream.stop_stream()
self.input_stream.close()
except Exception:
pass
self.input_stream = None
# 关闭输出流
if self.output_stream:
try:
if self.output_stream.is_active():
self.output_stream.stop_stream()
self.output_stream.close()
except Exception:
# logger.warning(f"关闭旧输出流时出错: {e}")
pass
# 在 MAC 上添加短暂延迟
if sys.platform in ("darwin", "linux"):
time.sleep(0.1)
self.output_stream = self.audio.open(
format=pyaudio.paInt16,
channels=AudioConfig.CHANNELS,
rate=AudioConfig.SAMPLE_RATE,
output=True,
frames_per_buffer=AudioConfig.FRAME_SIZE,
)
logger.info("音频输出流重新初始化成功")
except Exception as e:
logger.error(f"重新初始化音频输出流失败: {e}")
raise
def close(self):
"""关闭音频编解码器，确保资源正确释放"""
if self._is_closing: # 防止重复关闭
return
self._is_closing = True
logger.info("开始关闭音频编解码器...")
try:
# 等待并清理剩余音频数据
self.wait_for_audio_complete()
# 关闭输入流
if self.input_stream:
logger.debug("正在关闭输入流...")
try:
if self.input_stream.is_active():
self.input_stream.stop_stream()
self.input_stream.close()
except Exception as e:
logger.error(f"关闭输入流时出错: {e}")
self.input_stream = None
# 关闭输出流
if self.output_stream:
logger.debug("正在关闭输出流...")
try:
if self.output_stream.is_active():
self.output_stream.stop_stream()
self.output_stream.close()
except Exception as e:
logger.error(f"关闭输出流时出错: {e}")
self.output_stream = None
# 关闭 PyAudio 实例
if self.audio:
logger.debug("正在终止 PyAudio...")
try:
self.audio.terminate()
except Exception as e:
logger.error(f"终止 PyAudio 时出错: {e}")
except Exception:
pass
self.audio = None
# 清理编解码器
self.opus_encoder = None
self.opus_decoder = None
logger.info("音频编解码器关闭完成")
except Exception as e:
logger.error(f"关闭音频编解码器时发生错误: {e}")
except Exception:
pass
finally:
self._is_closing = False
def __del__(self):
"""析构函数，确保资源被释放"""
self.close()
examples/xiaozhi/xiaozhi/services/audio/kws/__init__.py
+73
View File
@@ -0,0 +1,73 @@
import asyncio
import os
import threading
import time
from config import APP_CONFIG
from xiaozhi.event import EventManager
from xiaozhi.ref import get_speaker, get_xiaoai, get_xiaozhi, set_kws
from xiaozhi.services.audio.kws.sherpa import SherpaOnnx
from xiaozhi.services.audio.stream import MyAudio
from xiaozhi.services.protocols.typing import AudioConfig, DeviceState
from xiaozhi.utils.base import get_env
class _KWS:
def __init__(self):
set_kws(self)
def start(self):
if not get_env("CLI"):
return
self.audio = MyAudio.create()
self.stream = self.audio.open(
format=AudioConfig.FORMAT,
channels=1,
rate=16000,
input=True,
frames_per_buffer=AudioConfig.FRAME_SIZE,
start=True,
)
# 启动 KWS 服务
self.thread = threading.Thread(target=self._detection_loop, daemon=True)
self.thread.start()
def get_file_path(self, file_name: str):
current_dir = os.path.dirname(os.path.abspath(__file__))
return os.path.join(current_dir, "../../../models", file_name)
def _detection_loop(self):
SherpaOnnx.start()
self.stream.start_stream()
while True:
# 读取缓冲区音频数据
frames = self.stream.read()
# 在说话和监听状态时，暂停 KWS
if not frames or get_xiaozhi().device_state in [
DeviceState.LISTENING,
DeviceState.SPEAKING,
]:
time.sleep(0.01)
continue
result = SherpaOnnx.kws(frames)
if result:
print(f"🔥 触发唤醒: {result}")
self.on_message(result)
def on_message(self, text: str):
asyncio.run_coroutine_threadsafe(
self._on_message(text), get_xiaoai().async_loop
)
async def _on_message(self, text: str):
before_wakeup = APP_CONFIG["wakeup"]["before_wakeup"]
wakeup = await before_wakeup(get_speaker(), text, "kws")
if wakeup:
EventManager.on_wakeup()
KWS = _KWS()
examples/xiaozhi/xiaozhi/services/audio/kws/keywords.py
+52
View File
@@ -0,0 +1,52 @@
import re
from sherpa_onnx import text2token
def init_project_context():
"""动态导入父模块"""
import os
import sys
project_root = os.path.abspath(
os.path.join(os.path.dirname(__file__), "../../../..")
)
if project_root not in sys.path:
sys.path.insert(0, project_root)
init_project_context()
from config import APP_CONFIG
from xiaozhi.utils.file import get_model_file_path
def get_args():
tokens_type = "cjkchar+bpe"
tokens = get_model_file_path("tokens.txt")
bpe_model = get_model_file_path("bpe.model")
output = get_model_file_path("keywords.txt")
keywords = APP_CONFIG["wakeup"]["keywords"]
texts = [f"{keyword.upper()}" for keyword in keywords]
return locals()
def main():
args = get_args()
encoded_texts = text2token(
args["texts"],
tokens=args["tokens"],
tokens_type=args["tokens_type"],
bpe_model=args["bpe_model"],
)
with open(args["output"], "w", encoding="utf8") as f:
for _, txt in enumerate(encoded_texts):
line = "".join(txt)
if re.match(r"^[▁A-Z\s]+$", line):
f.write(" ".join(txt) + "\n")
else:
f.write(" ".join(txt) + f" @{line}" + "\n")
if __name__ == "__main__":
main()
examples/xiaozhi/xiaozhi/services/audio/kws/sherpa.py
+36
View File
@@ -0,0 +1,36 @@
import numpy as np
import sherpa_onnx
from xiaozhi.utils.file import get_model_file_path
class _SherpaOnnx:
def start(self):
self.keyword_spotter = sherpa_onnx.KeywordSpotter(
provider="cpu",
num_threads=1,
max_active_paths=4,
keywords_score=2.0,
keywords_threshold=0.2,
num_trailing_blanks=1,
keywords_file=get_model_file_path("keywords.txt"),
tokens=get_model_file_path("tokens.txt"),
encoder=get_model_file_path("encoder.onnx"),
decoder=get_model_file_path("decoder.onnx"),
joiner=get_model_file_path("joiner.onnx"),
)
self.stream = self.keyword_spotter.create_stream()
def kws(self, frames):
samples = np.frombuffer(frames, dtype=np.int16)
samples = samples.astype(np.float32) / 32768.0
self.stream.accept_waveform(16000, samples)
while self.keyword_spotter.is_ready(self.stream):
self.keyword_spotter.decode_stream(self.stream)
result = self.keyword_spotter.get_result(self.stream)
if result:
self.keyword_spotter.reset_stream(self.stream)
return result.lower()
SherpaOnnx = _SherpaOnnx()
examples/xiaozhi/xiaozhi/services/audio/opus.dll
BIN
View File
Binary file not shown.
examples/xiaozhi/xiaozhi/services/audio/setup.py
-45
View File
@@ -1,45 +0,0 @@
import os
import sys
import subprocess
import logging
logger = logging.getLogger("Opus")
def setup_opus():
libs_dir = ""
lib_path = ""
logger.info("正在加载 Opus 动态库，请耐心等待...")
if sys.platform == "win32":
libs_dir = os.path.dirname(os.path.abspath(__file__))
lib_path = os.path.join(libs_dir, "opus.dll")
elif sys.platform == "darwin":
result = subprocess.check_output(
"brew list opus | grep libopus.dylib", shell=True
)
lib_path = result.decode("utf-8").strip()
if not lib_path.endswith("libopus.dylib"):
raise RuntimeError("请先安装 Opus： brew install opus")
libs_dir = os.path.dirname(lib_path)
else:
raise RuntimeError(f"暂不支持 {sys.platform} 平台")
import ctypes.util
original_find_library = ctypes.util.find_library
def patched_find_library(name):
if name == "opus":
return lib_path
return original_find_library(name)
ctypes.util.find_library = patched_find_library
if hasattr(os, "add_dll_directory"):
os.add_dll_directory(libs_dir)
os.environ["PATH"] = libs_dir + os.pathsep + os.environ.get("PATH", "")
ctypes.CDLL(lib_path)
examples/xiaozhi/xiaozhi/services/audio/stream.py
+101 -199
View File
@@ -1,159 +1,38 @@
from typing import ClassVar, Optional, Callable, Any
from threading import Lock
from collections import deque
import uuid
from typing import Any, Callable, ClassVar, Optional
import numpy as np
from config import APP_CONFIG
from xiaozhi.ref import get_xiaoai
class GlobalStream:
"""
全局音频缓冲区，用于存储和分发音频数据
class __GlobalStream:
def __init__(self):
self.readers = {}
self.on_output_data = None
用来将小爱音箱的音频输入输出流，适配到小智 AI 的音频输入输出流
"""
def register_reader(self, reader):
if reader.id not in self.readers:
self.readers[reader.id] = reader
_instance = None
_lock = Lock()
def unregister_reader(self, reader) -> None:
if reader.id in self.readers:
del self.readers[reader.id]
# 默认缓冲区大小（以字节为单位）
DEFAULT_BUFFER_SIZE = 1024 * 1024 # 1MB
def __new__(cls):
with cls._lock:
if cls._instance is None:
cls._instance = super(GlobalStream, cls).__new__(cls)
cls._instance._initialize()
return cls._instance
def _initialize(self):
"""初始化实例变量"""
self._max_buffer_size = self.DEFAULT_BUFFER_SIZE
self._input_buffer = deque(maxlen=self._max_buffer_size)
self._is_input_active = False
self._is_output_active = False
self._input_readers = {} # 跟踪每个流的读取位置
self._reader_counter = 0 # 为每个读取器分配唯一ID
self._buffer_overflow_count = 0 # 记录缓冲区溢出次数
self.on_output_data = None # 输入数据回调函数
def set_buffer_size(self, frames: int) -> None:
if frames <= 0:
raise ValueError("缓冲区大小必须大于0")
# 创建新的有限长度缓冲区
new_input_buffer = deque(self._input_buffer, maxlen=frames * 2)
# 替换旧缓冲区
self._input_buffer = new_input_buffer
self._max_buffer_size = frames * 2
# 重置读取位置，因为缓冲区可能已经改变
for reader_id in self._input_readers:
self._input_readers[reader_id] = 0
def register_reader(self) -> int:
"""注册一个新的读取器并返回其ID"""
reader_id = self._reader_counter
self._input_readers[reader_id] = 0 # 初始位置为0
self._reader_counter += 1
return reader_id
def unregister_reader(self, reader_id: int) -> None:
"""注销一个读取器"""
if reader_id in self._input_readers:
del self._input_readers[reader_id]
def read(self, reader_id: int, num_frames: int) -> bytes:
num_frames = num_frames * 2
if not self._is_input_active:
return bytes(num_frames)
if reader_id not in self._input_readers:
return bytes(num_frames)
# 将输入缓冲区转换为列表以便随机访问
buffer_list = list(self._input_buffer)
current_pos = self._input_readers[reader_id]
# 如果当前位置超出缓冲区大小，返回空字节
if current_pos >= len(buffer_list):
return bytes(num_frames)
# 读取数据
end_pos = min(current_pos + num_frames, len(buffer_list))
data = bytes(buffer_list[current_pos:end_pos])
# 更新读取位置
self._input_readers[reader_id] = end_pos
# 如果数据不足，用零填充
if len(data) < num_frames:
data += bytes(num_frames - len(data))
return data
# 转发输出音频流
def write(self, frames: bytes) -> None:
"""写入数据到输出缓冲区"""
if not self._is_output_active:
return
def input(self, data: bytes) -> None:
for key in self.readers:
self.readers[key].input(data)
def output(self, frames: bytes) -> None:
if self.on_output_data:
self.on_output_data(frames)
# 添加输入音频流
def add_input_data(self, data: bytes) -> None:
if not self._is_input_active:
return
"""添加输入数据到全局缓冲区"""
# 检查是否会溢出
if len(self._input_buffer) + len(data) > self._max_buffer_size:
self._buffer_overflow_count += 1
for b in data:
self._input_buffer.append(b)
# 如果有读取器的位置已经超出了缓冲区大小，需要调整
if len(self._input_buffer) >= self._max_buffer_size:
for reader_id in self._input_readers:
if self._input_readers[reader_id] > len(self._input_buffer) // 2:
# 将读取位置重置到缓冲区中间，避免读取器永远跟不上
self._input_readers[reader_id] = len(self._input_buffer) // 2
def start_input(self) -> None:
"""启动输入流"""
self._is_input_active = True
def stop_input(self) -> None:
"""停止输入流"""
self._is_input_active = False
self.clear()
def start_output(self) -> None:
"""启动输出流"""
self._is_output_active = True
def stop_output(self) -> None:
"""停止输出流"""
self._is_output_active = False
def is_input_active(self) -> bool:
"""检查输入流是否活跃"""
return self._is_input_active
def is_output_active(self) -> bool:
"""检查输出流是否活跃"""
return self._is_output_active
def clear(self) -> None:
"""清空缓冲区"""
self._input_buffer.clear()
for reader_id in self._input_readers:
self._input_readers[reader_id] = 0
GlobalStream = __GlobalStream()
class MyStream:
"""音频流类，用于读写音频数据"""
def __init__(
self,
rate: int,
@@ -164,6 +43,7 @@ class MyStream:
frames_per_buffer: int = 1024,
start: bool = True,
) -> None:
self.id = uuid.uuid4()
self._rate = rate
self._channels = channels
self._format = format
@@ -171,79 +51,112 @@ class MyStream:
self._is_input = input
self._is_output = output
self._is_active = False
self._is_closed = False
# 获取全局音频缓冲区
self._global_buffer = GlobalStream()
self.input_bytes: list[int] = []
# 注册读取器ID
self._reader_id = self._global_buffer.register_reader() if input else None
# 如果需要，启动流
if start:
self.start_stream()
def close(self) -> None:
"""关闭流"""
if not self._is_closed:
self.stop_stream()
if self._is_input and self._reader_id is not None:
self._global_buffer.unregister_reader(self._reader_id)
self._is_closed = True
self.stop_stream()
def is_active(self) -> bool:
"""检查流是否活跃"""
return self._is_active
def start_stream(self) -> None:
"""启动流"""
if not self._is_active and not self._is_closed:
if not self._is_active:
self._is_active = True
if self._is_input:
self._global_buffer.start_input()
if self._is_output:
self._global_buffer.start_output()
GlobalStream.register_reader(self)
def stop_stream(self) -> None:
"""停止流"""
if self._is_active:
self._is_active = False
if self._is_input:
self._global_buffer.stop_input()
if self._is_output:
self._global_buffer.stop_output()
def read(self, num_frames: int, exception_on_overflow=False) -> bytes:
"""从输入流读取数据"""
if (
not self._is_input
or self._is_closed
or not self._is_active
or self._reader_id is None
):
return bytes(num_frames)
return self._global_buffer.read(self._reader_id, num_frames)
GlobalStream.unregister_reader(self)
self.input_bytes = []
def write(self, frames: bytes) -> None:
"""写入数据到输出流"""
if not self._is_output or self._is_closed or not self._is_active:
# 发送输出音频流到扬声器
if not self._is_output or not self._is_active:
return
GlobalStream.output(frames)
def input(self, data: bytes):
# 收到麦克风输入音频流
if not self._is_input or not self._is_active:
return
self._global_buffer.write(frames)
if len(data) > 0:
samples = np.frombuffer(data, dtype=np.int16)
# 小爱音箱录音音量较小，需要后期放大一下
samples = samples * APP_CONFIG["vad"]["boost"]
self.input_bytes.extend(samples.tobytes())
def read(self, num_frames=None, exception_on_overflow=False) -> bytes:
if num_frames is None:
data = bytes(self.input_bytes)
self.input_bytes = []
return data
num_frames = num_frames * 2
if (
not self._is_input
or not self._is_active
# 达不到预期长度时，返回空字节，等待下一次读取
or len(self.input_bytes) < num_frames
):
return bytes([])
data = bytes(self.input_bytes[:num_frames])
self.input_bytes = self.input_bytes[num_frames:]
return data
class MyAudio:
"""PyAudio替代品，用于创建和管理音频流"""
"""PyAudio 替代品，用于创建和管理音频流"""
Stream: ClassVar[type] = MyStream
def __init__(self, buffer_size: int = GlobalStream.DEFAULT_BUFFER_SIZE) -> None:
# 初始化全局音频缓冲区
self._global_buffer = GlobalStream()
# 设置缓冲区大小
if buffer_size != GlobalStream.DEFAULT_BUFFER_SIZE:
self._global_buffer.set_buffer_size(buffer_size)
@classmethod
def create(cls):
if get_xiaoai().mode != "xiaozhi":
return MyAudio()
else:
from pyaudio import PyAudio
return PyAudio()
@classmethod
def get_input_device_index(cls, audio):
if get_xiaoai().mode != "xiaozhi":
return 0
try:
device = audio.get_default_input_device_info()
return device["index"]
except Exception:
for i in range(audio.get_device_count()):
dev = audio.get_device_info_by_index(i)
if dev["maxInputChannels"] > 0:
return i
return 0
@classmethod
def get_output_device_index(cls, audio):
if get_xiaoai().mode != "xiaozhi":
return 0
try:
device = audio.get_default_output_device_info()
return device["index"]
except Exception:
for i in range(audio.get_device_count()):
dev = audio.get_device_info_by_index(i)
if dev["maxOutputChannels"] > 0:
return i
return 0
def __init__(self) -> None:
self._is_terminated = False
def open(
@@ -261,17 +174,9 @@ class MyAudio:
output_host_api_specific_stream_info: Optional[Any] = None,
stream_callback: Optional[Callable] = None,
) -> MyStream:
"""打开一个新的音频流"""
if self._is_terminated:
raise RuntimeError("MyAudio instance has been terminated")
# 启动全局输入/输出流（如果需要）
if input:
self._global_buffer.start_input()
if output:
self._global_buffer.start_output()
# 创建并返回一个新的流实例
return MyStream(
rate=rate,
channels=channels,
@@ -283,8 +188,5 @@ class MyAudio:
)
def terminate(self) -> None:
"""终止MyAudio实例"""
if not self._is_terminated:
self._global_buffer.stop_input()
self._global_buffer.stop_output()
self._is_terminated = True
examples/xiaozhi/xiaozhi/services/audio/vad/__init__.py
+183
View File
@@ -0,0 +1,183 @@
import threading
import time
import numpy as np
from config import APP_CONFIG
from xiaozhi.event import EventManager
from xiaozhi.ref import set_vad
from xiaozhi.services.audio.stream import MyAudio
from xiaozhi.services.audio.vad.silero import VAD_MODEL
from xiaozhi.services.protocols.typing import AudioConfig
from xiaozhi.utils.base import get_env
class _VAD:
"""基于WebRTC VAD的语音活动检测器，用于检测用户打断"""
def __init__(self):
set_vad(self)
config = APP_CONFIG.get("vad", {})
# 参数设置
self.sample_rate = 16000
self.frame_size = 512
self.threshold = config.get("threshold", 0.01)
self.min_speech_duration = config.get("min_speech_duration", 250)
self.min_silence_duration = config.get("min_silence_duration", 500)
# 状态变量
self.paused = True
self.thread = None
self.speech_count = 0
self.silence_count = 0
# 创建独立的PyAudio实例和流，避免与主音频流冲突
self.audio = None
self.stream = None
# 暂存的语音片段
self.temp_frames = []
self.speech_buffer = []
self.target = None # 检测目标 speech/silence
def _reset_state(self):
"""重置状态"""
self.speech_count = 0
self.silence_count = 0
self.speech_buffer = []
self.temp_frames = []
def start(self):
"""启动VAD检测器"""
if not get_env("CLI"):
return
if self.thread and self.thread.is_alive():
return
self.paused = False
# 初始化PyAudio和流
self._initialize_audio_stream()
# 启动检测线程
self.thread = threading.Thread(target=self._detection_loop, daemon=True)
self.thread.start()
def pause(self):
"""暂停VAD检测"""
if not get_env("CLI"):
return
self.paused = True
self._reset_state()
self.stream.stop_stream()
def resume(self, target: str):
"""恢复VAD检测"""
if not get_env("CLI"):
return
self.paused = False
self.target = target
self.stream.start_stream()
def _handle_speech_frame(self, frames):
"""处理语音帧"""
self.speech_count += len(frames)
self.silence_count = 0
self.speech_buffer.extend(frames)
speech_bytes = bytes(self.speech_buffer)
if (
self.target == "speech"
and self.speech_count > self.min_speech_duration * self.sample_rate / 1000
):
self.pause()
EventManager.on_speech(speech_bytes)
def _handle_silence_frame(self, frames):
"""处理静音帧"""
self.silence_count += len(frames)
self.speech_count = 0
self.speech_buffer = []
if (
self.target == "silence"
and self.silence_count > self.min_silence_duration * self.sample_rate / 1000
):
self.pause()
EventManager.on_silence()
def _detect_speech(self, frames):
"""检测是否是语音"""
try:
audio_int16 = np.frombuffer(frames, dtype=np.int16)
audio_float32 = audio_int16.astype(np.float32) / 32768.0
speech_prob = VAD_MODEL(audio_float32, 16000).item()
is_speech = speech_prob >= self.threshold
return is_speech
except Exception:
return False
def _initialize_audio_stream(self):
"""初始化独立的音频流"""
try:
# 创建 PyAudio 实例
self.audio = MyAudio.create()
# 创建输入流
self.stream = self.audio.open(
format=AudioConfig.FORMAT,
channels=1,
rate=self.sample_rate,
input=True,
frames_per_buffer=self.frame_size,
start=True,
)
return True
except Exception:
return False
def _close_audio_stream(self):
"""关闭音频流"""
try:
if self.stream:
self.stream.stop_stream()
self.stream.close()
self.stream = None
if self.audio:
self.audio.terminate()
self.audio = None
except Exception:
pass
def _detection_loop(self):
"""VAD检测主循环"""
while True:
# 如果暂停或者音频流未初始化，则跳过
if self.paused or not self.stream:
time.sleep(0.1)
continue
# 读取缓冲区音频数据
frames = self.stream.read(self.frame_size)
if len(frames) != self.frame_size * 2:
time.sleep(0.01)
continue
# 检测是否是语音
is_speech = self._detect_speech(frames)
if is_speech:
self._handle_speech_frame(frames)
else:
self._handle_silence_frame(frames)
time.sleep(0.01)
VAD = _VAD()
examples/xiaozhi/xiaozhi/services/audio/vad/silero.py
+89
View File
@@ -0,0 +1,89 @@
import numpy as np
import onnxruntime as ort
from xiaozhi.utils.file import get_model_file_path
class OnnxWrapper:
def __init__(self, path):
opts = ort.SessionOptions()
opts.inter_op_num_threads = 1
opts.intra_op_num_threads = 1
self.session = ort.InferenceSession(
path, providers=["CPUExecutionProvider"], sess_options=opts
)
self.reset_states()
self.sample_rates = [8000, 16000]
def _validate_input(self, x, sr: int):
if len(x.shape) == 1:
x = np.expand_dims(x, 0)
if len(x.shape) > 2:
raise ValueError(
f"Too many dimensions for input audio chunk {len(x.shape)}"
)
if sr != 16000 and (sr % 16000 == 0):
step = sr // 16000
x = x[:, ::step]
sr = 16000
if sr not in self.sample_rates:
raise ValueError(
f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)"
)
if sr / x.shape[1] > 31.25:
raise ValueError("Input audio chunk is too short")
return x, sr
def reset_states(self, batch_size=1):
self._state = np.zeros((2, batch_size, 128), dtype=np.float32)
self._context = np.zeros(0, dtype=np.float32)
self._last_sr = 0
self._last_batch_size = 0
def __call__(self, x, sr: int):
x, sr = self._validate_input(x, sr)
num_samples = 512 if sr == 16000 else 256
if x.shape[-1] != num_samples:
raise ValueError(
f"Provided number of samples is {x.shape[-1]} (Supported values: 256 for 8000 sample rate, 512 for 16000)"
)
batch_size = x.shape[0]
context_size = 64 if sr == 16000 else 32
if not self._last_batch_size:
self.reset_states(batch_size)
if (self._last_sr) and (self._last_sr != sr):
self.reset_states(batch_size)
if (self._last_batch_size) and (self._last_batch_size != batch_size):
self.reset_states(batch_size)
if not len(self._context):
self._context = np.zeros((batch_size, context_size), dtype=np.float32)
x = np.concatenate([self._context, x], axis=1)
if sr in [8000, 16000]:
ort_inputs = {
"input": x,
"state": self._state,
"sr": np.array(sr, dtype="int64"),
}
ort_outs = self.session.run(None, ort_inputs)
out, state = ort_outs
self._state = state
else:
raise ValueError()
self._context = x[..., -context_size:]
self._last_sr = sr
self._last_batch_size = batch_size
return out
VAD_MODEL = OnnxWrapper(
path=get_model_file_path("silero_vad.onnx"),
)
examples/xiaozhi/xiaozhi/services/display/base_display.py
+15 -19
View File
@@ -1,32 +1,28 @@
from abc import ABC, abstractmethod
from typing import Optional, Callable
import logging
from typing import Callable, Optional
class BaseDisplay(ABC):
"""显示接口的抽象基类"""
def __init__(self):
self.logger = logging.getLogger(self.__class__.__name__)
self.current_volume = 70 # 默认音量
@abstractmethod
def set_callbacks(self,
press_callback: Optional[Callable] = None,
release_callback: Optional[Callable] = None,
status_callback: Optional[Callable] = None,
text_callback: Optional[Callable] = None,
emotion_callback: Optional[Callable] = None,
mode_callback: Optional[Callable] = None,
auto_callback: Optional[Callable] = None,
abort_callback: Optional[Callable] = None): # 添加打断回调参数
def set_callbacks(
self,
press_callback: Optional[Callable] = None,
release_callback: Optional[Callable] = None,
status_callback: Optional[Callable] = None,
text_callback: Optional[Callable] = None,
emotion_callback: Optional[Callable] = None,
mode_callback: Optional[Callable] = None,
auto_callback: Optional[Callable] = None,
abort_callback: Optional[Callable] = None,
):
"""设置回调函数"""
pass
@abstractmethod
def update_button_status(self, text: str):
"""更新按钮状态"""
pass
@abstractmethod
def update_status(self, status: str):
"""更新状态文本"""
@@ -50,4 +46,4 @@ class BaseDisplay(ABC):
@abstractmethod
def on_close(self):
"""关闭显示"""
pass
pass
examples/xiaozhi/xiaozhi/services/display/gui_display.py
+13 -85
View File
@@ -1,10 +1,9 @@
import queue
import threading
import time
import tkinter as tk
from tkinter import ttk
import queue
import logging
import time
from typing import Optional, Callable
from typing import Callable, Optional
from xiaozhi.services.display.base_display import BaseDisplay
@@ -13,14 +12,11 @@ class GuiDisplay(BaseDisplay):
def __init__(self):
super().__init__() # 调用父类初始化
"""创建 GUI 界面"""
# 初始化日志
self.logger = logging.getLogger("Display")
# 创建主窗口
self.root = tk.Tk()
self.root.title("小爱音箱接入小智 AI 演示")
self.root.geometry("520x360")
# 在窗口底部添加作者信息
self.author_label = ttk.Label(self.root, text="作者: https://del.wang")
self.author_label.pack(side=tk.BOTTOM, pady=5)
@@ -135,8 +131,8 @@ class GuiDisplay(BaseDisplay):
# 调用回调函数
if self.button_press_callback:
self.button_press_callback()
except Exception as e:
self.logger.error(f"按钮按下回调执行失败: {e}")
except Exception:
pass
def _on_manual_button_release(self, event):
"""手动模式按钮释放事件处理"""
@@ -147,67 +143,16 @@ class GuiDisplay(BaseDisplay):
# 调用回调函数
if self.button_release_callback:
self.button_release_callback()
except Exception as e:
self.logger.error(f"按钮释放回调执行失败: {e}")
def _on_auto_button_click(self):
"""自动模式按钮点击事件处理"""
try:
if self.auto_callback:
self.auto_callback()
except Exception as e:
self.logger.error(f"自动模式按钮回调执行失败: {e}")
except Exception:
pass
def _on_abort_button_click(self):
"""打断按钮点击事件处理"""
try:
if self.abort_callback:
self.abort_callback()
except Exception as e:
self.logger.error(f"打断按钮回调执行失败: {e}")
def _on_mode_button_click(self):
"""对话模式切换按钮点击事件"""
try:
# 检查是否可以切换模式（通过回调函数询问应用程序当前状态）
if self.mode_callback:
# 如果回调函数返回False，表示当前不能切换模式
if not self.mode_callback(not self.auto_mode):
return
# 切换模式
self.auto_mode = not self.auto_mode
# 更新按钮显示
if self.auto_mode:
# 切换到自动模式
self.update_mode_button_status("自动对话")
# 隐藏手动按钮，显示自动按钮
self.update_queue.put(lambda: self._switch_to_auto_mode())
else:
# 切换到手动模式
self.update_mode_button_status("手动对话")
# 隐藏自动按钮，显示手动按钮
self.update_queue.put(lambda: self._switch_to_manual_mode())
except Exception as e:
self.logger.error(f"模式切换按钮回调执行失败: {e}")
def _switch_to_auto_mode(self):
"""切换到自动模式的UI更新"""
self.manual_btn.pack_forget() # 移除手动按钮
self.auto_btn.pack(
side=tk.LEFT, padx=10, before=self.abort_btn
) # 显示自动按钮，放在打断按钮前面
def _switch_to_manual_mode(self):
"""切换到手动模式的UI更新"""
self.auto_btn.pack_forget() # 移除自动按钮
self.manual_btn.pack(
side=tk.LEFT, padx=10, before=self.abort_btn
) # 显示手动按钮，放在打断按钮前面
except Exception:
pass
def update_status(self, status: str):
"""更新状态文本"""
@@ -245,8 +190,8 @@ class GuiDisplay(BaseDisplay):
if emotion:
self.update_emotion(emotion)
except Exception as e:
self.logger.error(f"更新失败: {e}")
except Exception:
pass
time.sleep(0.1)
threading.Thread(target=update_loop, daemon=True).start()
@@ -262,23 +207,6 @@ class GuiDisplay(BaseDisplay):
# 启动更新线程
self.start_update_threads()
# 在主线程中运行主循环
self.logger.info("开始启动GUI主循环")
self.root.mainloop()
except Exception as e:
self.logger.error(f"GUI启动失败: {e}", exc_info=True)
# 尝试回退到CLI模式
print(f"GUI启动失败: {e}，请尝试使用CLI模式")
def update_mode_button_status(self, text: str):
"""更新模式按钮状态"""
self.update_queue.put(lambda: self.mode_btn.config(text=text))
def update_button_status(self, text: str):
"""更新按钮状态 - 保留此方法以满足抽象基类要求"""
# 根据当前模式更新相应的按钮
if self.auto_mode:
self.update_queue.put(lambda: self.auto_btn.config(text=text))
else:
# 在手动模式下，不通过此方法更新按钮文本
# 因为按钮文本由按下/释放事件直接控制
except Exception:
pass
examples/xiaozhi/xiaozhi/services/display/no_display.py
+38
View File
@@ -0,0 +1,38 @@
import time
from typing import Callable, Optional
from xiaozhi.services.display.base_display import BaseDisplay
class NoDisplay(BaseDisplay):
def set_callbacks(
self,
press_callback: Optional[Callable] = None,
release_callback: Optional[Callable] = None,
status_callback: Optional[Callable] = None,
text_callback: Optional[Callable] = None,
emotion_callback: Optional[Callable] = None,
mode_callback: Optional[Callable] = None,
auto_callback: Optional[Callable] = None,
abort_callback: Optional[Callable] = None,
):
pass
def update_status(self, status: str):
pass
def update_text(self, text: str):
pass
def update_emotion(self, emotion: str):
pass
def start_update_threads(self):
pass
def on_close(self):
pass
def start(self):
while True:
time.sleep(1)
examples/xiaozhi/xiaozhi/services/protocols/protocol.py
+3 -6
View File
@@ -1,6 +1,6 @@
import json
from xiaozhi.services.protocols.typing import AbortReason, ListeningMode
from xiaozhi.services.protocols.typing import ListeningMode
class Protocol:
@@ -38,12 +38,9 @@ class Protocol:
async def send_abort_speaking(self, reason):
"""发送中止语音的消息"""
message = {"session_id": self.session_id, "type": "abort"}
if reason == AbortReason.WAKE_WORD_DETECTED:
message["reason"] = "wake_word_detected"
message = {"session_id": self.session_id, "type": reason}
await self.send_text(json.dumps(message))
async def send_start_listening(self, mode):
"""发送开始监听的消息"""
mode_map = {
@@ -69,7 +66,7 @@ class Protocol:
message = {
"session_id": self.session_id,
"type": "iot",
"descriptors": json.loads(descriptors),
"descriptors": json.loads(descriptors),
}
await self.send_text(json.dumps(message))
examples/xiaozhi/xiaozhi/services/protocols/typing.py
+2 -2
View File
@@ -6,7 +6,7 @@ class ListeningMode:
class AbortReason:
"""中止原因"""
NONE = "none"
ABORT = "abort"
WAKE_WORD_DETECTED = "wake_word_detected"
class DeviceState:
@@ -20,10 +20,10 @@ class EventType:
"""事件类型"""
SCHEDULE_EVENT = "schedule_event"
AUDIO_INPUT_READY_EVENT = "audio_input_ready_event"
AUDIO_OUTPUT_READY_EVENT = "audio_output_ready_event"
class AudioConfig:
"""音频配置"""
FORMAT = 8
SAMPLE_RATE = 24000
CHANNELS = 1
FRAME_DURATION = 60 # ms
examples/xiaozhi/xiaozhi/services/protocols/websocket_protocol.py
+6 -30
View File
@@ -1,13 +1,10 @@
import asyncio
import json
import logging
import websockets
from xiaozhi.services.protocols.protocol import Protocol
from xiaozhi.utils.config_manager import ConfigManager
logger = logging.getLogger("WebsocketProtocol")
from xiaozhi.utils.config import ConfigManager
class WebsocketProtocol(Protocol):
@@ -66,16 +63,13 @@ class WebsocketProtocol(Protocol):
try:
await asyncio.wait_for(self.hello_received.wait(), timeout=10.0)
self.connected = True
logger.info("已连接到WebSocket服务器")
return True
except asyncio.TimeoutError:
logger.error("等待服务器hello响应超时")
if self.on_network_error:
self.on_network_error("等待响应超时")
return False
except Exception as e:
logger.error(f"WebSocket连接失败: {e}")
if self.on_network_error:
self.on_network_error(f"无法连接服务: {str(e)}")
return False
@@ -89,27 +83,21 @@ class WebsocketProtocol(Protocol):
data = json.loads(message)
msg_type = data.get("type")
if msg_type == "hello":
# 处理服务器 hello 消息
await self._handle_server_hello(data)
else:
if self.on_incoming_json:
self.on_incoming_json(data)
except json.JSONDecodeError as e:
logger.error(f"无效的JSON消息: {message}, 错误: {e}")
elif self.on_incoming_audio: # 使用 elif 更清晰
except json.JSONDecodeError:
pass
elif self.on_incoming_audio:
self.on_incoming_audio(message)
except websockets.ConnectionClosed:
logger.info("WebSocket连接已关闭")
self.connected = False
if self.on_audio_channel_closed:
# 使用 schedule 确保回调在主线程中执行
await self.on_audio_channel_closed()
except Exception as e:
logger.error(f"消息处理错误: {e}")
self.connected = False
if self.on_network_error:
# 使用 schedule 确保错误处理在主线程中执行
self.on_network_error(f"连接错误: {str(e)}")
async def send_audio(self, data: bytes):
@@ -120,7 +108,6 @@ class WebsocketProtocol(Protocol):
try:
await self.websocket.send(data)
except Exception as e:
logger.error(f"发送音频数据失败: {e}")
if self.on_network_error:
self.on_network_error(f"发送音频失败: {str(e)}")
@@ -161,7 +148,6 @@ class WebsocketProtocol(Protocol):
# 验证传输方式
transport = data.get("transport")
if not transport or transport != "websocket":
logger.error(f"不支持的传输方式: {transport}")
return
# 获取音频参数
@@ -171,13 +157,6 @@ class WebsocketProtocol(Protocol):
sample_rate = audio_params.get("sample_rate")
if sample_rate:
self.server_sample_rate = sample_rate
# 如果服务器采样率与本地不同，记录警告
if sample_rate != self.server_sample_rate:
logger.warning(
f"服务器的音频采样率 {sample_rate} "
f"与设备输出的采样率 {self.server_sample_rate} 不一致，"
"重采样后可能会失真"
)
# 设置 hello 接收事件
self.hello_received.set()
@@ -186,10 +165,7 @@ class WebsocketProtocol(Protocol):
if self.on_audio_channel_opened:
await self.on_audio_channel_opened()
logger.info("成功处理服务器 hello 消息")
except Exception as e:
logger.error(f"处理服务器 hello 消息时出错: {e}")
if self.on_network_error:
self.on_network_error(f"处理服务器响应失败: {str(e)}")
@@ -202,5 +178,5 @@ class WebsocketProtocol(Protocol):
self.connected = False
if self.on_audio_channel_closed:
await self.on_audio_channel_closed()
except Exception as e:
logger.error(f"关闭WebSocket连接失败: {e}")
except Exception:
pass
examples/xiaozhi/xiaozhi/services/speaker.py
+186
View File
@@ -0,0 +1,186 @@
from typing import Literal
from xiaozhi.ref import get_xiaoai, set_speaker
from xiaozhi.utils.base import json_decode, json_encode
class CommandResult:
def __init__(self, stdout: str, stderr: str, exit_code: int):
self.stdout = stdout
self.stderr = stderr
self.exit_code = exit_code
class SpeakerManager:
status: Literal["playing", "paused", "idle"] = "idle"
def __init__(self):
set_speaker(self)
async def get_playing(self, sync=False):
"""获取播放状态"""
if sync:
# 同步远端最新状态
res = await self.run_shell("mphelper mute_stat")
if "1" in res.stdout:
self.status = "playing"
elif "2" in res.stdout:
self.status = "paused"
return self.status
async def set_playing(self, playing=True):
"""播放/暂停"""
command = "mphelper play" if playing else "mphelper pause"
res = await self.run_shell(command)
return '"code": 0' in res.stdout
async def play(
self,
text=None,
url=None,
buffer=None,
blocking=True,
timeout=10 * 60 * 1000,
):
"""
播放文字、音频链接、音频流
参数:
text: 文字内容
url: 音频链接
buffer: 音频流
timeout: 超时时长（毫秒），默认10分钟
blocking: 是否阻塞运行(仅对播放文字、音频链接有效)
"""
if buffer is not None:
return get_xiaoai().on_output_data(buffer)
if blocking:
command = (
f"miplayer -f '{url}'"
if url
else f"/usr/sbin/tts_play.sh '{text.replace("'", "'\\''") or '你好'}'"
)
res = await self.run_shell(command, timeout=timeout)
return res.exit_code == 0
if url:
data = json_encode({"url": url, "type": 1})
command = f"ubus call mediaplayer player_play_url '{data}'"
else:
data = json_encode({"text": text or "你好", "save": 0})
command = f"ubus call mibrain text_to_speech '{data}'"
res = await self.run_shell(command, timeout=timeout)
return '"code": 0' in res.stdout if res else False
async def wake_up(self, awake=True, silent=True):
"""
（取消）唤醒小爱
参数:
awake: 是否唤醒
silent: 是否静默唤醒
"""
if awake:
if silent:
command = 'ubus call pnshelper event_notify \'{"src":1,"event":0}\''
else:
command = 'ubus call pnshelper event_notify \'{"src":0,"event":0}\''
else:
command = """
ubus call pnshelper event_notify '{"src":3, "event":7}'
sleep 0.1
ubus call pnshelper event_notify '{"src":3, "event":8}'
"""
res = await self.run_shell(command)
return '"code": 0' in res.stdout
async def ask_xiaoai(self, text: str, silent=False):
"""
把文字指令交给原来的小爱执行
参数:
text: 文字指令
silent: 是否静默执行
"""
data = {"nlp": 1, "nlp_text": text}
if not silent:
data["tts"] = 1
command = f"ubus call mibrain ai_service '{json_encode(data)}'"
res = await self.run_shell(command)
return '"code": 0' in res.stdout
async def abort_xiaoai(self):
"""
中断原来小爱的运行
注意：重启需要大约 1-2s 的时间，在此期间无法使用小爱音箱自带的 TTS 服务
"""
res = await self.run_shell("/etc/init.d/mico_aivs_lab restart >/dev/null 2>&1")
return res.exit_code == 0
async def get_boot(self):
"""获取启动分区"""
res = await self.run_shell("echo $(fw_env -g boot_part)")
return res.stdout.strip()
async def set_boot(self, boot_part: Literal["boot0", "boot1"]):
"""设置启动分区"""
command = f"fw_env -s boot_part {boot_part} >/dev/null 2>&1 && echo $(fw_env -g boot_part)"
res = await self.run_shell(command)
return boot_part in res.stdout
async def get_device(self):
"""获取设备型号、序列号信息"""
res = await self.run_shell("echo $(micocfg_model) $(micocfg_sn)")
info = res.stdout.strip().split(" ")
return {
"model": info[0] if len(info) > 0 else "unknown",
"sn": info[1] if len(info) > 1 else "unknown",
}
async def get_mic(self):
"""获取麦克风状态"""
res = await self.run_shell("[ ! -f /tmp/mipns/mute ] && echo on || echo off")
status = "off"
if "on" in res.stdout:
status = "on"
return status
async def set_mic(self, on=True):
"""打开/关闭麦克风"""
if on:
command = (
'ubus -t1 -S call pnshelper event_notify \'{"src":3, "event":7}\' 2>&1'
)
else:
command = (
'ubus -t1 -S call pnshelper event_notify \'{"src":3, "event":8}\' 2>&1'
)
res = await self.run_shell(command)
return '"code":0' in res.stdout
async def run_shell(self, script: str, timeout=10000):
"""
执行脚本
参数:
script: 脚本内容
timeout: 超时时间（毫秒）
"""
res = "unknown"
try:
res = await get_xiaoai().run_shell(script, timeout=timeout)
data = json_decode(res)
if data:
return CommandResult(
data.get("stdout", ""),
data.get("stderr", ""),
data.get("exit_code", 0),
)
except Exception:
return CommandResult("error", res, -1)
examples/xiaozhi/xiaozhi/utils/base.py
+12
View File
@@ -1,4 +1,10 @@
import json
import os
import random
def get_env(key: str, default_value: str | None = None):
return os.environ.get(key, default_value)
def to_set(data):
@@ -7,6 +13,12 @@ def to_set(data):
return data
def pick_one(data: list):
if len(data) == 0:
return None
return data[random.randint(0, len(data) - 1)]
def json_encode(obj, pretty=False):
try:
return json.dumps(obj, ensure_ascii=False, indent=4 if pretty else None)
examples/xiaozhi/xiaozhi/utils/config_manager.py → examples/xiaozhi/xiaozhi/utils/config.py
+32 -37
View File
@@ -1,5 +1,4 @@
import json
import logging
import re
import socket
import threading
import uuid
@@ -7,9 +6,8 @@ from typing import Any, Optional
import requests
from config import XIAOZHI_CONFIG
logger = logging.getLogger("ConfigManager")
from config import APP_CONFIG
from xiaozhi.utils.file import read_file, write_file
class ConfigManager:
@@ -26,7 +24,6 @@ class ConfigManager:
def __init__(self):
"""初始化配置管理器"""
self.logger = logger
if hasattr(self, "_initialized"):
return
self._initialized = True
@@ -34,9 +31,9 @@ class ConfigManager:
# 加载配置
self._config = {
"CLIENT_ID": None,
"DEVICE_ID": None,
"DEVICE_ID": APP_CONFIG["xiaozhi"]["DEVICE_ID"],
"NETWORK": APP_CONFIG.get("xiaozhi"),
"MQTT_INFO": None,
"NETWORK": XIAOZHI_CONFIG,
}
self._initialize_client_id()
@@ -78,10 +75,22 @@ class ConfigManager:
current = current.setdefault(part, {})
current[last] = value
return True
except Exception as e:
logger.error(f"Error updating config {path}: {e}")
except Exception:
return False
def update_config_file(self, path: str, value: Any):
"""
更新 config.py 文件中的特定配置项
"""
write_file(
"config.py",
re.sub(
r'"{}"\s*:\s*"[^"]*"'.format(path),
f'"{path}": "{value}"',
read_file("config.py"),
),
)
@classmethod
def instance(cls):
"""获取配置管理器实例（线程安全）"""
@@ -111,24 +120,23 @@ class ConfigManager:
"""确保存在客户端ID"""
if not self._config["CLIENT_ID"]:
client_id = self.generate_uuid()
success = self.update_config("CLIENT_ID", client_id)
if success:
logger.info(f"Generated new CLIENT_ID: {client_id}")
else:
logger.error("Failed to save new CLIENT_ID")
self.update_config("CLIENT_ID", client_id)
def _initialize_device_id(self):
"""确保存在设备ID"""
if self._config["DEVICE_ID"]:
# 检查设备 ID 是否符合 MAC 地址格式(如 a6:85:b4:9c:09:66)
mac_pattern = re.compile(r"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$")
if not mac_pattern.match(self._config["DEVICE_ID"]):
self._config["DEVICE_ID"] = None
if not self._config["DEVICE_ID"]:
try:
device_hash = self.get_mac_address()
success = self.update_config("DEVICE_ID", device_hash)
if success:
logger.info(f"Generated new DEVICE_ID: {device_hash}")
else:
logger.error("Failed to save new DEVICE_ID")
except Exception as e:
logger.error(f"Error generating DEVICE_ID: {e}")
self.update_config("DEVICE_ID", device_hash)
self.update_config_file("DEVICE_ID", device_hash)
except Exception:
pass
def refresh_mqtt_info(self):
"""刷新 MQTT 信息"""
@@ -140,13 +148,10 @@ class ConfigManager:
mqtt_info = self._get_ota_version()
if mqtt_info:
self.update_config("MQTT_INFO", mqtt_info)
self.logger.info("MQTT信息已成功更新")
return mqtt_info
else:
self.logger.warning("获取MQTT信息失败，使用已保存的配置")
return self.get_config("MQTT_INFO")
except Exception as e:
self.logger.error(f"初始化MQTT信息失败: {e}")
except Exception:
return self.get_config("MQTT_INFO")
def _get_ota_version(self):
@@ -200,26 +205,16 @@ class ConfigManager:
# 检查HTTP状态码
if response.status_code != 200:
self.logger.error(f"OTA服务器错误: HTTP {response.status_code}")
raise ValueError(f"OTA服务器返回错误状态码: {response.status_code}")
# 解析JSON数据
response_data = response.json()
self.logger.debug(
f"OTA服务器返回数据: {json.dumps(response_data, indent=4, ensure_ascii=False)}"
)
if "mqtt" in response_data:
self.logger.info("MQTT服务器信息已更新")
return response_data["mqtt"]
else:
self.logger.error("OTA服务器返回的数据无效: MQTT信息缺失")
raise ValueError("OTA服务器返回的数据无效，请检查服务器状态或MAC地址！")
except requests.Timeout:
self.logger.error("OTA请求超时，请检查网络或服务器状态")
raise ValueError("OTA请求超时！请稍后重试。")
except requests.RequestException as e:
self.logger.error(f"OTA请求失败: {e}")
except requests.RequestException:
raise ValueError("无法连接到OTA服务器，请检查网络连接！")
examples/xiaozhi/xiaozhi/utils/file.py
+16
View File
@@ -0,0 +1,16 @@
import os
def get_model_file_path(file_name: str):
current_dir = os.path.dirname(os.path.abspath(__file__))
return os.path.abspath(os.path.join(current_dir, "../models", file_name))
def read_file(file_path: str):
with open(file_path, "r", encoding="utf-8") as f:
return f.read()
def write_file(file_path: str, content: str):
with open(file_path, "w", encoding="utf-8") as f:
f.write(content)
examples/xiaozhi/xiaozhi/utils/logging_config.py
-30
View File
@@ -1,30 +0,0 @@
import logging
def setup_logging():
"""配置日志系统"""
# 创建根日志记录器
root_logger = logging.getLogger()
root_logger.setLevel(logging.INFO) # 设置根日志级别
# 清除已有的处理器（避免重复添加）
if root_logger.handlers:
root_logger.handlers.clear()
# 创建控制台处理器
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.INFO)
# 创建格式化器
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
console_handler.setFormatter(formatter)
# 添加处理器到根日志记录器
root_logger.addHandler(console_handler)
# 设置特定模块的日志级别
logging.getLogger("XiaoZhi").setLevel(logging.INFO)
logging.getLogger("WebsocketProtocol").setLevel(logging.INFO)
examples/xiaozhi/xiaozhi/xiaoai.py
+38 -20
View File
@@ -5,17 +5,31 @@ import threading
import numpy as np
import open_xiaoai_server
from config import APP_CONFIG
from xiaozhi.event import EventManager
from xiaozhi.ref import get_speaker, set_xiaoai
from xiaozhi.services.audio.stream import GlobalStream
from xiaozhi.services.speaker import SpeakerManager
from xiaozhi.utils.base import json_decode
ASCII_BANNER = """
▄▖ ▖▖▘ ▄▖▄▖
▌▌▛▌█▌▛▌▚▘▌▀▌▛▌▌▌▐
▙▌▙▌▙▖▌▌▌▌▌█▌▙▌▛▌▟▖
▌
v1.0.0 by: https://del.wang
"""
class XiaoAI:
mode = "xiaoai"
sync_loop: asyncio.AbstractEventLoop = None
speaker = SpeakerManager()
async_loop: asyncio.AbstractEventLoop = None
@classmethod
def setup_mode(cls):
set_xiaoai(cls)
parser = argparse.ArgumentParser(
description="小爱音箱接入小智 AI | by: https://del.wang"
)
@@ -33,19 +47,21 @@ class XiaoAI:
@classmethod
def on_input_data(cls, data: bytes):
audio_array = np.frombuffer(data, dtype=np.uint16)
GlobalStream().add_input_data(audio_array.tobytes())
GlobalStream.input(audio_array.tobytes())
@classmethod
def on_output_data(cls, data: bytes):
async def on_output_data_async(data: bytes):
return await open_xiaoai_server.on_output_data(data)
future = on_output_data_async(data)
cls.sync_loop.run_until_complete(future)
asyncio.run_coroutine_threadsafe(
on_output_data_async(data),
cls.async_loop,
)
@classmethod
async def run_shell(cls, script: str, timeout_millis: float = 10 * 1000):
return await open_xiaoai_server.run_shell(script, timeout_millis)
async def run_shell(cls, script: str, timeout: float = 10 * 1000):
return await open_xiaoai_server.run_shell(script, timeout)
@classmethod
async def on_event(cls, event: str):
@@ -53,18 +69,16 @@ class XiaoAI:
event_data = event_json.get("data", {})
event_type = event_json.get("event")
async def on_wakeup(text, source):
before_wakeup = APP_CONFIG["wakeup"]["before_wakeup"]
wakeup = await before_wakeup(get_speaker(), text, source)
if wakeup:
EventManager.on_wakeup()
if not event_json.get("event"):
print(f"❌ Event 解析失败: {event}")
return
if event_type == "kws":
if event_data == "Started":
print("🔥 自定义唤醒词已开启")
await cls.run_shell("/usr/sbin/tts_play.sh '自定义唤醒词已开启'")
else:
keyword = event_data.get("Keyword")
print(f"🔥 自定义唤醒词: {keyword}")
elif event_type == "instruction" and event_data.get("NewLine"):
if event_type == "instruction" and event_data.get("NewLine"):
line = json_decode(event_data.get("NewLine"))
if (
line
@@ -72,15 +86,18 @@ class XiaoAI:
and line.get("header", {}).get("name") == "RecognizeResult"
):
text = line.get("payload", {}).get("results")[0].get("text")
if not text:
print("🔥 唤醒小爱同学")
if not text and not line.get("payload", {}).get("is_vad_begin"):
print(f"🔥 唤醒小爱: {line}")
EventManager.on_interrupt()
elif text and line.get("payload", {}).get("is_final"):
print(f"🔥 收到用户指令: {text}")
print(f"🔥 收到指令: {text}")
await on_wakeup(text, "xiaoai")
elif event_type == "playing":
get_speaker().status = event_data.lower()
@classmethod
def __init_background_event_loop(cls):
def run_event_loop():
cls.sync_loop = asyncio.new_event_loop()
cls.async_loop = asyncio.new_event_loop()
asyncio.set_event_loop(cls.async_loop)
cls.async_loop.run_forever()
@@ -97,8 +114,9 @@ class XiaoAI:
@classmethod
async def init_xiaoai(cls):
GlobalStream().on_output_data = cls.on_output_data
GlobalStream.on_output_data = cls.on_output_data
open_xiaoai_server.register_fn("on_input_data", cls.on_input_data)
open_xiaoai_server.register_fn("on_event", cls.__on_event)
cls.__init_background_event_loop()
print(ASCII_BANNER)
await open_xiaoai_server.start_server()
examples/xiaozhi/xiaozhi/xiaozhi.py
+91 -315
View File
@@ -1,11 +1,13 @@
import asyncio
import json
import logging
import re
import threading
import time
from xiaozhi.services.display import gui_display
from xiaozhi.event import EventManager
from xiaozhi.ref import set_xiaozhi
from xiaozhi.services.audio.kws import KWS
from xiaozhi.services.audio.vad import VAD
from xiaozhi.services.protocols.typing import (
AbortReason,
AudioConfig,
@@ -14,12 +16,10 @@ from xiaozhi.services.protocols.typing import (
ListeningMode,
)
from xiaozhi.services.protocols.websocket_protocol import WebsocketProtocol
from xiaozhi.utils.config_manager import ConfigManager
from xiaozhi.utils.base import get_env
from xiaozhi.utils.config import ConfigManager
from xiaozhi.xiaoai import XiaoAI
# 配置日志
logger = logging.getLogger("XiaoZhi")
class XiaoZhi:
"""智能音箱应用程序主类"""
@@ -46,8 +46,6 @@ class XiaoZhi:
# 状态变量
self.device_state = DeviceState.IDLE
self.voice_detected = False
self.keep_listening = False
self.aborted = False
self.current_text = ""
self.current_emotion = "neutral"
@@ -73,11 +71,11 @@ class XiaoZhi:
self.events = {
EventType.SCHEDULE_EVENT: threading.Event(),
EventType.AUDIO_INPUT_READY_EVENT: threading.Event(),
EventType.AUDIO_OUTPUT_READY_EVENT: threading.Event(),
}
# 创建显示界面
self.display = None
set_xiaozhi(self)
def run(self):
self.protocol = WebsocketProtocol()
@@ -99,6 +97,9 @@ class XiaoZhi:
main_loop_thread.daemon = True
main_loop_thread.start()
VAD.start()
KWS.start()
# 启动 GUI
self._initialize_display()
self.display.start()
@@ -110,10 +111,6 @@ class XiaoZhi:
async def _initialize_without_connect(self):
"""初始化应用程序组件（不建立连接）"""
logger.info("正在初始化应用程序...")
# 设置设备状态为待命
self.set_device_state(DeviceState.IDLE)
# 初始化音频编解码器
self._initialize_audio()
@@ -125,7 +122,8 @@ class XiaoZhi:
self.protocol.on_audio_channel_opened = self._on_audio_channel_opened
self.protocol.on_audio_channel_closed = self._on_audio_channel_closed
logger.info("应用程序初始化完成")
# 设置设备状态为待命
self.set_device_state(DeviceState.IDLE)
def _initialize_audio(self):
"""初始化音频设备和编解码器"""
@@ -133,14 +131,19 @@ class XiaoZhi:
from xiaozhi.services.audio.codec import AudioCodec
self.audio_codec = AudioCodec()
logger.info("音频编解码器初始化成功")
except Exception as e:
logger.error(f"初始化音频设备失败: {e}")
self.alert("错误", f"初始化音频设备失败: {e}")
def _initialize_display(self):
"""初始化显示界面"""
self.display = gui_display.GuiDisplay()
if get_env("CLI"):
from xiaozhi.services.display import no_display
self.display = no_display.NoDisplay()
else:
from xiaozhi.services.display import gui_display
self.display = gui_display.GuiDisplay()
# 设置回调函数
self.display.set_callbacks(
@@ -156,7 +159,6 @@ class XiaoZhi:
def _main_loop(self):
"""应用程序主循环"""
logger.info("主循环已启动")
self.running = True
while self.running:
@@ -167,12 +169,9 @@ class XiaoZhi:
if event_type == EventType.AUDIO_INPUT_READY_EVENT:
self._handle_input_audio()
elif event_type == EventType.AUDIO_OUTPUT_READY_EVENT:
self._handle_output_audio()
elif event_type == EventType.SCHEDULE_EVENT:
self._process_scheduled_tasks()
# 短暂休眠以避免CPU占用过高
time.sleep(0.01)
def _process_scheduled_tasks(self):
@@ -184,8 +183,8 @@ class XiaoZhi:
for task in tasks:
try:
task()
except Exception as e:
logger.error(f"执行调度任务时出错: {e}")
except Exception:
pass
def schedule(self, callback):
"""调度任务到主循环"""
@@ -209,19 +208,10 @@ class XiaoZhi:
self.protocol.send_audio(encoded_data), self.loop
)
def _handle_output_audio(self):
"""处理音频输出"""
if self.device_state != DeviceState.SPEAKING:
return
self.audio_codec.play_audio()
def _on_network_error(self, message):
"""网络错误回调"""
self.keep_listening = False
self.set_device_state(DeviceState.IDLE)
if self.device_state != DeviceState.CONNECTING:
logger.info("检测到连接断开")
self.set_device_state(DeviceState.IDLE)
# 关闭现有连接
@@ -233,7 +223,6 @@ class XiaoZhi:
def _attempt_reconnect(self):
"""尝试重新连接服务器"""
if self.device_state != DeviceState.CONNECTING:
logger.info("检测到连接断开，尝试重新连接...")
self.set_device_state(DeviceState.CONNECTING)
# 关闭现有连接
@@ -264,16 +253,13 @@ class XiaoZhi:
max_retries = 3
while retry_count < max_retries:
logger.info(f"尝试重新连接 (尝试 {retry_count + 1}/{max_retries})...")
if await self.protocol.connect():
logger.info("重新连接成功")
self.set_device_state(DeviceState.IDLE)
return True
retry_count += 1
await asyncio.sleep(2) # 等待2秒后重试
logger.error(f"重新连接失败，已尝试 {max_retries} 次")
self.schedule(lambda: self.alert("连接错误", "无法重新连接到服务器"))
self.set_device_state(DeviceState.IDLE)
return False
@@ -282,7 +268,6 @@ class XiaoZhi:
"""接收音频数据回调"""
if self.device_state == DeviceState.SPEAKING:
self.audio_codec.write_audio(data)
self.events[EventType.AUDIO_OUTPUT_READY_EVENT].set()
def _on_incoming_json(self, json_data):
"""接收JSON数据回调"""
@@ -304,48 +289,33 @@ class XiaoZhi:
self._handle_stt_message(data)
elif msg_type == "llm":
self._handle_llm_message(data)
else:
logger.warning(f"收到未知类型的消息: {msg_type}")
except Exception as e:
logger.error(f"处理JSON消息时出错: {e}")
except Exception:
pass
def _handle_tts_message(self, data):
"""处理TTS消息"""
state = data.get("state", "")
if state == "start":
EventManager.on_tts_start()
self.schedule(lambda: self._handle_tts_start())
elif state == "stop":
EventManager.on_tts_end()
self.schedule(lambda: self._handle_tts_stop())
elif state == "sentence_start":
text = data.get("text", "")
if text:
logger.info(f"<< {text}")
print(f"🤖 小智：{text}")
need_verification_code = re.search(r"验证码.*\d+", text)
verification_tips = (
"\n🔥 注意：绑定成功后，需要重新运行本应用才会生效"
if need_verification_code
else ""
)
if need_verification_code:
logger.info(verification_tips)
self.schedule(
lambda: self.set_chat_message(
"assistant",
text + verification_tips,
verification_code = re.findall(r"验证码.*\d+", text)
if verification_code:
self.config.update_config_file(
"VERIFICATION_CODE", verification_code[0]
)
)
self.schedule(lambda: self.set_chat_message("assistant", text))
def _handle_tts_start(self):
"""处理TTS开始事件"""
self.aborted = False
# 清空可能存在的旧音频数据
self.audio_codec.clear_audio_queue()
if (
self.device_state == DeviceState.IDLE
or self.device_state == DeviceState.LISTENING
@@ -354,30 +324,13 @@ class XiaoZhi:
def _handle_tts_stop(self):
"""处理TTS停止事件"""
if self.device_state == DeviceState.SPEAKING:
# 给音频播放一个缓冲时间，确保所有音频都播放完毕
def delayed_state_change():
# 等待音频队列清空
self.audio_codec.wait_for_audio_complete()
# 状态转换
if self.keep_listening:
asyncio.run_coroutine_threadsafe(
self.protocol.send_start_listening(ListeningMode.AUTO_STOP),
self.loop,
)
self.set_device_state(DeviceState.LISTENING)
else:
self.set_device_state(DeviceState.IDLE)
# 安排延迟执行
threading.Thread(target=delayed_state_change, daemon=True).start()
pass
def _handle_stt_message(self, data):
"""处理STT消息"""
text = data.get("text", "")
if text:
logger.info(f">> {text}")
print(f"💬 我说：{text}")
self.schedule(lambda: self.set_chat_message("user", text))
def _handle_llm_message(self, data):
@@ -388,26 +341,19 @@ class XiaoZhi:
async def _on_audio_channel_opened(self):
"""音频通道打开回调"""
logger.info("音频通道已打开")
self.schedule(lambda: self._start_audio_streams())
def _start_audio_streams(self):
"""启动音频流"""
try:
# 确保流已关闭后再重新打开
if (
self.audio_codec.input_stream
and self.audio_codec.input_stream.is_active()
):
if self.audio_codec.input_stream.is_active():
self.audio_codec.input_stream.stop_stream()
# 重新打开流
self.audio_codec.input_stream.start_stream()
if (
self.audio_codec.output_stream
and self.audio_codec.output_stream.is_active()
):
if self.audio_codec.output_stream.is_active():
self.audio_codec.output_stream.stop_stream()
# 重新打开流
@@ -417,141 +363,65 @@ class XiaoZhi:
threading.Thread(
target=self._audio_input_event_trigger, daemon=True
).start()
threading.Thread(
target=self._audio_output_event_trigger, daemon=True
).start()
logger.info("音频流已启动")
except Exception as e:
logger.error(f"启动音频流失败: {e}")
except Exception:
pass
def _audio_input_event_trigger(self):
"""音频输入事件触发器"""
while self.running:
try:
if (
self.audio_codec.input_stream
and self.audio_codec.input_stream.is_active()
):
if self.audio_codec.input_stream.is_active():
self.events[EventType.AUDIO_INPUT_READY_EVENT].set()
except OSError as e:
logger.error(f"音频输入流错误: {e}")
# 如果流已关闭，尝试重新打开或者退出循环
if "Stream not open" in str(e):
break
except Exception as e:
logger.error(f"音频输入事件触发器错误: {e}")
except Exception:
pass
time.sleep(AudioConfig.FRAME_DURATION / 1000) # 按帧时长触发
def _audio_output_event_trigger(self):
"""音频输出事件触发器"""
while (
self.running
and self.audio_codec.output_stream
and self.audio_codec.output_stream.is_active()
):
# 当队列中有数据时才触发事件
if (
not self.audio_codec.audio_decode_queue.empty()
): # 修改为使用 audio_codec 的队列
self.events[EventType.AUDIO_OUTPUT_READY_EVENT].set()
time.sleep(0.02) # 稍微延长检查间隔
async def _on_audio_channel_closed(self):
"""音频通道关闭回调"""
logger.info("音频通道已关闭")
self.set_device_state(DeviceState.IDLE)
self.keep_listening = False
self.schedule(lambda: self._stop_audio_streams())
def _stop_audio_streams(self):
"""停止音频流"""
try:
if (
self.audio_codec.input_stream
and self.audio_codec.input_stream.is_active()
):
self.audio_codec.input_stream.stop_stream()
if (
self.audio_codec.output_stream
and self.audio_codec.output_stream.is_active()
):
self.audio_codec.output_stream.stop_stream()
logger.info("音频流已停止")
except Exception as e:
logger.error(f"停止音频流失败: {e}")
self.audio_codec.stop_streams()
def set_device_state(self, state):
"""设置设备状态"""
if self.device_state == state:
return
old_state = self.device_state
# 如果从 SPEAKING 状态切换出去，确保音频播放完成
if old_state == DeviceState.SPEAKING:
self.audio_codec.wait_for_audio_complete()
self.device_state = state
logger.info(f"状态变更: {old_state} -> {state}")
# 根据状态执行相应操作
VAD.pause() # 停用 VAD
self.audio_codec.stop_streams() # 停用输入输出流
if state == DeviceState.IDLE:
self.display.update_status("待命")
self.display.update_emotion("😶")
if (
self.audio_codec.output_stream
and self.audio_codec.output_stream.is_active()
):
try:
self.audio_codec.output_stream.stop_stream()
except Exception as e:
logger.warning(f"停止输出流时出错: {e}")
elif state == DeviceState.CONNECTING:
self.display.update_status("连接中...")
elif state == DeviceState.LISTENING:
self.display.update_status("聆听中...")
self.display.update_emotion("🙂")
if (
self.audio_codec.input_stream
and not self.audio_codec.input_stream.is_active()
):
try:
self.audio_codec.input_stream.start_stream()
except Exception as e:
logger.warning(f"启动输入流时出错: {e}")
# 使用 AudioCodec 类中的方法重新初始化
self.audio_codec._reinitialize_input_stream()
# 停止输出流
if self.audio_codec.output_stream.is_active():
self.audio_codec.output_stream.stop_stream()
# 打开输入流
if not self.audio_codec.input_stream.is_active():
self.audio_codec.input_stream.start_stream()
elif state == DeviceState.SPEAKING:
self.display.update_status("说话中...")
# 确保输出流处于活跃状态
if self.audio_codec.output_stream:
if not self.audio_codec.output_stream.is_active():
try:
self.audio_codec.output_stream.start_stream()
except Exception as e:
logger.warning(f"启动输出流时出错: {e}")
# 使用 AudioCodec 类中的方法重新初始化
self.audio_codec._reinitialize_output_stream()
# 停止输入流
if (
self.audio_codec.input_stream
and self.audio_codec.input_stream.is_active()
):
try:
self.audio_codec.input_stream.stop_stream()
except Exception as e:
logger.warning(f"停止输入流时出错: {e}")
if self.audio_codec.input_stream.is_active():
self.audio_codec.input_stream.stop_stream()
# 打开输出流
if not self.audio_codec.output_stream.is_active():
self.audio_codec.output_stream.start_stream()
# 通知状态变化
for callback in self.on_state_changed_callbacks:
try:
callback(state)
except Exception as e:
logger.error(f"执行状态变化回调时出错: {e}")
except Exception:
pass
def _get_status_text(self):
"""获取当前状态文本"""
@@ -615,105 +485,33 @@ class XiaoZhi:
def _start_listening_impl(self):
"""开始监听的实现"""
if not self.protocol:
logger.error("协议未初始化")
return
self.keep_listening = False
self.set_device_state(DeviceState.IDLE)
asyncio.run_coroutine_threadsafe(
self.protocol.send_abort_speaking(AbortReason.ABORT),
self.loop,
)
if self.device_state == DeviceState.IDLE:
# 尝试打开音频通道
if not self.protocol.is_audio_channel_opened():
self.set_device_state(DeviceState.CONNECTING) # 设置设备状态为连接中
try:
# 等待异步操作完成
future = asyncio.run_coroutine_threadsafe(
self.protocol.open_audio_channel(), self.loop
)
# 等待操作完成并获取结果
assert future.result(timeout=10.0) # 添加超时时间
except Exception as e:
self.alert("错误", f"打开音频通道失败: {str(e)}")
self.set_device_state(DeviceState.IDLE)
return
# 尝试打开音频通道
if not self.protocol.is_audio_channel_opened():
try:
# 等待异步操作完成
future = asyncio.run_coroutine_threadsafe(
self.protocol.open_audio_channel(), self.loop
)
# 等待操作完成并获取结果
success = future.result(timeout=10.0) # 添加超时时间
if not success:
self.alert("错误", "打开音频通道失败") # 弹出错误提示
self.set_device_state(DeviceState.IDLE) # 设置设备状态为空闲
return
except Exception as e:
logger.error(f"打开音频通道时发生错误: {e}")
self.alert("错误", f"打开音频通道失败: {str(e)}")
self.set_device_state(DeviceState.IDLE)
return
asyncio.run_coroutine_threadsafe(
self.protocol.send_start_listening(ListeningMode.MANUAL), self.loop
)
self.set_device_state(DeviceState.LISTENING) # 设置设备状态为监听中
elif self.device_state == DeviceState.SPEAKING:
if not self.aborted:
self.abort_speaking(AbortReason.WAKE_WORD_DETECTED)
async def _open_audio_channel_and_start_manual_listening(self):
"""打开音频通道并开始手动监听"""
if not await self.protocol.open_audio_channel():
self.set_device_state(DeviceState.IDLE)
self.alert("错误", "打开音频通道失败")
return
await self.protocol.send_start_listening(ListeningMode.MANUAL)
self.set_device_state(DeviceState.LISTENING)
def toggle_chat_state(self):
"""切换聊天状态"""
self.schedule(self._toggle_chat_state_impl)
def _toggle_chat_state_impl(self):
"""切换聊天状态的具体实现"""
# 检查协议是否已初始化
if not self.protocol:
logger.error("协议未初始化")
return
# 如果设备当前处于空闲状态，尝试连接并开始监听
if self.device_state == DeviceState.IDLE:
self.set_device_state(DeviceState.CONNECTING) # 设置设备状态为连接中
# 尝试打开音频通道
if not self.protocol.is_audio_channel_opened():
try:
# 等待异步操作完成
future = asyncio.run_coroutine_threadsafe(
self.protocol.open_audio_channel(), self.loop
)
# 等待操作完成并获取结果
success = future.result(timeout=10.0) # 添加超时时间
if not success:
self.alert("错误", "打开音频通道失败") # 弹出错误提示
self.set_device_state(DeviceState.IDLE) # 设置设备状态为空闲
return
except Exception as e:
logger.error(f"打开音频通道时发生错误: {e}")
self.alert("错误", f"打开音频通道失败: {str(e)}")
self.set_device_state(DeviceState.IDLE)
return
self.keep_listening = True # 开始监听
# 启动自动停止的监听模式
asyncio.run_coroutine_threadsafe(
self.protocol.send_start_listening(ListeningMode.AUTO_STOP), self.loop
)
self.set_device_state(DeviceState.LISTENING) # 设置设备状态为监听中
# 如果设备正在说话，停止当前说话
elif self.device_state == DeviceState.SPEAKING:
self.abort_speaking(AbortReason.NONE) # 中止说话
# 如果设备正在监听，关闭音频通道
elif self.device_state == DeviceState.LISTENING:
asyncio.run_coroutine_threadsafe(
self.protocol.close_audio_channel(), self.loop
)
asyncio.run_coroutine_threadsafe(
self.protocol.send_start_listening(ListeningMode.MANUAL), self.loop
)
self.set_device_state(DeviceState.LISTENING) # 设置设备状态为监听中
def stop_listening(self):
"""停止监听"""
@@ -721,35 +519,19 @@ class XiaoZhi:
def _stop_listening_impl(self):
"""停止监听的实现"""
if self.device_state == DeviceState.LISTENING:
asyncio.run_coroutine_threadsafe(
self.protocol.send_stop_listening(), self.loop
)
self.set_device_state(DeviceState.IDLE)
self.set_device_state(DeviceState.IDLE)
asyncio.run_coroutine_threadsafe(self.protocol.send_stop_listening(), self.loop)
def abort_speaking(self, reason):
"""中止语音输出"""
logger.info(f"中止语音输出，原因: {reason}")
self.aborted = True
asyncio.run_coroutine_threadsafe(
self.protocol.send_abort_speaking(reason), self.loop
)
self.set_device_state(DeviceState.IDLE)
# 添加此代码：当用户主动打断时自动进入录音模式
if reason == AbortReason.WAKE_WORD_DETECTED and self.keep_listening:
# 短暂延迟确保abort命令被处理
def start_listening_after_abort():
time.sleep(0.2) # 短暂延迟
self.set_device_state(DeviceState.IDLE)
self.schedule(lambda: self.toggle_chat_state())
threading.Thread(target=start_listening_after_abort, daemon=True).start()
asyncio.run_coroutine_threadsafe(
self.protocol.send_abort_speaking(AbortReason.ABORT),
self.loop,
)
def alert(self, title, message):
"""显示警告信息"""
logger.warning(f"警告: {title}, {message}")
# 在GUI上显示警告
if self.display:
self.display.update_text(f"{title}: {message}")
@@ -759,7 +541,6 @@ class XiaoZhi:
def shutdown(self):
"""关闭应用程序"""
logger.info("正在关闭应用程序...")
self.running = False
# 关闭音频编解码器
@@ -780,15 +561,10 @@ class XiaoZhi:
if self.loop_thread and self.loop_thread.is_alive():
self.loop_thread.join(timeout=1.0)
logger.info("应用程序已关闭")
def toggle_chat_state(self):
"""切换聊天状态"""
pass
def _on_mode_changed(self, auto_mode):
"""处理对话模式变更"""
# 只有在IDLE状态下才允许切换模式
if self.device_state != DeviceState.IDLE:
self.alert("提示", "只有在待命状态下才能切换对话模式")
return False
self.keep_listening = auto_mode
logger.info(f"对话模式已切换为: {'自动' if auto_mode else '手动'}")
return True
pass