mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-28 01:53:53 +08:00
update:test迁移重命名为digital-human
This commit is contained in:
3030332422
@@ -0,0 +1 @@
|
||||
"""Test-side wakeword runtime package."""
|
||||
@@ -0,0 +1 @@
|
||||
from .event_bridge import WakewordEventBridge
|
||||
@@ -0,0 +1,106 @@
|
||||
import json
|
||||
import logging
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WakewordEventBridge:
|
||||
def __init__(self) -> None:
|
||||
self._clients: list[queue.Queue[str]] = []
|
||||
self._clients_lock = threading.Lock()
|
||||
self._running = True
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
return self._running
|
||||
|
||||
def build_ready_message(self) -> str:
|
||||
return self.build_message(
|
||||
"bridge_connected",
|
||||
{"status": "ready"},
|
||||
)
|
||||
|
||||
def publish_detected(self, wake_word: str) -> None:
|
||||
self.publish(
|
||||
"wake_word_detected",
|
||||
{
|
||||
"wake_word": wake_word,
|
||||
"timestamp": time.time(),
|
||||
},
|
||||
)
|
||||
|
||||
def publish(self, event_type: str, payload: dict[str, Any] | None = None) -> None:
|
||||
if not self._running:
|
||||
return
|
||||
|
||||
message = self.build_message(event_type, payload or {})
|
||||
with self._clients_lock:
|
||||
clients = list(self._clients)
|
||||
|
||||
stale_clients: list[queue.Queue[str]] = []
|
||||
for client_queue in clients:
|
||||
try:
|
||||
client_queue.put_nowait(message)
|
||||
except queue.Full:
|
||||
stale_clients.append(client_queue)
|
||||
|
||||
if stale_clients:
|
||||
with self._clients_lock:
|
||||
for client_queue in stale_clients:
|
||||
if client_queue in self._clients:
|
||||
self._clients.remove(client_queue)
|
||||
|
||||
def add_client(self) -> queue.Queue[str]:
|
||||
client_queue: queue.Queue[str] = queue.Queue(maxsize=16)
|
||||
with self._clients_lock:
|
||||
self._clients.append(client_queue)
|
||||
return client_queue
|
||||
|
||||
def build_message(
|
||||
self,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
request_id: str | None = None,
|
||||
success: bool = True,
|
||||
error: str | None = None,
|
||||
) -> str:
|
||||
message: dict[str, Any] = {
|
||||
"type": event_type,
|
||||
"requestId": request_id,
|
||||
"success": success,
|
||||
"payload": payload or {},
|
||||
}
|
||||
if error:
|
||||
message["error"] = error
|
||||
return json.dumps(message, ensure_ascii=False)
|
||||
|
||||
def remove_client(self, client_queue: queue.Queue[str]) -> None:
|
||||
with self._clients_lock:
|
||||
if client_queue in self._clients:
|
||||
self._clients.remove(client_queue)
|
||||
|
||||
def publish_service_ready(self) -> None:
|
||||
self.publish("service_ready", {"status": "ready"})
|
||||
|
||||
def publish_service_stopping(self) -> None:
|
||||
self.publish("service_stopping", {"status": "stopping"})
|
||||
|
||||
def close(self) -> None:
|
||||
if not self._running:
|
||||
return
|
||||
|
||||
self.publish_service_stopping()
|
||||
self._running = False
|
||||
with self._clients_lock:
|
||||
clients = list(self._clients)
|
||||
self._clients.clear()
|
||||
|
||||
for client_queue in clients:
|
||||
try:
|
||||
client_queue.put_nowait("__bridge_closed__")
|
||||
except queue.Full:
|
||||
pass
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"wakeword": {
|
||||
"enabled": true
|
||||
},
|
||||
"model_dir": "models",
|
||||
"audio": {
|
||||
"input_device": null,
|
||||
"sample_rate": 16000,
|
||||
"channels": 1
|
||||
},
|
||||
"detector": {
|
||||
"num_threads": 4,
|
||||
"provider": "cpu",
|
||||
"max_active_paths": 2,
|
||||
"keywords_score": 1.8,
|
||||
"keywords_threshold": 0.1,
|
||||
"num_trailing_blanks": 1,
|
||||
"cooldown_seconds": 1.5
|
||||
},
|
||||
"logging": {
|
||||
"level": "INFO",
|
||||
"dir": "logs",
|
||||
"file": "wakeword-runtime.log"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
from .config_loader import RuntimeConfig, load_config
|
||||
from .logging_setup import setup_logging
|
||||
@@ -0,0 +1,153 @@
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass
|
||||
class WakewordSettings:
|
||||
enabled: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioSettings:
|
||||
input_device: str | int | None = None
|
||||
sample_rate: int = 16000
|
||||
channels: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class DetectorSettings:
|
||||
num_threads: int = 4
|
||||
provider: str = "cpu"
|
||||
max_active_paths: int = 2
|
||||
keywords_score: float = 1.8
|
||||
keywords_threshold: float = 0.2
|
||||
num_trailing_blanks: int = 1
|
||||
cooldown_seconds: float = 1.5
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoggingSettings:
|
||||
level: str = "INFO"
|
||||
directory: str = "logs"
|
||||
file_name: str = "wakeword-runtime.log"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RuntimeConfig:
|
||||
runtime_root: Path
|
||||
wakeword: WakewordSettings
|
||||
wake_words: list[str]
|
||||
model_dir: Path
|
||||
audio: AudioSettings
|
||||
detector: DetectorSettings
|
||||
logging: LoggingSettings
|
||||
|
||||
def validate(self) -> None:
|
||||
if self.wakeword.enabled and not self.wake_words:
|
||||
raise ValueError("keywords.txt cannot be empty when wakeword is enabled")
|
||||
|
||||
if self.audio.sample_rate <= 0:
|
||||
raise ValueError("audio.sample_rate must be greater than 0")
|
||||
|
||||
if self.audio.channels <= 0:
|
||||
raise ValueError("audio.channels must be greater than 0")
|
||||
|
||||
if self.detector.num_threads <= 0:
|
||||
raise ValueError("detector.num_threads must be greater than 0")
|
||||
|
||||
if self.detector.cooldown_seconds < 0:
|
||||
raise ValueError("detector.cooldown_seconds cannot be negative")
|
||||
|
||||
if not self.logging.level:
|
||||
raise ValueError("logging.level cannot be empty")
|
||||
|
||||
@property
|
||||
def wakeword_enabled(self) -> bool:
|
||||
return self.wakeword.enabled
|
||||
|
||||
@property
|
||||
def log_dir(self) -> Path:
|
||||
return (self.runtime_root / self.logging.directory).resolve()
|
||||
|
||||
@property
|
||||
def log_file(self) -> Path:
|
||||
return self.log_dir / self.logging.file_name
|
||||
|
||||
@property
|
||||
def log_level(self) -> str:
|
||||
return self.logging.level.upper()
|
||||
|
||||
|
||||
def load_config(runtime_root: Path) -> RuntimeConfig:
|
||||
config_path = runtime_root / "config.json"
|
||||
raw = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
wakeword_cfg = dict(raw.get("wakeword", {}))
|
||||
raw_model_dir = Path(str(raw.get("model_dir", "models")))
|
||||
model_dir = raw_model_dir.resolve() if raw_model_dir.is_absolute() else (runtime_root / raw_model_dir).resolve()
|
||||
wake_words = _load_wake_words_from_keywords_file(model_dir)
|
||||
|
||||
audio_cfg = dict(raw.get("audio", {}))
|
||||
detector_cfg = dict(raw.get("detector", {}))
|
||||
logging_cfg = dict(raw.get("logging", {}))
|
||||
|
||||
config = RuntimeConfig(
|
||||
runtime_root=runtime_root,
|
||||
wakeword=WakewordSettings(enabled=bool(wakeword_cfg.get("enabled", False))),
|
||||
wake_words=wake_words,
|
||||
model_dir=model_dir,
|
||||
audio=AudioSettings(
|
||||
input_device=audio_cfg.get("input_device"),
|
||||
sample_rate=int(audio_cfg.get("sample_rate", 16000)),
|
||||
channels=int(audio_cfg.get("channels", 1)),
|
||||
),
|
||||
detector=DetectorSettings(
|
||||
num_threads=int(detector_cfg.get("num_threads", 4)),
|
||||
provider=str(detector_cfg.get("provider", "cpu")),
|
||||
max_active_paths=int(detector_cfg.get("max_active_paths", 2)),
|
||||
keywords_score=float(detector_cfg.get("keywords_score", 1.8)),
|
||||
keywords_threshold=float(detector_cfg.get("keywords_threshold", 0.2)),
|
||||
num_trailing_blanks=int(detector_cfg.get("num_trailing_blanks", 1)),
|
||||
cooldown_seconds=float(detector_cfg.get("cooldown_seconds", 1.5)),
|
||||
),
|
||||
logging=LoggingSettings(
|
||||
level=str(logging_cfg.get("level", "INFO")).upper(),
|
||||
directory=str(logging_cfg.get("dir", "logs")),
|
||||
file_name=str(logging_cfg.get("file", "wakeword-runtime.log")),
|
||||
),
|
||||
)
|
||||
config.validate()
|
||||
return config
|
||||
|
||||
|
||||
DEFAULT_WAKE_WORDS = [
|
||||
"你好小智",
|
||||
"你好小志",
|
||||
"小爱同学",
|
||||
"你好小鑫",
|
||||
"你好小新",
|
||||
"小美同学",
|
||||
"小龙小龙",
|
||||
"喵喵同学",
|
||||
"小滨小滨",
|
||||
"小冰小冰",
|
||||
"嘿你好呀",
|
||||
]
|
||||
|
||||
|
||||
def _load_wake_words_from_keywords_file(model_dir: Path) -> list[str]:
|
||||
keywords_file = model_dir / "keywords.txt"
|
||||
if not keywords_file.exists():
|
||||
return DEFAULT_WAKE_WORDS.copy()
|
||||
|
||||
wake_words: list[str] = []
|
||||
for line in keywords_file.read_text(encoding="utf-8").splitlines():
|
||||
text = line.strip()
|
||||
if not text or text.startswith("#") or "@" not in text:
|
||||
continue
|
||||
|
||||
wake_word = text.split("@", 1)[1].strip()
|
||||
if wake_word:
|
||||
wake_words.append(wake_word)
|
||||
|
||||
return wake_words if wake_words else DEFAULT_WAKE_WORDS.copy()
|
||||
@@ -0,0 +1,23 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def setup_logging(log_file: Path, level: str = "INFO") -> None:
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(getattr(logging, level.upper(), logging.INFO))
|
||||
logger.handlers.clear()
|
||||
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s [%(levelname)s] %(name)s - %(message)s"
|
||||
)
|
||||
|
||||
stream_handler = logging.StreamHandler()
|
||||
stream_handler.setFormatter(formatter)
|
||||
|
||||
file_handler = logging.FileHandler(log_file, encoding="utf-8")
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
logger.addHandler(stream_handler)
|
||||
logger.addHandler(file_handler)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .detector_assets import DetectorAssets, DetectorAssetsBuilder
|
||||
from .detector import WakewordDetector
|
||||
from .microphone import MicrophoneListener
|
||||
@@ -0,0 +1,191 @@
|
||||
import queue
|
||||
import threading
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Callable
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ..config import RuntimeConfig
|
||||
from .detector_assets import DetectorAssetsBuilder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WakewordDetector:
|
||||
def __init__(self, config: RuntimeConfig) -> None:
|
||||
self.config = config
|
||||
self.enabled = config.wakeword_enabled
|
||||
self.assets_builder = DetectorAssetsBuilder(config)
|
||||
self.audio_source = None
|
||||
self.keyword_spotter: Any | None = None
|
||||
self.stream: Any | None = None
|
||||
self.is_running_flag = False
|
||||
self.paused = False
|
||||
self.on_detected_callback: Callable[[str, str], None] | None = None
|
||||
self.on_error: Callable[[Exception], None] | None = None
|
||||
self._audio_queue: queue.Queue[np.ndarray] = queue.Queue(maxsize=100)
|
||||
self._worker_thread: threading.Thread | None = None
|
||||
self.last_detection_time = 0.0
|
||||
self.detection_cooldown = self.config.detector.cooldown_seconds
|
||||
|
||||
def initialize(self) -> None:
|
||||
if not self.enabled:
|
||||
raise RuntimeError("wakeword detector is disabled")
|
||||
|
||||
try:
|
||||
import sherpa_onnx
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"Missing dependency: sherpa-onnx. Install runtime dependencies before initializing detector."
|
||||
) from exc
|
||||
|
||||
assets = self.assets_builder.prepare()
|
||||
detector_cfg = self.config.detector
|
||||
|
||||
self.keyword_spotter = sherpa_onnx.KeywordSpotter(
|
||||
tokens=str(assets.tokens_file),
|
||||
encoder=str(assets.encoder_file),
|
||||
decoder=str(assets.decoder_file),
|
||||
joiner=str(assets.joiner_file),
|
||||
keywords_file=str(assets.keywords_file),
|
||||
num_threads=detector_cfg.num_threads,
|
||||
sample_rate=self.config.audio.sample_rate,
|
||||
feature_dim=80,
|
||||
max_active_paths=detector_cfg.max_active_paths,
|
||||
keywords_score=detector_cfg.keywords_score,
|
||||
keywords_threshold=detector_cfg.keywords_threshold,
|
||||
num_trailing_blanks=detector_cfg.num_trailing_blanks,
|
||||
provider=detector_cfg.provider,
|
||||
)
|
||||
self.stream = self.keyword_spotter.create_stream()
|
||||
logger.info("detector initialized")
|
||||
logger.info("detector model root: %s", assets.model_root)
|
||||
logger.info("detector keywords file: %s", assets.keywords_file)
|
||||
|
||||
def on_detected(self, callback: Callable[[str, str], None]) -> None:
|
||||
self.on_detected_callback = callback
|
||||
|
||||
def on_audio_data(self, audio_data: np.ndarray) -> None:
|
||||
if not self.enabled or not self.is_running_flag or self.paused:
|
||||
return
|
||||
|
||||
try:
|
||||
self._audio_queue.put_nowait(audio_data.copy())
|
||||
except queue.Full:
|
||||
try:
|
||||
self._audio_queue.get_nowait()
|
||||
self._audio_queue.put_nowait(audio_data.copy())
|
||||
except queue.Empty:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug("audio data enqueue failed: %s", exc)
|
||||
|
||||
def start(self, audio_source) -> None:
|
||||
if not self.enabled:
|
||||
logger.info("wakeword detector disabled")
|
||||
return
|
||||
|
||||
if self.keyword_spotter is None or self.stream is None:
|
||||
self.initialize()
|
||||
|
||||
if self.is_running_flag:
|
||||
return
|
||||
|
||||
self.audio_source = audio_source
|
||||
self.audio_source.add_audio_listener(self)
|
||||
self.is_running_flag = True
|
||||
self.paused = False
|
||||
self._worker_thread = threading.Thread(
|
||||
target=self._detection_loop,
|
||||
name="wakeword-detector",
|
||||
daemon=True,
|
||||
)
|
||||
self._worker_thread.start()
|
||||
logger.info("wakeword detector started")
|
||||
|
||||
def stop(self) -> None:
|
||||
if not self.is_running_flag and self.audio_source is None and self._worker_thread is None:
|
||||
return
|
||||
|
||||
self.is_running_flag = False
|
||||
|
||||
if self.audio_source is not None:
|
||||
self.audio_source.remove_audio_listener(self)
|
||||
self.audio_source = None
|
||||
|
||||
if self._worker_thread is not None:
|
||||
self._worker_thread.join(timeout=1.0)
|
||||
self._worker_thread = None
|
||||
|
||||
while not self._audio_queue.empty():
|
||||
try:
|
||||
self._audio_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
logger.info("wakeword detector stopped")
|
||||
|
||||
def _detection_loop(self) -> None:
|
||||
error_count = 0
|
||||
max_errors = 5
|
||||
|
||||
while self.is_running_flag:
|
||||
try:
|
||||
if self.paused:
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
|
||||
audio_data = self._audio_queue.get(timeout=0.1)
|
||||
result = self.process_audio_chunk(audio_data)
|
||||
if result and self.on_detected_callback is not None:
|
||||
self.on_detected_callback(result, result)
|
||||
error_count = 0
|
||||
except queue.Empty:
|
||||
continue
|
||||
except Exception as exc:
|
||||
error_count += 1
|
||||
logger.error("wakeword detection loop error(%s/%s): %s", error_count, max_errors, exc)
|
||||
if self.on_error is not None:
|
||||
try:
|
||||
self.on_error(exc)
|
||||
except Exception:
|
||||
logger.exception("wakeword error callback failed")
|
||||
if error_count >= max_errors:
|
||||
logger.critical("too many wakeword detection errors, stopping detector")
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
self.is_running_flag = False
|
||||
|
||||
def process_audio_chunk(self, audio_data: np.ndarray) -> str | None:
|
||||
if self.keyword_spotter is None or self.stream is None:
|
||||
raise RuntimeError("detector is not initialized")
|
||||
|
||||
if audio_data is None or len(audio_data) == 0:
|
||||
return None
|
||||
|
||||
if audio_data.dtype == np.int16:
|
||||
samples = audio_data.astype(np.float32) / 32768.0
|
||||
else:
|
||||
samples = audio_data.astype(np.float32)
|
||||
|
||||
sample_rate = self.config.audio.sample_rate
|
||||
self.stream.accept_waveform(sample_rate=sample_rate, waveform=samples)
|
||||
|
||||
if not self.keyword_spotter.is_ready(self.stream):
|
||||
return None
|
||||
|
||||
self.keyword_spotter.decode_stream(self.stream)
|
||||
result = self.keyword_spotter.get_result(self.stream)
|
||||
if not result:
|
||||
return None
|
||||
|
||||
self.keyword_spotter.reset_stream(self.stream)
|
||||
|
||||
current_time = time.time()
|
||||
if current_time - self.last_detection_time < self.detection_cooldown:
|
||||
return None
|
||||
|
||||
self.last_detection_time = current_time
|
||||
return str(result)
|
||||
@@ -0,0 +1,94 @@
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ..config import RuntimeConfig
|
||||
|
||||
REQUIRED_MODEL_FILES = (
|
||||
"encoder.onnx",
|
||||
"decoder.onnx",
|
||||
"joiner.onnx",
|
||||
"tokens.txt",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DetectorAssets:
|
||||
model_root: Path
|
||||
tokens_file: Path
|
||||
encoder_file: Path
|
||||
decoder_file: Path
|
||||
joiner_file: Path
|
||||
keywords_file: Path
|
||||
|
||||
|
||||
class DetectorAssetsBuilder:
|
||||
def __init__(self, config: RuntimeConfig) -> None:
|
||||
self.config = config
|
||||
|
||||
def prepare(self) -> DetectorAssets:
|
||||
model_root = self._resolve_model_root()
|
||||
keywords_file = self._write_keywords_file(model_root)
|
||||
return DetectorAssets(
|
||||
model_root=model_root,
|
||||
tokens_file=model_root / "tokens.txt",
|
||||
encoder_file=model_root / "encoder.onnx",
|
||||
decoder_file=model_root / "decoder.onnx",
|
||||
joiner_file=model_root / "joiner.onnx",
|
||||
keywords_file=keywords_file,
|
||||
)
|
||||
|
||||
def _resolve_model_root(self) -> Path:
|
||||
preferred = self.config.model_dir
|
||||
if self._has_required_files(preferred):
|
||||
return preferred
|
||||
|
||||
raise FileNotFoundError(
|
||||
"No valid model directory found. Expected configured model files to exist."
|
||||
)
|
||||
|
||||
def _write_keywords_file(self, model_root: Path) -> Path:
|
||||
try:
|
||||
from pypinyin import Style, pinyin
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"Missing dependency: pypinyin. Install runtime dependencies before generating keywords."
|
||||
) from exc
|
||||
|
||||
wake_words = self.config.wake_words
|
||||
if not wake_words:
|
||||
raise ValueError("keywords.txt cannot be empty")
|
||||
|
||||
keywords_path = model_root / "keywords.txt"
|
||||
lines: list[str] = []
|
||||
for keyword_text in wake_words:
|
||||
initials = pinyin(keyword_text, style=Style.INITIALS, strict=False)
|
||||
finals = pinyin(
|
||||
keyword_text,
|
||||
style=Style.FINALS_TONE,
|
||||
strict=False,
|
||||
neutral_tone_with_five=True,
|
||||
)
|
||||
|
||||
tokens: list[str] = []
|
||||
for initial_parts, final_parts in zip(initials, finals):
|
||||
initial = initial_parts[0].strip()
|
||||
final = final_parts[0].strip()
|
||||
if initial:
|
||||
tokens.append(initial)
|
||||
if final:
|
||||
tokens.append(final)
|
||||
|
||||
if not tokens:
|
||||
raise ValueError(
|
||||
f"failed to generate pinyin tokens for wake word: {keyword_text}"
|
||||
)
|
||||
|
||||
lines.append(f"{' '.join(tokens)} @{keyword_text}")
|
||||
|
||||
keywords_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
return keywords_path
|
||||
|
||||
def _has_required_files(self, directory: Path) -> bool:
|
||||
if not directory.exists():
|
||||
return False
|
||||
return all((directory / file_name).exists() for file_name in REQUIRED_MODEL_FILES)
|
||||
@@ -0,0 +1,91 @@
|
||||
import logging
|
||||
from typing import Protocol
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ..config import RuntimeConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AudioListener(Protocol):
|
||||
def on_audio_data(self, audio_data: np.ndarray) -> None:
|
||||
...
|
||||
|
||||
|
||||
class MicrophoneListener:
|
||||
|
||||
def __init__(self, config: RuntimeConfig) -> None:
|
||||
self.config = config
|
||||
self._stream = None
|
||||
self._running = False
|
||||
self._listeners: list[AudioListener] = []
|
||||
self._sample_rate = self.config.audio.sample_rate
|
||||
self._channels = self.config.audio.channels
|
||||
self._device = self.config.audio.input_device
|
||||
self._block_duration_ms = 30
|
||||
self._block_size = int(self._sample_rate * (self._block_duration_ms / 1000))
|
||||
|
||||
def add_audio_listener(self, listener: AudioListener) -> None:
|
||||
if listener not in self._listeners:
|
||||
self._listeners.append(listener)
|
||||
|
||||
def remove_audio_listener(self, listener: AudioListener) -> None:
|
||||
if listener in self._listeners:
|
||||
self._listeners.remove(listener)
|
||||
|
||||
def start(self) -> None:
|
||||
try:
|
||||
import sounddevice as sd
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"Missing dependency: sounddevice. Install runtime dependencies before starting microphone listener."
|
||||
) from exc
|
||||
|
||||
self._stream = sd.InputStream(
|
||||
device=self._device,
|
||||
samplerate=self._sample_rate,
|
||||
channels=self._channels,
|
||||
dtype="int16",
|
||||
blocksize=self._block_size,
|
||||
callback=self._input_callback,
|
||||
latency="low",
|
||||
)
|
||||
self._stream.start()
|
||||
self._running = True
|
||||
|
||||
logger.info("microphone listener started")
|
||||
logger.info("microphone sample rate: %s", self._sample_rate)
|
||||
logger.info("microphone channels: %s", self._channels)
|
||||
logger.info("microphone device: %s", self._device if self._device is not None else "default")
|
||||
logger.info("microphone block size: %s", self._block_size)
|
||||
|
||||
def stop(self) -> None:
|
||||
if not self._running and self._stream is None:
|
||||
return
|
||||
|
||||
self._running = False
|
||||
if self._stream is not None:
|
||||
try:
|
||||
self._stream.stop()
|
||||
finally:
|
||||
self._stream.close()
|
||||
self._stream = None
|
||||
logger.info("microphone listener stopped")
|
||||
|
||||
def _input_callback(self, indata, frames, time_info, status) -> None:
|
||||
_ = frames, time_info
|
||||
if status and "overflow" not in str(status).lower():
|
||||
logger.warning("microphone status: %s", status)
|
||||
|
||||
audio = np.copy(indata)
|
||||
if audio.ndim > 1:
|
||||
audio = audio[:, 0]
|
||||
else:
|
||||
audio = audio.reshape(-1)
|
||||
|
||||
for listener in list(self._listeners):
|
||||
try:
|
||||
listener.on_audio_data(audio)
|
||||
except Exception:
|
||||
logger.exception("audio listener callback failed")
|
||||
@@ -0,0 +1,4 @@
|
||||
from .audio import AudioPlugin
|
||||
from .base import Plugin
|
||||
from .manager import PluginManager
|
||||
from .wake_word import WakeWordPlugin
|
||||
@@ -0,0 +1,37 @@
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from ..core import MicrophoneListener
|
||||
from .base import Plugin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AudioPlugin(Plugin):
|
||||
name = "audio"
|
||||
priority = 10
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.app = None
|
||||
self.source: MicrophoneListener | None = None
|
||||
|
||||
def setup(self, app: Any) -> None:
|
||||
self.app = app
|
||||
self.source = MicrophoneListener(app.config)
|
||||
self.app.audio_source = self.source
|
||||
|
||||
def start(self) -> None:
|
||||
if self.source is None:
|
||||
logger.warning("audio source not initialized")
|
||||
return
|
||||
self.source.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
if self.source is not None:
|
||||
self.source.stop()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.app is not None:
|
||||
self.app.audio_source = None
|
||||
self.source = None
|
||||
self.app = None
|
||||
@@ -0,0 +1,18 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
class Plugin:
|
||||
name: str = "plugin"
|
||||
priority: int = 50
|
||||
|
||||
def setup(self, app: Any) -> None:
|
||||
return None
|
||||
|
||||
def start(self) -> None:
|
||||
return None
|
||||
|
||||
def stop(self) -> None:
|
||||
return None
|
||||
|
||||
def shutdown(self) -> None:
|
||||
return None
|
||||
@@ -0,0 +1,37 @@
|
||||
from typing import Any
|
||||
|
||||
from .base import Plugin
|
||||
|
||||
|
||||
class PluginManager:
|
||||
def __init__(self) -> None:
|
||||
self._plugins: list[Plugin] = []
|
||||
self._by_name: dict[str, Plugin] = {}
|
||||
|
||||
def register(self, *plugins: Plugin) -> None:
|
||||
for plugin in sorted(plugins, key=lambda item: getattr(item, "priority", 50)):
|
||||
if plugin in self._plugins:
|
||||
continue
|
||||
self._plugins.append(plugin)
|
||||
name = getattr(plugin, "name", "")
|
||||
if isinstance(name, str) and name:
|
||||
self._by_name[name] = plugin
|
||||
|
||||
def get_plugin(self, name: str) -> Plugin | None:
|
||||
return self._by_name.get(name)
|
||||
|
||||
def setup_all(self, app: Any) -> None:
|
||||
for plugin in list(self._plugins):
|
||||
plugin.setup(app)
|
||||
|
||||
def start_all(self) -> None:
|
||||
for plugin in list(self._plugins):
|
||||
plugin.start()
|
||||
|
||||
def stop_all(self) -> None:
|
||||
for plugin in reversed(self._plugins):
|
||||
plugin.stop()
|
||||
|
||||
def shutdown_all(self) -> None:
|
||||
for plugin in reversed(self._plugins):
|
||||
plugin.shutdown()
|
||||
@@ -0,0 +1,53 @@
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from ..core import WakewordDetector
|
||||
from .base import Plugin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WakeWordPlugin(Plugin):
|
||||
name = "wake_word"
|
||||
priority = 30
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.app = None
|
||||
self.detector: WakewordDetector | None = None
|
||||
|
||||
def setup(self, app: Any) -> None:
|
||||
self.app = app
|
||||
self.detector = WakewordDetector(app.config)
|
||||
if not self.detector.enabled:
|
||||
self.detector = None
|
||||
return
|
||||
self.detector.on_detected(self._on_detected)
|
||||
self.detector.on_error = self._on_error
|
||||
|
||||
def start(self) -> None:
|
||||
if self.detector is None:
|
||||
return
|
||||
|
||||
audio_plugin = self.app.plugins.get_plugin("audio") if self.app else None
|
||||
audio_source = getattr(audio_plugin, "source", None)
|
||||
if audio_source is None:
|
||||
logger.warning("audio source unavailable, wakeword plugin not started")
|
||||
return
|
||||
|
||||
self.detector.start(audio_source)
|
||||
|
||||
def stop(self) -> None:
|
||||
if self.detector is not None:
|
||||
self.detector.stop()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.detector = None
|
||||
self.app = None
|
||||
|
||||
def _on_detected(self, wake_word: str, full_text: str) -> None:
|
||||
if self.app is None:
|
||||
return
|
||||
self.app.handle_wake_word_detected(wake_word, full_text)
|
||||
|
||||
def _on_error(self, error: Exception) -> None:
|
||||
logger.error("wakeword detection error: %s", error)
|
||||
@@ -0,0 +1,3 @@
|
||||
sherpa-onnx==1.12.29
|
||||
sounddevice>=0.4.4
|
||||
pypinyin==0.55.0
|
||||
@@ -0,0 +1,2 @@
|
||||
from .app import TestRuntimeApplication
|
||||
from .http_server import TestRuntimeHttpServer
|
||||
@@ -0,0 +1,70 @@
|
||||
import logging
|
||||
from typing import Protocol
|
||||
|
||||
from ..config import RuntimeConfig
|
||||
from ..plugins import AudioPlugin, PluginManager, WakeWordPlugin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EventPublisher(Protocol):
|
||||
def publish_service_ready(self) -> None:
|
||||
...
|
||||
|
||||
def publish_service_stopping(self) -> None:
|
||||
...
|
||||
|
||||
def publish_detected(self, wake_word: str) -> None:
|
||||
...
|
||||
|
||||
|
||||
class TestRuntimeApplication:
|
||||
def __init__(self, config: RuntimeConfig, event_publisher: EventPublisher) -> None:
|
||||
self.config = config
|
||||
self.event_bridge = event_publisher
|
||||
self.plugins = PluginManager()
|
||||
self.audio_source = None
|
||||
self._is_setup = False
|
||||
self._is_running = False
|
||||
|
||||
def setup(self) -> None:
|
||||
if self._is_setup:
|
||||
return
|
||||
|
||||
if self.config.wakeword_enabled:
|
||||
self.plugins.register(
|
||||
AudioPlugin(),
|
||||
WakeWordPlugin(),
|
||||
)
|
||||
self.plugins.setup_all(self)
|
||||
self._is_setup = True
|
||||
|
||||
def start(self) -> None:
|
||||
if self._is_running:
|
||||
return
|
||||
|
||||
self.setup()
|
||||
self.plugins.start_all()
|
||||
if self.config.wakeword_enabled:
|
||||
self.event_bridge.publish_service_ready()
|
||||
self._is_running = True
|
||||
logger.info("test runtime application started")
|
||||
|
||||
def stop(self) -> None:
|
||||
if not self._is_running:
|
||||
return
|
||||
|
||||
self.event_bridge.publish_service_stopping()
|
||||
self.plugins.stop_all()
|
||||
self._is_running = False
|
||||
logger.info("test runtime application stopped")
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.stop()
|
||||
if self._is_setup:
|
||||
self.plugins.shutdown_all()
|
||||
|
||||
def handle_wake_word_detected(self, wake_word: str, full_text: str) -> None:
|
||||
_ = full_text
|
||||
logger.info("wake word detected: %s", wake_word)
|
||||
self.event_bridge.publish_detected(wake_word)
|
||||
@@ -0,0 +1,352 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import queue
|
||||
import socket
|
||||
import threading
|
||||
from http import HTTPStatus
|
||||
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from ..bridge import WakewordEventBridge
|
||||
from ..config.config_loader import load_config
|
||||
|
||||
|
||||
class TestRuntimeHttpServer:
|
||||
def __init__(self, test_root: Path, host: str = "0.0.0.0", port: int = 8006) -> None:
|
||||
self.test_root = test_root
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.event_bridge = WakewordEventBridge()
|
||||
self._restart_handler: Callable[[], None] | None = None
|
||||
self._restart_lock = threading.Lock()
|
||||
self._server = self._build_server()
|
||||
|
||||
@property
|
||||
def page_url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.port}/index.html"
|
||||
|
||||
@property
|
||||
def bridge_url(self) -> str:
|
||||
return f"ws://127.0.0.1:{self.port}/wakeword-ws"
|
||||
|
||||
def serve_forever(self) -> None:
|
||||
self._server.serve_forever()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._server.shutdown()
|
||||
self.event_bridge.close()
|
||||
self._server.server_close()
|
||||
|
||||
def set_restart_handler(self, handler: Callable[[], None]) -> None:
|
||||
self._restart_handler = handler
|
||||
|
||||
def request_runtime_restart(self) -> None:
|
||||
with self._restart_lock:
|
||||
handler = self._restart_handler
|
||||
|
||||
if handler is None:
|
||||
raise RuntimeError("restart handler is not configured")
|
||||
|
||||
threading.Thread(
|
||||
target=self._run_restart_handler,
|
||||
name="test-runtime-restart",
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
def _run_restart_handler(self) -> None:
|
||||
handler = self._restart_handler
|
||||
if handler is None:
|
||||
return
|
||||
handler()
|
||||
|
||||
def _build_server(self) -> ThreadingHTTPServer:
|
||||
test_root = self.test_root
|
||||
event_bridge = self.event_bridge
|
||||
schedule_restart = self.request_runtime_restart
|
||||
|
||||
class TestRuntimeHandler(SimpleHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, directory=str(test_root), **kwargs)
|
||||
|
||||
def handle(self) -> None:
|
||||
try:
|
||||
super().handle()
|
||||
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
|
||||
pass
|
||||
|
||||
def do_GET(self) -> None:
|
||||
if self.path == "/wakeword-ws":
|
||||
self._handle_websocket(event_bridge)
|
||||
return
|
||||
|
||||
if self.path == "/health":
|
||||
body = json.dumps({"status": "ok"}).encode("utf-8")
|
||||
self.send_response(HTTPStatus.OK)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Cache-Control", "no-cache")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
return
|
||||
|
||||
super().do_GET()
|
||||
|
||||
def log_message(self, format: str, *args) -> None:
|
||||
return
|
||||
|
||||
def _handle_websocket(self, bridge: WakewordEventBridge) -> None:
|
||||
if self.headers.get("Upgrade", "").lower() != "websocket":
|
||||
self.send_error(HTTPStatus.BAD_REQUEST, "expected websocket upgrade")
|
||||
return
|
||||
|
||||
websocket_key = self.headers.get("Sec-WebSocket-Key")
|
||||
if not websocket_key:
|
||||
self.send_error(HTTPStatus.BAD_REQUEST, "missing Sec-WebSocket-Key")
|
||||
return
|
||||
|
||||
accept_source = websocket_key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
|
||||
accept_value = base64.b64encode(
|
||||
hashlib.sha1(accept_source.encode("utf-8")).digest()
|
||||
).decode("ascii")
|
||||
|
||||
client_queue = bridge.add_client()
|
||||
self.send_response(HTTPStatus.SWITCHING_PROTOCOLS)
|
||||
self.send_header("Upgrade", "websocket")
|
||||
self.send_header("Connection", "Upgrade")
|
||||
self.send_header("Sec-WebSocket-Accept", accept_value)
|
||||
self.end_headers()
|
||||
|
||||
try:
|
||||
self.connection.settimeout(0.2)
|
||||
self._send_websocket_text(bridge.build_ready_message())
|
||||
self._send_websocket_text(self._build_wakeword_config_message(bridge))
|
||||
|
||||
while bridge.is_running:
|
||||
inbound_message = self._receive_websocket_message()
|
||||
if inbound_message is not None:
|
||||
response_message = self._handle_bridge_request(bridge, inbound_message)
|
||||
if response_message:
|
||||
self._send_websocket_text(response_message)
|
||||
|
||||
try:
|
||||
message = client_queue.get(timeout=0.2)
|
||||
if message == "__bridge_closed__":
|
||||
break
|
||||
self._send_websocket_text(message)
|
||||
except queue.Empty:
|
||||
if not bridge.is_running:
|
||||
break
|
||||
continue
|
||||
except socket.timeout:
|
||||
pass
|
||||
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
|
||||
pass
|
||||
finally:
|
||||
bridge.remove_client(client_queue)
|
||||
|
||||
def _build_wakeword_config_message(self, bridge: WakewordEventBridge) -> str:
|
||||
try:
|
||||
runtime_root = test_root / "wakeword_runtime"
|
||||
config = load_config(runtime_root)
|
||||
payload = {
|
||||
"enabled": config.wakeword_enabled,
|
||||
"wakeWords": config.wake_words,
|
||||
}
|
||||
return bridge.build_message("wakeword_config", payload)
|
||||
except Exception as exc:
|
||||
return bridge.build_message(
|
||||
"wakeword_config",
|
||||
{},
|
||||
success=False,
|
||||
error=f"读取唤醒词配置失败: {exc}",
|
||||
)
|
||||
|
||||
def _handle_bridge_request(self, bridge: WakewordEventBridge, raw_message: str) -> str | None:
|
||||
try:
|
||||
message = json.loads(raw_message)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
message_type = str(message.get("type", "")).strip()
|
||||
request_id = message.get("requestId")
|
||||
payload = message.get("payload") or {}
|
||||
result_type = f"{message_type}_result" if message_type else "bridge_request_result"
|
||||
|
||||
if message_type == "set_wakeword_config":
|
||||
try:
|
||||
result_payload = self._save_wakeword_config(payload)
|
||||
bridge.publish("wakeword_config", result_payload)
|
||||
return bridge.build_message(
|
||||
"set_wakeword_config_result",
|
||||
result_payload,
|
||||
request_id=request_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
return bridge.build_message(
|
||||
"set_wakeword_config_result",
|
||||
{},
|
||||
request_id=request_id,
|
||||
success=False,
|
||||
error=f"保存唤醒词配置失败: {exc}",
|
||||
)
|
||||
|
||||
if message_type == "restart_wakeword_service":
|
||||
schedule_restart()
|
||||
return bridge.build_message(
|
||||
"restart_wakeword_service_result",
|
||||
{"restarting": True},
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
return bridge.build_message(
|
||||
result_type,
|
||||
{},
|
||||
request_id=request_id,
|
||||
success=False,
|
||||
error=f"unsupported message type: {message_type}",
|
||||
)
|
||||
|
||||
def _save_wakeword_config(self, payload: dict) -> dict:
|
||||
runtime_root = test_root / "wakeword_runtime"
|
||||
config_path = runtime_root / "config.json"
|
||||
model_root = runtime_root / "models"
|
||||
keywords_path = model_root / "keywords.txt"
|
||||
|
||||
enabled = bool(payload.get("enabled", True))
|
||||
wake_words = payload.get("wakeWords") or []
|
||||
normalized_wake_words = []
|
||||
for item in wake_words:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
text = item.strip()
|
||||
if text and text not in normalized_wake_words:
|
||||
normalized_wake_words.append(text)
|
||||
|
||||
if enabled and not normalized_wake_words:
|
||||
raise ValueError("wakeWords cannot be empty when wakeword is enabled")
|
||||
|
||||
raw_config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
raw_config.setdefault("wakeword", {})["enabled"] = enabled
|
||||
config_path.write_text(
|
||||
json.dumps(raw_config, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
keywords_lines = [self._build_keyword_line(item) for item in normalized_wake_words]
|
||||
keywords_path.write_text(
|
||||
("\n".join(keywords_lines) + "\n") if keywords_lines else "",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return {
|
||||
"enabled": enabled,
|
||||
"wakeWords": normalized_wake_words,
|
||||
}
|
||||
|
||||
def _build_keyword_line(self, keyword_text: str) -> str:
|
||||
from pypinyin import Style, pinyin
|
||||
|
||||
initials = pinyin(keyword_text, style=Style.INITIALS, strict=False)
|
||||
finals = pinyin(
|
||||
keyword_text,
|
||||
style=Style.FINALS_TONE,
|
||||
strict=False,
|
||||
neutral_tone_with_five=True,
|
||||
)
|
||||
|
||||
tokens: list[str] = []
|
||||
for initial_parts, final_parts in zip(initials, finals):
|
||||
initial = initial_parts[0].strip()
|
||||
final = final_parts[0].strip()
|
||||
if initial:
|
||||
tokens.append(initial)
|
||||
if final:
|
||||
tokens.append(final)
|
||||
|
||||
if not tokens:
|
||||
raise ValueError(f"failed to generate pinyin tokens for wake word: {keyword_text}")
|
||||
|
||||
return f"{' '.join(tokens)} @{keyword_text}"
|
||||
|
||||
def _receive_websocket_message(self) -> str | None:
|
||||
try:
|
||||
header = self._read_exact(2)
|
||||
except socket.timeout:
|
||||
return None
|
||||
|
||||
if not header:
|
||||
return None
|
||||
|
||||
first_byte, second_byte = header[0], header[1]
|
||||
opcode = first_byte & 0x0F
|
||||
masked = (second_byte & 0x80) != 0
|
||||
payload_length = second_byte & 0x7F
|
||||
|
||||
if payload_length == 126:
|
||||
payload_length = int.from_bytes(self._read_exact(2), "big")
|
||||
elif payload_length == 127:
|
||||
payload_length = int.from_bytes(self._read_exact(8), "big")
|
||||
|
||||
masking_key = self._read_exact(4) if masked else b""
|
||||
payload = self._read_exact(payload_length) if payload_length else b""
|
||||
|
||||
if masked and payload:
|
||||
payload = bytes(
|
||||
byte ^ masking_key[index % 4]
|
||||
for index, byte in enumerate(payload)
|
||||
)
|
||||
|
||||
if opcode == 0x8:
|
||||
raise ConnectionAbortedError("websocket closed by client")
|
||||
|
||||
if opcode == 0x9:
|
||||
self._send_websocket_frame(0xA, payload)
|
||||
return None
|
||||
|
||||
if opcode == 0xA:
|
||||
return None
|
||||
|
||||
if opcode != 0x1:
|
||||
return None
|
||||
|
||||
return payload.decode("utf-8")
|
||||
|
||||
def _read_exact(self, size: int) -> bytes:
|
||||
if size <= 0:
|
||||
return b""
|
||||
|
||||
chunks = bytearray()
|
||||
while len(chunks) < size:
|
||||
chunk = self.connection.recv(size - len(chunks))
|
||||
if not chunk:
|
||||
raise ConnectionResetError("websocket connection closed")
|
||||
chunks.extend(chunk)
|
||||
return bytes(chunks)
|
||||
|
||||
def _send_websocket_text(self, message: str) -> None:
|
||||
self._send_websocket_frame(0x1, message.encode("utf-8"))
|
||||
|
||||
def _send_websocket_frame(self, opcode: int, payload: bytes) -> None:
|
||||
header = bytearray()
|
||||
header.append(0x80 | opcode)
|
||||
|
||||
payload_length = len(payload)
|
||||
if payload_length < 126:
|
||||
header.append(payload_length)
|
||||
elif payload_length < 65536:
|
||||
header.append(126)
|
||||
header.extend(payload_length.to_bytes(2, "big"))
|
||||
else:
|
||||
header.append(127)
|
||||
header.extend(payload_length.to_bytes(8, "big"))
|
||||
|
||||
self.wfile.write(bytes(header) + payload)
|
||||
self.wfile.flush()
|
||||
|
||||
server = ThreadingHTTPServer((self.host, self.port), TestRuntimeHandler)
|
||||
server.daemon_threads = True
|
||||
return server
|
||||
Reference in New Issue
Block a user