diff --git a/.gitignore b/.gitignore index 698d638c..a25b3401 100644 --- a/.gitignore +++ b/.gitignore @@ -141,8 +141,8 @@ music/ # Cython debug symbols cython_debug/ *.iml -model.pt tmp +.history .DS_Store main/xiaozhi-server/data main/manager-web/node_modules @@ -151,3 +151,6 @@ main/manager-web/node_modules .private_config.yaml .env.development +# model files +main/xiaozhi-server/models/SenseVoiceSmall/model.pt +main/xiaozhi-server/models/sherpa-onnx* diff --git a/README.md b/README.md index 121dfbe6..efc45101 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,7 @@ server: | 类型 | 平台名称 | 使用方式 | 收费模式 | 备注 | |:---:|:---------:|:----:|:----:|:--:| | ASR | FunASR | 本地使用 | 免费 | | +| ASR | SherpaASR | 本地使用 | 免费 | | | ASR | DoubaoASR | 接口调用 | 收费 | | --- diff --git a/README_en.md b/README_en.md index 8c251a44..88306675 100644 --- a/README_en.md +++ b/README_en.md @@ -165,6 +165,7 @@ In fact, any LLM that supports OpenAI API calls can be integrated. | Type | Platform Name | Usage Method | Pricing Model | Remarks | |:----:|:-------------------:|:------------:|:-------------:|:-------:| | ASR | FunASR | Local | Free | | +| ASR | SherpaASR | Local | Free | | | ASR | DoubaoASR | API call | Paid | | --- diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 7fa8a24a..31210e9e 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -120,6 +120,10 @@ ASR: type: fun_local model_dir: models/SenseVoiceSmall output_dir: tmp/ + SherpaASR: + type: sherpa_onnx_local + model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17 + output_dir: tmp/ DoubaoASR: type: doubao appid: 你的火山引擎语音合成服务appid diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py new file mode 100644 index 00000000..3dfa3162 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -0,0 +1,164 @@ +import time +import wave +import os +import sys +import io +from config.logger import setup_logging +from typing import Optional, Tuple, List +import uuid +import opuslib_next +from core.providers.asr.base import ASRProviderBase + +import numpy as np +import sherpa_onnx + +from modelscope.hub.file_download import model_file_download + +TAG = __name__ +logger = setup_logging() + + +# 捕获标准输出 +class CaptureOutput: + def __enter__(self): + self._output = io.StringIO() + self._original_stdout = sys.stdout + sys.stdout = self._output + + def __exit__(self, exc_type, exc_value, traceback): + sys.stdout = self._original_stdout + self.output = self._output.getvalue() + self._output.close() + + # 将捕获到的内容通过 logger 输出 + if self.output: + logger.bind(tag=TAG).info(self.output.strip()) + + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool): + self.model_dir = config.get("model_dir") + self.output_dir = config.get("output_dir") + self.delete_audio_file = delete_audio_file + + # 确保输出目录存在 + os.makedirs(self.output_dir, exist_ok=True) + + # 初始化模型文件路径 + model_files = { + "model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"), + "tokens.txt": os.path.join(self.model_dir, "tokens.txt") + } + + # 下载并检查模型文件 + try: + for file_name, file_path in model_files.items(): + if not os.path.isfile(file_path): + logger.bind(tag=TAG).info(f"正在下载模型文件: {file_name}") + model_file_download( + model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue", + file_path=file_name, + local_dir=self.model_dir + ) + + if not os.path.isfile(file_path): + raise FileNotFoundError(f"模型文件下载失败: {file_path}") + + self.model_path = model_files["model.int8.onnx"] + self.tokens_path = model_files["tokens.txt"] + + except Exception as e: + logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}") + raise + + with CaptureOutput(): + self.model = sherpa_onnx.OfflineRecognizer.from_sense_voice( + model=self.model_path, + tokens=self.tokens_path, + num_threads=2, + sample_rate=16000, + feature_dim=80, + decoding_method="greedy_search", + debug=False, + use_itn=True, + ) + + def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: + """将Opus音频数据解码并保存为WAV文件""" + file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" + file_path = os.path.join(self.output_dir, file_name) + + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + with wave.open(file_path, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) # 2 bytes = 16-bit + wf.setframerate(16000) + wf.writeframes(b"".join(pcm_data)) + + return file_path + + def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]: + """ + Args: + wave_filename: + Path to a wave file. It should be single channel and each sample should + be 16-bit. Its sample rate does not need to be 16kHz. + Returns: + Return a tuple containing: + - A 1-D array of dtype np.float32 containing the samples, which are + normalized to the range [-1, 1]. + - sample rate of the wave file + """ + + with wave.open(wave_filename) as f: + assert f.getnchannels() == 1, f.getnchannels() + assert f.getsampwidth() == 2, f.getsampwidth() # it is in bytes + num_samples = f.getnframes() + samples = f.readframes(num_samples) + samples_int16 = np.frombuffer(samples, dtype=np.int16) + samples_float32 = samples_int16.astype(np.float32) + + samples_float32 = samples_float32 / 32768 + return samples_float32, f.getframerate() + + async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + """语音转文本主处理逻辑""" + file_path = None + try: + # 保存音频文件 + start_time = time.time() + file_path = self.save_audio_to_file(opus_data, session_id) + logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}") + + # 语音识别 + start_time = time.time() + s = self.model.create_stream() + samples, sample_rate = self.read_wave(file_path) + s.accept_waveform(sample_rate, samples) + self.model.decode_stream(s) + text = s.result.text + logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}") + + return text, file_path + + except Exception as e: + logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) + return "", None + + finally: + # 文件清理逻辑 + if self.delete_audio_file and file_path and os.path.exists(file_path): + try: + os.remove(file_path) + logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") + except Exception as e: + logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") diff --git a/main/xiaozhi-server/requirements.txt b/main/xiaozhi-server/requirements.txt index bcd6b4ee..58362e02 100755 --- a/main/xiaozhi-server/requirements.txt +++ b/main/xiaozhi-server/requirements.txt @@ -19,4 +19,6 @@ loguru==0.7.3 requests==2.32.3 cozepy==0.12.0 mem0ai==0.1.62 -bs4==0.0.2 \ No newline at end of file +bs4==0.0.2 +modelscope==1.23.2 +sherpa_onnx==1.11.0 \ No newline at end of file