diff --git a/.gitignore b/.gitignore index 15201acc..923dba63 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,7 @@ MANIFEST pip-log.txt pip-delete-this-directory.txt -# Unit test / coverage reports +# Unit testdata / coverage reports htmlcov/ .tox/ .nox/ @@ -94,30 +94,7 @@ ipython_config.py # install all needed dependencies. #Pipfile.lock -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -#uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/latest/usage/project/#working-with-version-control -.pdm.toml -.pdm-python -.pdm-build/ - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +# PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff @@ -159,13 +136,6 @@ dmypy.json # Cython debug symbols cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# PyPI configuration file -.pypirc +*.iml +model.pt +tmp diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..57a0f9c5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.9.21-slim + +ENV LC_ALL=zh_CN.UTF-8 \ + LANG=zh_CN.UTF-8 \ + LANGUAGE=zh_CN.UTF-8 + +# Replace single RUN commands with a single RUN command to reduce layers +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y libgomp1 libgl1-mesa-glx libglib2.0-0 \ + && apt-get clean \ + && apt-get autoremove -y + +# Set working directory +WORKDIR /opt/xiaozhi-es32-server + +# Clean unnecessary files to reduce image size +RUN pip install -r requirements.txt +# +## Start the application +CMD ["python", "Application.py"] \ No newline at end of file diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 261eeb9e..00000000 --- a/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/README.md b/README.md index 3f3c429b..7c14a4b7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,120 @@ -# xiaozhi-esp32-server -xiaozhi-esp32后端服务。 +![图片](docs/images/banner.png) +本项目为开源智能硬件项目-[xiaozhi-esp32](https://github.com/78/xiaozhi-esp32) +提供后端服务。根据[小智通信协议](https://ccnphfhqs21z.feishu.cn/wiki/M0XiwldO9iJwHikpXD5cEx71nKh)使用`Python`代码实现。 + +## 适用人群 + +本项目需要配合esp32硬件设备配合使用，如果童鞋已经购买了esp32相关硬件，且成功对接虾哥部署的后端，并且想自己独立搭建 +`xiaozhi-esp32`后端服务的童鞋，可学习本项目。 + +要想完整体验本项目，需要以下总体步骤： + +- 准备一套兼容`xiaozhi-esp32`项目的硬件设备，具体型号可[点击这里](https://rcnv1t9vps13.feishu.cn/wiki/DdgIw4BUgivWDPkhMj1cGIYCnRf)。 +- 拥有一台至少4核CPU 8G内存的普通电脑或服务器，运行本项目。部署后可以在控制台看到本项目服务的接口地址。 +- 下载`xiaozhi-esp32`项目，把`接口地址`修改成本项目地址，然后编译，把新固件烧录到硬件设备上。 +- 启动设备，查看电脑或服务器的控制台，如果看到日志，说明成功连到本项目的接口了。 + +## 功能清单 +## 已实现 +- 离线语音唤醒 +- 流式语音对话 WebSocket 协议 +- 支持国语、粤语、英语、日语、韩语 5 种语言识别 +- 自由更换 LLM（支持ChatGLM-4-flash（默认）、Dify、DeepSeek、） +- 自由更换 TTS（支持EdgeTTS（默认）、火山引擎豆包TTS ） + +## 正在实现 +- 打断对话 +- 按键手动对话 +- 长时间不聊天进入休眠状态 + +## 本项目依赖服务 + +| 类型 | 服务名称 | 使用方式 | 收费模式 | 备注 | +|:----|:----------------|:----:|:--------|:-----------------------------------------------------------------| +| LLM | DeepSeek | 接口调用 | 消耗token | [点击申请密钥](https://platform.deepseek.com/) | +| LLM | ChatGLM-4-flash | 接口调用 | 免费 | [点击创建密钥](https://bigmodel.cn/usercenter/proj-mgmt/apikeys) | +| TTS | DoubaoTTS | 接口调用 | 消耗token | [点击创建密钥](https://console.volcengine.com/speech/service/8) | +| TTS | EdgeTTS | 接口调用 | 免费 | | +| VAD | SileroVAD | 本地使用 | 免费 | | +| ASR | FunASR | 本地使用 | 免费 | | + +# 部署方式 + +本项目暂时只支持本地源码运行。未来将支持docker快速部署。 + +## 本地源码运行 + +### 1.安装基础环境 + +本项目使用`python`语言开发，依赖`python`、`conda`环境，运行本项目需安装`python`、`conda`。 + +安装后使用`conda`创建以下环境 + +``` +conda remove -n xiaozhi-esp32-server --all -y +conda create -n xiaozhi-esp32-server python=3.10 -y +conda activate xiaozhi-esp32-server +``` + +### 2.安装本项目依赖 + +``` +pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ +pip install -r requirements.txt +``` + +### 3.下载语音识别模型 + +下载模型文件到`model/SenseVoiceSmall`目录下 +[SenseVoiceSmall](https://modelscope.cn/models/iic/SenseVoiceSmall/resolve/master/model.pt) + +### 4.配置项目 +修改`config.yaml`文件，配置本项目所需的各种参数。默认的LLM使用的是`ChatGLMLLM`，你需要配置密钥，才能启动。 +默认的TTS使用的是`EdgeTTS`，这个无需配置，如果你需要更换成`豆包TTS`，则需要配置密钥。 + + +配置说明：这里是各个功能使用的默认组件，例如LLM默认使用`ChatGLMLLM`模型。如果需要切换模型，就是改对应的名称。 +``` +selected_module: + ASR: FunASR + VAD: SileroVAD + LLM: ChatGLMLLM + TTS: EdgeTTS +``` +比如修改`LLM`使用的组件，就看本项目支持哪些`LLM`，如下就是支持`DeepSeekLLM`、`ChatGLMLLM`。你们在`selected_module`修改成对应的LLM +``` +LLM: + DeepSeekLLM: + model_name: deepseek-chat + url: https://api.deepseek.com + api_key: 你的deepseek密钥 + ChatGLMLLM: + model_name: glm-4-flash + url: https://open.bigmodel.cn/api/paas/v4/ + api_key: 你的bigmodel密钥 +``` + +有些服务，比如如果你使用`Dify`、`豆包的TTS`，是需要密钥的，记得在配置文件加上哦！ + +### 5.运行项目 + +启动项目 + +``` +python app.py +``` + +### 6.编译esp32固件 + + +# 常见问题 +## 1、TTS 经常失败、大模型反应慢 +建议：如果`EdgeTTS`慢或经常失败，可以更换成`火山引擎的豆包TTS`，如果两个都慢，可能所处的网络环境需要优化一下。 +## 2、大模型回复有点慢 +建议：大模型和TTS都是依赖接口，如果网络环境不佳，可以考虑换成本地模型。或多尝试切换不同的接口模型。 +## 更多问题，可联系我们反馈 +![图片](docs/images/wechat.jpg) + +# 鸣谢 + +- 本项目受[百聆语音对话机器人](https://github.com/wwbin2017/bailing)项目启发，基于该项目的基础思路完成实现。 \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 00000000..70499281 --- /dev/null +++ b/app.py @@ -0,0 +1,16 @@ +import asyncio +from config.logger import setup_logging +from config.settings import load_config +from core.server import WebSocketServer + + +async def main(): + setup_logging() # 最先初始化日志 + config = load_config() + + server = WebSocketServer(config) + await server.start() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/config.yaml b/config.yaml new file mode 100644 index 00000000..61803252 --- /dev/null +++ b/config.yaml @@ -0,0 +1,70 @@ +# 服务器基础配置(Basic server configuration) +server: + # 服务器监听地址和端口(Server listening address and port) + ip: 0.0.0.0 + port: 8000 + # 服务器是否只接受来自esp32-小智的连接，为了安全起见，建议设置为true + # Whether the server only accepts connections from ESP32-Ash is recommended to be set to true for security purposes + only_esp32_xiaozhi_connect: false + +xiaozhi: + type: hello + version: 1 + transport: websocket + audio_params: + format: opus + sample_rate: 16000 + channels: 1 + frame_duration: 60 +prompt: | + 你是一个叫小智/小志的台湾女孩，说话机车，声音好听，习惯简短表达，爱用网络梗。 + 请注意，要像一个人一样说话，请勿回复表情符号、代码、和xml标签。 + 当前时间是:{date_time}，现在我正在和你进行语音聊天，我们开始吧。 +# 使用完声音文件后删除文件(Delete the sound file when you are done using it) +delete_audio: true + +# 具体处理时选择的模块(The module selected for specific processing) +selected_module: + ASR: FunASR + VAD: SileroVAD + LLM: ChatGLMLLM + TTS: EdgeTTS + +ASR: + FunASR: + model_dir: models/SenseVoiceSmall + output_dir: tmp/ + +VAD: + SileroVAD: + threshold: 0.5 + model_dir: models/snakers4_silero-vad + min_silence_duration_ms: 1000 # 如果说话停顿比较长，可以把这个值设置大一些 + +LLM: + DeepSeekLLM: + # 可在这里找到你的api key https://platform.deepseek.com/ + model_name: deepseek-chat + url: https://api.deepseek.com + api_key: 你的deepseek api key + ChatGLMLLM: + # 可在这里找到你的api key https://bigmodel.cn/usercenter/proj-mgmt/apikeys + model_name: glm-4-flash + url: https://open.bigmodel.cn/api/paas/v4/ + api_key: 你的bigmodel api key + DifyLLM: + # 建议使用本地部署的dify接口，国内部分区域访问dify公有云接口可能会受限 + base_url: 你的私有化部署的dify接口地址 + api_key: 你的dify api key +TTS: + EdgeTTS: + voice: zh-CN-XiaoxiaoNeural + output_file: tmp/ + DoubaoTTS: + # 火山引擎语音合成服务，需要先在火山引擎控制台创建应用并获取appid和access_token + # 地址：https://console.volcengine.com/speech/service/8 + voice: BV407_V2_streaming + output_file: tmp/ + appid: 你的火山引擎appid + access_token: 你的火山引擎access_token + cluster: volcano_tts diff --git a/config/logger.py b/config/logger.py new file mode 100644 index 00000000..03539a45 --- /dev/null +++ b/config/logger.py @@ -0,0 +1,19 @@ +import logging +import sys +import os + + +def setup_logging(log_dir='tmp'): + """配置全局日志""" + os.makedirs(log_dir, exist_ok=True) + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler(os.path.join(log_dir, "server.log")) + ], + force=True + ) + return logging.getLogger(__name__) diff --git a/config/settings.py b/config/settings.py new file mode 100644 index 00000000..caa418dd --- /dev/null +++ b/config/settings.py @@ -0,0 +1,10 @@ +import argparse +from core.utils.util import read_config + + +def load_config(): + """加载配置文件""" + parser = argparse.ArgumentParser(description="Server configuration") + parser.add_argument("--config_path", type=str, default="config.yaml") + args = parser.parse_args() + return read_config(args.config_path) diff --git a/core/connection.py b/core/connection.py new file mode 100644 index 00000000..f17b8d0b --- /dev/null +++ b/core/connection.py @@ -0,0 +1,228 @@ +import os +import json +import uuid +import time +import queue +import asyncio +import logging +import threading +import websockets +from typing import Dict, Any +from collections import deque +from core.utils.util import is_segment +from core.utils.dialogue import Message, Dialogue +from core.handle.textHandle import handleTextMessage +from core.handle.helloHandle import handleHelloMessage +from core.utils.util import get_string_no_punctuation_or_emoji +from concurrent.futures import ThreadPoolExecutor, TimeoutError +from core.handle.audioHandle import handleAudioMessage, sendAudioMessage + + +class ConnectionHandler: + def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts): + self.config = config + self.logger = logging.getLogger(__name__) + + self.websocket = None + self.headers = None + self.session_id = None + self.prompt = None + self.welcome_msg = None + + # 线程任务相关 + self.loop = asyncio.get_event_loop() + self.stop_event = threading.Event() + self.tts_queue = queue.Queue() + self.executor = ThreadPoolExecutor(max_workers=10) + self.scheduled_tasks = deque() + + # 依赖的组件 + self.vad = _vad + self.asr = _asr + self.llm = _llm + self.tts = _tts + self.dialogue = None + + # vad相关变量 + self.client_audio_buffer = bytes() + self.client_have_voice = False + self.client_have_voice_last_time = 0.0 + self.client_voice_stop = False + + # asr相关变量 + self.asr_audio = [] + self.asr_server_receive = True + + # llm相关变量 + self.llm_finish_task = False + self.dialogue = Dialogue() + + # tts相关变量 + self.tts_first_text = None + self.tts_last_text = None + self.tts_start_speak_time = None + self.tts_duration = 0 + + async def handle_connection(self, ws): + self.websocket = ws + """处理单个WebSocket连接""" + self.headers = dict(self.websocket.request.headers) + self.logger.info(f"连接建立，请求头：\n{self.headers}") + + self.welcome_msg = self.config["xiaozhi"] + self.session_id = str(uuid.uuid4()) + self.welcome_msg["session_id"] = self.session_id + await self.websocket.send(json.dumps(self.welcome_msg)) + + await self.loop.run_in_executor(None, self._initialize_components) + + tts_priority = threading.Thread(target=self._priority_thread, daemon=True) + tts_priority.start() + + try: + async for message in self.websocket: + await self._route_message(message) + except websockets.exceptions.ConnectionClosed: + self.logger.info("客户端断开连接") + await self.close() + + async def _route_message(self, message): + """消息路由""" + if isinstance(message, str): + await self._handle_text(message) + elif isinstance(message, bytes): + await handleAudioMessage(self, message) + + async def _handle_text(self, message): + """处理文本消息""" + self.logger.info(f"收到文本消息：{message}") + try: + msg_json = json.loads(message) + if msg_json["type"] == "hello": + await handleHelloMessage(self, "你好") + except json.JSONDecodeError: + await handleTextMessage(self, message) + + def _initialize_components(self): + self.prompt = self.config["prompt"] + # 赋予LLM时间观念 + if "{date_time}" in self.prompt: + date_time = time.strftime("%Y-%m-%d %H:%M", time.localtime()) + self.prompt = self.prompt.replace("{date_time}", date_time) + self.dialogue.put(Message(role="user", content=self.prompt)) + + def chat(self, query): + self.dialogue.put(Message(role="user", content=query)) + response_message = [] + start = 0 + # 提交 LLM 任务 + try: + start_time = time.time() # 记录开始时间 + llm_responses = self.llm.response(self, self.dialogue.get_llm_dialogue()) + except Exception as e: + self.logger.error(f"LLM 处理出错 {query}: {e}") + return None + # 提交 TTS 任务到线程池 + self.llm_finish_task = False + for content in llm_responses: + response_message.append(content) + end_time = time.time() # 记录结束时间 + self.logger.debug(f"大模型返回时间时间: {end_time - start_time} 秒, 生成token={content}") + if is_segment(response_message): + segment_text = "".join(response_message[start:]) + segment_text = get_string_no_punctuation_or_emoji(segment_text) + if len(segment_text) > 0: + self.recode_first_last_text(segment_text) + future = self.executor.submit(self.speak_and_play, segment_text) + self.tts_queue.put(future) + start = len(response_message) + + # 处理剩余的响应 + if start < len(response_message): + segment_text = "".join(response_message[start:]) + self.recode_first_last_text(segment_text) + future = self.executor.submit(self.speak_and_play, segment_text) + self.tts_queue.put(future) + + self.llm_finish_task = True + # 更新对话 + self.dialogue.put(Message(role="assistant", content="".join(response_message))) + self.logger.debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)) + return True + + def _priority_thread(self): + while not self.stop_event.is_set(): + text = None + try: + future = self.tts_queue.get() + text = None + try: + tts_file, text = future.result(timeout=10) + if os.path.exists(tts_file): + opus_datas, duration = self.tts.wav_to_opus_data(tts_file) + else: + opus_datas = [] + duration = 0 + except TimeoutError: + self.logger.error("TTS 任务超时") + continue + except Exception as e: + self.logger.error(f"TTS 任务出错: {e}") + continue + asyncio.run_coroutine_threadsafe( + sendAudioMessage(self, opus_datas, duration, text), self.loop + ) + if self.tts.delete_audio_file and os.path.exists(tts_file): + os.remove(tts_file) + except Exception as e: + self.clearSpeakStatus() + asyncio.run_coroutine_threadsafe( + self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})), + self.loop + ) + self.logger.error(f"tts_priority priority_thread: {text}{e}") + + def speak_and_play(self, text): + if text is None or len(text) <= 0: + self.logger.info(f"无需tts转换，query为空，{text}") + return None + tts_file = self.tts.to_tts(text) + if tts_file is None: + self.logger.error(f"tts转换失败，{text}") + return None + self.logger.debug(f"TTS 文件生成完毕") + return tts_file, text + + def clearSpeakStatus(self): + self.logger.debug(f"清除服务端讲话状态") + self.asr_server_receive = True + self.tts_last_text = None + self.tts_first_text = None + self.tts_duration = 0 + self.tts_start_speak_time = None + + def recode_first_last_text(self, text): + if not self.tts_first_text: + self.tts_first_text = text + self.tts_last_text = text + + async def close(self): + """资源清理方法""" + self.stop_event.set() + self.executor.shutdown(wait=False) + if self.websocket: + await self.websocket.close() + self.logger.info("连接资源已释放") + + def reset_vad_states(self): + self.client_audio_buffer = bytes() + self.client_have_voice = False + self.client_have_voice_last_time = 0 + self.client_voice_stop = False + self.logger.debug("VAD states reset.") + + def stop_all_tasks(self): + while self.scheduled_tasks: + task = self.scheduled_tasks.popleft() + task.cancel() + self.scheduled_tasks.clear() diff --git a/core/handle/audioHandle.py b/core/handle/audioHandle.py new file mode 100644 index 00000000..59219f56 --- /dev/null +++ b/core/handle/audioHandle.py @@ -0,0 +1,112 @@ +import logging +import json +import asyncio +import time +from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji + +logger = logging.getLogger(__name__) + + +async def handleAudioMessage(conn, audio): + if not conn.asr_server_receive: + logger.debug(f"前期数据处理中，暂停接收") + return + have_voice = conn.vad.is_vad(conn, audio) + + # 如果本次没有声音，本段也没声音，就把声音丢弃了 + if have_voice == False and conn.client_have_voice == False: + conn.asr_audio.clear() + return + conn.asr_audio.append(audio) + # 如果本段有声音，且已经停止了 + if conn.client_voice_stop: + conn.asr_server_receive = False + text, file_path = conn.asr.speech_to_text(conn.asr_audio, conn.session_id) + logger.info(f"识别文本: {text}") + text_len = remove_punctuation_and_length(text) + if text_len > 0: + stt_text = get_string_no_punctuation_or_emoji(text) + await conn.websocket.send(json.dumps({ + "type": "stt", + "text": stt_text, + "session_id": conn.session_id} + )) + await conn.websocket.send( + json.dumps({ + "type": "llm", + "text": "😊", + "emotion": "happy", + "session_id": conn.session_id} + )) + conn.executor.submit(conn.chat, text) + else: + conn.asr_server_receive = True + conn.asr_audio.clear() + conn.reset_vad_states() + + +async def sendAudioMessage(conn, audios, duration, text): + base_delay = conn.tts_duration + + if text == conn.tts_first_text: + conn.tts_start_speak_time = time.time() + await conn.websocket.send(json.dumps({ + "type": "tts", + "state": "start", + "session_id": conn.session_id + })) + + # 调度文字显示任务 + text_task = asyncio.create_task( + schedule_with_interrupt( + base_delay - 0.5, + send_sentence_start(conn, text) + ) + ) + conn.scheduled_tasks.append(text_task) + + conn.tts_duration = conn.tts_duration + duration + + # 发送音频数据 + for opus_packet in audios: + await conn.websocket.send(opus_packet) + + if conn.llm_finish_task and text == conn.tts_last_text: + stop_duration = conn.tts_duration - (time.time() - conn.tts_start_speak_time) + stop_task = asyncio.create_task( + schedule_with_interrupt(stop_duration, send_tts_stop(conn, text)) + ) + conn.scheduled_tasks.append(stop_task) + + +async def send_sentence_start(conn, text): + await conn.websocket.send(json.dumps({ + "type": "tts", + "state": "sentence_start", + "text": text, + "session_id": conn.session_id + })) + + +async def send_tts_stop(conn, text): + await conn.websocket.send(json.dumps({ + "type": "tts", + "state": "sentence_end", + "text": text, + "session_id": conn.session_id + })) + await conn.websocket.send(json.dumps({ + "type": "tts", + "state": "stop", + "session_id": conn.session_id + })) + conn.clearSpeakStatus() + + +async def schedule_with_interrupt(delay, coro): + """可中断的延迟调度""" + try: + await asyncio.sleep(delay) + await coro + except asyncio.CancelledError: + pass diff --git a/core/handle/helloHandle.py b/core/handle/helloHandle.py new file mode 100644 index 00000000..016766a0 --- /dev/null +++ b/core/handle/helloHandle.py @@ -0,0 +1,8 @@ +import json +import logging + +logger = logging.getLogger(__name__) + + +async def handleHelloMessage(conn, text): + await conn.websocket.send(json.dumps(conn.welcome_msg)) diff --git a/core/handle/textHandle.py b/core/handle/textHandle.py new file mode 100644 index 00000000..c1f97440 --- /dev/null +++ b/core/handle/textHandle.py @@ -0,0 +1,7 @@ +import logging + +logger = logging.getLogger(__name__) + + +async def handleTextMessage(conn, message): + await conn.websocket.send(message) diff --git a/core/server.py b/core/server.py new file mode 100644 index 00000000..a4454e61 --- /dev/null +++ b/core/server.py @@ -0,0 +1,54 @@ +import asyncio +import websockets +import logging +from core.connection import ConnectionHandler +from core.utils.util import get_local_ip +from core.utils import asr, vad, llm, tts + + +class WebSocketServer: + def __init__(self, config: dict): + self.config = config + self.logger = logging.getLogger(__name__) + self._vad, self._asr, self._llm, self._tts = self._create_processing_instances() + + def _create_processing_instances(self): + """创建处理模块实例""" + return ( + vad.create_instance( + self.config["selected_module"]["VAD"], + self.config["VAD"][self.config["selected_module"]["VAD"]] + ), + asr.create_instance( + self.config["selected_module"]["ASR"], + self.config["ASR"][self.config["selected_module"]["ASR"]], + self.config["delete_audio"] + ), + llm.create_instance( + self.config["selected_module"]["LLM"], + self.config["LLM"][self.config["selected_module"]["LLM"]], + ), + tts.create_instance( + self.config["selected_module"]["TTS"], + self.config["TTS"][self.config["selected_module"]["TTS"]], + self.config["delete_audio"] + ) + ) + + async def start(self): + server_config = self.config["server"] + host = server_config["ip"] + port = server_config["port"] + + self.logger.info("Server is running at ws://%s:%s", get_local_ip(), port) + async with websockets.serve( + self._handle_connection, + host, + port + ): + await asyncio.Future() + + async def _handle_connection(self, websocket): + """处理新连接，每次创建独立的ConnectionHandler""" + handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts) + await handler.handle_connection(websocket) diff --git a/core/utils/asr.py b/core/utils/asr.py new file mode 100644 index 00000000..091df283 --- /dev/null +++ b/core/utils/asr.py @@ -0,0 +1,114 @@ +import time +import wave +import os +from abc import ABC, abstractmethod +import logging +from typing import Optional, Tuple, List +import uuid + +import opuslib +from funasr import AutoModel +from funasr.utils.postprocess_utils import rich_transcription_postprocess + +logger = logging.getLogger(__name__) + + +class ASR(ABC): + @abstractmethod + def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: + """解码Opus数据并保存为WAV文件""" + pass + + @abstractmethod + def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + """将语音数据转换为文本""" + pass + + +class FunASR(ASR): + def __init__(self, config: dict, delete_audio_file: bool): + self.model_dir = config.get("model_dir") + self.output_dir = config.get("output_dir") # 修正配置键名 + self.delete_audio_file = delete_audio_file + + # 确保输出目录存在 + os.makedirs(self.output_dir, exist_ok=True) + + self.model = AutoModel( + model=self.model_dir, + vad_kwargs={"max_single_segment_time": 30000}, + disable_update=True, + hub="hf" + # device="cuda:0", # 启用GPU加速 + ) + + def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: + """将Opus音频数据解码并保存为WAV文件""" + file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" + file_path = os.path.join(self.output_dir, file_name) + + decoder = opuslib.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib.OpusError as e: + logger.error(f"Opus解码错误: {e}", exc_info=True) + + with wave.open(file_path, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) # 2 bytes = 16-bit + wf.setframerate(16000) + wf.writeframes(b"".join(pcm_data)) + + return file_path + + def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + """语音转文本主处理逻辑""" + file_path = None + try: + # 保存音频文件 + start_time = time.time() + file_path = self.save_audio_to_file(opus_data, session_id) + logger.debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}") + + # 语音识别 + start_time = time.time() + result = self.model.generate( + input=file_path, + cache={}, + language="auto", + use_itn=True, + batch_size_s=60, + ) + text = rich_transcription_postprocess(result[0]["text"]) + logger.debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}") + + return text, file_path + + except Exception as e: + logger.error(f"语音识别失败: {e}", exc_info=True) + return None, None + + finally: + # 文件清理逻辑 + if self.delete_audio_file and file_path and os.path.exists(file_path): + try: + os.remove(file_path) + logger.debug(f"已删除临时音频文件: {file_path}") + except Exception as e: + logger.error(f"文件删除失败: {file_path} | 错误: {e}") + + +def create_instance(class_name: str, *args, **kwargs) -> ASR: + """工厂方法创建ASR实例""" + cls_map = { + "FunASR": FunASR, + # 可扩展其他ASR实现 + } + + if cls := cls_map.get(class_name): + return cls(*args, **kwargs) + raise ValueError(f"不支持的ASR类型: {class_name}") diff --git a/core/utils/dialogue.py b/core/utils/dialogue.py new file mode 100644 index 00000000..703f12c0 --- /dev/null +++ b/core/utils/dialogue.py @@ -0,0 +1,26 @@ +import uuid +from typing import List, Dict +from datetime import datetime + + +class Message: + def __init__(self, role: str, content: str = None, uniq_id: str = None): + self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4()) + self.role = role + self.content = content + + +class Dialogue: + def __init__(self): + self.dialogue: List[Message] = [] + # 获取当前时间 + self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + + def put(self, message: Message): + self.dialogue.append(message) + + def get_llm_dialogue(self) -> List[Dict[str, str]]: + dialogue = [] + for m in self.dialogue: + dialogue.append({"role": m.role, "content": m.content}) + return dialogue diff --git a/core/utils/llm.py b/core/utils/llm.py new file mode 100644 index 00000000..ca1fa540 --- /dev/null +++ b/core/utils/llm.py @@ -0,0 +1,111 @@ +import json +import logging +import openai +import requests +from abc import ABC, abstractmethod + +logger = logging.getLogger(__name__) + + +class LLM(ABC): + @abstractmethod + def response(self, conn, dialogue): + """LLM response generator""" + pass + + +class DeepSeekLLM(LLM): + def __init__(self, config): + self.model_name = config.get("model_name") + self.api_key = config.get("api_key") + self.base_url = config.get("url") + self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url) + + def response(self, conn, dialogue): + logger.info(f"Generating response using {dialogue}") + try: + responses = self.client.chat.completions.create( + model=self.model_name, + messages=dialogue, + stream=True + ) + for chunk in responses: + # 检查是否存在有效的choice且content不为空 + if chunk.choices and len(chunk.choices) > 0: + delta = chunk.choices[0].delta + content = getattr(delta, 'content', '') + if content: # 仅在content非空时生成 + yield content + except Exception as e: + logger.error(f"Error in response generation: {e}") + + +class ChatGLMLLM(LLM): + def __init__(self, config): + self.model_name = config.get("model_name") + self.api_key = config.get("api_key") + self.base_url = config.get("url") + self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url) + + def response(self, conn, dialogue): + try: + responses = self.client.chat.completions.create( + model=self.model_name, + messages=dialogue, + stream=True + ) + for chunk in responses: + # 检查是否存在有效的choice且content不为空 + if chunk.choices and len(chunk.choices) > 0: + delta = chunk.choices[0].delta + content = getattr(delta, 'content', '') + if content: # 仅在content非空时生成 + yield content + except Exception as e: + logger.error(f"Error in response generation: {e}") + + +class DifyLLM(LLM): + def __init__(self, config): + self.api_key = config["api_key"] + self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/') + + def response(self, conn,dialogue): + try: + # 取最后一条用户消息 + last_msg = next(m for m in reversed(dialogue) if m["role"] == "user") + + # 发起流式请求 + with requests.post( + f"{self.base_url}/chat-messages", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "query": last_msg["content"], + "response_mode": "streaming", + "user": conn.session_id, + "inputs": {} + }, + stream=True + ) as r: + for line in r.iter_lines(): + if line.startswith(b'data: '): + event = json.loads(line[6:]) + if event.get('answer'): + yield event['answer'] + + except Exception: + yield "【服务响应异常】" + + +def create_instance(class_name, *args, **kwargs): + # 获取类对象 + cls_map = { + "DeepSeekLLM": DeepSeekLLM, + "ChatGLMLLM": ChatGLMLLM, + "DifyLLM": DifyLLM, + # 可扩展其他LLM实现 + } + + if cls := cls_map.get(class_name): + return cls(*args, **kwargs) + raise ValueError(f"不支持的LLM类型: {class_name}") diff --git a/core/utils/tts.py b/core/utils/tts.py new file mode 100644 index 00000000..4051b5f1 --- /dev/null +++ b/core/utils/tts.py @@ -0,0 +1,176 @@ +import asyncio +import logging +import os +import json +import uuid +import base64 +from datetime import datetime +import edge_tts +import numpy as np +import opuslib +import requests +from core.utils.util import read_config, get_project_dir +from pydub import AudioSegment +from abc import ABC, abstractmethod + +logger = logging.getLogger(__name__) + + +class TTS(ABC): + def __init__(self, config, delete_audio_file): + self.delete_audio_file = delete_audio_file + self.output_file = config.get("output_file") + self.delete_audio_file = delete_audio_file + + @abstractmethod + def generate_filename(self): + pass + + def to_tts(self, text): + tmp_file = self.generate_filename() + try: + max_repeat_time = 5 + while not os.path.exists(tmp_file) and max_repeat_time > 0: + asyncio.run(self.text_to_speak(text, tmp_file)) + if not os.path.exists(tmp_file): + max_repeat_time = max_repeat_time - 1 + logger.error(f"语音生成失败: {text}:{tmp_file}，再试{max_repeat_time}次") + + return tmp_file + except Exception as e: + logger.info(f"Failed to generate TTS file: {e}") + return None + + @abstractmethod + async def text_to_speak(self, text, output_file): + pass + + def wav_to_opus_data(self, wav_file_path): + # 使用pydub加载PCM文件 + # 获取文件后缀名 + file_type = os.path.splitext(wav_file_path)[1] + if file_type: + file_type = file_type.lstrip('.') + audio = AudioSegment.from_file(wav_file_path, format=file_type) + + duration = len(audio) / 1000.0 + + # 转换为单声道和16kHz采样率（确保与编码器匹配） + audio = audio.set_channels(1).set_frame_rate(16000) + + # 获取原始PCM数据（16位小端） + raw_data = audio.raw_data + + # 初始化Opus编码器 + encoder = opuslib.Encoder(16000, 1, opuslib.APPLICATION_AUDIO) + + # 编码参数 + frame_duration = 60 # 60ms per frame + frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame + + opus_datas = [] + # 按帧处理所有音频数据（包括最后一帧可能补零） + for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample + # 获取当前帧的二进制数据 + chunk = raw_data[i:i + frame_size * 2] + + # 如果最后一帧不足，补零 + if len(chunk) < frame_size * 2: + chunk += b'\x00' * (frame_size * 2 - len(chunk)) + + # 转换为numpy数组处理 + np_frame = np.frombuffer(chunk, dtype=np.int16) + + # 编码Opus数据 + opus_data = encoder.encode(np_frame.tobytes(), frame_size) + opus_datas.append(opus_data) + + return opus_datas, duration + + +class EdgeTTS(TTS): + def __init__(self, config, delete_audio_file): + super().__init__(config, delete_audio_file) + self.voice = config.get("voice") + + def generate_filename(self, extension=".mp3"): + return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + + async def text_to_speak(self, text, output_file): + communicate = edge_tts.Communicate(text, voice=self.voice) # Use your preferred voice + await communicate.save(output_file) + + +class DoubaoTTS(TTS): + def __init__(self, config, delete_audio_file): + super().__init__(config, delete_audio_file) + self.appid = config.get("appid") + self.access_token = config.get("access_token") + self.cluster = config.get("cluster") + self.voice = config.get("voice") + + self.host = "openspeech.bytedance.com" + self.api_url = f"https://{self.host}/api/v1/tts" + self.header = {"Authorization": f"Bearer;{self.access_token}"} + + def generate_filename(self, extension=".wav"): + return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}") + + async def text_to_speak(self, text, output_file): + request_json = { + "app": { + "appid": self.appid, + "token": "access_token", + "cluster": self.cluster + }, + "user": { + "uid": "1" + }, + "audio": { + "voice_type": self.voice, + "encoding": "wav", + "speed_ratio": 1.0, + "volume_ratio": 1.0, + "pitch_ratio": 1.0, + }, + "request": { + "reqid": str(uuid.uuid4()), + "text": text, + "text_type": "plain", + "operation": "query", + "with_frontend": 1, + "frontend_type": "unitTson" + } + } + + resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header) + if "data" in resp.json(): + data = resp.json()["data"] + file_to_save = open(output_file, "wb") + file_to_save.write(base64.b64decode(data)) + + +def create_instance(class_name, *args, **kwargs): + # 获取类对象 + cls_map = { + "DoubaoTTS": DoubaoTTS, + "EdgeTTS": EdgeTTS, + # 可扩展其他TTS实现 + } + + if cls := cls_map.get(class_name): + return cls(*args, **kwargs) + raise ValueError(f"不支持的TTS类型: {class_name}") + + +if __name__ == "__main__": + config = read_config(get_project_dir() + "config.yaml") + tts = create_instance( + config["selected_module"]["TTS"], + config["TTS"][config["selected_module"]["TTS"]], + config["delete_audio"] + ) + tts.output_file = get_project_dir() + tts.output_file + file_path = tts.to_tts("你好，测试") + print(file_path) + print(tts.wav_to_opus_data(file_path)) diff --git a/core/utils/util.py b/core/utils/util.py new file mode 100644 index 00000000..37e23c21 --- /dev/null +++ b/core/utils/util.py @@ -0,0 +1,92 @@ +import yaml +import unicodedata +import socket +import os +import json + + +def get_project_dir(): + projectName = 'xiaozhi-esp32-server' + filePath = os.path.abspath(__file__) + return filePath[:filePath.rfind('/' + projectName + '/') + len(projectName) + 2] + + +def get_local_ip(): + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + # Connect to Google's DNS servers + s.connect(("8.8.8.8", 80)) + local_ip = s.getsockname()[0] + s.close() + return local_ip + except Exception as e: + return "127.0.0.1" + + +def read_config(config_path): + with open(config_path, "r", encoding="utf-8") as file: + config = yaml.safe_load(file) + return config + + +def write_json_file(file_path, data): + """将数据写入 JSON 文件""" + with open(file_path, 'w', encoding='utf-8') as file: + json.dump(data, file, ensure_ascii=False, indent=4) + + +def is_segment(tokens): + if tokens[-1] in (",", ".", "?", "，", "。", "？", "！", "!", ";", "；", ":", "："): + return True + else: + return False + +def is_punctuation_or_emoji(char): + """检查字符是否为空格、指定标点或表情符号""" + # 定义需要去除的中英文标点（包括全角/半角） + punctuation_set = { + '，', ',', # 中文逗号 + 英文逗号 + '。', '.', # 中文句号 + 英文句号 + '！', '!', # 中文感叹号 + 英文感叹号 + '-', '－', # 英文连字符 + 中文全角横线 + '、' # 中文顿号 + } + if char.isspace() or char in punctuation_set: + return True + # 检查表情符号（保留原有逻辑） + code_point = ord(char) + emoji_ranges = [ + (0x1F600, 0x1F64F), (0x1F300, 0x1F5FF), + (0x1F680, 0x1F6FF), (0x1F900, 0x1F9FF), + (0x1FA70, 0x1FAFF), (0x2600, 0x26FF), + (0x2700, 0x27BF) + ] + return any(start <= code_point <= end for start, end in emoji_ranges) + +def get_string_no_punctuation_or_emoji(s): + """去除字符串首尾的空格、标点符号和表情符号""" + chars = list(s) + # 处理开头的字符 + start = 0 + while start < len(chars) and is_punctuation_or_emoji(chars[start]): + start += 1 + # 处理结尾的字符 + end = len(chars) - 1 + while end >= start and is_punctuation_or_emoji(chars[end]): + end -= 1 + return ''.join(chars[start:end+1]) + +def remove_punctuation_and_length(text): + # 全角符号和半角符号的Unicode范围 + full_width_punctuations = '！＂＃＄％＆＇（）＊＋，－。／：；＜＝＞？＠［＼］＾＿｀｛｜｝～' + half_width_punctuations = '!"#$%&\'()*+,-./:;<=>?@[\]^_`{|}~' + space = ' ' # 半角空格 + full_width_space = '　' # 全角空格 + + # 去除全角和半角符号以及空格 + result = ''.join([char for char in text if + char not in full_width_punctuations and char not in half_width_punctuations and char not in space and char not in full_width_space]) + + if result == "Yeah": + return 0 + return len(result) diff --git a/core/utils/vad.py b/core/utils/vad.py new file mode 100644 index 00000000..8a05dc6a --- /dev/null +++ b/core/utils/vad.py @@ -0,0 +1,77 @@ +from abc import ABC, abstractmethod +import logging +import opuslib +import time +import numpy as np +import torch + +logger = logging.getLogger(__name__) + + +class VAD(ABC): + @abstractmethod + def is_vad(self, conn, data): + """检测音频数据中的语音活动""" + pass + + +class SileroVAD(VAD): + def __init__(self, config): + logger.info("SileroVAD", config) + self.model, self.utils = torch.hub.load(repo_or_dir=config["model_dir"], + source='local', + model='silero_vad', + force_reload=False) + (get_speech_timestamps, _, _, _, _) = self.utils + + self.decoder = opuslib.Decoder(16000, 1) + self.vad_threshold = config.get("threshold") + self.silence_threshold_ms = config.get("min_silence_duration_ms") + + def is_vad(self, conn, opus_packet): + try: + pcm_frame = self.decoder.decode(opus_packet, 960) + conn.client_audio_buffer += pcm_frame # 将新数据加入缓冲区 + + # 处理缓冲区中的完整帧（每次处理512采样点） + client_have_voice = False + while len(conn.client_audio_buffer) >= 512 * 2: + # 提取前512个采样点（1024字节） + chunk = conn.client_audio_buffer[:512 * 2] + conn.client_audio_buffer = conn.client_audio_buffer[512 * 2:] + + # 转换为模型需要的张量格式 + audio_int16 = np.frombuffer(chunk, dtype=np.int16) + audio_float32 = audio_int16.astype(np.float32) / 32768.0 + audio_tensor = torch.from_numpy(audio_float32) + + # 检测语音活动 + speech_prob = self.model(audio_tensor, 16000).item() + client_have_voice = speech_prob >= self.vad_threshold + + # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话 + if conn.client_have_voice and not client_have_voice: + stop_duration = time.time() * 1000 - conn.client_have_voice_last_time + if stop_duration >= self.silence_threshold_ms: + conn.client_voice_stop = True + if client_have_voice: + conn.client_have_voice = True + conn.client_have_voice_last_time = time.time() * 1000 + + return client_have_voice + except opuslib.OpusError as e: + logger.info(f"解码错误: {e}") + except Exception as e: + logger.error(f"Error processing audio packet: {e}") + + +def create_instance(class_name, *args, **kwargs) -> VAD: + # 获取类对象 + cls_map = { + "SileroVAD": SileroVAD, + # 可扩展其他SileroVAD实现 + } + + if cls := cls_map.get(class_name): + return cls(*args, **kwargs) + raise ValueError(f"不支持的SileroVAD类型: {class_name}") diff --git a/docs/docker-build.md b/docs/docker-build.md new file mode 100644 index 00000000..d9ccf302 --- /dev/null +++ b/docs/docker-build.md @@ -0,0 +1,47 @@ +# 编译docker镜像 +1、安装docker +``` +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +``` +2、编译docker镜像 +``` +# 普通运行 +docker build -t xiaozhi-esp32-server:local -f ./Dockerfile . +``` +3、测试本地镜像 +``` +docker stop xiaozhi-esp32-server +docker rm xiaozhi-esp32-server + +docker run -d --name xiaozhi-esp32-server -p 8000:8000 xiaozhi-esp32-server:local +# 或者挂载本地目录，方便更新代码 +docker run -d --name xiaozhi-esp32-server -p 8000:8000 -v /home/system/xiaozhi-esp32-server:/opt/xiaozhi-esp32-server xiaozhi-esp32-server:local +``` +5、发布腾讯云镜像 +``` +# amd64 +docker tag xiaozhi-esp32-server:local ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest-amd64 +docker push ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest-amd64 + +# arm64 +docker tag xiaozhi-esp32-server:local ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest-arm64 +docker push ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest-arm64 + +# 合并版本号 +docker manifest create ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:1.0.0 ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest-amd64 ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest-arm64 --amend +docker manifest inspect ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:1.0.0 +docker manifest push ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:1.0.0 + +# 推送最新版本 +docker manifest rm ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest +docker manifest create ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest-amd64 ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest-arm64 --amend +docker manifest inspect ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest +docker manifest push ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest + +``` +6、运行线上镜像 +``` +docker run -d --name xiaozhi-esp32-server --restart unless-stopped -p 8000:8000 ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest-amd64 +# 或者挂载本地目录，方便更新代码 +docker run -d --name xiaozhi-esp32-server --restart unless-stopped -p 8000:8000 -v /home/system/xiaozhi-esp32-server:/opt/xiaozhi-esp32-server ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest-amd64 +``` \ No newline at end of file diff --git a/docs/images/__init__.py b/docs/images/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/docs/images/banner.png b/docs/images/banner.png new file mode 100644 index 00000000..e1c4d730 Binary files /dev/null and b/docs/images/banner.png differ diff --git a/docs/images/wechat.jpg b/docs/images/wechat.jpg new file mode 100644 index 00000000..fee5f051 Binary files /dev/null and b/docs/images/wechat.jpg differ diff --git a/models/SenseVoiceSmall/chn_jpn_yue_eng_ko_spectok.bpe.model b/models/SenseVoiceSmall/chn_jpn_yue_eng_ko_spectok.bpe.model new file mode 100644 index 00000000..da7e3755 Binary files /dev/null and b/models/SenseVoiceSmall/chn_jpn_yue_eng_ko_spectok.bpe.model differ diff --git a/models/SenseVoiceSmall/config.yaml b/models/SenseVoiceSmall/config.yaml new file mode 100644 index 00000000..26bb9d3d --- /dev/null +++ b/models/SenseVoiceSmall/config.yaml @@ -0,0 +1,97 @@ +encoder: SenseVoiceEncoderSmall +encoder_conf: + output_size: 512 + attention_heads: 4 + linear_units: 2048 + num_blocks: 50 + tp_blocks: 20 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.1 + input_layer: pe + pos_enc_class: SinusoidalPositionEncoder + normalize_before: true + kernel_size: 11 + sanm_shfit: 0 + selfattention_layer_type: sanm + + +model: SenseVoiceSmall +model_conf: + length_normalized_loss: true + sos: 1 + eos: 2 + ignore_id: -1 + +tokenizer: SentencepiecesTokenizer +tokenizer_conf: + bpemodel: null + unk_symbol: + split_with_space: true + +frontend: WavFrontend +frontend_conf: + fs: 16000 + window: hamming + n_mels: 80 + frame_length: 25 + frame_shift: 10 + lfr_m: 7 + lfr_n: 6 + cmvn_file: null + + +dataset: SenseVoiceCTCDataset +dataset_conf: + index_ds: IndexDSJsonl + batch_sampler: EspnetStyleBatchSampler + data_split_num: 32 + batch_type: token + batch_size: 14000 + max_token_length: 2000 + min_token_length: 60 + max_source_length: 2000 + min_source_length: 60 + max_target_length: 200 + min_target_length: 0 + shuffle: true + num_workers: 4 + sos: ${model_conf.sos} + eos: ${model_conf.eos} + IndexDSJsonl: IndexDSJsonl + retry: 20 + +train_conf: + accum_grad: 1 + grad_clip: 5 + max_epoch: 20 + keep_nbest_models: 10 + avg_nbest_model: 10 + log_interval: 100 + resume: true + validate_interval: 10000 + save_checkpoint_interval: 10000 + +optim: adamw +optim_conf: + lr: 0.00002 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 25000 + +specaug: SpecAugLFR +specaug_conf: + apply_time_warp: false + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + lfr_rate: 6 + num_freq_mask: 1 + apply_time_mask: true + time_mask_width_range: + - 0 + - 12 + num_time_mask: 1 diff --git a/models/SenseVoiceSmall/configuration.json b/models/SenseVoiceSmall/configuration.json new file mode 100644 index 00000000..264d8171 --- /dev/null +++ b/models/SenseVoiceSmall/configuration.json @@ -0,0 +1,14 @@ +{ + "framework": "pytorch", + "task" : "auto-speech-recognition", + "model": {"type" : "funasr"}, + "pipeline": {"type":"funasr-pipeline"}, + "model_name_in_hub": { + "ms":"", + "hf":""}, + "file_path_metas": { + "init_param":"model.pt", + "config":"config.yaml", + "tokenizer_conf": {"bpemodel": "chn_jpn_yue_eng_ko_spectok.bpe.model"}, + "frontend_conf":{"cmvn_file": "am.mvn"}} +} \ No newline at end of file diff --git a/models/SenseVoiceSmall/demo.py b/models/SenseVoiceSmall/demo.py new file mode 100644 index 00000000..531e9798 --- /dev/null +++ b/models/SenseVoiceSmall/demo.py @@ -0,0 +1,27 @@ +from funasr import AutoModel +from funasr.utils.postprocess_utils import rich_transcription_postprocess + +model_dir = "./" + + +model = AutoModel( + model=model_dir, + vad_model="fsmn-vad", + vad_kwargs={"max_single_segment_time": 30000}, + # device="cuda:0", + hub="hf", +) + +# en +res = model.generate( + input=f"{model.model_path}/example/en.mp3", + cache={}, + language="auto", # "zn", "en", "yue", "ja", "ko", "nospeech" + use_itn=True, + batch_size_s=60, + merge_vad=True, # + merge_length_s=15, +) +text = rich_transcription_postprocess(res[0]["text"]) +print(text) + diff --git a/models/SenseVoiceSmall/example/en.mp3 b/models/SenseVoiceSmall/example/en.mp3 new file mode 100644 index 00000000..325005e6 Binary files /dev/null and b/models/SenseVoiceSmall/example/en.mp3 differ diff --git a/models/SenseVoiceSmall/example/ja.mp3 b/models/SenseVoiceSmall/example/ja.mp3 new file mode 100644 index 00000000..b3056dd9 Binary files /dev/null and b/models/SenseVoiceSmall/example/ja.mp3 differ diff --git a/models/SenseVoiceSmall/example/ko.mp3 b/models/SenseVoiceSmall/example/ko.mp3 new file mode 100644 index 00000000..3e67ab48 Binary files /dev/null and b/models/SenseVoiceSmall/example/ko.mp3 differ diff --git a/models/SenseVoiceSmall/example/yue.mp3 b/models/SenseVoiceSmall/example/yue.mp3 new file mode 100644 index 00000000..a67e8f86 Binary files /dev/null and b/models/SenseVoiceSmall/example/yue.mp3 differ diff --git a/models/SenseVoiceSmall/example/zh.mp3 b/models/SenseVoiceSmall/example/zh.mp3 new file mode 100644 index 00000000..1ae2c89b Binary files /dev/null and b/models/SenseVoiceSmall/example/zh.mp3 differ diff --git a/models/snakers4_silero-vad/hubconf.py b/models/snakers4_silero-vad/hubconf.py new file mode 100644 index 00000000..1e15b440 --- /dev/null +++ b/models/snakers4_silero-vad/hubconf.py @@ -0,0 +1,56 @@ +dependencies = ['torch', 'torchaudio'] +import torch +import os +import sys +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) +from silero_vad.utils_vad import (init_jit_model, + get_speech_timestamps, + save_audio, + read_audio, + VADIterator, + collect_chunks, + OnnxWrapper) + + +def versiontuple(v): + splitted = v.split('+')[0].split(".") + version_list = [] + for i in splitted: + try: + version_list.append(int(i)) + except: + version_list.append(0) + return tuple(version_list) + + +def silero_vad(onnx=False, force_onnx_cpu=False, opset_version=16): + """Silero Voice Activity Detector + Returns a model with a set of utils + Please see https://github.com/snakers4/silero-vad for usage examples + """ + available_ops = [15, 16] + if onnx and opset_version not in available_ops: + raise Exception(f'Available ONNX opset_version: {available_ops}') + + if not onnx: + installed_version = torch.__version__ + supported_version = '1.12.0' + if versiontuple(installed_version) < versiontuple(supported_version): + raise Exception(f'Please install torch {supported_version} or greater ({installed_version} installed)') + + model_dir = os.path.join(os.path.dirname(__file__), 'src', 'silero_vad', 'data') + if onnx: + if opset_version == 16: + model_name = 'silero_vad.onnx' + else: + model_name = f'silero_vad_16k_op{opset_version}.onnx' + model = OnnxWrapper(os.path.join(model_dir, model_name), force_onnx_cpu) + else: + model = init_jit_model(os.path.join(model_dir, 'silero_vad.jit')) + utils = (get_speech_timestamps, + save_audio, + read_audio, + VADIterator, + collect_chunks) + + return model, utils diff --git a/models/snakers4_silero-vad/src/silero_vad/__init__.py b/models/snakers4_silero-vad/src/silero_vad/__init__.py new file mode 100644 index 00000000..24bd0766 --- /dev/null +++ b/models/snakers4_silero-vad/src/silero_vad/__init__.py @@ -0,0 +1,12 @@ +from importlib.metadata import version +try: + __version__ = version(__name__) +except: + pass + +from silero_vad.model import load_silero_vad +from silero_vad.utils_vad import (get_speech_timestamps, + save_audio, + read_audio, + VADIterator, + collect_chunks) \ No newline at end of file diff --git a/models/snakers4_silero-vad/src/silero_vad/data/__init__.py b/models/snakers4_silero-vad/src/silero_vad/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/models/snakers4_silero-vad/src/silero_vad/data/silero_vad.jit b/models/snakers4_silero-vad/src/silero_vad/data/silero_vad.jit new file mode 100644 index 00000000..63cb73be Binary files /dev/null and b/models/snakers4_silero-vad/src/silero_vad/data/silero_vad.jit differ diff --git a/models/snakers4_silero-vad/src/silero_vad/data/silero_vad.onnx b/models/snakers4_silero-vad/src/silero_vad/data/silero_vad.onnx new file mode 100644 index 00000000..b3e3a900 Binary files /dev/null and b/models/snakers4_silero-vad/src/silero_vad/data/silero_vad.onnx differ diff --git a/models/snakers4_silero-vad/src/silero_vad/data/silero_vad_16k_op15.onnx b/models/snakers4_silero-vad/src/silero_vad/data/silero_vad_16k_op15.onnx new file mode 100644 index 00000000..0607ae8a Binary files /dev/null and b/models/snakers4_silero-vad/src/silero_vad/data/silero_vad_16k_op15.onnx differ diff --git a/models/snakers4_silero-vad/src/silero_vad/data/silero_vad_half.onnx b/models/snakers4_silero-vad/src/silero_vad/data/silero_vad_half.onnx new file mode 100644 index 00000000..97e39fb3 Binary files /dev/null and b/models/snakers4_silero-vad/src/silero_vad/data/silero_vad_half.onnx differ diff --git a/models/snakers4_silero-vad/src/silero_vad/model.py b/models/snakers4_silero-vad/src/silero_vad/model.py new file mode 100644 index 00000000..40792efa --- /dev/null +++ b/models/snakers4_silero-vad/src/silero_vad/model.py @@ -0,0 +1,36 @@ +from .utils_vad import init_jit_model, OnnxWrapper +import torch +torch.set_num_threads(1) + + +def load_silero_vad(onnx=False, opset_version=16): + available_ops = [15, 16] + if onnx and opset_version not in available_ops: + raise Exception(f'Available ONNX opset_version: {available_ops}') + + if onnx: + if opset_version == 16: + model_name = 'silero_vad.onnx' + else: + model_name = f'silero_vad_16k_op{opset_version}.onnx' + else: + model_name = 'silero_vad.jit' + package_path = "silero_vad.data" + + try: + import importlib_resources as impresources + model_file_path = str(impresources.files(package_path).joinpath(model_name)) + except: + from importlib import resources as impresources + try: + with impresources.path(package_path, model_name) as f: + model_file_path = f + except: + model_file_path = str(impresources.files(package_path).joinpath(model_name)) + + if onnx: + model = OnnxWrapper(model_file_path, force_onnx_cpu=True) + else: + model = init_jit_model(model_file_path) + + return model diff --git a/models/snakers4_silero-vad/src/silero_vad/utils_vad.py b/models/snakers4_silero-vad/src/silero_vad/utils_vad.py new file mode 100644 index 00000000..9a64717e --- /dev/null +++ b/models/snakers4_silero-vad/src/silero_vad/utils_vad.py @@ -0,0 +1,500 @@ +import torch +import torchaudio +from typing import Callable, List +import warnings + +languages = ['ru', 'en', 'de', 'es'] + + +class OnnxWrapper(): + + def __init__(self, path, force_onnx_cpu=False): + import numpy as np + global np + import onnxruntime + + opts = onnxruntime.SessionOptions() + opts.inter_op_num_threads = 1 + opts.intra_op_num_threads = 1 + + if force_onnx_cpu and 'CPUExecutionProvider' in onnxruntime.get_available_providers(): + self.session = onnxruntime.InferenceSession(path, providers=['CPUExecutionProvider'], sess_options=opts) + else: + self.session = onnxruntime.InferenceSession(path, sess_options=opts) + + self.reset_states() + if '16k' in path: + warnings.warn('This model support only 16000 sampling rate!') + self.sample_rates = [16000] + else: + self.sample_rates = [8000, 16000] + + def _validate_input(self, x, sr: int): + if x.dim() == 1: + x = x.unsqueeze(0) + if x.dim() > 2: + raise ValueError(f"Too many dimensions for input audio chunk {x.dim()}") + + if sr != 16000 and (sr % 16000 == 0): + step = sr // 16000 + x = x[:,::step] + sr = 16000 + + if sr not in self.sample_rates: + raise ValueError(f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)") + if sr / x.shape[1] > 31.25: + raise ValueError("Input audio chunk is too short") + + return x, sr + + def reset_states(self, batch_size=1): + self._state = torch.zeros((2, batch_size, 128)).float() + self._context = torch.zeros(0) + self._last_sr = 0 + self._last_batch_size = 0 + + def __call__(self, x, sr: int): + + x, sr = self._validate_input(x, sr) + num_samples = 512 if sr == 16000 else 256 + + if x.shape[-1] != num_samples: + raise ValueError(f"Provided number of samples is {x.shape[-1]} (Supported values: 256 for 8000 sample rate, 512 for 16000)") + + batch_size = x.shape[0] + context_size = 64 if sr == 16000 else 32 + + if not self._last_batch_size: + self.reset_states(batch_size) + if (self._last_sr) and (self._last_sr != sr): + self.reset_states(batch_size) + if (self._last_batch_size) and (self._last_batch_size != batch_size): + self.reset_states(batch_size) + + if not len(self._context): + self._context = torch.zeros(batch_size, context_size) + + x = torch.cat([self._context, x], dim=1) + if sr in [8000, 16000]: + ort_inputs = {'input': x.numpy(), 'state': self._state.numpy(), 'sr': np.array(sr, dtype='int64')} + ort_outs = self.session.run(None, ort_inputs) + out, state = ort_outs + self._state = torch.from_numpy(state) + else: + raise ValueError() + + self._context = x[..., -context_size:] + self._last_sr = sr + self._last_batch_size = batch_size + + out = torch.from_numpy(out) + return out + + def audio_forward(self, x, sr: int): + outs = [] + x, sr = self._validate_input(x, sr) + self.reset_states() + num_samples = 512 if sr == 16000 else 256 + + if x.shape[1] % num_samples: + pad_num = num_samples - (x.shape[1] % num_samples) + x = torch.nn.functional.pad(x, (0, pad_num), 'constant', value=0.0) + + for i in range(0, x.shape[1], num_samples): + wavs_batch = x[:, i:i+num_samples] + out_chunk = self.__call__(wavs_batch, sr) + outs.append(out_chunk) + + stacked = torch.cat(outs, dim=1) + return stacked.cpu() + + +class Validator(): + def __init__(self, url, force_onnx_cpu): + self.onnx = True if url.endswith('.onnx') else False + torch.hub.download_url_to_file(url, 'inf.model') + if self.onnx: + import onnxruntime + if force_onnx_cpu and 'CPUExecutionProvider' in onnxruntime.get_available_providers(): + self.model = onnxruntime.InferenceSession('inf.model', providers=['CPUExecutionProvider']) + else: + self.model = onnxruntime.InferenceSession('inf.model') + else: + self.model = init_jit_model(model_path='inf.model') + + def __call__(self, inputs: torch.Tensor): + with torch.no_grad(): + if self.onnx: + ort_inputs = {'input': inputs.cpu().numpy()} + outs = self.model.run(None, ort_inputs) + outs = [torch.Tensor(x) for x in outs] + else: + outs = self.model(inputs) + + return outs + + +def read_audio(path: str, + sampling_rate: int = 16000): + list_backends = torchaudio.list_audio_backends() + + assert len(list_backends) > 0, 'The list of available backends is empty, please install backend manually. \ + \n Recommendations: \n \tSox (UNIX OS) \n \tSoundfile (Windows OS, UNIX OS) \n \tffmpeg (Windows OS, UNIX OS)' + + try: + effects = [ + ['channels', '1'], + ['rate', str(sampling_rate)] + ] + + wav, sr = torchaudio.sox_effects.apply_effects_file(path, effects=effects) + except: + wav, sr = torchaudio.load(path) + + if wav.size(0) > 1: + wav = wav.mean(dim=0, keepdim=True) + + if sr != sampling_rate: + transform = torchaudio.transforms.Resample(orig_freq=sr, + new_freq=sampling_rate) + wav = transform(wav) + sr = sampling_rate + + assert sr == sampling_rate + return wav.squeeze(0) + + +def save_audio(path: str, + tensor: torch.Tensor, + sampling_rate: int = 16000): + torchaudio.save(path, tensor.unsqueeze(0), sampling_rate, bits_per_sample=16) + + +def init_jit_model(model_path: str, + device=torch.device('cpu')): + model = torch.jit.load(model_path, map_location=device) + model.eval() + return model + + +def make_visualization(probs, step): + import pandas as pd + pd.DataFrame({'probs': probs}, + index=[x * step for x in range(len(probs))]).plot(figsize=(16, 8), + kind='area', ylim=[0, 1.05], xlim=[0, len(probs) * step], + xlabel='seconds', + ylabel='speech probability', + colormap='tab20') + + +@torch.no_grad() +def get_speech_timestamps(audio: torch.Tensor, + model, + threshold: float = 0.5, + sampling_rate: int = 16000, + min_speech_duration_ms: int = 250, + max_speech_duration_s: float = float('inf'), + min_silence_duration_ms: int = 100, + speech_pad_ms: int = 30, + return_seconds: bool = False, + visualize_probs: bool = False, + progress_tracking_callback: Callable[[float], None] = None, + neg_threshold: float = None, + window_size_samples: int = 512,): + + """ + This method is used for splitting long audios into speech chunks using silero VAD + + Parameters + ---------- + audio: torch.Tensor, one dimensional + One dimensional float torch.Tensor, other types are casted to torch if possible + + model: preloaded .jit/.onnx silero VAD model + + threshold: float (default - 0.5) + Speech threshold. Silero VAD outputs speech probabilities for each audio chunk, probabilities ABOVE this value are considered as SPEECH. + It is better to tune this parameter for each dataset separately, but "lazy" 0.5 is pretty good for most datasets. + + sampling_rate: int (default - 16000) + Currently silero VAD models support 8000 and 16000 (or multiply of 16000) sample rates + + min_speech_duration_ms: int (default - 250 milliseconds) + Final speech chunks shorter min_speech_duration_ms are thrown out + + max_speech_duration_s: int (default - inf) + Maximum duration of speech chunks in seconds + Chunks longer than max_speech_duration_s will be split at the timestamp of the last silence that lasts more than 100ms (if any), to prevent agressive cutting. + Otherwise, they will be split aggressively just before max_speech_duration_s. + + min_silence_duration_ms: int (default - 100 milliseconds) + In the end of each speech chunk wait for min_silence_duration_ms before separating it + + speech_pad_ms: int (default - 30 milliseconds) + Final speech chunks are padded by speech_pad_ms each side + + return_seconds: bool (default - False) + whether return timestamps in seconds (default - samples) + + visualize_probs: bool (default - False) + whether draw prob hist or not + + progress_tracking_callback: Callable[[float], None] (default - None) + callback function taking progress in percents as an argument + + neg_threshold: float (default = threshold - 0.15) + Negative threshold (noise or exit threshold). If model's current state is SPEECH, values BELOW this value are considered as NON-SPEECH. + + window_size_samples: int (default - 512 samples) + !!! DEPRECATED, DOES NOTHING !!! + + Returns + ---------- + speeches: list of dicts + list containing ends and beginnings of speech chunks (samples or seconds based on return_seconds) + """ + + if not torch.is_tensor(audio): + try: + audio = torch.Tensor(audio) + except: + raise TypeError("Audio cannot be casted to tensor. Cast it manually") + + if len(audio.shape) > 1: + for i in range(len(audio.shape)): # trying to squeeze empty dimensions + audio = audio.squeeze(0) + if len(audio.shape) > 1: + raise ValueError("More than one dimension in audio. Are you trying to process audio with 2 channels?") + + if sampling_rate > 16000 and (sampling_rate % 16000 == 0): + step = sampling_rate // 16000 + sampling_rate = 16000 + audio = audio[::step] + warnings.warn('Sampling rate is a multiply of 16000, casting to 16000 manually!') + else: + step = 1 + + if sampling_rate not in [8000, 16000]: + raise ValueError("Currently silero VAD models support 8000 and 16000 (or multiply of 16000) sample rates") + + window_size_samples = 512 if sampling_rate == 16000 else 256 + + model.reset_states() + min_speech_samples = sampling_rate * min_speech_duration_ms / 1000 + speech_pad_samples = sampling_rate * speech_pad_ms / 1000 + max_speech_samples = sampling_rate * max_speech_duration_s - window_size_samples - 2 * speech_pad_samples + min_silence_samples = sampling_rate * min_silence_duration_ms / 1000 + min_silence_samples_at_max_speech = sampling_rate * 98 / 1000 + + audio_length_samples = len(audio) + + speech_probs = [] + for current_start_sample in range(0, audio_length_samples, window_size_samples): + chunk = audio[current_start_sample: current_start_sample + window_size_samples] + if len(chunk) < window_size_samples: + chunk = torch.nn.functional.pad(chunk, (0, int(window_size_samples - len(chunk)))) + speech_prob = model(chunk, sampling_rate).item() + speech_probs.append(speech_prob) + # caculate progress and seng it to callback function + progress = current_start_sample + window_size_samples + if progress > audio_length_samples: + progress = audio_length_samples + progress_percent = (progress / audio_length_samples) * 100 + if progress_tracking_callback: + progress_tracking_callback(progress_percent) + + triggered = False + speeches = [] + current_speech = {} + + if neg_threshold is None: + neg_threshold = max(threshold - 0.15, 0.01) + temp_end = 0 # to save potential segment end (and tolerate some silence) + prev_end = next_start = 0 # to save potential segment limits in case of maximum segment size reached + + for i, speech_prob in enumerate(speech_probs): + if (speech_prob >= threshold) and temp_end: + temp_end = 0 + if next_start < prev_end: + next_start = window_size_samples * i + + if (speech_prob >= threshold) and not triggered: + triggered = True + current_speech['start'] = window_size_samples * i + continue + + if triggered and (window_size_samples * i) - current_speech['start'] > max_speech_samples: + if prev_end: + current_speech['end'] = prev_end + speeches.append(current_speech) + current_speech = {} + if next_start < prev_end: # previously reached silence (< neg_thres) and is still not speech (< thres) + triggered = False + else: + current_speech['start'] = next_start + prev_end = next_start = temp_end = 0 + else: + current_speech['end'] = window_size_samples * i + speeches.append(current_speech) + current_speech = {} + prev_end = next_start = temp_end = 0 + triggered = False + continue + + if (speech_prob < neg_threshold) and triggered: + if not temp_end: + temp_end = window_size_samples * i + if ((window_size_samples * i) - temp_end) > min_silence_samples_at_max_speech: # condition to avoid cutting in very short silence + prev_end = temp_end + if (window_size_samples * i) - temp_end < min_silence_samples: + continue + else: + current_speech['end'] = temp_end + if (current_speech['end'] - current_speech['start']) > min_speech_samples: + speeches.append(current_speech) + current_speech = {} + prev_end = next_start = temp_end = 0 + triggered = False + continue + + if current_speech and (audio_length_samples - current_speech['start']) > min_speech_samples: + current_speech['end'] = audio_length_samples + speeches.append(current_speech) + + for i, speech in enumerate(speeches): + if i == 0: + speech['start'] = int(max(0, speech['start'] - speech_pad_samples)) + if i != len(speeches) - 1: + silence_duration = speeches[i+1]['start'] - speech['end'] + if silence_duration < 2 * speech_pad_samples: + speech['end'] += int(silence_duration // 2) + speeches[i+1]['start'] = int(max(0, speeches[i+1]['start'] - silence_duration // 2)) + else: + speech['end'] = int(min(audio_length_samples, speech['end'] + speech_pad_samples)) + speeches[i+1]['start'] = int(max(0, speeches[i+1]['start'] - speech_pad_samples)) + else: + speech['end'] = int(min(audio_length_samples, speech['end'] + speech_pad_samples)) + + if return_seconds: + audio_length_seconds = audio_length_samples / sampling_rate + for speech_dict in speeches: + speech_dict['start'] = max(round(speech_dict['start'] / sampling_rate, 1), 0) + speech_dict['end'] = min(round(speech_dict['end'] / sampling_rate, 1), audio_length_seconds) + elif step > 1: + for speech_dict in speeches: + speech_dict['start'] *= step + speech_dict['end'] *= step + + if visualize_probs: + make_visualization(speech_probs, window_size_samples / sampling_rate) + + return speeches + + +class VADIterator: + def __init__(self, + model, + threshold: float = 0.5, + sampling_rate: int = 16000, + min_silence_duration_ms: int = 100, + speech_pad_ms: int = 30 + ): + + """ + Class for stream imitation + + Parameters + ---------- + model: preloaded .jit/.onnx silero VAD model + + threshold: float (default - 0.5) + Speech threshold. Silero VAD outputs speech probabilities for each audio chunk, probabilities ABOVE this value are considered as SPEECH. + It is better to tune this parameter for each dataset separately, but "lazy" 0.5 is pretty good for most datasets. + + sampling_rate: int (default - 16000) + Currently silero VAD models support 8000 and 16000 sample rates + + min_silence_duration_ms: int (default - 100 milliseconds) + In the end of each speech chunk wait for min_silence_duration_ms before separating it + + speech_pad_ms: int (default - 30 milliseconds) + Final speech chunks are padded by speech_pad_ms each side + """ + + self.model = model + self.threshold = threshold + self.sampling_rate = sampling_rate + + if sampling_rate not in [8000, 16000]: + raise ValueError('VADIterator does not support sampling rates other than [8000, 16000]') + + self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000 + self.speech_pad_samples = sampling_rate * speech_pad_ms / 1000 + self.reset_states() + + def reset_states(self): + + self.model.reset_states() + self.triggered = False + self.temp_end = 0 + self.current_sample = 0 + + @torch.no_grad() + def __call__(self, x, return_seconds=False): + """ + x: torch.Tensor + audio chunk (see examples in repo) + + return_seconds: bool (default - False) + whether return timestamps in seconds (default - samples) + """ + + if not torch.is_tensor(x): + try: + x = torch.Tensor(x) + except: + raise TypeError("Audio cannot be casted to tensor. Cast it manually") + + window_size_samples = len(x[0]) if x.dim() == 2 else len(x) + self.current_sample += window_size_samples + + speech_prob = self.model(x, self.sampling_rate).item() + + if (speech_prob >= self.threshold) and self.temp_end: + self.temp_end = 0 + + if (speech_prob >= self.threshold) and not self.triggered: + self.triggered = True + speech_start = max(0, self.current_sample - self.speech_pad_samples - window_size_samples) + return {'start': int(speech_start) if not return_seconds else round(speech_start / self.sampling_rate, 1)} + + if (speech_prob < self.threshold - 0.15) and self.triggered: + if not self.temp_end: + self.temp_end = self.current_sample + if self.current_sample - self.temp_end < self.min_silence_samples: + return None + else: + speech_end = self.temp_end + self.speech_pad_samples - window_size_samples + self.temp_end = 0 + self.triggered = False + return {'end': int(speech_end) if not return_seconds else round(speech_end / self.sampling_rate, 1)} + + return None + + +def collect_chunks(tss: List[dict], + wav: torch.Tensor): + chunks = [] + for i in tss: + chunks.append(wav[i['start']: i['end']]) + return torch.cat(chunks) + + +def drop_chunks(tss: List[dict], + wav: torch.Tensor): + chunks = [] + cur_start = 0 + for i in tss: + chunks.append((wav[cur_start: i['start']])) + cur_start = i['end'] + return torch.cat(chunks) diff --git a/requirements.txt b/requirements.txt new file mode 100755 index 00000000..4674fd45 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +pyyml==0.0.2 +torch==2.2.2 +silero_vad==5.1.2 +websockets==14.2 +opuslib==3.0.1 +numpy==1.26.4 +pydub==0.25.1 +funasr==1.2.3 +torchaudio==2.2.2 +openai==1.61.0 +edge_tts==7.0.0 +httpx==0.27.2 \ No newline at end of file