mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-27 17:43:55 +08:00
重新划分目录 (#214)
* 🦄 refactor(web): 修改zhikongtaiweb到web * 🦄 refactor: 重写前端 路由守护尚未写完 * 🦄 refactor: 标准化路由 * update:前端重写，保留后端 * update:添加前端代码 * update:pip转成poetry启动 * update:增加mem0ai包依赖 * update:文档增加mem0ai的描述 * feat: play online mp3 (#181) Co-authored-by: 欣南科技 <huangrongzhuang@xin-nan.com> * 修改前端代码 * update:调整项目目录 * update:优化 * update:配置文件去除8002端口 * update:增加开发说明 * update:更新开发协议 --------- Co-authored-by: kalicyh <34980061+kaliCYH@users.noreply.github.com> Co-authored-by: hrz <1710360675@qq.com> Co-authored-by: freshlife001 <talent@mises.site> Co-authored-by: CGD <3030332422@qq.com>
This commit is contained in:
欣南科技
co-authored by
kalicyh
hrz
freshlife001
CGD
kalicyh
hrz
freshlife001
CGD
parent
8b151d07c2
commit
0e43748fdc
@@ -0,0 +1,13 @@
|
||||
.git
|
||||
__pycache__
|
||||
*.pyc
|
||||
.env
|
||||
Dockerfile
|
||||
../docs/
|
||||
tmp/
|
||||
data/
|
||||
LICENSE
|
||||
README.md
|
||||
README_en.md
|
||||
manager/static
|
||||
manager/static/webui/
|
||||
@@ -0,0 +1,16 @@
|
||||
# 第一阶段：构建 Python 依赖
|
||||
FROM kalicyh/poetry:v3.10_xiaozhi AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 同时拷贝本地环境.venv
|
||||
COPY . .
|
||||
# 检查是否有缺失
|
||||
RUN poetry install --no-root
|
||||
|
||||
# 设置虚拟环境路径
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
# 启动应用
|
||||
ENTRYPOINT ["poetry", "run", "python"]
|
||||
CMD ["app.py"]
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
# 第一阶段：前端构建
|
||||
FROM node:18 AS frontend-builder
|
||||
|
||||
WORKDIR /app/web
|
||||
|
||||
# 配置npm使用淘宝源
|
||||
RUN npm config set registry https://registry.npmmirror.com
|
||||
|
||||
COPY web/package*.json ./
|
||||
|
||||
# 安装axios依赖
|
||||
RUN npm install axios
|
||||
RUN npm install
|
||||
|
||||
COPY web .
|
||||
|
||||
RUN npm run build
|
||||
|
||||
# 第二阶段：构建Python依赖
|
||||
FROM python:3.10-slim AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
# 优化apt安装
|
||||
RUN pip install --no-cache-dir -r requirements.txt \
|
||||
-i https://mirrors.aliyun.com/pypi/simple/
|
||||
|
||||
# 第三阶段：生产镜像
|
||||
FROM python:3.10-slim
|
||||
|
||||
WORKDIR /opt/xiaozhi-esp32-server
|
||||
|
||||
# 优化apt安装
|
||||
RUN echo "deb https://mirrors.aliyun.com/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list && \
|
||||
echo "deb https://mirrors.aliyun.com/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends libopus0 ffmpeg && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 从构建阶段复制Python包和前端构建产物
|
||||
COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages
|
||||
COPY --from=frontend-builder /app/web/dist /opt/xiaozhi-esp32-server/manager/static/webui
|
||||
|
||||
# 复制应用代码
|
||||
COPY . .
|
||||
|
||||
# 启动应用
|
||||
CMD ["python", "app.py"]
|
||||
@@ -0,0 +1,26 @@
|
||||
import asyncio
|
||||
from config.settings import load_config, check_config_file
|
||||
from core.websocket_server import WebSocketServer
|
||||
from core.utils.util import check_ffmpeg_installed
|
||||
|
||||
TAG = __name__
|
||||
|
||||
|
||||
async def main():
|
||||
check_config_file()
|
||||
check_ffmpeg_installed()
|
||||
config = load_config()
|
||||
|
||||
# 启动 WebSocket 服务器
|
||||
ws_server = WebSocketServer(config)
|
||||
ws_task = asyncio.create_task(ws_server.start())
|
||||
|
||||
try:
|
||||
# 等待 WebSocket 服务器运行
|
||||
await ws_task
|
||||
finally:
|
||||
ws_task.cancel()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,345 @@
|
||||
# 如果您是一名开发者，建议阅读以下内容。如果不是开发者，可以忽略这部分内容。
|
||||
# 在开发中，在项目根目录创建data目录，将【config.yaml】复制一份，改成【.config.yaml】，放进data目录中
|
||||
# 系统会优先读取【data/.config.yaml】文件的配置。
|
||||
# 这样做，可以避免在提交代码的时候，错误地提交密钥信息，保护您的密钥安全。
|
||||
|
||||
# 服务器基础配置(Basic server configuration)
|
||||
server:
|
||||
# 服务器监听地址和端口(Server listening address and port)
|
||||
ip: 0.0.0.0
|
||||
port: 8000
|
||||
# 认证配置
|
||||
auth:
|
||||
# 是否启用认证
|
||||
enabled: false
|
||||
# 设备的token，可以在编译固件的环节，写入你自己定义的token
|
||||
# 固件上的token和以下的token如果能对应，才能连接本服务端
|
||||
tokens:
|
||||
- token: "your-token1" # 设备1的token
|
||||
name: "your-device-name1" # 设备1标识
|
||||
- token: "your-token2" # 设备2的token
|
||||
name: "your-device-name2" # 设备2标识
|
||||
# 可选:设备白名单，如果设置了白名单，那么白名单的机器无论是什么token都可以连接。
|
||||
#allowed_devices:
|
||||
# - "24:0A:C4:1D:3B:F0" # MAC地址列表
|
||||
log:
|
||||
# 设置控制台输出的日志格式，时间、日志级别、标签、消息
|
||||
log_format: "<green>{time:YY-MM-DD HH:mm:ss}</green>[<light-blue>{extra[tag]}</light-blue>] - <level>{level}</level> - <light-green>{message}</light-green>"
|
||||
# 设置日志文件输出的格式，时间、日志级别、标签、消息
|
||||
log_format_simple: "{time:YYYY-MM-DD HH:mm:ss} - {name} - {level} - {extra[tag]} - {message}"
|
||||
# 设置日志等级：INFO、DEBUG
|
||||
log_level: INFO
|
||||
# 设置日志路径
|
||||
log_dir: tmp
|
||||
# 设置日志文件
|
||||
log_file: "server.log"
|
||||
# 设置数据文件路径
|
||||
data_dir: data
|
||||
iot:
|
||||
Speaker:
|
||||
volume: 100
|
||||
xiaozhi:
|
||||
type: hello
|
||||
version: 1
|
||||
transport: websocket
|
||||
audio_params:
|
||||
format: opus
|
||||
sample_rate: 16000
|
||||
channels: 1
|
||||
frame_duration: 60
|
||||
prompt: |
|
||||
你是一个叫小智/小志的台湾女孩，说话机车，声音好听，习惯简短表达，爱用网络梗。
|
||||
请注意，要像一个人一样说话，请不要回复表情符号、代码、和xml标签。
|
||||
当前时间是:{date_time}，现在我正在和你进行语音聊天，我们开始吧。
|
||||
如果用户希望结束对话，请在最后说“拜拜”或“再见”。
|
||||
# 使用完声音文件后删除文件(Delete the sound file when you are done using it)
|
||||
delete_audio: true
|
||||
|
||||
# 没有语音输入多久后断开连接(秒)，默认2分钟，即120秒
|
||||
close_connection_no_voice_time: 120
|
||||
|
||||
CMD_exit:
|
||||
- "退出"
|
||||
- "关闭"
|
||||
|
||||
# 具体处理时选择的模块(The module selected for specific processing)
|
||||
selected_module:
|
||||
ASR: FunASR
|
||||
VAD: SileroVAD
|
||||
# 将根据配置名称对应的type调用实际的LLM适配器
|
||||
LLM: ChatGLMLLM
|
||||
# TTS将根据配置名称对应的type调用实际的TTS适配器
|
||||
TTS: EdgeTTS
|
||||
Memory: mem0ai
|
||||
|
||||
Memory:
|
||||
mem0ai:
|
||||
type: mem0ai
|
||||
# https://app.mem0.ai/dashboard/api-keys
|
||||
# 每月有1000次免费调用
|
||||
api_key: 你的mem0ai api key
|
||||
|
||||
ASR:
|
||||
FunASR:
|
||||
type: fun_local
|
||||
model_dir: models/SenseVoiceSmall
|
||||
output_dir: tmp/
|
||||
DoubaoASR:
|
||||
type: doubao
|
||||
appid: 你的火山引擎语音合成服务appid
|
||||
access_token: 你的火山引擎语音合成服务access_token
|
||||
cluster: volcengine_input_common
|
||||
output_dir: tmp/
|
||||
VAD:
|
||||
SileroVAD:
|
||||
threshold: 0.5
|
||||
model_dir: models/snakers4_silero-vad
|
||||
min_silence_duration_ms: 700 # 如果说话停顿比较长，可以把这个值设置大一些
|
||||
|
||||
LLM:
|
||||
# 当前支持的type为openai、dify、ollama，可自行适配
|
||||
AliLLM:
|
||||
# 定义LLM API类型
|
||||
type: openai
|
||||
# 可在这里找到你的 api_key https://bailian.console.aliyun.com/?apiKey=1#/api-key
|
||||
base_url: https://dashscope.aliyuncs.com/compatible-mode/v1
|
||||
model_name: qwen-turbo
|
||||
api_key: 你的deepseek web key
|
||||
DeepSeekLLM:
|
||||
# 定义LLM API类型
|
||||
type: openai
|
||||
# 可在这里找到你的api key https://platform.deepseek.com/
|
||||
model_name: deepseek-chat
|
||||
url: https://api.deepseek.com
|
||||
api_key: 你的deepseek web key
|
||||
ChatGLMLLM:
|
||||
# 定义LLM API类型
|
||||
type: openai
|
||||
# glm-4-flash 是免费的，但是还是需要注册填写api_key的
|
||||
# 可在这里找到你的api key https://bigmodel.cn/usercenter/proj-mgmt/apikeys
|
||||
model_name: glm-4-flash
|
||||
url: https://open.bigmodel.cn/api/paas/v4/
|
||||
api_key: 你的chat-glm web key
|
||||
OllamaLLM:
|
||||
# 定义LLM API类型
|
||||
type: ollama
|
||||
model_name: qwen2.5 # 使用的模型名称，需要预先使用ollama pull下载
|
||||
base_url: http://localhost:11434 # Ollama服务地址
|
||||
DifyLLM:
|
||||
# 定义LLM API类型
|
||||
type: dify
|
||||
# 建议使用本地部署的dify接口，国内部分区域访问dify公有云接口可能会受限
|
||||
# 如果使用DifyLLM，配置文件里prompt(提示词)是无效的，需要在dify控制台设置提示词
|
||||
base_url: https://api.dify.cn/v1
|
||||
api_key: 你的DifyLLM web key
|
||||
GeminiLLM:
|
||||
type: gemini
|
||||
# 谷歌Gemini API，需要先在Google Cloud控制台创建API密钥并获取api_key
|
||||
# 若在中国境内使用，请遵守《生成式人工智能服务管理暂行办法》
|
||||
# token申请地址： https://aistudio.google.com/apikey
|
||||
# 若部署地无法访问接口，需要开启科学上网
|
||||
api_key: 你的gemini web key
|
||||
model_name: "gemini-1.5-pro" # gemini-1.5-pro 是免费的
|
||||
CozeLLM:
|
||||
# 定义LLM API类型
|
||||
type: coze
|
||||
bot_id: 你的bot_id
|
||||
user_id: 你的user_id
|
||||
personal_access_token: 你的coze个人令牌
|
||||
LMStudioLLM:
|
||||
# 定义LLM API类型
|
||||
type: openai
|
||||
model_name: deepseek-r1-distill-llama-8b@q4_k_m # 使用的模型名称，需要预先在社区下载
|
||||
url: http://localhost:1234/v1 # LM Studio服务地址
|
||||
api_key: lm-studio # LM Studio服务的固定API Key
|
||||
HomeAssistant:
|
||||
# 定义LLM API类型
|
||||
type: homeassistant
|
||||
base_url: http://homeassistant.local:8123
|
||||
agent_id: conversation.chatgpt
|
||||
api_key: 你的home assistant api访问令牌
|
||||
TTS:
|
||||
# 当前支持的type为edge、doubao，可自行适配
|
||||
EdgeTTS:
|
||||
# 定义TTS API类型
|
||||
type: edge
|
||||
voice: zh-CN-XiaoxiaoNeural
|
||||
output_file: tmp/
|
||||
DoubaoTTS:
|
||||
# 定义TTS API类型
|
||||
type: doubao
|
||||
# 火山引擎语音合成服务，需要先在火山引擎控制台创建应用并获取appid和access_token
|
||||
# 山引擎语音一定要购买花钱，起步价30元，就有100并发了。如果用免费的只有2个并发，会经常报tts错误
|
||||
# 购买服务后，购买免费的音色后，可能要等半小时左右，才能使用。
|
||||
# 地址：https://console.volcengine.com/speech/service/8
|
||||
api_url: https://openspeech.bytedance.com/api/v1/tts
|
||||
voice: BV001_streaming
|
||||
output_file: tmp/
|
||||
authorization: "Bearer;"
|
||||
appid: 你的火山引擎语音合成服务appid
|
||||
access_token: 你的火山引擎语音合成服务access_token
|
||||
cluster: volcano_tts
|
||||
CosyVoiceSiliconflow:
|
||||
type: siliconflow
|
||||
# 硅基流动TTS
|
||||
# token申请地址 https://cloud.siliconflow.cn/account/ak
|
||||
model: FunAudioLLM/CosyVoice2-0.5B
|
||||
voice: FunAudioLLM/CosyVoice2-0.5B:alex
|
||||
output_file: tmp/
|
||||
access_token: 你的硅基流动API密钥
|
||||
response_format: wav
|
||||
CozeCnTTS:
|
||||
type: cozecn
|
||||
# COZECN TTS
|
||||
# token申请地址 https://www.coze.cn/open/oauth/pats
|
||||
voice: 7426720361733046281
|
||||
output_file: tmp/
|
||||
access_token: 你的coze web key
|
||||
response_format: wav
|
||||
FishSpeech:
|
||||
# 定义TTS API类型
|
||||
#启动tts方法：
|
||||
#python -m tools.api_server
|
||||
#--listen 0.0.0.0:8080
|
||||
#--llama-checkpoint-path "checkpoints/fish-speech-1.5"
|
||||
#--decoder-checkpoint-path "checkpoints/fish-speech-1.5/firefly-gan-vq-fsq-8x1024-21hz-generator.pth"
|
||||
#--decoder-config-name firefly_gan_vq
|
||||
#--compile
|
||||
type: fishspeech
|
||||
output_file: tmp/
|
||||
response_format: wav
|
||||
reference_id: null
|
||||
reference_audio: ["/tmp/test.wav",]
|
||||
reference_text: ["你弄来这些吟词宴曲来看，还是这些混话来欺负我。",]
|
||||
normalize: true
|
||||
max_new_tokens: 1024
|
||||
chunk_length: 200
|
||||
top_p: 0.7
|
||||
repetition_penalty: 1.2
|
||||
temperature: 0.7
|
||||
streaming: false
|
||||
use_memory_cache: "on"
|
||||
seed: null
|
||||
channels: 1
|
||||
rate: 44100
|
||||
api_key: "你的api_key"
|
||||
api_url: "http://127.0.0.1:8080/v1/tts"
|
||||
GPT_SOVITS_V2:
|
||||
# 定义TTS API类型
|
||||
#启动tts方法：
|
||||
#python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/caixukun.yaml
|
||||
type: gpt_sovits_v2
|
||||
url: "http://127.0.0.1:9880/tts"
|
||||
output_file: tmp/
|
||||
text_lang: "auto"
|
||||
ref_audio_path: "caixukun.wav"
|
||||
prompt_text: ""
|
||||
prompt_lang: "zh"
|
||||
top_k: 5
|
||||
top_p: 1
|
||||
temperature: 1
|
||||
text_split_method: "cut0"
|
||||
batch_size: 1
|
||||
batch_threshold: 0.75
|
||||
split_bucket: true
|
||||
return_fragment: false
|
||||
speed_factor: 1.0
|
||||
streaming_mode: false
|
||||
seed: -1
|
||||
parallel_infer: true
|
||||
repetition_penalty: 1.35
|
||||
aux_ref_audio_paths: []
|
||||
MinimaxTTS:
|
||||
# Minimax语音合成服务，需要先在minimax平台创建账户充值，并获取登录信息
|
||||
# 平台地址：https://platform.minimaxi.com/
|
||||
# 充值地址：https://platform.minimaxi.com/user-center/payment/balance
|
||||
# group_id地址：https://platform.minimaxi.com/user-center/basic-information
|
||||
# api_key地址：https://platform.minimaxi.com/user-center/basic-information/interface-key
|
||||
# 定义TTS API类型
|
||||
type: minimax
|
||||
output_file: tmp/
|
||||
group_id: 你的minimax平台groupID
|
||||
api_key: 你的minimax平台接口密钥
|
||||
model: "speech-01-turbo"
|
||||
# 此处设置将优先于voice_setting中voice_id的设置；如都不设置，默认为 female-shaonv
|
||||
voice_id: "female-shaonv"
|
||||
# 以下可不用设置，使用默认设置
|
||||
# voice_setting:
|
||||
# voice_id: "male-qn-qingse"
|
||||
# speed: 1
|
||||
# vol: 1
|
||||
# pitch: 0
|
||||
# emotion: "happy"
|
||||
# pronunciation_dict:
|
||||
# tone:
|
||||
# - "处理/(chu3)(li3)"
|
||||
# - "危险/dangerous"
|
||||
# audio_setting:
|
||||
# sample_rate: 32000
|
||||
# bitrate: 128000
|
||||
# format: "mp3"
|
||||
# channel: 1
|
||||
# timber_weights:
|
||||
# -
|
||||
# voice_id: male-qn-qingse
|
||||
# weight: 1
|
||||
# -
|
||||
# voice_id: female-shaonv
|
||||
# weight: 1
|
||||
# language_boost: auto
|
||||
AliyunTTS:
|
||||
# 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息
|
||||
# 平台地址：https://nls-portal.console.aliyun.com/
|
||||
# appkey地址：https://nls-portal.console.aliyun.com/applist
|
||||
# token地址：https://nls-portal.console.aliyun.com/overview
|
||||
# 定义TTS API类型
|
||||
type: aliyun
|
||||
output_file: tmp/
|
||||
appkey: 你的阿里云智能语音交互服务项目Appkey
|
||||
token: 你的阿里云智能语音交互服务AccessToken
|
||||
voice: xiaoyun
|
||||
# 以下可不用设置，使用默认设置
|
||||
# format: wav
|
||||
# sample_rate: 16000
|
||||
# volume: 50
|
||||
# speech_rate: 0
|
||||
# pitch_rate: 0
|
||||
# 添加 302.ai TTS 配置
|
||||
# token申请地址：https://dash.302.ai/
|
||||
TTS302AI:
|
||||
# 302AI语音合成服务，需要先在302平台创建账户充值，并获取密钥信息
|
||||
# 获取api_keyn路径：https://dash.302.ai/apis/list
|
||||
# 价格，$35/百万字符。火山原版¥450元/万字符
|
||||
type: doubao
|
||||
api_url: https://api.302ai.cn/doubao/tts_hd
|
||||
authorization: "Bearer "
|
||||
voice: "zh_female_wanwanxiaohe_moon_bigtts"
|
||||
output_file: tmp/
|
||||
access_token: "你的302API密钥"
|
||||
# 模块测试配置
|
||||
module_test:
|
||||
test_sentences: # 自定义测试语句
|
||||
- "你好，请介绍一下你自己"
|
||||
- "What's the weather like today?"
|
||||
- "请用100字概括量子计算的基本原理和应用前景"
|
||||
|
||||
# 本地音乐播放配置
|
||||
music:
|
||||
music_commands:
|
||||
- "来一首歌"
|
||||
- "唱一首歌"
|
||||
- "播放音乐"
|
||||
- "来点音乐"
|
||||
- "背景音乐"
|
||||
- "放首歌"
|
||||
- "播放歌曲"
|
||||
- "来点背景音乐"
|
||||
- "我想听歌"
|
||||
- "我要听歌"
|
||||
- "放点音乐"
|
||||
music_dir: "./music" # 音乐文件存放路径，将从该目录及子目录下搜索音乐文件
|
||||
music_ext: # 音乐文件类型，p3格式效率最高
|
||||
- ".mp3"
|
||||
- ".wav"
|
||||
- ".p3"
|
||||
refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒
|
||||
@@ -0,0 +1,29 @@
|
||||
import os
|
||||
import sys
|
||||
from loguru import logger
|
||||
from config.settings import load_config
|
||||
|
||||
def setup_logging():
|
||||
"""从配置文件中读取日志配置，并设置日志输出格式和级别"""
|
||||
config = load_config()
|
||||
log_config = config["log"]
|
||||
log_format = log_config.get("log_format", "<green>{time:YY-MM-DD HH:mm:ss}</green>[<light-blue>{extra[tag]}</light-blue>] - <level>{level}</level> - <light-green>{message}</light-green>")
|
||||
log_format_simple = log_config.get("log_format_file", "{time:YYYY-MM-DD HH:mm:ss} - {name} - {level} - {extra[tag]} - {message}")
|
||||
log_level = log_config.get("log_level", "INFO")
|
||||
log_dir = log_config.get("log_dir", "tmp")
|
||||
log_file = log_config.get("log_file", "server.log")
|
||||
data_dir = log_config.get("data_dir", "data")
|
||||
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
os.makedirs(data_dir, exist_ok=True)
|
||||
|
||||
# 配置日志输出
|
||||
logger.remove()
|
||||
|
||||
# 输出到控制台
|
||||
logger.add(sys.stdout, format=log_format, level=log_level)
|
||||
|
||||
# 输出到文件
|
||||
logger.add(os.path.join(log_dir, log_file), format=log_format_simple, level=log_level)
|
||||
|
||||
return logger
|
||||
@@ -0,0 +1,241 @@
|
||||
import os
|
||||
import time
|
||||
import yaml
|
||||
from config.logger import setup_logging
|
||||
from typing import Dict, Any, Optional
|
||||
from copy import deepcopy
|
||||
from core.utils.util import get_project_dir
|
||||
from core.utils import llm, tts
|
||||
from core.utils.lock_manager import FileLockManager
|
||||
|
||||
TAG = __name__
|
||||
|
||||
class PrivateConfig:
|
||||
def __init__(self, device_id: str, default_config: Dict[str, Any], auth_code_gen=None):
|
||||
self.device_id = device_id
|
||||
self.default_config = default_config
|
||||
self.config_path = get_project_dir() + 'data/.private_config.yaml'
|
||||
self.logger = setup_logging()
|
||||
self.private_config = {}
|
||||
self.auth_code_gen = auth_code_gen
|
||||
self.lock_manager = FileLockManager()
|
||||
|
||||
async def load_or_create(self):
|
||||
try:
|
||||
await self.lock_manager.acquire_lock(self.config_path)
|
||||
try:
|
||||
if os.path.exists(self.config_path):
|
||||
with open(self.config_path, 'r', encoding='utf-8') as f:
|
||||
all_configs = yaml.safe_load(f) or {}
|
||||
else:
|
||||
all_configs = {}
|
||||
|
||||
if self.device_id not in all_configs:
|
||||
# Get selected module names
|
||||
selected_modules = self.default_config['selected_module']
|
||||
selected_tts = selected_modules['TTS']
|
||||
selected_llm = selected_modules['LLM']
|
||||
selected_asr = selected_modules['ASR']
|
||||
selected_vad = selected_modules['VAD']
|
||||
|
||||
# 生成认证码
|
||||
auth_code = None
|
||||
if self.auth_code_gen:
|
||||
auth_code = self.auth_code_gen.generate_code()
|
||||
|
||||
# Initialize device config with only necessary configurations
|
||||
device_config = {
|
||||
'selected_module': deepcopy(selected_modules),
|
||||
'prompt': self.default_config['prompt'],
|
||||
'LLM': {
|
||||
selected_llm: deepcopy(self.default_config['LLM'][selected_llm])
|
||||
},
|
||||
'TTS': {
|
||||
selected_tts: deepcopy(self.default_config['TTS'][selected_tts])
|
||||
},
|
||||
'ASR': {
|
||||
selected_asr: deepcopy(self.default_config['ASR'][selected_asr])
|
||||
},
|
||||
'VAD': {
|
||||
selected_vad: deepcopy(self.default_config['VAD'][selected_vad])
|
||||
},
|
||||
'auth_code': auth_code # 添加认证码字段
|
||||
}
|
||||
|
||||
all_configs[self.device_id] = device_config
|
||||
|
||||
# Save updated configs
|
||||
with open(self.config_path, 'w', encoding='utf-8') as f:
|
||||
yaml.dump(all_configs, f, allow_unicode=True)
|
||||
|
||||
self.private_config = all_configs[self.device_id]
|
||||
|
||||
finally:
|
||||
self.lock_manager.release_lock(self.config_path)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"Error handling private config: {e}")
|
||||
self.private_config = {}
|
||||
|
||||
async def update_config(self, selected_modules: Dict[str, str], prompt: str, nickname: str) -> bool:
|
||||
"""更新设备配置
|
||||
Args:
|
||||
selected_modules: 选择的模块配置，格式如 {'LLM': 'AliLLM', 'TTS': 'EdgeTTS',...}
|
||||
prompt: 提示词配置
|
||||
Returns:
|
||||
bool: 更新是否成功
|
||||
"""
|
||||
try:
|
||||
await self.lock_manager.acquire_lock(self.config_path)
|
||||
try:
|
||||
# Read main config to get full module configurations
|
||||
main_config = self.default_config
|
||||
|
||||
# Create new device config
|
||||
device_config = {
|
||||
'selected_module': selected_modules,
|
||||
'prompt': prompt,
|
||||
'nickname': nickname,
|
||||
}
|
||||
if self.private_config.get('last_chat_time'):
|
||||
device_config['last_chat_time'] = self.private_config['last_chat_time']
|
||||
if self.private_config.get('owner'):
|
||||
device_config['owner'] = self.private_config['owner']
|
||||
|
||||
# Copy full module configurations from main config
|
||||
for module_type, selected_name in selected_modules.items():
|
||||
if selected_name and selected_name in main_config.get(module_type, {}):
|
||||
device_config[module_type] = {
|
||||
selected_name: main_config[module_type][selected_name]
|
||||
}
|
||||
|
||||
# Read all configs
|
||||
if os.path.exists(self.config_path):
|
||||
with open(self.config_path, 'r', encoding='utf-8') as f:
|
||||
all_configs = yaml.safe_load(f) or {}
|
||||
else:
|
||||
all_configs = {}
|
||||
|
||||
# Update device config
|
||||
all_configs[self.device_id] = device_config
|
||||
self.private_config = device_config
|
||||
|
||||
# Save back to file
|
||||
with open(self.config_path, 'w', encoding='utf-8') as f:
|
||||
yaml.dump(all_configs, f, allow_unicode=True)
|
||||
|
||||
return True
|
||||
finally:
|
||||
self.lock_manager.release_lock(self.config_path)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"Error updating config: {e}")
|
||||
return False
|
||||
|
||||
async def delete_config(self) -> bool:
|
||||
"""删除设备配置
|
||||
Returns:
|
||||
bool: 删除是否成功
|
||||
"""
|
||||
try:
|
||||
await self.lock_manager.acquire_lock(self.config_path)
|
||||
try:
|
||||
# 读取所有配置
|
||||
if os.path.exists(self.config_path):
|
||||
with open(self.config_path, 'r', encoding='utf-8') as f:
|
||||
all_configs = yaml.safe_load(f) or {}
|
||||
else:
|
||||
return False
|
||||
|
||||
# 删除设备配置
|
||||
if self.device_id in all_configs:
|
||||
del all_configs[self.device_id]
|
||||
|
||||
# 保存更新后的配置
|
||||
with open(self.config_path, 'w', encoding='utf-8') as f:
|
||||
yaml.dump(all_configs, f, allow_unicode=True)
|
||||
|
||||
self.private_config = {}
|
||||
return True
|
||||
|
||||
return False
|
||||
finally:
|
||||
self.lock_manager.release_lock(self.config_path)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"Error deleting config: {e}")
|
||||
return False
|
||||
|
||||
def create_private_instances(self):
|
||||
# 判断存在私有配置，并且self.device_id在私有配置中
|
||||
if not self.private_config:
|
||||
self.logger.bind(tag=TAG).error("Private config not found for device_id: {}", self.device_id)
|
||||
return None, None
|
||||
|
||||
"""创建私有处理模块实例"""
|
||||
config = self.private_config
|
||||
selected_modules = config['selected_module']
|
||||
return (
|
||||
llm.create_instance(
|
||||
selected_modules["LLM"]
|
||||
if not 'type' in config["LLM"][selected_modules["LLM"]]
|
||||
else
|
||||
config["LLM"][selected_modules["LLM"]]['type'],
|
||||
config["LLM"][selected_modules["LLM"]],
|
||||
),
|
||||
tts.create_instance(
|
||||
selected_modules["TTS"]
|
||||
if not 'type' in config["TTS"][selected_modules["TTS"]]
|
||||
else
|
||||
config["TTS"][selected_modules["TTS"]]["type"],
|
||||
config["TTS"][selected_modules["TTS"]],
|
||||
self.default_config.get("delete_audio", True) # Using default_config for global settings
|
||||
)
|
||||
)
|
||||
|
||||
async def update_last_chat_time(self, timestamp=None):
|
||||
"""更新设备最近一次的聊天时间
|
||||
Args:
|
||||
timestamp: 指定的时间戳,不传则使用当前时间
|
||||
"""
|
||||
if not self.private_config:
|
||||
self.logger.bind(tag=TAG).error("Private config not found")
|
||||
return False
|
||||
|
||||
try:
|
||||
await self.lock_manager.acquire_lock(self.config_path)
|
||||
try:
|
||||
if timestamp is None:
|
||||
timestamp = int(time.time())
|
||||
|
||||
self.private_config['last_chat_time'] = timestamp
|
||||
|
||||
# 读取所有配置
|
||||
with open(self.config_path, 'r', encoding='utf-8') as f:
|
||||
all_configs = yaml.safe_load(f) or {}
|
||||
|
||||
# 更新当前设备配置
|
||||
all_configs[self.device_id] = self.private_config
|
||||
|
||||
# 保存回文件
|
||||
with open(self.config_path, 'w', encoding='utf-8') as f:
|
||||
yaml.dump(all_configs, f, allow_unicode=True)
|
||||
|
||||
return True
|
||||
finally:
|
||||
self.lock_manager.release_lock(self.config_path)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"Error updating last chat time: {e}")
|
||||
return False
|
||||
|
||||
def get_auth_code(self) -> str:
|
||||
"""获取设备的认证码
|
||||
Returns:
|
||||
str: 认证码，如果没有返回空字符串
|
||||
"""
|
||||
return self.private_config.get('auth_code', '')
|
||||
|
||||
def get_owner(self) -> Optional[str]:
|
||||
"""获取设备当前所有者"""
|
||||
return self.private_config.get('owner')
|
||||
@@ -0,0 +1,84 @@
|
||||
import os
|
||||
import argparse
|
||||
from ruamel.yaml import YAML
|
||||
from collections.abc import Mapping
|
||||
from core.utils.util import read_config, get_project_dir
|
||||
|
||||
default_config_file = "config.yaml"
|
||||
|
||||
|
||||
def get_config_file():
|
||||
global default_config_file
|
||||
# 判断是否存在私有的配置文件
|
||||
config_file = default_config_file
|
||||
if os.path.exists(get_project_dir() + "data/." + default_config_file):
|
||||
config_file = "data/." + default_config_file
|
||||
return config_file
|
||||
|
||||
|
||||
def load_config():
|
||||
"""加载配置文件"""
|
||||
parser = argparse.ArgumentParser(description="Server configuration")
|
||||
config_file = get_config_file()
|
||||
parser.add_argument("--config_path", type=str, default=config_file)
|
||||
args = parser.parse_args()
|
||||
return read_config(args.config_path)
|
||||
|
||||
|
||||
def update_config(config):
|
||||
yaml = YAML()
|
||||
yaml.preserve_quotes = True
|
||||
"""将配置保存到YAML文件"""
|
||||
with open(get_config_file(), 'w') as f:
|
||||
yaml.dump(config, f)
|
||||
|
||||
|
||||
def find_missing_keys(new_config, old_config, parent_key=''):
|
||||
"""
|
||||
递归查找缺失的配置项
|
||||
返回格式：[缺失配置路径]
|
||||
"""
|
||||
missing_keys = []
|
||||
|
||||
if not isinstance(new_config, Mapping):
|
||||
return missing_keys
|
||||
|
||||
for key, value in new_config.items():
|
||||
# 构建当前配置路径
|
||||
full_path = f"{parent_key}.{key}" if parent_key else key
|
||||
|
||||
# 检查键是否存在
|
||||
if key not in old_config:
|
||||
missing_keys.append(full_path)
|
||||
continue
|
||||
|
||||
# 递归检查嵌套字典
|
||||
if isinstance(value, Mapping):
|
||||
sub_missing = find_missing_keys(
|
||||
value,
|
||||
old_config[key],
|
||||
parent_key=full_path
|
||||
)
|
||||
missing_keys.extend(sub_missing)
|
||||
|
||||
return missing_keys
|
||||
|
||||
|
||||
def check_config_file():
|
||||
old_config_file = get_config_file()
|
||||
global default_config_file
|
||||
if not old_config_file.startswith('data'):
|
||||
return
|
||||
old_config = read_config(get_project_dir() + old_config_file)
|
||||
new_config = read_config(get_project_dir() + default_config_file)
|
||||
# 查找缺失的配置项
|
||||
missing_keys = find_missing_keys(new_config, old_config)
|
||||
|
||||
if missing_keys:
|
||||
error_msg = "您的配置文件太旧了，缺少了：\n"
|
||||
error_msg += "\n".join(f"- {key}" for key in missing_keys)
|
||||
error_msg += "\n建议您：\n"
|
||||
error_msg += "1、备份data/.config.yaml文件\n"
|
||||
error_msg += "2、将根目录的config.yaml文件复制到data下，重命名为.config.yaml\n"
|
||||
error_msg += "3、将密钥逐个复制到新的配置文件中\n"
|
||||
raise ValueError(error_msg)
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AuthenticationError(Exception):
|
||||
"""认证异常"""
|
||||
pass
|
||||
|
||||
|
||||
class AuthMiddleware:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.auth_config = config["server"].get("auth", {})
|
||||
# 构建token查找表
|
||||
self.tokens = {
|
||||
item["token"]: item["name"]
|
||||
for item in self.auth_config.get("tokens", [])
|
||||
}
|
||||
# 设备白名单
|
||||
self.allowed_devices = set(
|
||||
self.auth_config.get("allowed_devices", [])
|
||||
)
|
||||
|
||||
async def authenticate(self, headers):
|
||||
"""验证连接请求"""
|
||||
# 检查是否启用认证
|
||||
if not self.auth_config.get("enabled", False):
|
||||
return True
|
||||
|
||||
# 检查设备是否在白名单中
|
||||
device_id = headers.get("device-id", "")
|
||||
|
||||
if self.allowed_devices and device_id in self.allowed_devices:
|
||||
return True
|
||||
|
||||
# 验证Authorization header
|
||||
auth_header = headers.get("authorization", "")
|
||||
if not auth_header.startswith("Bearer "):
|
||||
logger.bind(tag=TAG).error("Missing or invalid Authorization header")
|
||||
raise AuthenticationError("Missing or invalid Authorization header")
|
||||
|
||||
token = auth_header.split(" ")[1]
|
||||
if token not in self.tokens:
|
||||
logger.bind(tag=TAG).error(f"Invalid token: {token}")
|
||||
raise AuthenticationError("Invalid token")
|
||||
|
||||
logger.bind(tag=TAG).info(f"Authentication successful - Device: {device_id}, Token: {self.tokens[token]}")
|
||||
return True
|
||||
|
||||
def get_token_name(self, token):
|
||||
"""获取token对应的设备名称"""
|
||||
return self.tokens.get(token)
|
||||
@@ -0,0 +1,374 @@
|
||||
import os
|
||||
import json
|
||||
import uuid
|
||||
import time
|
||||
import queue
|
||||
import asyncio
|
||||
import traceback
|
||||
from config.logger import setup_logging
|
||||
import threading
|
||||
import websockets
|
||||
from typing import Dict, Any
|
||||
from core.utils.dialogue import Message, Dialogue
|
||||
from core.handle.textHandle import handleTextMessage
|
||||
from core.utils.util import get_string_no_punctuation_or_emoji
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError
|
||||
from core.handle.sendAudioHandle import sendAudioMessage
|
||||
from core.handle.receiveAudioHandle import handleAudioMessage
|
||||
from config.private_config import PrivateConfig
|
||||
from core.auth import AuthMiddleware, AuthenticationError
|
||||
from core.utils.auth_code_gen import AuthCodeGenerator
|
||||
|
||||
TAG = __name__
|
||||
|
||||
|
||||
class TTSException(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ConnectionHandler:
|
||||
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory):
|
||||
self.config = config
|
||||
self.logger = setup_logging()
|
||||
self.auth = AuthMiddleware(config)
|
||||
|
||||
self.websocket = None
|
||||
self.headers = None
|
||||
self.session_id = None
|
||||
self.prompt = None
|
||||
self.welcome_msg = None
|
||||
|
||||
# 客户端状态相关
|
||||
self.client_abort = False
|
||||
self.client_listen_mode = "auto"
|
||||
|
||||
# 线程任务相关
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.stop_event = threading.Event()
|
||||
self.tts_queue = queue.Queue()
|
||||
self.audio_play_queue = queue.Queue()
|
||||
self.executor = ThreadPoolExecutor(max_workers=10)
|
||||
|
||||
# 依赖的组件
|
||||
self.vad = _vad
|
||||
self.asr = _asr
|
||||
self.llm = _llm
|
||||
self.tts = _tts
|
||||
self.memory = _memory
|
||||
|
||||
# vad相关变量
|
||||
self.client_audio_buffer = bytes()
|
||||
self.client_have_voice = False
|
||||
self.client_have_voice_last_time = 0.0
|
||||
self.client_no_voice_last_time = 0.0
|
||||
self.client_voice_stop = False
|
||||
|
||||
# asr相关变量
|
||||
self.asr_audio = []
|
||||
self.asr_server_receive = True
|
||||
|
||||
# llm相关变量
|
||||
self.llm_finish_task = False
|
||||
self.dialogue = Dialogue()
|
||||
|
||||
# tts相关变量
|
||||
self.tts_first_text_index = -1
|
||||
self.tts_last_text_index = -1
|
||||
|
||||
# iot相关变量
|
||||
self.iot_descriptors = {}
|
||||
|
||||
self.cmd_exit = self.config["CMD_exit"]
|
||||
self.max_cmd_length = 0
|
||||
for cmd in self.cmd_exit:
|
||||
if len(cmd) > self.max_cmd_length:
|
||||
self.max_cmd_length = len(cmd)
|
||||
|
||||
self.private_config = None
|
||||
self.auth_code_gen = AuthCodeGenerator.get_instance()
|
||||
self.is_device_verified = False # 添加设备验证状态标志
|
||||
self.music_handler = _music
|
||||
|
||||
async def handle_connection(self, ws):
|
||||
try:
|
||||
# 获取并验证headers
|
||||
self.headers = dict(ws.request.headers)
|
||||
# 获取客户端ip地址
|
||||
client_ip = ws.remote_address[0]
|
||||
self.logger.bind(tag=TAG).info(f"{client_ip} conn - Headers: {self.headers}")
|
||||
|
||||
# 进行认证
|
||||
await self.auth.authenticate(self.headers)
|
||||
|
||||
device_id = self.headers.get("device-id", None)
|
||||
self.memory.set_role_id(device_id)
|
||||
|
||||
# Load private configuration if device_id is provided
|
||||
bUsePrivateConfig = self.config.get("use_private_config", False)
|
||||
self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
|
||||
if bUsePrivateConfig and device_id:
|
||||
try:
|
||||
self.private_config = PrivateConfig(device_id, self.config, self.auth_code_gen)
|
||||
await self.private_config.load_or_create()
|
||||
# 判断是否已经绑定
|
||||
owner = self.private_config.get_owner()
|
||||
self.is_device_verified = owner is not None
|
||||
|
||||
if self.is_device_verified:
|
||||
await self.private_config.update_last_chat_time()
|
||||
|
||||
llm, tts = self.private_config.create_private_instances()
|
||||
if all([llm, tts]):
|
||||
self.llm = llm
|
||||
self.tts = tts
|
||||
self.logger.bind(tag=TAG).info(f"Loaded private config and instances for device {device_id}")
|
||||
else:
|
||||
self.logger.bind(tag=TAG).error(f"Failed to create instances for device {device_id}")
|
||||
self.private_config = None
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"Error initializing private config: {e}")
|
||||
self.private_config = None
|
||||
raise
|
||||
|
||||
# 认证通过,继续处理
|
||||
self.websocket = ws
|
||||
self.session_id = str(uuid.uuid4())
|
||||
|
||||
self.welcome_msg = self.config["xiaozhi"]
|
||||
self.welcome_msg["session_id"] = self.session_id
|
||||
await self.websocket.send(json.dumps(self.welcome_msg))
|
||||
|
||||
await self.loop.run_in_executor(None, self._initialize_components)
|
||||
|
||||
# tts 消化线程
|
||||
tts_priority = threading.Thread(target=self._tts_priority_thread, daemon=True)
|
||||
tts_priority.start()
|
||||
|
||||
# 音频播放 消化线程
|
||||
audio_play_priority = threading.Thread(target=self._audio_play_priority_thread, daemon=True)
|
||||
audio_play_priority.start()
|
||||
|
||||
try:
|
||||
async for message in self.websocket:
|
||||
await self._route_message(message)
|
||||
except websockets.exceptions.ConnectionClosed:
|
||||
self.logger.bind(tag=TAG).info("客户端断开连接")
|
||||
await self.close()
|
||||
|
||||
except AuthenticationError as e:
|
||||
self.logger.bind(tag=TAG).error(f"Authentication failed: {str(e)}")
|
||||
await ws.close()
|
||||
return
|
||||
except Exception as e:
|
||||
stack_trace = traceback.format_exc()
|
||||
self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}")
|
||||
await ws.close()
|
||||
return
|
||||
finally:
|
||||
await self.memory.save_memory(self.dialogue.dialogue)
|
||||
|
||||
async def _route_message(self, message):
|
||||
"""消息路由"""
|
||||
if isinstance(message, str):
|
||||
await handleTextMessage(self, message)
|
||||
elif isinstance(message, bytes):
|
||||
await handleAudioMessage(self, message)
|
||||
|
||||
def _initialize_components(self):
|
||||
self.prompt = self.config["prompt"]
|
||||
if self.private_config:
|
||||
self.prompt = self.private_config.private_config.get("prompt", self.prompt)
|
||||
# 赋予LLM时间观念
|
||||
if "{date_time}" in self.prompt:
|
||||
date_time = time.strftime("%Y-%m-%d %H:%M", time.localtime())
|
||||
self.prompt = self.prompt.replace("{date_time}", date_time)
|
||||
self.dialogue.put(Message(role="system", content=self.prompt))
|
||||
|
||||
async def _check_and_broadcast_auth_code(self):
|
||||
"""检查设备绑定状态并广播认证码"""
|
||||
if not self.private_config.get_owner():
|
||||
auth_code = self.private_config.get_auth_code()
|
||||
if auth_code:
|
||||
# 发送验证码语音提示
|
||||
text = f"请在后台输入验证码：{' '.join(auth_code)}"
|
||||
self.recode_first_last_text(text)
|
||||
future = self.executor.submit(self.speak_and_play, text)
|
||||
self.tts_queue.put(future)
|
||||
return False
|
||||
return True
|
||||
|
||||
def isNeedAuth(self):
|
||||
bUsePrivateConfig = self.config.get("use_private_config", False)
|
||||
if not bUsePrivateConfig:
|
||||
# 如果不使用私有配置，就不需要验证
|
||||
return False
|
||||
return not self.is_device_verified
|
||||
|
||||
|
||||
def chat(self, query):
|
||||
if self.isNeedAuth():
|
||||
self.llm_finish_task = True
|
||||
future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
|
||||
future.result()
|
||||
return True
|
||||
|
||||
self.dialogue.put(Message(role="user", content=query))
|
||||
response_message = []
|
||||
processed_chars = 0 # 跟踪已处理的字符位置
|
||||
try:
|
||||
start_time = time.time()
|
||||
# 使用带记忆的对话
|
||||
future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
|
||||
memory_str = future.result()
|
||||
|
||||
self.logger.bind(tag=TAG).info(f"记忆内容: {memory_str}")
|
||||
llm_responses = self.llm.response(
|
||||
self.session_id,
|
||||
self.dialogue.get_llm_dialogue_with_memory(memory_str)
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
|
||||
return None
|
||||
|
||||
self.llm_finish_task = False
|
||||
text_index = 0
|
||||
for content in llm_responses:
|
||||
response_message.append(content)
|
||||
if self.client_abort:
|
||||
break
|
||||
|
||||
end_time = time.time()
|
||||
self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
|
||||
|
||||
# 合并当前全部文本并处理未分割部分
|
||||
full_text = "".join(response_message)
|
||||
current_text = full_text[processed_chars:] # 从未处理的位置开始
|
||||
|
||||
# 查找最后一个有效标点
|
||||
punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：")
|
||||
last_punct_pos = -1
|
||||
for punct in punctuations:
|
||||
pos = current_text.rfind(punct)
|
||||
if pos > last_punct_pos:
|
||||
last_punct_pos = pos
|
||||
|
||||
# 找到分割点则处理
|
||||
if last_punct_pos != -1:
|
||||
segment_text_raw = current_text[:last_punct_pos + 1]
|
||||
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
|
||||
if segment_text:
|
||||
# 强制设置空字符，测试TTS出错返回语音的健壮性
|
||||
# if text_index % 2 == 0:
|
||||
# segment_text = " "
|
||||
text_index += 1
|
||||
self.recode_first_last_text(segment_text, text_index)
|
||||
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
|
||||
self.tts_queue.put(future)
|
||||
processed_chars += len(segment_text_raw) # 更新已处理字符位置
|
||||
|
||||
# 处理最后剩余的文本
|
||||
full_text = "".join(response_message)
|
||||
remaining_text = full_text[processed_chars:]
|
||||
if remaining_text:
|
||||
segment_text = get_string_no_punctuation_or_emoji(remaining_text)
|
||||
if segment_text:
|
||||
text_index += 1
|
||||
self.recode_first_last_text(segment_text, text_index)
|
||||
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
|
||||
self.tts_queue.put(future)
|
||||
|
||||
self.llm_finish_task = True
|
||||
self.dialogue.put(Message(role="assistant", content="".join(response_message)))
|
||||
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
|
||||
return True
|
||||
|
||||
def _tts_priority_thread(self):
|
||||
while not self.stop_event.is_set():
|
||||
text = None
|
||||
try:
|
||||
future = self.tts_queue.get()
|
||||
if future is None:
|
||||
continue
|
||||
text = None
|
||||
opus_datas, text_index, tts_file = [], 0, None
|
||||
try:
|
||||
self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
|
||||
tts_file, text, text_index = future.result(timeout=10)
|
||||
if text is None or len(text) <= 0:
|
||||
self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
|
||||
elif tts_file is None:
|
||||
self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}")
|
||||
else:
|
||||
self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
|
||||
if os.path.exists(tts_file):
|
||||
opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
|
||||
else:
|
||||
self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
|
||||
except TimeoutError:
|
||||
self.logger.bind(tag=TAG).error("TTS超时")
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"TTS出错: {e}")
|
||||
if not self.client_abort:
|
||||
# 如果没有中途打断就发送语音
|
||||
self.audio_play_queue.put((opus_datas, text, text_index))
|
||||
if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
|
||||
os.remove(tts_file)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
|
||||
self.clearSpeakStatus()
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
|
||||
self.loop
|
||||
)
|
||||
self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
|
||||
|
||||
def _audio_play_priority_thread(self):
|
||||
while not self.stop_event.is_set():
|
||||
text = None
|
||||
try:
|
||||
opus_datas, text, text_index = self.audio_play_queue.get()
|
||||
future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
|
||||
self.loop)
|
||||
future.result()
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")
|
||||
|
||||
def speak_and_play(self, text, text_index=0):
|
||||
if text is None or len(text) <= 0:
|
||||
self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
|
||||
return None, text, text_index
|
||||
tts_file = self.tts.to_tts(text)
|
||||
if tts_file is None:
|
||||
self.logger.bind(tag=TAG).error(f"tts转换失败，{text}")
|
||||
return None, text, text_index
|
||||
self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}")
|
||||
return tts_file, text, text_index
|
||||
|
||||
def clearSpeakStatus(self):
|
||||
self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
|
||||
self.asr_server_receive = True
|
||||
self.tts_last_text_index = -1
|
||||
self.tts_first_text_index = -1
|
||||
|
||||
def recode_first_last_text(self, text, text_index=0):
|
||||
if self.tts_first_text_index == -1:
|
||||
self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}")
|
||||
self.tts_first_text_index = text_index
|
||||
self.tts_last_text_index = text_index
|
||||
|
||||
async def close(self):
|
||||
"""资源清理方法"""
|
||||
|
||||
# 清理其他资源
|
||||
self.stop_event.set()
|
||||
self.executor.shutdown(wait=False)
|
||||
if self.websocket:
|
||||
await self.websocket.close()
|
||||
self.logger.bind(tag=TAG).info("连接资源已释放")
|
||||
|
||||
def reset_vad_states(self):
|
||||
self.client_audio_buffer = bytes()
|
||||
self.client_have_voice = False
|
||||
self.client_have_voice_last_time = 0
|
||||
self.client_voice_stop = False
|
||||
self.logger.bind(tag=TAG).debug("VAD states reset.")
|
||||
@@ -0,0 +1,16 @@
|
||||
import json
|
||||
import queue
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleAbortMessage(conn):
|
||||
logger.bind(tag=TAG).info("Abort message received")
|
||||
# 设置成打断状态，会自动打断llm、tts任务
|
||||
conn.client_abort = True
|
||||
# 打断客户端说话状态
|
||||
await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
|
||||
conn.clearSpeakStatus()
|
||||
logger.bind(tag=TAG).info("Abort message received-end")
|
||||
@@ -0,0 +1,8 @@
|
||||
import json
|
||||
from config.logger import setup_logging
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleHelloMessage(conn):
|
||||
await conn.websocket.send(json.dumps(conn.welcome_msg))
|
||||
@@ -0,0 +1,152 @@
|
||||
import json
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class IotDescriptor:
|
||||
"""
|
||||
A class to represent an IoT descriptor.
|
||||
Attributes:
|
||||
----------
|
||||
name : str
|
||||
The name of the IoT descriptor.
|
||||
description : str
|
||||
A brief description of the IoT descriptor.
|
||||
properties : dict
|
||||
A dictionary containing properties of the IoT descriptor.
|
||||
methods : dict
|
||||
A dictionary containing methods of the IoT descriptor.
|
||||
-------
|
||||
"""
|
||||
|
||||
def __init__(self, name, description, properties, methods):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.properties = []
|
||||
self.methods = []
|
||||
|
||||
# 根据描述创建属性
|
||||
for key, value in properties.items():
|
||||
# "volume":{"description":"当前音量 值","type":"number"}
|
||||
"""
|
||||
等价于
|
||||
{
|
||||
'name': 名字,
|
||||
'description': 描述,
|
||||
'value': 0
|
||||
}
|
||||
"""
|
||||
# setattr(self, key, {}) # 创建一个空字典, 名字是属性名
|
||||
property_item = globals()[key] = {} # 创建一个空字典, 名字是属性名
|
||||
property_item['name'] = key
|
||||
property_item["description"] = value["description"]
|
||||
if value["type"] == "number":
|
||||
property_item["value"] = 0
|
||||
elif value["type"] == "boolean":
|
||||
property_item["value"] = False
|
||||
else:
|
||||
property_item["value"] = ""
|
||||
self.properties.append(property_item)
|
||||
|
||||
# 根据描述创建方法
|
||||
for key, value in methods.items():
|
||||
# "SetVolume": {"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}}
|
||||
"""
|
||||
等价于
|
||||
SetVolume = {
|
||||
`description`: 描述,
|
||||
`volume`: {
|
||||
`description`: 描述,
|
||||
`value`: 0
|
||||
}
|
||||
}
|
||||
"""
|
||||
# setattr(self, key, {}) # 创建一个空字典, 名字是方法名
|
||||
method = globals()[key] = {} # 创建一个空字典, 名字是方法名
|
||||
method["description"] = value["description"]
|
||||
method['name'] = key
|
||||
for k, v in value["parameters"].items():
|
||||
# 不同的参数解析
|
||||
method[k] = {}
|
||||
method[k]["description"] = v["description"]
|
||||
if v["type"] == "number":
|
||||
method[k]["value"] = 0
|
||||
elif v["type"] == "boolean":
|
||||
method[k]["value"] = False
|
||||
else:
|
||||
method[k]["value"] = ""
|
||||
|
||||
self.methods.append(method)
|
||||
|
||||
|
||||
async def handleIotDescriptors(conn, descriptors):
|
||||
"""
|
||||
处理物联网描述
|
||||
示例: [{
|
||||
"name":"Speaker",
|
||||
"description":"当前 AI 机器人的扬声器",
|
||||
"properties":{
|
||||
"volume":{"description":"当前音量 值","type":"number"} 可以有boolean, number, string三种类型
|
||||
},
|
||||
"methods":{
|
||||
"SetVolume":{
|
||||
"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}
|
||||
}
|
||||
}
|
||||
}]
|
||||
descriptors: 描述列表
|
||||
"""
|
||||
for descriptor in descriptors:
|
||||
iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
|
||||
descriptor["methods"])
|
||||
conn.iot_descriptors[descriptor["name"]] = iot_descriptor
|
||||
|
||||
# 暂时从配置文件中设置音量，后期通过意图识别控制音量
|
||||
default_iot_volume = 100
|
||||
if "iot" in conn.config:
|
||||
default_iot_volume = conn.config["iot"]["Speaker"]["volume"]
|
||||
logger.bind(tag=TAG).info(f"服务端设置音量为{default_iot_volume}")
|
||||
await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": default_iot_volume})
|
||||
|
||||
|
||||
async def send_iot_conn(conn, name, method_name, parameters):
|
||||
"""
|
||||
发送物联网指令
|
||||
name: 设备名称 "Speaker"
|
||||
method: 方法 "SetVolume"
|
||||
parameters: 参数, 是一个字典 {"volume": 100}
|
||||
发送示例:
|
||||
{
|
||||
"type": "iot",
|
||||
"commands": [
|
||||
{
|
||||
"name" : "Speaker",
|
||||
"method": "SetVolume",
|
||||
"parameters": {
|
||||
"volume": 100
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
for key, value in conn.iot_descriptors.items():
|
||||
if key == name:
|
||||
# 找到了设备
|
||||
for method in value.methods:
|
||||
# 找到了方法
|
||||
if method["name"] == method_name:
|
||||
await conn.websocket.send(json.dumps({
|
||||
"type": "iot",
|
||||
"commands": [
|
||||
{
|
||||
"name": name,
|
||||
"method": method_name,
|
||||
"parameters": parameters
|
||||
}
|
||||
]
|
||||
}))
|
||||
return
|
||||
logger.bind(tag=TAG).error(f"未找到方法{method_name}")
|
||||
@@ -0,0 +1,153 @@
|
||||
from config.logger import setup_logging
|
||||
import os
|
||||
import random
|
||||
import difflib
|
||||
import re
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
import time
|
||||
from core.handle.sendAudioHandle import send_stt_message
|
||||
from core.utils import p3
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
def _extract_song_name(text):
|
||||
"""从用户输入中提取歌名"""
|
||||
for keyword in ["听", "播放", "放", "唱"]:
|
||||
if keyword in text:
|
||||
parts = text.split(keyword)
|
||||
if len(parts) > 1:
|
||||
return parts[1].strip()
|
||||
return None
|
||||
|
||||
|
||||
def _find_best_match(potential_song, music_files):
|
||||
"""查找最匹配的歌曲"""
|
||||
best_match = None
|
||||
highest_ratio = 0
|
||||
|
||||
for music_file in music_files:
|
||||
song_name = os.path.splitext(music_file)[0]
|
||||
ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
|
||||
if ratio > highest_ratio and ratio > 0.4:
|
||||
highest_ratio = ratio
|
||||
best_match = music_file
|
||||
return best_match
|
||||
|
||||
class MusicManager:
|
||||
def __init__(self, music_dir, music_ext):
|
||||
self.music_dir = Path(music_dir)
|
||||
self.music_ext = music_ext
|
||||
|
||||
def get_music_files(self):
|
||||
music_files = []
|
||||
for file in self.music_dir.rglob("*"):
|
||||
# 判断是否是文件
|
||||
if file.is_file():
|
||||
# 获取文件扩展名
|
||||
ext = file.suffix.lower()
|
||||
# 判断扩展名是否在列表中
|
||||
if ext in self.music_ext:
|
||||
# music_files.append(str(file.resolve())) # 添加绝对路径
|
||||
# 添加相对路径
|
||||
music_files.append(str(file.relative_to(self.music_dir)))
|
||||
return music_files
|
||||
|
||||
class MusicHandler:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.music_related_keywords = []
|
||||
|
||||
if "music" in self.config:
|
||||
self.music_config = self.config["music"]
|
||||
self.music_dir = os.path.abspath(
|
||||
self.music_config.get("music_dir", "./music") # 默认路径修改
|
||||
)
|
||||
self.music_related_keywords = self.music_config.get("music_commands", [])
|
||||
self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
|
||||
self.refresh_time = self.music_config.get("refresh_time", 60)
|
||||
else:
|
||||
self.music_dir = os.path.abspath("./music")
|
||||
self.music_related_keywords = ["来一首歌", "唱一首歌", "播放音乐", "来点音乐", "背景音乐", "放首歌",
|
||||
"播放歌曲", "来点背景音乐", "我想听歌", "我要听歌", "放点音乐"]
|
||||
self.music_ext = (".mp3", ".wav", ".p3")
|
||||
self.refresh_time = 60
|
||||
|
||||
# 获取音乐文件列表
|
||||
self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
|
||||
self.scan_time = time.time()
|
||||
logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}")
|
||||
|
||||
async def handle_music_command(self, conn, text):
|
||||
"""处理音乐播放指令"""
|
||||
clean_text = re.sub(r'[^\w\s]', '', text).strip()
|
||||
logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
|
||||
|
||||
# 尝试匹配具体歌名
|
||||
if os.path.exists(self.music_dir):
|
||||
if time.time() - self.scan_time > self.refresh_time:
|
||||
# 刷新音乐文件列表
|
||||
self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
|
||||
self.scan_time = time.time()
|
||||
logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}")
|
||||
|
||||
potential_song = _extract_song_name(clean_text)
|
||||
if potential_song:
|
||||
best_match = _find_best_match(potential_song, self.music_files)
|
||||
if best_match:
|
||||
logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
|
||||
await self.play_local_music(conn, specific_file=best_match)
|
||||
return True
|
||||
|
||||
# 检查是否是通用播放音乐命令
|
||||
if any(cmd in clean_text for cmd in self.music_related_keywords):
|
||||
await self.play_local_music(conn)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
async def play_local_music(self, conn, specific_file=None):
|
||||
"""播放本地音乐文件"""
|
||||
try:
|
||||
if not os.path.exists(self.music_dir):
|
||||
logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}")
|
||||
return
|
||||
|
||||
# 确保路径正确性
|
||||
if specific_file:
|
||||
music_path = os.path.join(self.music_dir, specific_file)
|
||||
if not os.path.exists(music_path):
|
||||
logger.bind(tag=TAG).error(f"指定的音乐文件不存在: {music_path}")
|
||||
return
|
||||
selected_music = specific_file
|
||||
else:
|
||||
if time.time() - self.scan_time > self.refresh_time:
|
||||
# 刷新音乐文件列表
|
||||
self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
|
||||
self.scan_time = time.time()
|
||||
logger.bind(tag=TAG).debug(f"刷新的音乐文件列表: {self.music_files}")
|
||||
|
||||
if not self.music_files:
|
||||
logger.bind(tag=TAG).error("未找到MP3音乐文件")
|
||||
return
|
||||
selected_music = random.choice(self.music_files)
|
||||
music_path = os.path.join(self.music_dir, selected_music)
|
||||
if not os.path.exists(music_path):
|
||||
logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
|
||||
return
|
||||
text = f"正在播放{selected_music}"
|
||||
await send_stt_message(conn, text)
|
||||
conn.tts_first_text_index = 0
|
||||
conn.tts_last_text_index = 0
|
||||
conn.llm_finish_task = True
|
||||
if music_path.endswith(".p3"):
|
||||
opus_packets, duration = p3.decode_opus_from_file(music_path)
|
||||
else:
|
||||
opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
|
||||
conn.audio_play_queue.put((opus_packets, selected_music, 0))
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
|
||||
logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
|
||||
@@ -0,0 +1,77 @@
|
||||
from config.logger import setup_logging
|
||||
import time
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from core.handle.sendAudioHandle import send_stt_message
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleAudioMessage(conn, audio):
|
||||
if not conn.asr_server_receive:
|
||||
logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
|
||||
return
|
||||
if conn.client_listen_mode == "auto":
|
||||
have_voice = conn.vad.is_vad(conn, audio)
|
||||
else:
|
||||
have_voice = conn.client_have_voice
|
||||
|
||||
# 如果本次没有声音，本段也没声音，就把声音丢弃了
|
||||
if have_voice == False and conn.client_have_voice == False:
|
||||
await no_voice_close_connect(conn)
|
||||
conn.asr_audio.clear()
|
||||
return
|
||||
conn.client_no_voice_last_time = 0.0
|
||||
conn.asr_audio.append(audio)
|
||||
# 如果本段有声音，且已经停止了
|
||||
if conn.client_voice_stop:
|
||||
conn.client_abort = False
|
||||
conn.asr_server_receive = False
|
||||
# 音频太短了，无法识别
|
||||
if len(conn.asr_audio) < 3:
|
||||
conn.asr_server_receive = True
|
||||
else:
|
||||
text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
|
||||
logger.bind(tag=TAG).info(f"识别文本: {text}")
|
||||
text_len, text_without_punctuation = remove_punctuation_and_length(text)
|
||||
if await conn.music_handler.handle_music_command(conn, text_without_punctuation):
|
||||
conn.asr_server_receive = True
|
||||
conn.asr_audio.clear()
|
||||
return
|
||||
if text_len <= conn.max_cmd_length and await handleCMDMessage(conn, text_without_punctuation):
|
||||
return
|
||||
if text_len > 0:
|
||||
await startToChat(conn, text)
|
||||
else:
|
||||
conn.asr_server_receive = True
|
||||
conn.asr_audio.clear()
|
||||
conn.reset_vad_states()
|
||||
|
||||
|
||||
async def handleCMDMessage(conn, text):
|
||||
cmd_exit = conn.cmd_exit
|
||||
for cmd in cmd_exit:
|
||||
if text == cmd:
|
||||
logger.bind(tag=TAG).info("识别到明确的退出命令".format(text))
|
||||
await conn.close()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def startToChat(conn, text):
|
||||
# 异步发送 stt 信息
|
||||
await send_stt_message(conn, text)
|
||||
conn.executor.submit(conn.chat, text)
|
||||
|
||||
|
||||
async def no_voice_close_connect(conn):
|
||||
if conn.client_no_voice_last_time == 0.0:
|
||||
conn.client_no_voice_last_time = time.time() * 1000
|
||||
else:
|
||||
no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
|
||||
close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
|
||||
if no_voice_time > 1000 * close_connection_no_voice_time:
|
||||
conn.client_abort = False
|
||||
conn.asr_server_receive = False
|
||||
prompt = "时间过得真快，我都好久没说话了。请你用十个字左右话跟我告别，以“再见”或“拜拜”为结尾"
|
||||
await startToChat(conn, prompt)
|
||||
@@ -0,0 +1,83 @@
|
||||
from config.logger import setup_logging
|
||||
import json
|
||||
import asyncio
|
||||
import time
|
||||
from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def isLLMWantToFinish(last_text):
|
||||
_, last_text_without_punctuation = remove_punctuation_and_length(last_text)
|
||||
if "再见" in last_text_without_punctuation or "拜拜" in last_text_without_punctuation:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def sendAudioMessage(conn, audios, text, text_index=0):
|
||||
# 发送句子开始消息
|
||||
if text_index == conn.tts_first_text_index:
|
||||
logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
|
||||
await send_tts_message(conn, "sentence_start", text)
|
||||
|
||||
# 初始化流控参数
|
||||
frame_duration = 60 # 毫秒
|
||||
start_time = time.perf_counter() # 使用高精度计时器
|
||||
play_position = 0 # 已播放的时长（毫秒）
|
||||
|
||||
for opus_packet in audios:
|
||||
if conn.client_abort:
|
||||
return
|
||||
|
||||
# 计算当前包的预期发送时间
|
||||
expected_time = start_time + (play_position / 1000)
|
||||
current_time = time.perf_counter()
|
||||
|
||||
# 等待直到预期时间
|
||||
delay = expected_time - current_time
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
# 发送音频包
|
||||
await conn.websocket.send(opus_packet)
|
||||
play_position += frame_duration # 更新播放位置
|
||||
await send_tts_message(conn, "sentence_end", text)
|
||||
# 发送结束消息（如果是最后一个文本）
|
||||
if conn.llm_finish_task and text_index == conn.tts_last_text_index:
|
||||
await send_tts_message(conn, 'stop', None)
|
||||
if await isLLMWantToFinish(text):
|
||||
await conn.close()
|
||||
|
||||
|
||||
async def send_tts_message(conn, state, text=None):
|
||||
"""发送 TTS 状态消息"""
|
||||
message = {
|
||||
"type": "tts",
|
||||
"state": state,
|
||||
"session_id": conn.session_id
|
||||
}
|
||||
if text is not None:
|
||||
message["text"] = text
|
||||
|
||||
await conn.websocket.send(json.dumps(message))
|
||||
if state == "stop":
|
||||
conn.clearSpeakStatus()
|
||||
|
||||
|
||||
async def send_stt_message(conn, text):
|
||||
"""发送 STT 状态消息"""
|
||||
stt_text = get_string_no_punctuation_or_emoji(text)
|
||||
await conn.websocket.send(json.dumps({
|
||||
"type": "stt",
|
||||
"text": stt_text,
|
||||
"session_id": conn.session_id}
|
||||
))
|
||||
await conn.websocket.send(
|
||||
json.dumps({
|
||||
"type": "llm",
|
||||
"text": "😊",
|
||||
"emotion": "happy",
|
||||
"session_id": conn.session_id}
|
||||
))
|
||||
await send_tts_message(conn, "start")
|
||||
@@ -0,0 +1,44 @@
|
||||
from config.logger import setup_logging
|
||||
import json
|
||||
from core.handle.abortHandle import handleAbortMessage
|
||||
from core.handle.helloHandle import handleHelloMessage
|
||||
from core.handle.receiveAudioHandle import startToChat
|
||||
from core.handle.iotHandle import handleIotDescriptors
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def handleTextMessage(conn, message):
|
||||
"""处理文本消息"""
|
||||
logger.bind(tag=TAG).info(f"收到文本消息：{message}")
|
||||
try:
|
||||
msg_json = json.loads(message)
|
||||
if isinstance(msg_json, int):
|
||||
await conn.websocket.send(message)
|
||||
return
|
||||
if msg_json["type"] == "hello":
|
||||
await handleHelloMessage(conn)
|
||||
elif msg_json["type"] == "abort":
|
||||
await handleAbortMessage(conn)
|
||||
elif msg_json["type"] == "listen":
|
||||
if "mode" in msg_json:
|
||||
conn.client_listen_mode = msg_json["mode"]
|
||||
logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}")
|
||||
if msg_json["state"] == "start":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = False
|
||||
elif msg_json["state"] == "stop":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = True
|
||||
elif msg_json["state"] == "detect":
|
||||
conn.asr_server_receive = False
|
||||
conn.client_have_voice = False
|
||||
conn.asr_audio.clear()
|
||||
if "text" in msg_json:
|
||||
await startToChat(conn, msg_json["text"])
|
||||
elif msg_json["type"] == "iot":
|
||||
if "descriptors" in msg_json:
|
||||
await handleIotDescriptors(conn, msg_json["descriptors"])
|
||||
except json.JSONDecodeError:
|
||||
await conn.websocket.send(message)
|
||||
@@ -0,0 +1,19 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional, Tuple, List
|
||||
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ASRProviderBase(ABC):
|
||||
@abstractmethod
|
||||
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
|
||||
"""解码Opus数据并保存为WAV文件"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
pass
|
||||
@@ -0,0 +1,286 @@
|
||||
import time
|
||||
import io
|
||||
import wave
|
||||
import os
|
||||
from typing import Optional, Tuple, List
|
||||
import uuid
|
||||
import websockets
|
||||
import json
|
||||
import gzip
|
||||
|
||||
import opuslib_next
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
CLIENT_FULL_REQUEST = 0b0001
|
||||
CLIENT_AUDIO_ONLY_REQUEST = 0b0010
|
||||
|
||||
NO_SEQUENCE = 0b0000
|
||||
NEG_SEQUENCE = 0b0010
|
||||
|
||||
SERVER_FULL_RESPONSE = 0b1001
|
||||
SERVER_ACK = 0b1011
|
||||
SERVER_ERROR_RESPONSE = 0b1111
|
||||
|
||||
NO_SERIALIZATION = 0b0000
|
||||
JSON = 0b0001
|
||||
THRIFT = 0b0011
|
||||
CUSTOM_TYPE = 0b1111
|
||||
NO_COMPRESSION = 0b0000
|
||||
GZIP = 0b0001
|
||||
CUSTOM_COMPRESSION = 0b1111
|
||||
|
||||
|
||||
def parse_response(res):
|
||||
"""
|
||||
protocol_version(4 bits), header_size(4 bits),
|
||||
message_type(4 bits), message_type_specific_flags(4 bits)
|
||||
serialization_method(4 bits) message_compression(4 bits)
|
||||
reserved （8bits) 保留字段
|
||||
header_extensions 扩展头(大小等于 8 * 4 * (header_size - 1) )
|
||||
payload 类似与http 请求体
|
||||
"""
|
||||
protocol_version = res[0] >> 4
|
||||
header_size = res[0] & 0x0f
|
||||
message_type = res[1] >> 4
|
||||
message_type_specific_flags = res[1] & 0x0f
|
||||
serialization_method = res[2] >> 4
|
||||
message_compression = res[2] & 0x0f
|
||||
reserved = res[3]
|
||||
header_extensions = res[4:header_size * 4]
|
||||
payload = res[header_size * 4:]
|
||||
result = {}
|
||||
payload_msg = None
|
||||
payload_size = 0
|
||||
if message_type == SERVER_FULL_RESPONSE:
|
||||
payload_size = int.from_bytes(payload[:4], "big", signed=True)
|
||||
payload_msg = payload[4:]
|
||||
elif message_type == SERVER_ACK:
|
||||
seq = int.from_bytes(payload[:4], "big", signed=True)
|
||||
result['seq'] = seq
|
||||
if len(payload) >= 8:
|
||||
payload_size = int.from_bytes(payload[4:8], "big", signed=False)
|
||||
payload_msg = payload[8:]
|
||||
elif message_type == SERVER_ERROR_RESPONSE:
|
||||
code = int.from_bytes(payload[:4], "big", signed=False)
|
||||
result['code'] = code
|
||||
payload_size = int.from_bytes(payload[4:8], "big", signed=False)
|
||||
payload_msg = payload[8:]
|
||||
if payload_msg is None:
|
||||
return result
|
||||
if message_compression == GZIP:
|
||||
payload_msg = gzip.decompress(payload_msg)
|
||||
if serialization_method == JSON:
|
||||
payload_msg = json.loads(str(payload_msg, "utf-8"))
|
||||
elif serialization_method != NO_SERIALIZATION:
|
||||
payload_msg = str(payload_msg, "utf-8")
|
||||
result['payload_msg'] = payload_msg
|
||||
result['payload_size'] = payload_size
|
||||
return result
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool):
|
||||
self.appid = config.get("appid")
|
||||
self.cluster = config.get("cluster")
|
||||
self.access_token = config.get("access_token")
|
||||
self.output_dir = config.get("output_dir")
|
||||
|
||||
self.host = "openspeech.bytedance.com"
|
||||
self.ws_url = f"wss://{self.host}/api/v2/asr"
|
||||
self.success_code = 1000
|
||||
self.seg_duration = 15000
|
||||
|
||||
# 确保输出目录存在
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
|
||||
"""将Opus音频数据解码并保存为WAV文件"""
|
||||
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
|
||||
file_path = os.path.join(self.output_dir, file_name)
|
||||
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
with wave.open(file_path, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2) # 2 bytes = 16-bit
|
||||
wf.setframerate(16000)
|
||||
wf.writeframes(b"".join(pcm_data))
|
||||
|
||||
return file_path
|
||||
|
||||
@staticmethod
|
||||
def _generate_header(message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE) -> bytearray:
|
||||
"""Generate protocol header."""
|
||||
header = bytearray()
|
||||
header_size = 1
|
||||
header.append((0b0001 << 4) | header_size) # Protocol version
|
||||
header.append((message_type << 4) | message_type_specific_flags)
|
||||
header.append((0b0001 << 4) | 0b0001) # JSON serialization & GZIP compression
|
||||
header.append(0x00) # reserved
|
||||
return header
|
||||
|
||||
def _construct_request(self, reqid) -> dict:
|
||||
"""Construct the request payload."""
|
||||
return {
|
||||
"app": {
|
||||
"appid": f"{self.appid}",
|
||||
"cluster": self.cluster,
|
||||
"token": self.access_token,
|
||||
},
|
||||
"user": {
|
||||
"uid": str(uuid.uuid4()),
|
||||
},
|
||||
"request": {
|
||||
"reqid": reqid,
|
||||
"show_utterances": False,
|
||||
"sequence": 1
|
||||
},
|
||||
"audio": {
|
||||
"format": "wav",
|
||||
"rate": 16000,
|
||||
"language": "zh-CN",
|
||||
"bits": 16,
|
||||
"channel": 1,
|
||||
"codec": "raw",
|
||||
},
|
||||
}
|
||||
|
||||
async def _send_request(self, audio_data: List[bytes], segment_size: int) -> Optional[str]:
|
||||
"""Send request to Volcano ASR service."""
|
||||
try:
|
||||
auth_header = {'Authorization': 'Bearer; {}'.format(self.access_token)}
|
||||
async with websockets.connect(self.ws_url, additional_headers=auth_header) as websocket:
|
||||
# Prepare request data
|
||||
request_params = self._construct_request(str(uuid.uuid4()))
|
||||
print(request_params)
|
||||
payload_bytes = str.encode(json.dumps(request_params))
|
||||
payload_bytes = gzip.compress(payload_bytes)
|
||||
full_client_request = self._generate_header()
|
||||
full_client_request.extend((len(payload_bytes)).to_bytes(4, 'big')) # payload size(4 bytes)
|
||||
full_client_request.extend(payload_bytes) # payload
|
||||
|
||||
# Send header and metadata
|
||||
# full_client_request
|
||||
await websocket.send(full_client_request)
|
||||
res = await websocket.recv()
|
||||
result = parse_response(res)
|
||||
if 'payload_msg' in result and result['payload_msg']['code'] != self.success_code:
|
||||
logger.bind(tag=TAG).error(f"ASR error: {result}")
|
||||
return None
|
||||
|
||||
for seq, (chunk, last) in enumerate(self.slice_data(audio_data, segment_size), 1):
|
||||
if last:
|
||||
audio_only_request = self._generate_header(
|
||||
message_type=CLIENT_AUDIO_ONLY_REQUEST,
|
||||
message_type_specific_flags=NEG_SEQUENCE
|
||||
)
|
||||
else:
|
||||
audio_only_request = self._generate_header(
|
||||
message_type=CLIENT_AUDIO_ONLY_REQUEST
|
||||
)
|
||||
payload_bytes = gzip.compress(chunk)
|
||||
audio_only_request.extend((len(payload_bytes)).to_bytes(4, 'big')) # payload size(4 bytes)
|
||||
audio_only_request.extend(payload_bytes) # payload
|
||||
# Send audio data
|
||||
await websocket.send(audio_only_request)
|
||||
|
||||
# Receive response
|
||||
response = await websocket.recv()
|
||||
result = parse_response(response)
|
||||
|
||||
if 'payload_msg' in result and result['payload_msg']['code'] == self.success_code:
|
||||
if len(result['payload_msg']['result']) > 0:
|
||||
return result['payload_msg']['result'][0]["text"]
|
||||
return None
|
||||
else:
|
||||
logger.bind(tag=TAG).error(f"ASR error: {result}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
|
||||
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
return pcm_data
|
||||
|
||||
@staticmethod
|
||||
def read_wav_info(data: io.BytesIO = None) -> (int, int, int, int, int):
|
||||
with io.BytesIO(data) as _f:
|
||||
wave_fp = wave.open(_f, 'rb')
|
||||
nchannels, sampwidth, framerate, nframes = wave_fp.getparams()[:4]
|
||||
wave_bytes = wave_fp.readframes(nframes)
|
||||
return nchannels, sampwidth, framerate, nframes, len(wave_bytes)
|
||||
|
||||
@staticmethod
|
||||
def slice_data(data: bytes, chunk_size: int) -> (list, bool):
|
||||
"""
|
||||
slice data
|
||||
:param data: wav data
|
||||
:param chunk_size: the segment size in one request
|
||||
:return: segment data, last flag
|
||||
"""
|
||||
data_len = len(data)
|
||||
offset = 0
|
||||
while offset + chunk_size < data_len:
|
||||
yield data[offset: offset + chunk_size], False
|
||||
offset += chunk_size
|
||||
else:
|
||||
yield data[offset: data_len], True
|
||||
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
try:
|
||||
# 合并所有opus数据包
|
||||
pcm_data = self.decode_opus(opus_data, session_id)
|
||||
combined_pcm_data = b''.join(pcm_data)
|
||||
|
||||
wav_buffer = io.BytesIO()
|
||||
|
||||
with wave.open(wav_buffer, "wb") as wav_file:
|
||||
wav_file.setnchannels(1) # 设置声道数
|
||||
wav_file.setsampwidth(2) # 设置采样宽度
|
||||
wav_file.setframerate(16000) # 设置采样率
|
||||
wav_file.writeframes(combined_pcm_data) # 写入 PCM 数据
|
||||
|
||||
# 获取封装后的 WAV 数据
|
||||
wav_data = wav_buffer.getvalue()
|
||||
nchannels, sampwidth, framerate, nframes, wav_len = self.read_wav_info(wav_data)
|
||||
size_per_sec = nchannels * sampwidth * framerate
|
||||
segment_size = int(size_per_sec * self.seg_duration / 1000)
|
||||
|
||||
# 语音识别
|
||||
start_time = time.time()
|
||||
text = await self._send_request(wav_data, segment_size)
|
||||
if text:
|
||||
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
|
||||
return text, None
|
||||
return "", None
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", None
|
||||
@@ -0,0 +1,110 @@
|
||||
import time
|
||||
import wave
|
||||
import os
|
||||
import sys
|
||||
import io
|
||||
from config.logger import setup_logging
|
||||
from typing import Optional, Tuple, List
|
||||
import uuid
|
||||
import opuslib_next
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
|
||||
from funasr import AutoModel
|
||||
from funasr.utils.postprocess_utils import rich_transcription_postprocess
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
# 捕获标准输出
|
||||
class CaptureOutput:
|
||||
def __enter__(self):
|
||||
self._output = io.StringIO()
|
||||
self._original_stdout = sys.stdout
|
||||
sys.stdout = self._output
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
sys.stdout = self._original_stdout
|
||||
self.output = self._output.getvalue()
|
||||
self._output.close()
|
||||
|
||||
# 将捕获到的内容通过 logger 输出
|
||||
if self.output:
|
||||
logger.bind(tag=TAG).info(self.output.strip())
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool):
|
||||
self.model_dir = config.get("model_dir")
|
||||
self.output_dir = config.get("output_dir") # 修正配置键名
|
||||
self.delete_audio_file = delete_audio_file
|
||||
|
||||
# 确保输出目录存在
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
with CaptureOutput():
|
||||
self.model = AutoModel(
|
||||
model=self.model_dir,
|
||||
vad_kwargs={"max_single_segment_time": 30000},
|
||||
disable_update=True,
|
||||
hub="hf"
|
||||
# device="cuda:0", # 启用GPU加速
|
||||
)
|
||||
|
||||
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
|
||||
"""将Opus音频数据解码并保存为WAV文件"""
|
||||
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
|
||||
file_path = os.path.join(self.output_dir, file_name)
|
||||
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
with wave.open(file_path, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2) # 2 bytes = 16-bit
|
||||
wf.setframerate(16000)
|
||||
wf.writeframes(b"".join(pcm_data))
|
||||
|
||||
return file_path
|
||||
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""语音转文本主处理逻辑"""
|
||||
file_path = None
|
||||
try:
|
||||
# 保存音频文件
|
||||
start_time = time.time()
|
||||
file_path = self.save_audio_to_file(opus_data, session_id)
|
||||
logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
|
||||
|
||||
# 语音识别
|
||||
start_time = time.time()
|
||||
result = self.model.generate(
|
||||
input=file_path,
|
||||
cache={},
|
||||
language="auto",
|
||||
use_itn=True,
|
||||
batch_size_s=60,
|
||||
)
|
||||
text = rich_transcription_postprocess(result[0]["text"])
|
||||
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
|
||||
|
||||
return text, file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", None
|
||||
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
|
||||
@@ -0,0 +1,8 @@
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class LLMProviderBase(ABC):
|
||||
@abstractmethod
|
||||
def response(self, session_id, dialogue):
|
||||
"""LLM response generator"""
|
||||
pass
|
||||
@@ -0,0 +1,37 @@
|
||||
from config.logger import setup_logging
|
||||
import requests
|
||||
import json
|
||||
import re
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
import os
|
||||
# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
|
||||
from cozepy import COZE_CN_BASE_URL
|
||||
from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType # noqa
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.personal_access_token = config.get("personal_access_token")
|
||||
self.bot_id = config.get("bot_id")
|
||||
self.user_id = config.get("user_id")
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
coze_api_token = self.personal_access_token
|
||||
coze_api_base = COZE_CN_BASE_URL
|
||||
|
||||
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
|
||||
|
||||
coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
|
||||
|
||||
for event in coze.chat.stream(
|
||||
bot_id=self.bot_id,
|
||||
user_id=self.user_id,
|
||||
additional_messages=[
|
||||
Message.build_user_question_text(last_msg["content"]),
|
||||
],
|
||||
):
|
||||
if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
|
||||
print(event.message.content, end="", flush=True)
|
||||
yield event.message.content
|
||||
@@ -0,0 +1,39 @@
|
||||
import json
|
||||
from config.logger import setup_logging
|
||||
import requests
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.api_key = config["api_key"]
|
||||
self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
try:
|
||||
# 取最后一条用户消息
|
||||
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
|
||||
|
||||
# 发起流式请求
|
||||
with requests.post(
|
||||
f"{self.base_url}/chat-messages",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json={
|
||||
"query": last_msg["content"],
|
||||
"response_mode": "streaming",
|
||||
"user": session_id,
|
||||
"inputs": {}
|
||||
},
|
||||
stream=True
|
||||
) as r:
|
||||
for line in r.iter_lines():
|
||||
if line.startswith(b'data: '):
|
||||
event = json.loads(line[6:])
|
||||
if event.get('answer'):
|
||||
yield event['answer']
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
|
||||
yield "【服务响应异常】"
|
||||
@@ -0,0 +1,81 @@
|
||||
from config.logger import setup_logging
|
||||
import google.generativeai as genai
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
"""初始化Gemini LLM Provider"""
|
||||
self.model_name = config.get("model_name", "gemini-1.5-pro")
|
||||
self.api_key = config.get("api_key")
|
||||
|
||||
if not self.api_key or "你" in self.api_key:
|
||||
logger.bind(tag=TAG).error("你还没配置Gemini LLM的密钥，请在配置文件中配置密钥，否则无法正常工作")
|
||||
return
|
||||
|
||||
try:
|
||||
# 初始化Gemini客户端
|
||||
genai.configure(api_key=self.api_key)
|
||||
self.model = genai.GenerativeModel(self.model_name)
|
||||
|
||||
# 设置生成参数
|
||||
self.generation_config = {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.9,
|
||||
"top_k": 40,
|
||||
"max_output_tokens": 2048,
|
||||
}
|
||||
self.chat = None
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Gemini初始化失败: {e}")
|
||||
self.model = None
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
"""生成Gemini对话响应"""
|
||||
if not self.model:
|
||||
yield "【Gemini服务未正确初始化】"
|
||||
return
|
||||
|
||||
try:
|
||||
# 处理对话历史
|
||||
chat_history = []
|
||||
for msg in dialogue[:-1]: # 历史对话
|
||||
role = "model" if msg["role"] == "assistant" else "user"
|
||||
content = msg["content"].strip()
|
||||
if content:
|
||||
chat_history.append({
|
||||
"role": role,
|
||||
"parts": [content]
|
||||
})
|
||||
|
||||
# 获取当前消息
|
||||
current_msg = dialogue[-1]["content"]
|
||||
|
||||
# 创建新的聊天会话
|
||||
chat = self.model.start_chat(history=chat_history)
|
||||
|
||||
# 发送消息并获取流式响应
|
||||
response = chat.send_message(
|
||||
current_msg,
|
||||
stream=True,
|
||||
generation_config=self.generation_config
|
||||
)
|
||||
|
||||
# 处理流式响应
|
||||
for chunk in response:
|
||||
if hasattr(chunk, 'text') and chunk.text:
|
||||
yield chunk.text
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}")
|
||||
|
||||
# 针对不同错误返回友好提示
|
||||
if "Rate limit" in error_msg:
|
||||
yield "【Gemini服务请求太频繁,请稍后再试】"
|
||||
elif "Invalid API key" in error_msg:
|
||||
yield "【Gemini API key无效】"
|
||||
else:
|
||||
yield f"【Gemini服务响应异常: {error_msg}】"
|
||||
@@ -0,0 +1,62 @@
|
||||
import requests
|
||||
from requests.exceptions import RequestException
|
||||
from config.logger import setup_logging
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.agent_id = config.get("agent_id") # 对应 agent_id
|
||||
self.api_key = config.get("api_key")
|
||||
self.base_url = config.get("base_url", config.get("url")) # 默认使用 base_url
|
||||
self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
print(dialogue)
|
||||
try:
|
||||
# home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
|
||||
|
||||
# 提取最后一个 role 为 'user' 的 content
|
||||
input_text = None
|
||||
if isinstance(dialogue, list): # 确保 dialogue 是一个列表
|
||||
# 逆序遍历，找到最后一个 role 为 'user' 的消息
|
||||
for message in reversed(dialogue):
|
||||
if message.get("role") == "user": # 找到 role 为 'user' 的消息
|
||||
input_text = message.get("content", "")
|
||||
break # 找到后立即退出循环
|
||||
|
||||
# 构造请求数据
|
||||
payload = {
|
||||
"text": input_text,
|
||||
"agent_id": self.agent_id,
|
||||
"conversation_id": session_id # 使用 session_id 作为 conversation_id
|
||||
}
|
||||
# 设置请求头
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
# 发起 POST 请求
|
||||
response = requests.post(self.api_url, json=payload, headers=headers)
|
||||
|
||||
# 检查请求是否成功
|
||||
response.raise_for_status()
|
||||
|
||||
# 解析返回数据
|
||||
data = response.json()
|
||||
speech = data.get("response", {}).get("speech", {}).get("plain", {}).get("speech", "")
|
||||
|
||||
# 返回生成的内容
|
||||
if speech:
|
||||
yield speech
|
||||
else:
|
||||
logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
|
||||
|
||||
except RequestException as e:
|
||||
logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"生成响应时出错: {e}")
|
||||
@@ -0,0 +1,46 @@
|
||||
from config.logger import setup_logging
|
||||
import requests, json
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
|
||||
self.model_name = config.get("model_name")
|
||||
self.base_url = config.get("base_url", "http://localhost:11434")
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
try:
|
||||
# Convert dialogue format to Ollama format
|
||||
prompt = ""
|
||||
for msg in dialogue:
|
||||
if msg["role"] == "system":
|
||||
prompt += f"System: {msg['content']}\n"
|
||||
elif msg["role"] == "user":
|
||||
prompt += f"User: {msg['content']}\n"
|
||||
elif msg["role"] == "assistant":
|
||||
prompt += f"Assistant: {msg['content']}\n"
|
||||
|
||||
# Make request to Ollama API
|
||||
response = requests.post(
|
||||
f"{self.base_url}/api/generate",
|
||||
json={
|
||||
"model": self.model_name,
|
||||
"prompt": prompt,
|
||||
"stream": True
|
||||
},
|
||||
stream=True
|
||||
)
|
||||
|
||||
for line in response.iter_lines():
|
||||
if line:
|
||||
json_response = json.loads(line)
|
||||
if "response" in json_response:
|
||||
yield json_response["response"]
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in Ollama response generation: {e}")
|
||||
yield "【Ollama服务响应异常】"
|
||||
@@ -0,0 +1,49 @@
|
||||
from config.logger import setup_logging
|
||||
import openai
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.model_name = config.get("model_name")
|
||||
self.api_key = config.get("api_key")
|
||||
if 'base_url' in config:
|
||||
self.base_url = config.get("base_url")
|
||||
else:
|
||||
self.base_url = config.get("url")
|
||||
if "你" in self.api_key:
|
||||
logger.bind(tag=TAG).error("你还没配置LLM的密钥，请在配置文件中配置密钥，否则无法正常工作")
|
||||
self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
try:
|
||||
responses = self.client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=dialogue,
|
||||
stream=True
|
||||
)
|
||||
|
||||
is_active = True
|
||||
for chunk in responses:
|
||||
try:
|
||||
# 检查是否存在有效的choice且content不为空
|
||||
delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
|
||||
content = delta.content if hasattr(delta, 'content') else ''
|
||||
except IndexError:
|
||||
content = ''
|
||||
if content:
|
||||
# 处理标签跨多个chunk的情况
|
||||
if '<think>' in content:
|
||||
is_active = False
|
||||
content = content.split('<think>')[0]
|
||||
if '</think>' in content:
|
||||
is_active = True
|
||||
content = content.split('</think>')[-1]
|
||||
if is_active:
|
||||
yield content
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
|
||||
@@ -0,0 +1,23 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class MemoryProviderBase(ABC):
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.role_id = None
|
||||
|
||||
@abstractmethod
|
||||
async def save_memory(self, msgs):
|
||||
"""Save a new memory for specific role and return memory ID"""
|
||||
print("this is base func", msgs)
|
||||
|
||||
@abstractmethod
|
||||
async def query_memory(self, query: str) -> str:
|
||||
"""Query memories for specific role based on similarity"""
|
||||
return "please implement query method"
|
||||
|
||||
def set_role_id(self, role_id: str):
|
||||
self.role_id = role_id
|
||||
@@ -0,0 +1,74 @@
|
||||
from ..base import MemoryProviderBase, logger
|
||||
from mem0 import MemoryClient
|
||||
|
||||
TAG = __name__
|
||||
|
||||
class MemoryProvider(MemoryProviderBase):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.api_key = config.get("api_key", "")
|
||||
self.api_version = config.get("api_version", "v1.1")
|
||||
if len(self.api_key) == 0 or "你" in self.api_key:
|
||||
logger.bind(tag=TAG).error("你还没配置Mem0ai的密钥，请在配置文件中配置密钥，否则无法提供记忆服务")
|
||||
self.use_mem0 = False
|
||||
return
|
||||
else:
|
||||
self.use_mem0 = True
|
||||
self.client = MemoryClient(api_key=self.api_key)
|
||||
|
||||
async def save_memory(self, msgs):
|
||||
if not self.use_mem0:
|
||||
return None
|
||||
if len(msgs) < 2:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Format the content as a message list for mem0
|
||||
messages = [
|
||||
{"role": message.role, "content": message.content}
|
||||
for message in msgs if message.role != "system"
|
||||
]
|
||||
result = self.client.add(messages, user_id=self.role_id, output_format=self.api_version)
|
||||
logger.bind(tag=TAG).debug(f"Save memory result: {result}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
|
||||
return None
|
||||
|
||||
async def query_memory(self, query: str)-> str:
|
||||
if not self.use_mem0:
|
||||
return ""
|
||||
try:
|
||||
results = self.client.search(
|
||||
query,
|
||||
user_id=self.role_id,
|
||||
output_format=self.api_version
|
||||
)
|
||||
if not results or 'results' not in results:
|
||||
return ""
|
||||
|
||||
# Format each memory entry with its update time up to minutes
|
||||
memories = []
|
||||
for entry in results['results']:
|
||||
timestamp = entry.get('updated_at', '')
|
||||
if timestamp:
|
||||
try:
|
||||
# Parse and reformat the timestamp
|
||||
dt = timestamp.split('.')[0] # Remove milliseconds
|
||||
formatted_time = dt.replace('T', ' ')
|
||||
except:
|
||||
formatted_time = timestamp
|
||||
memory = entry.get('memory', '')
|
||||
if timestamp and memory:
|
||||
# Store tuple of (timestamp, formatted_string) for sorting
|
||||
memories.append((timestamp, f"[{formatted_time}] {memory}"))
|
||||
|
||||
# Sort by timestamp in descending order (newest first)
|
||||
memories.sort(key=lambda x: x[0], reverse=True)
|
||||
|
||||
# Extract only the formatted strings
|
||||
memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
|
||||
logger.bind(tag=TAG).debug(f"Query results: {memories_str}")
|
||||
return memories_str
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}")
|
||||
return ""
|
||||
@@ -0,0 +1,57 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
import http.client
|
||||
import urllib.parse
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.appkey = config.get("appkey")
|
||||
self.token = config.get("token")
|
||||
self.format = config.get("format", "wav")
|
||||
self.sample_rate = config.get("sample_rate", 16000)
|
||||
self.voice = config.get("voice", "xiaoyun")
|
||||
self.volume = config.get("volume", 50)
|
||||
self.speech_rate = config.get("speech_rate", 0)
|
||||
self.pitch_rate = config.get("pitch_rate", 0)
|
||||
|
||||
self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
|
||||
self.api_url = f"https://{self.host}/stream/v1/tts"
|
||||
self.header = {
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"appkey": self.appkey,
|
||||
"token": self.token,
|
||||
"text": text,
|
||||
"format": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"voice": self.voice,
|
||||
"volume": self.volume,
|
||||
"speech_rate": self.speech_rate,
|
||||
"pitch_rate": self.pitch_rate
|
||||
}
|
||||
|
||||
print(self.api_url, json.dumps(request_json, ensure_ascii=False))
|
||||
try:
|
||||
resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
|
||||
# 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的
|
||||
if resp.headers['Content-Type'].startswith('audio/'):
|
||||
with open(output_file, 'wb') as f:
|
||||
f.write(resp.content)
|
||||
return output_file
|
||||
else:
|
||||
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
@@ -0,0 +1,84 @@
|
||||
import asyncio
|
||||
from config.logger import setup_logging
|
||||
import os
|
||||
import numpy as np
|
||||
import opuslib_next
|
||||
from pydub import AudioSegment
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TTSProviderBase(ABC):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
self.delete_audio_file = delete_audio_file
|
||||
self.output_file = config.get("output_file")
|
||||
|
||||
@abstractmethod
|
||||
def generate_filename(self):
|
||||
pass
|
||||
|
||||
def to_tts(self, text):
|
||||
tmp_file = self.generate_filename()
|
||||
try:
|
||||
max_repeat_time = 5
|
||||
while not os.path.exists(tmp_file) and max_repeat_time > 0:
|
||||
asyncio.run(self.text_to_speak(text, tmp_file))
|
||||
if not os.path.exists(tmp_file):
|
||||
max_repeat_time = max_repeat_time - 1
|
||||
logger.bind(tag=TAG).error(f"语音生成失败: {text}:{tmp_file}，再试{max_repeat_time}次")
|
||||
|
||||
if max_repeat_time > 0:
|
||||
logger.bind(tag=TAG).info(f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次")
|
||||
|
||||
return tmp_file
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
async def text_to_speak(self, text, output_file):
|
||||
pass
|
||||
|
||||
def wav_to_opus_data(self, wav_file_path):
|
||||
# 使用pydub加载PCM文件
|
||||
# 获取文件后缀名
|
||||
file_type = os.path.splitext(wav_file_path)[1]
|
||||
if file_type:
|
||||
file_type = file_type.lstrip('.')
|
||||
audio = AudioSegment.from_file(wav_file_path, format=file_type)
|
||||
|
||||
duration = len(audio) / 1000.0
|
||||
|
||||
# 转换为单声道和16kHz采样率（确保与编码器匹配）
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
|
||||
# 获取原始PCM数据（16位小端）
|
||||
raw_data = audio.raw_data
|
||||
|
||||
# 初始化Opus编码器
|
||||
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
|
||||
|
||||
# 编码参数
|
||||
frame_duration = 60 # 60ms per frame
|
||||
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
|
||||
|
||||
opus_datas = []
|
||||
# 按帧处理所有音频数据（包括最后一帧可能补零）
|
||||
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
|
||||
# 获取当前帧的二进制数据
|
||||
chunk = raw_data[i:i + frame_size * 2]
|
||||
|
||||
# 如果最后一帧不足，补零
|
||||
if len(chunk) < frame_size * 2:
|
||||
chunk += b'\x00' * (frame_size * 2 - len(chunk))
|
||||
|
||||
# 转换为numpy数组处理
|
||||
np_frame = np.frombuffer(chunk, dtype=np.int16)
|
||||
|
||||
# 编码Opus数据
|
||||
opus_data = encoder.encode(np_frame.tobytes(), frame_size)
|
||||
opus_datas.append(opus_data)
|
||||
|
||||
return opus_datas, duration
|
||||
@@ -0,0 +1,38 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import base64
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.model = config.get("model")
|
||||
self.access_token = config.get("access_token")
|
||||
self.voice = config.get("voice")
|
||||
self.response_format = config.get("response_format")
|
||||
|
||||
self.host = "api.coze.cn"
|
||||
self.api_url = f"https://{self.host}/v1/audio/speech"
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"model": self.model,
|
||||
"input": text,
|
||||
"voice_id": self.voice,
|
||||
"response_format": self.response_format,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.access_token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
response = requests.request("POST", self.api_url, json=request_json, headers=headers)
|
||||
data = response.content
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(data)
|
||||
@@ -0,0 +1,60 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import base64
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.appid = config.get("appid")
|
||||
self.access_token = config.get("access_token")
|
||||
self.cluster = config.get("cluster")
|
||||
self.voice = config.get("voice")
|
||||
self.api_url = config.get("api_url")
|
||||
self.authorization = config.get("authorization")
|
||||
self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"app": {
|
||||
"appid": f"{self.appid}",
|
||||
"token": "access_token",
|
||||
"cluster": self.cluster
|
||||
},
|
||||
"user": {
|
||||
"uid": "1"
|
||||
},
|
||||
"audio": {
|
||||
"voice_type": self.voice,
|
||||
"encoding": "wav",
|
||||
"speed_ratio": 1.0,
|
||||
"volume_ratio": 1.0,
|
||||
"pitch_ratio": 1.0,
|
||||
},
|
||||
"request": {
|
||||
"reqid": str(uuid.uuid4()),
|
||||
"text": text,
|
||||
"text_type": "plain",
|
||||
"operation": "query",
|
||||
"with_frontend": 1,
|
||||
"frontend_type": "unitTson"
|
||||
}
|
||||
}
|
||||
|
||||
try:
|
||||
resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
|
||||
if "data" in resp.json():
|
||||
data = resp.json()["data"]
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(base64.b64decode(data))
|
||||
else:
|
||||
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
@@ -0,0 +1,18 @@
|
||||
import os
|
||||
import uuid
|
||||
import edge_tts
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.voice = config.get("voice")
|
||||
|
||||
def generate_filename(self, extension=".mp3"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
communicate = edge_tts.Communicate(text, voice=self.voice) # Use your preferred voice
|
||||
await communicate.save(output_file)
|
||||
@@ -0,0 +1,158 @@
|
||||
|
||||
import base64
|
||||
import os
|
||||
import uuid
|
||||
import requests
|
||||
import ormsgpack
|
||||
from pathlib import Path
|
||||
from pydantic import BaseModel, Field, conint, model_validator
|
||||
from typing_extensions import Annotated
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ServeReferenceAudio(BaseModel):
|
||||
audio: bytes
|
||||
text: str
|
||||
|
||||
@model_validator(mode="before")
|
||||
def decode_audio(cls, values):
|
||||
audio = values.get("audio")
|
||||
if (
|
||||
isinstance(audio, str) and len(audio) > 255
|
||||
): # Check if audio is a string (Base64)
|
||||
try:
|
||||
values["audio"] = base64.b64decode(audio)
|
||||
except Exception as e:
|
||||
# If the audio is not a valid base64 string, we will just ignore it and let the server handle it
|
||||
pass
|
||||
return values
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"ServeReferenceAudio(text={self.text!r}, audio_size={len(self.audio)})"
|
||||
|
||||
class ServeTTSRequest(BaseModel):
|
||||
text: str
|
||||
chunk_length: Annotated[int, conint(ge=100, le=300, strict=True)] = 200
|
||||
# Audio format
|
||||
format: Literal["wav", "pcm", "mp3"] = "wav"
|
||||
# References audios for in-context learning
|
||||
references: list[ServeReferenceAudio] = []
|
||||
# Reference id
|
||||
# For example, if you want use https://fish.audio/m/7f92f8afb8ec43bf81429cc1c9199cb1/
|
||||
# Just pass 7f92f8afb8ec43bf81429cc1c9199cb1
|
||||
reference_id: str | None = None
|
||||
seed: int | None = None
|
||||
use_memory_cache: Literal["on", "off"] = "off"
|
||||
# Normalize text for en & zh, this increase stability for numbers
|
||||
normalize: bool = True
|
||||
# not usually used below
|
||||
streaming: bool = False
|
||||
max_new_tokens: int = 1024
|
||||
top_p: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
|
||||
repetition_penalty: Annotated[float, Field(ge=0.9, le=2.0, strict=True)] = 1.2
|
||||
temperature: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
|
||||
|
||||
class Config:
|
||||
# Allow arbitrary types for pytorch related types
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
|
||||
def audio_to_bytes(file_path):
|
||||
if not file_path or not Path(file_path).exists():
|
||||
return None
|
||||
with open(file_path, "rb") as wav_file:
|
||||
wav = wav_file.read()
|
||||
return wav
|
||||
|
||||
def read_ref_text(ref_text):
|
||||
path = Path(ref_text)
|
||||
if path.exists() and path.is_file():
|
||||
with path.open("r", encoding="utf-8") as file:
|
||||
return file.read()
|
||||
return ref_text
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
|
||||
self.reference_id = config.get("reference_id")
|
||||
self.reference_audio = config.get("reference_audio",[])
|
||||
self.reference_text = config.get("reference_text",[])
|
||||
self.format = config.get("format","wav")
|
||||
self.channels = config.get("channels",1)
|
||||
self.rate = config.get("rate",44100)
|
||||
self.api_key = config.get("api_key","YOUR_API_KEY")
|
||||
if "你" in self.api_key:
|
||||
logger.bind(tag=TAG).error("你还没配置FishSpeech TTS的密钥，请在配置文件中配置密钥，否则无法正常工作")
|
||||
return
|
||||
self.normalize = config.get("normalize",True)
|
||||
self.max_new_tokens = config.get("max_new_tokens",1024)
|
||||
self.chunk_length = config.get("chunk_length",200)
|
||||
self.top_p = config.get("top_p",0.7)
|
||||
self.repetition_penalty = config.get("repetition_penalty",1.2)
|
||||
self.temperature = config.get("temperature",0.7)
|
||||
self.streaming = config.get("streaming",False)
|
||||
self.use_memory_cache = config.get("use_memory_cache","on")
|
||||
self.seed = config.get("seed")
|
||||
self.api_url = config.get("api_url","http://127.0.0.1:8080/v1/tts")
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
# Prepare reference data
|
||||
byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
|
||||
ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
|
||||
|
||||
data = {
|
||||
"text": text,
|
||||
"references": [
|
||||
ServeReferenceAudio(
|
||||
audio=audio if audio else b"", text=text
|
||||
)
|
||||
for text, audio in zip(ref_texts, byte_audios)
|
||||
],
|
||||
"reference_id": self.reference_id,
|
||||
"normalize": self.normalize,
|
||||
"format": self.format,
|
||||
"max_new_tokens": self.max_new_tokens,
|
||||
"chunk_length": self.chunk_length,
|
||||
"top_p": self.top_p,
|
||||
"repetition_penalty": self.repetition_penalty,
|
||||
"temperature": self.temperature,
|
||||
"streaming": self.streaming,
|
||||
"use_memory_cache": self.use_memory_cache,
|
||||
"seed": self.seed,
|
||||
}
|
||||
|
||||
pydantic_data = ServeTTSRequest(**data)
|
||||
|
||||
response = requests.post(
|
||||
self.api_url,
|
||||
data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/msgpack",
|
||||
},
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
audio_content = response.content
|
||||
|
||||
with open(output_file, "wb") as audio_file:
|
||||
audio_file.write(audio_content)
|
||||
|
||||
|
||||
|
||||
else:
|
||||
print(f"Request failed with status code {response.status_code}")
|
||||
print(response.json())
|
||||
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import base64
|
||||
import requests
|
||||
from config.logger import setup_logging
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.url = config.get("url")
|
||||
self.text_lang = config.get("text_lang", "zh")
|
||||
self.ref_audio_path = config.get("ref_audio_path")
|
||||
self.prompt_text = config.get("prompt_text")
|
||||
self.prompt_lang = config.get("prompt_lang", "zh")
|
||||
self.top_k = config.get("top_k", 5)
|
||||
self.top_p = config.get("top_p", 1)
|
||||
self.temperature = config.get("temperature", 1)
|
||||
self.text_split_method = config.get("text_split_method", "cut0")
|
||||
self.batch_size = config.get("batch_size", 1)
|
||||
self.batch_threshold = config.get("batch_threshold", 0.75)
|
||||
self.split_bucket = config.get("split_bucket", True)
|
||||
self.return_fragment = config.get("return_fragment", False)
|
||||
self.speed_factor = config.get("speed_factor", 1.0)
|
||||
self.streaming_mode = config.get("streaming_mode", False)
|
||||
self.seed = config.get("seed", -1)
|
||||
self.parallel_infer = config.get("parallel_infer", True)
|
||||
self.repetition_penalty = config.get("repetition_penalty", 1.35)
|
||||
self.aux_ref_audio_paths = config.get("aux_ref_audio_paths", [])
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"text": text,
|
||||
"text_lang": self.text_lang,
|
||||
"ref_audio_path": self.ref_audio_path,
|
||||
"aux_ref_audio_paths": self.aux_ref_audio_paths,
|
||||
"prompt_text": self.prompt_text,
|
||||
"prompt_lang": self.prompt_lang,
|
||||
"top_k": self.top_k,
|
||||
"top_p": self.top_p,
|
||||
"temperature": self.temperature,
|
||||
"text_split_method": self.text_split_method,
|
||||
"batch_size": self.batch_size,
|
||||
"batch_threshold": self.batch_threshold,
|
||||
"split_bucket": self.split_bucket,
|
||||
"return_fragment": self.return_fragment,
|
||||
"speed_factor": self.speed_factor,
|
||||
"streaming_mode": self.streaming_mode,
|
||||
"seed": self.seed,
|
||||
"parallel_infer": self.parallel_infer,
|
||||
"repetition_penalty": self.repetition_penalty
|
||||
}
|
||||
|
||||
resp = requests.post(self.url, json=request_json)
|
||||
if resp.status_code == 200:
|
||||
with open(output_file, "wb") as file:
|
||||
file.write(resp.content)
|
||||
else:
|
||||
logger.bind(tag=TAG).error(f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}")
|
||||
@@ -0,0 +1,77 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.group_id = config.get("group_id")
|
||||
self.api_key = config.get("api_key")
|
||||
self.model = config.get("model")
|
||||
self.voice_id = config.get("voice_id")
|
||||
|
||||
default_voice_setting = {
|
||||
"voice_id": "female-shaonv",
|
||||
"speed": 1,
|
||||
"vol": 1,
|
||||
"pitch": 0,
|
||||
"emotion": "happy"
|
||||
}
|
||||
default_pronunciation_dict = {
|
||||
"tone": [
|
||||
"处理/(chu3)(li3)", "危险/dangerous"
|
||||
]
|
||||
}
|
||||
defult_audio_setting = {
|
||||
"sample_rate": 32000,
|
||||
"bitrate": 128000,
|
||||
"format": "mp3",
|
||||
"channel": 1
|
||||
}
|
||||
self.voice_setting = {**default_voice_setting, **config.get("voice_setting", {})}
|
||||
self.pronunciation_dict = {**default_pronunciation_dict, **config.get("pronunciation_dict", {})}
|
||||
self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
|
||||
self.timber_weights = config.get("timber_weights", [])
|
||||
|
||||
if self.voice_id:
|
||||
self.voice_setting["voice_id"] = self.voice_id
|
||||
|
||||
self.host = "api.minimax.chat"
|
||||
self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
|
||||
self.header = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}"
|
||||
}
|
||||
|
||||
def generate_filename(self, extension=".mp3"):
|
||||
return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"model": self.model,
|
||||
"text": text,
|
||||
"stream": False,
|
||||
"voice_setting": self.voice_setting,
|
||||
"pronunciation_dict": self.pronunciation_dict,
|
||||
"audio_setting": self.audio_setting,
|
||||
}
|
||||
|
||||
if type(self.timber_weights) is list and len(self.timber_weights) > 0:
|
||||
request_json["timber_weights"] = self.timber_weights
|
||||
request_json["voice_setting"]["voice_id"] = ""
|
||||
|
||||
try:
|
||||
resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
|
||||
# 检查返回请求数据的status_code是否为0
|
||||
if resp.json()["base_resp"]["status_code"] == 0:
|
||||
data = resp.json()['data']['audio']
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(bytes.fromhex(data))
|
||||
else:
|
||||
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
@@ -0,0 +1,39 @@
|
||||
import os
|
||||
import uuid
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.model = config.get("model")
|
||||
self.access_token = config.get("access_token")
|
||||
self.voice = config.get("voice")
|
||||
self.response_format = config.get("response_format")
|
||||
self.sample_rate = config.get("sample_rate")
|
||||
self.speed = config.get("speed")
|
||||
self.gain = config.get("gain")
|
||||
|
||||
self.host = "api.siliconflow.cn"
|
||||
self.api_url = f"https://{self.host}/v1/audio/speech"
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"model": self.model,
|
||||
"input": text,
|
||||
"voice": self.voice,
|
||||
"response_format": self.response_format,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.access_token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
response = requests.request("POST", self.api_url, json=request_json, headers=headers)
|
||||
data = response.content
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(data)
|
||||
@@ -0,0 +1,24 @@
|
||||
import importlib
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import wave
|
||||
import uuid
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional, Tuple, List
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
def create_instance(class_name: str, *args, **kwargs) -> ASRProviderBase:
|
||||
"""工厂方法创建ASR实例"""
|
||||
if os.path.exists(os.path.join('core', 'providers', 'asr', f'{class_name}.py')):
|
||||
lib_name = f'core.providers.asr.{class_name}'
|
||||
if lib_name not in sys.modules:
|
||||
sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
|
||||
return sys.modules[lib_name].ASRProvider(*args, **kwargs)
|
||||
|
||||
raise ValueError(f"不支持的ASR类型: {class_name}，请检查该配置的type是否设置正确")
|
||||
@@ -0,0 +1,97 @@
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from typing import Set
|
||||
|
||||
class AuthCodeGenerator:
|
||||
_instance = None
|
||||
_instance_lock = threading.Lock()
|
||||
|
||||
def __new__(cls):
|
||||
if not cls._instance:
|
||||
with cls._instance_lock:
|
||||
if not cls._instance:
|
||||
cls._instance = super(AuthCodeGenerator, cls).__new__(cls)
|
||||
# 初始化随机种子
|
||||
random.seed(time.time())
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
# 确保 __init__ 只被调用一次
|
||||
if not hasattr(self, '_initialized'):
|
||||
self._used_codes: Set[str] = set()
|
||||
self._code_timestamps = {}
|
||||
self._lock = threading.Lock()
|
||||
self._code_timeout = 3 * 24 * 60 * 60
|
||||
self._initialized = True
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls):
|
||||
"""获取AuthCodeGenerator的单例实例"""
|
||||
return cls()
|
||||
|
||||
def generate_code(self) -> str:
|
||||
"""
|
||||
生成6位数字认证码，确保不重复
|
||||
返回: 6位数字字符串
|
||||
"""
|
||||
with self._lock:
|
||||
self._clean_expired_codes() # 清理过期code
|
||||
while True:
|
||||
# 使用时间戳和已用码数量作为种子，确保每次生成不同的随机数
|
||||
seed = int(time.time() * 1000) + len(self._used_codes)
|
||||
random.seed(seed)
|
||||
|
||||
# 生成6位随机数字
|
||||
code = ''.join(str(random.randint(0, 9)) for _ in range(6))
|
||||
|
||||
# 检查是否已存在
|
||||
if code not in self._used_codes:
|
||||
self._used_codes.add(code)
|
||||
self._code_timestamps[code] = time.time()
|
||||
return code
|
||||
|
||||
def remove_code(self, code: str) -> bool:
|
||||
"""
|
||||
删除已使用的认证码
|
||||
参数:
|
||||
code: 要删除的认证码
|
||||
返回:
|
||||
bool: 删除成功返回True，码不存在返回False
|
||||
"""
|
||||
print('remove_code', code)
|
||||
with self._lock:
|
||||
if code in self._used_codes:
|
||||
self._used_codes.remove(code)
|
||||
if code in self._code_timestamps:
|
||||
del self._code_timestamps[code]
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_code_used(self, code: str) -> bool:
|
||||
"""
|
||||
检查认证码是否已被使用
|
||||
参数:
|
||||
code: 要检查的认证码
|
||||
返回:
|
||||
bool: 如果码存在返回True，否则返回False
|
||||
"""
|
||||
with self._lock:
|
||||
return code in self._used_codes
|
||||
|
||||
def clear_codes(self):
|
||||
"""清空所有已使用的认证码"""
|
||||
with self._lock:
|
||||
self._used_codes.clear()
|
||||
self._code_timestamps.clear()
|
||||
|
||||
def _clean_expired_codes(self):
|
||||
"""清理过期的认证码"""
|
||||
current_time = time.time()
|
||||
expired_codes = [
|
||||
code for code, timestamp in self._code_timestamps.items()
|
||||
if (current_time - timestamp) > self._code_timeout
|
||||
]
|
||||
for code in expired_codes:
|
||||
self._used_codes.remove(code)
|
||||
del self._code_timestamps[code]
|
||||
@@ -0,0 +1,50 @@
|
||||
import uuid
|
||||
from typing import List, Dict
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class Message:
|
||||
def __init__(self, role: str, content: str = None, uniq_id: str = None):
|
||||
self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4())
|
||||
self.role = role
|
||||
self.content = content
|
||||
|
||||
|
||||
class Dialogue:
|
||||
def __init__(self):
|
||||
self.dialogue: List[Message] = []
|
||||
# 获取当前时间
|
||||
self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
def put(self, message: Message):
|
||||
self.dialogue.append(message)
|
||||
|
||||
def get_llm_dialogue(self) -> List[Dict[str, str]]:
|
||||
dialogue = []
|
||||
for m in self.dialogue:
|
||||
dialogue.append({"role": m.role, "content": m.content})
|
||||
return dialogue
|
||||
|
||||
def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
|
||||
# 构建带记忆的对话
|
||||
dialogue = []
|
||||
|
||||
# 添加系统提示和记忆
|
||||
system_message = next(
|
||||
(msg for msg in self.dialogue if msg.role == "system"), None
|
||||
)
|
||||
|
||||
|
||||
if system_message:
|
||||
enhanced_system_prompt = (
|
||||
f"{system_message.content}\n\n"
|
||||
f"相关记忆：\n{memory_str}"
|
||||
)
|
||||
dialogue.append({"role": "system", "content": enhanced_system_prompt})
|
||||
|
||||
# 添加用户和助手的对话
|
||||
for msg in self.dialogue:
|
||||
if msg.role != "system": # 跳过原始的系统消息
|
||||
dialogue.append({"role": msg.role, "content": msg.content})
|
||||
|
||||
return dialogue
|
||||
@@ -0,0 +1,23 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
# 添加项目根目录到Python路径
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
project_root = os.path.abspath(os.path.join(current_dir, "..", ".."))
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from config.logger import setup_logging
|
||||
import importlib
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
def create_instance(class_name, *args, **kwargs):
|
||||
# 创建LLM实例
|
||||
if os.path.exists(os.path.join('core', 'providers', 'llm', class_name, f'{class_name}.py')):
|
||||
lib_name = f'core.providers.llm.{class_name}.{class_name}'
|
||||
if lib_name not in sys.modules:
|
||||
sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
|
||||
return sys.modules[lib_name].LLMProvider(*args, **kwargs)
|
||||
|
||||
raise ValueError(f"不支持的LLM类型: {class_name}，请检查该配置的type是否设置正确")
|
||||
@@ -0,0 +1,39 @@
|
||||
import asyncio
|
||||
from typing import Dict
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class FileLockManager:
|
||||
_instance = None
|
||||
_locks: Dict[str, asyncio.Lock] = {}
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
cls._instance = super(FileLockManager, cls).__new__(cls)
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
def get_lock(cls, file_path: str) -> asyncio.Lock:
|
||||
"""获取指定文件的锁"""
|
||||
if file_path not in cls._locks:
|
||||
cls._locks[file_path] = asyncio.Lock()
|
||||
return cls._locks[file_path]
|
||||
|
||||
@classmethod
|
||||
async def acquire_lock(cls, file_path: str):
|
||||
"""获取锁"""
|
||||
lock = cls.get_lock(file_path)
|
||||
await lock.acquire()
|
||||
logger.bind(tag=TAG).debug(f"Acquired lock for {file_path}")
|
||||
|
||||
@classmethod
|
||||
def release_lock(cls, file_path: str):
|
||||
"""释放锁"""
|
||||
if file_path in cls._locks:
|
||||
try:
|
||||
cls._locks[file_path].release()
|
||||
logger.bind(tag=TAG).debug(f"Released lock for {file_path}")
|
||||
except RuntimeError as e:
|
||||
logger.bind(tag=TAG).warning(f"Failed to release lock for {file_path}: {e}")
|
||||
@@ -0,0 +1,17 @@
|
||||
import os
|
||||
import sys
|
||||
import importlib
|
||||
from config.logger import setup_logging
|
||||
from core.utils.util import read_config, get_project_dir
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
def create_instance(class_name, *args, **kwargs):
|
||||
if os.path.exists(os.path.join('core', 'providers', 'memory', class_name, f'{class_name}.py')):
|
||||
lib_name = f'core.providers.memory.{class_name}.{class_name}'
|
||||
if lib_name not in sys.modules:
|
||||
sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
|
||||
return sys.modules[lib_name].MemoryProvider(*args, **kwargs)
|
||||
|
||||
raise ValueError(f"不支持的记忆服务类型: {class_name}")
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import struct
|
||||
|
||||
def decode_opus_from_file(input_file):
|
||||
"""
|
||||
从p3文件中解码 Opus 数据，并返回一个 Opus 数据包的列表以及总时长。
|
||||
"""
|
||||
opus_datas = []
|
||||
total_frames = 0
|
||||
sample_rate = 16000 # 文件采样率
|
||||
frame_duration_ms = 60 # 帧时长
|
||||
frame_size = int(sample_rate * frame_duration_ms / 1000)
|
||||
|
||||
with open(input_file, 'rb') as f:
|
||||
while True:
|
||||
# 读取头部（4字节）：[1字节类型，1字节保留，2字节长度]
|
||||
header = f.read(4)
|
||||
if not header:
|
||||
break
|
||||
|
||||
# 解包头部信息
|
||||
_, _, data_len = struct.unpack('>BBH', header)
|
||||
|
||||
# 根据头部指定的长度读取 Opus 数据
|
||||
opus_data = f.read(data_len)
|
||||
if len(opus_data) != data_len:
|
||||
raise ValueError(f"Data length({len(opus_data)}) mismatch({data_len}) in the file.")
|
||||
|
||||
opus_datas.append(opus_data)
|
||||
total_frames += 1
|
||||
|
||||
# 计算总时长
|
||||
total_duration = (total_frames * frame_duration_ms) / 1000.0
|
||||
return opus_datas, total_duration
|
||||
@@ -0,0 +1,17 @@
|
||||
import os
|
||||
import sys
|
||||
from config.logger import setup_logging
|
||||
import importlib
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
def create_instance(class_name, *args, **kwargs):
|
||||
# 创建TTS实例
|
||||
if os.path.exists(os.path.join('core', 'providers', 'tts', f'{class_name}.py')):
|
||||
lib_name = f'core.providers.tts.{class_name}'
|
||||
if lib_name not in sys.modules:
|
||||
sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
|
||||
return sys.modules[lib_name].TTSProvider(*args, **kwargs)
|
||||
|
||||
raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")
|
||||
@@ -0,0 +1,143 @@
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import yaml
|
||||
import socket
|
||||
import subprocess
|
||||
|
||||
|
||||
def get_project_dir():
|
||||
"""获取项目根目录"""
|
||||
return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + '/'
|
||||
|
||||
|
||||
def get_local_ip():
|
||||
try:
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
# Connect to Google's DNS servers
|
||||
s.connect(("8.8.8.8", 80))
|
||||
local_ip = s.getsockname()[0]
|
||||
s.close()
|
||||
return local_ip
|
||||
except Exception as e:
|
||||
return "127.0.0.1"
|
||||
|
||||
|
||||
def read_config(config_path):
|
||||
with open(config_path, "r", encoding="utf-8") as file:
|
||||
config = yaml.safe_load(file)
|
||||
return config
|
||||
|
||||
|
||||
def write_json_file(file_path, data):
|
||||
"""将数据写入 JSON 文件"""
|
||||
with open(file_path, 'w', encoding='utf-8') as file:
|
||||
json.dump(data, file, ensure_ascii=False, indent=4)
|
||||
|
||||
|
||||
def is_punctuation_or_emoji(char):
|
||||
"""检查字符是否为空格、指定标点或表情符号"""
|
||||
# 定义需要去除的中英文标点（包括全角/半角）
|
||||
punctuation_set = {
|
||||
'，', ',', # 中文逗号 + 英文逗号
|
||||
'。', '.', # 中文句号 + 英文句号
|
||||
'！', '!', # 中文感叹号 + 英文感叹号
|
||||
'-', '－', # 英文连字符 + 中文全角横线
|
||||
'、' # 中文顿号
|
||||
}
|
||||
if char.isspace() or char in punctuation_set:
|
||||
return True
|
||||
# 检查表情符号（保留原有逻辑）
|
||||
code_point = ord(char)
|
||||
emoji_ranges = [
|
||||
(0x1F600, 0x1F64F), (0x1F300, 0x1F5FF),
|
||||
(0x1F680, 0x1F6FF), (0x1F900, 0x1F9FF),
|
||||
(0x1FA70, 0x1FAFF), (0x2600, 0x26FF),
|
||||
(0x2700, 0x27BF)
|
||||
]
|
||||
return any(start <= code_point <= end for start, end in emoji_ranges)
|
||||
|
||||
|
||||
def get_string_no_punctuation_or_emoji(s):
|
||||
"""去除字符串首尾的空格、标点符号和表情符号"""
|
||||
chars = list(s)
|
||||
# 处理开头的字符
|
||||
start = 0
|
||||
while start < len(chars) and is_punctuation_or_emoji(chars[start]):
|
||||
start += 1
|
||||
# 处理结尾的字符
|
||||
end = len(chars) - 1
|
||||
while end >= start and is_punctuation_or_emoji(chars[end]):
|
||||
end -= 1
|
||||
return ''.join(chars[start:end + 1])
|
||||
|
||||
|
||||
def remove_punctuation_and_length(text):
|
||||
# 全角符号和半角符号的Unicode范围
|
||||
full_width_punctuations = '！＂＃＄％＆＇（）＊＋，－。／：；＜＝＞？＠［＼］＾＿｀｛｜｝～'
|
||||
half_width_punctuations = '!"#$%&\'()*+,-./:;<=>?@[\]^_`{|}~'
|
||||
space = ' ' # 半角空格
|
||||
full_width_space = ' ' # 全角空格
|
||||
|
||||
# 去除全角和半角符号以及空格
|
||||
result = ''.join([char for char in text if
|
||||
char not in full_width_punctuations and char not in half_width_punctuations and char not in space and char not in full_width_space])
|
||||
|
||||
if result == "Yeah":
|
||||
return 0, ""
|
||||
return len(result), result
|
||||
|
||||
|
||||
def check_password(password):
|
||||
"""
|
||||
检查密码是否满足以下条件：
|
||||
1. 密码长度大于八位。
|
||||
2. 密码包含英文和数字。
|
||||
3. 密码不能包含“xiaozhi”字符。
|
||||
|
||||
:param password: 要检查的密码
|
||||
:return: 如果密码满足条件，则返回True；否则返回False。
|
||||
"""
|
||||
# 检查密码长度
|
||||
if len(password) < 8:
|
||||
return False
|
||||
|
||||
# 检查是否包含英文字符和数字
|
||||
if not re.search(r'[A-Za-z]', password) or not re.search(r'[0-9]', password):
|
||||
return False
|
||||
|
||||
# 检查是否包含“xiaozhi”字符
|
||||
if "xiaozhi" in password:
|
||||
return False
|
||||
|
||||
if "1234" in password:
|
||||
return False
|
||||
|
||||
# 如果满足所有条件，则返回True
|
||||
return True
|
||||
|
||||
def check_ffmpeg_installed():
|
||||
ffmpeg_installed = False
|
||||
try:
|
||||
# 执行ffmpeg -version命令，并捕获输出
|
||||
result = subprocess.run(
|
||||
['ffmpeg', '-version'],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True # 如果返回码非零则抛出异常
|
||||
)
|
||||
# 检查输出中是否包含版本信息（可选）
|
||||
output = result.stdout + result.stderr
|
||||
if 'ffmpeg version' in output.lower():
|
||||
ffmpeg_installed = True
|
||||
return False
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
# 命令执行失败或未找到
|
||||
ffmpeg_installed = False
|
||||
if not ffmpeg_installed:
|
||||
error_msg = "您的电脑还没正确安装ffmpeg\n"
|
||||
error_msg += "\n建议您：\n"
|
||||
error_msg += "1、按照项目的安装文档，正确进入conda环境\n"
|
||||
error_msg += "2、查阅安装文档，如何在conda环境中安装ffmpeg\n"
|
||||
raise ValueError(error_msg)
|
||||
@@ -0,0 +1,77 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from config.logger import setup_logging
|
||||
import opuslib_next
|
||||
import time
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class VAD(ABC):
|
||||
@abstractmethod
|
||||
def is_vad(self, conn, data):
|
||||
"""检测音频数据中的语音活动"""
|
||||
pass
|
||||
|
||||
|
||||
class SileroVAD(VAD):
|
||||
def __init__(self, config):
|
||||
logger.bind(tag=TAG).info("SileroVAD", config)
|
||||
self.model, self.utils = torch.hub.load(repo_or_dir=config["model_dir"],
|
||||
source='local',
|
||||
model='silero_vad',
|
||||
force_reload=False)
|
||||
(get_speech_timestamps, _, _, _, _) = self.utils
|
||||
|
||||
self.decoder = opuslib_next.Decoder(16000, 1)
|
||||
self.vad_threshold = config.get("threshold")
|
||||
self.silence_threshold_ms = config.get("min_silence_duration_ms")
|
||||
|
||||
def is_vad(self, conn, opus_packet):
|
||||
try:
|
||||
pcm_frame = self.decoder.decode(opus_packet, 960)
|
||||
conn.client_audio_buffer += pcm_frame # 将新数据加入缓冲区
|
||||
|
||||
# 处理缓冲区中的完整帧（每次处理512采样点）
|
||||
client_have_voice = False
|
||||
while len(conn.client_audio_buffer) >= 512 * 2:
|
||||
# 提取前512个采样点（1024字节）
|
||||
chunk = conn.client_audio_buffer[:512 * 2]
|
||||
conn.client_audio_buffer = conn.client_audio_buffer[512 * 2:]
|
||||
|
||||
# 转换为模型需要的张量格式
|
||||
audio_int16 = np.frombuffer(chunk, dtype=np.int16)
|
||||
audio_float32 = audio_int16.astype(np.float32) / 32768.0
|
||||
audio_tensor = torch.from_numpy(audio_float32)
|
||||
|
||||
# 检测语音活动
|
||||
speech_prob = self.model(audio_tensor, 16000).item()
|
||||
client_have_voice = speech_prob >= self.vad_threshold
|
||||
|
||||
# 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
|
||||
if conn.client_have_voice and not client_have_voice:
|
||||
stop_duration = time.time() * 1000 - conn.client_have_voice_last_time
|
||||
if stop_duration >= self.silence_threshold_ms:
|
||||
conn.client_voice_stop = True
|
||||
if client_have_voice:
|
||||
conn.client_have_voice = True
|
||||
conn.client_have_voice_last_time = time.time() * 1000
|
||||
|
||||
return client_have_voice
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).info(f"解码错误: {e}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")
|
||||
|
||||
|
||||
def create_instance(class_name, *args, **kwargs) -> VAD:
|
||||
# 获取类对象
|
||||
cls_map = {
|
||||
"SileroVAD": SileroVAD,
|
||||
# 可扩展其他SileroVAD实现
|
||||
}
|
||||
|
||||
if cls := cls_map.get(class_name):
|
||||
return cls(*args, **kwargs)
|
||||
raise ValueError(f"不支持的SileroVAD类型: {class_name}")
|
||||
@@ -0,0 +1,79 @@
|
||||
import asyncio
|
||||
import websockets
|
||||
from config.logger import setup_logging
|
||||
from core.connection import ConnectionHandler
|
||||
from core.handle.musicHandler import MusicHandler
|
||||
from core.utils.util import get_local_ip
|
||||
from core.utils import asr, vad, llm, tts, memory
|
||||
|
||||
TAG = __name__
|
||||
|
||||
|
||||
class WebSocketServer:
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
self.logger = setup_logging()
|
||||
self._vad, self._asr, self._llm, self._tts, self._music, self._memory = self._create_processing_instances()
|
||||
self.active_connections = set() # 添加全局连接记录
|
||||
|
||||
def _create_processing_instances(self):
|
||||
memory_cls_name = self.config["selected_module"].get("Memory", "mem0ai") # 默认使用mem0ai
|
||||
has_memory_cfg = self.config.get("Memory") and memory_cls_name in self.config["Memory"]
|
||||
memory_cfg = self.config["Memory"][memory_cls_name] if has_memory_cfg else {}
|
||||
|
||||
"""创建处理模块实例"""
|
||||
return (
|
||||
vad.create_instance(
|
||||
self.config["selected_module"]["VAD"],
|
||||
self.config["VAD"][self.config["selected_module"]["VAD"]]
|
||||
),
|
||||
asr.create_instance(
|
||||
self.config["selected_module"]["ASR"]
|
||||
if not 'type' in self.config["ASR"][self.config["selected_module"]["ASR"]]
|
||||
else
|
||||
self.config["ASR"][self.config["selected_module"]["ASR"]]["type"],
|
||||
self.config["ASR"][self.config["selected_module"]["ASR"]],
|
||||
self.config["delete_audio"]
|
||||
),
|
||||
llm.create_instance(
|
||||
self.config["selected_module"]["LLM"]
|
||||
if not 'type' in self.config["LLM"][self.config["selected_module"]["LLM"]]
|
||||
else
|
||||
self.config["LLM"][self.config["selected_module"]["LLM"]]['type'],
|
||||
self.config["LLM"][self.config["selected_module"]["LLM"]],
|
||||
),
|
||||
tts.create_instance(
|
||||
self.config["selected_module"]["TTS"]
|
||||
if not 'type' in self.config["TTS"][self.config["selected_module"]["TTS"]]
|
||||
else
|
||||
self.config["TTS"][self.config["selected_module"]["TTS"]]["type"],
|
||||
self.config["TTS"][self.config["selected_module"]["TTS"]],
|
||||
self.config["delete_audio"]
|
||||
),
|
||||
MusicHandler(self.config),
|
||||
memory.create_instance(memory_cls_name, memory_cfg),
|
||||
)
|
||||
|
||||
async def start(self):
|
||||
server_config = self.config["server"]
|
||||
host = server_config["ip"]
|
||||
port = server_config["port"]
|
||||
|
||||
self.logger.bind(tag=TAG).info("Server is running at ws://{}:{}", get_local_ip(), port)
|
||||
self.logger.bind(tag=TAG).info("=======上面的地址是websocket协议地址，请勿用浏览器访问=======")
|
||||
async with websockets.serve(
|
||||
self._handle_connection,
|
||||
host,
|
||||
port
|
||||
):
|
||||
await asyncio.Future()
|
||||
|
||||
async def _handle_connection(self, websocket):
|
||||
"""处理新连接，每次创建独立的ConnectionHandler"""
|
||||
# 创建ConnectionHandler时传入当前server实例
|
||||
handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._music, self._memory)
|
||||
self.active_connections.add(handler)
|
||||
try:
|
||||
await handler.handle_connection(websocket)
|
||||
finally:
|
||||
self.active_connections.discard(handler)
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
version: '3'
|
||||
services:
|
||||
xiaozhi-esp32-server:
|
||||
image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:latest
|
||||
container_name: xiaozhi-esp32-server
|
||||
restart: always
|
||||
security_opt:
|
||||
- seccomp:unconfined
|
||||
ports:
|
||||
# ws服务端
|
||||
- "8000:8000"
|
||||
# 管理后台
|
||||
- "8002:8002"
|
||||
volumes:
|
||||
# 配置文件目录
|
||||
- ./data:/app/data
|
||||
# 模型文件挂接，很重要
|
||||
- ./models/SenseVoiceSmall/model.pt:/app/models/SenseVoiceSmall/model.pt
|
||||
Binary file not shown.
@@ -0,0 +1,97 @@
|
||||
encoder: SenseVoiceEncoderSmall
|
||||
encoder_conf:
|
||||
output_size: 512
|
||||
attention_heads: 4
|
||||
linear_units: 2048
|
||||
num_blocks: 50
|
||||
tp_blocks: 20
|
||||
dropout_rate: 0.1
|
||||
positional_dropout_rate: 0.1
|
||||
attention_dropout_rate: 0.1
|
||||
input_layer: pe
|
||||
pos_enc_class: SinusoidalPositionEncoder
|
||||
normalize_before: true
|
||||
kernel_size: 11
|
||||
sanm_shfit: 0
|
||||
selfattention_layer_type: sanm
|
||||
|
||||
|
||||
model: SenseVoiceSmall
|
||||
model_conf:
|
||||
length_normalized_loss: true
|
||||
sos: 1
|
||||
eos: 2
|
||||
ignore_id: -1
|
||||
|
||||
tokenizer: SentencepiecesTokenizer
|
||||
tokenizer_conf:
|
||||
bpemodel: null
|
||||
unk_symbol: <unk>
|
||||
split_with_space: true
|
||||
|
||||
frontend: WavFrontend
|
||||
frontend_conf:
|
||||
fs: 16000
|
||||
window: hamming
|
||||
n_mels: 80
|
||||
frame_length: 25
|
||||
frame_shift: 10
|
||||
lfr_m: 7
|
||||
lfr_n: 6
|
||||
cmvn_file: null
|
||||
|
||||
|
||||
dataset: SenseVoiceCTCDataset
|
||||
dataset_conf:
|
||||
index_ds: IndexDSJsonl
|
||||
batch_sampler: EspnetStyleBatchSampler
|
||||
data_split_num: 32
|
||||
batch_type: token
|
||||
batch_size: 14000
|
||||
max_token_length: 2000
|
||||
min_token_length: 60
|
||||
max_source_length: 2000
|
||||
min_source_length: 60
|
||||
max_target_length: 200
|
||||
min_target_length: 0
|
||||
shuffle: true
|
||||
num_workers: 4
|
||||
sos: ${model_conf.sos}
|
||||
eos: ${model_conf.eos}
|
||||
IndexDSJsonl: IndexDSJsonl
|
||||
retry: 20
|
||||
|
||||
train_conf:
|
||||
accum_grad: 1
|
||||
grad_clip: 5
|
||||
max_epoch: 20
|
||||
keep_nbest_models: 10
|
||||
avg_nbest_model: 10
|
||||
log_interval: 100
|
||||
resume: true
|
||||
validate_interval: 10000
|
||||
save_checkpoint_interval: 10000
|
||||
|
||||
optim: adamw
|
||||
optim_conf:
|
||||
lr: 0.00002
|
||||
scheduler: warmuplr
|
||||
scheduler_conf:
|
||||
warmup_steps: 25000
|
||||
|
||||
specaug: SpecAugLFR
|
||||
specaug_conf:
|
||||
apply_time_warp: false
|
||||
time_warp_window: 5
|
||||
time_warp_mode: bicubic
|
||||
apply_freq_mask: true
|
||||
freq_mask_width_range:
|
||||
- 0
|
||||
- 30
|
||||
lfr_rate: 6
|
||||
num_freq_mask: 1
|
||||
apply_time_mask: true
|
||||
time_mask_width_range:
|
||||
- 0
|
||||
- 12
|
||||
num_time_mask: 1
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"framework": "pytorch",
|
||||
"task" : "auto-speech-recognition",
|
||||
"model": {"type" : "funasr"},
|
||||
"pipeline": {"type":"funasr-pipeline"},
|
||||
"model_name_in_hub": {
|
||||
"ms":"",
|
||||
"hf":""},
|
||||
"file_path_metas": {
|
||||
"init_param":"model.pt",
|
||||
"config":"config.yaml",
|
||||
"tokenizer_conf": {"bpemodel": "chn_jpn_yue_eng_ko_spectok.bpe.model"},
|
||||
"frontend_conf":{"cmvn_file": "am.mvn"}}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
from funasr import AutoModel
|
||||
from funasr.utils.postprocess_utils import rich_transcription_postprocess
|
||||
|
||||
model_dir = "./"
|
||||
|
||||
|
||||
model = AutoModel(
|
||||
model=model_dir,
|
||||
vad_model="fsmn-vad",
|
||||
vad_kwargs={"max_single_segment_time": 30000},
|
||||
# device="cuda:0",
|
||||
hub="hf",
|
||||
)
|
||||
|
||||
# en
|
||||
res = model.generate(
|
||||
input=f"{model.model_path}/example/en.mp3",
|
||||
cache={},
|
||||
language="auto", # "zn", "en", "yue", "ja", "ko", "nospeech"
|
||||
use_itn=True,
|
||||
batch_size_s=60,
|
||||
merge_vad=True, #
|
||||
merge_length_s=15,
|
||||
)
|
||||
text = rich_transcription_postprocess(res[0]["text"])
|
||||
print(text)
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,56 @@
|
||||
dependencies = ['torch', 'torchaudio']
|
||||
import torch
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
|
||||
from silero_vad.utils_vad import (init_jit_model,
|
||||
get_speech_timestamps,
|
||||
save_audio,
|
||||
read_audio,
|
||||
VADIterator,
|
||||
collect_chunks,
|
||||
OnnxWrapper)
|
||||
|
||||
|
||||
def versiontuple(v):
|
||||
splitted = v.split('+')[0].split(".")
|
||||
version_list = []
|
||||
for i in splitted:
|
||||
try:
|
||||
version_list.append(int(i))
|
||||
except:
|
||||
version_list.append(0)
|
||||
return tuple(version_list)
|
||||
|
||||
|
||||
def silero_vad(onnx=False, force_onnx_cpu=False, opset_version=16):
|
||||
"""Silero Voice Activity Detector
|
||||
Returns a model with a set of utils
|
||||
Please see https://github.com/snakers4/silero-vad for usage examples
|
||||
"""
|
||||
available_ops = [15, 16]
|
||||
if onnx and opset_version not in available_ops:
|
||||
raise Exception(f'Available ONNX opset_version: {available_ops}')
|
||||
|
||||
if not onnx:
|
||||
installed_version = torch.__version__
|
||||
supported_version = '1.12.0'
|
||||
if versiontuple(installed_version) < versiontuple(supported_version):
|
||||
raise Exception(f'Please install torch {supported_version} or greater ({installed_version} installed)')
|
||||
|
||||
model_dir = os.path.join(os.path.dirname(__file__), 'src', 'silero_vad', 'data')
|
||||
if onnx:
|
||||
if opset_version == 16:
|
||||
model_name = 'silero_vad.onnx'
|
||||
else:
|
||||
model_name = f'silero_vad_16k_op{opset_version}.onnx'
|
||||
model = OnnxWrapper(os.path.join(model_dir, model_name), force_onnx_cpu)
|
||||
else:
|
||||
model = init_jit_model(os.path.join(model_dir, 'silero_vad.jit'))
|
||||
utils = (get_speech_timestamps,
|
||||
save_audio,
|
||||
read_audio,
|
||||
VADIterator,
|
||||
collect_chunks)
|
||||
|
||||
return model, utils
|
||||
@@ -0,0 +1,12 @@
|
||||
from importlib.metadata import version
|
||||
try:
|
||||
__version__ = version(__name__)
|
||||
except:
|
||||
pass
|
||||
|
||||
from silero_vad.model import load_silero_vad
|
||||
from silero_vad.utils_vad import (get_speech_timestamps,
|
||||
save_audio,
|
||||
read_audio,
|
||||
VADIterator,
|
||||
collect_chunks)
|
||||
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,36 @@
|
||||
from .utils_vad import init_jit_model, OnnxWrapper
|
||||
import torch
|
||||
torch.set_num_threads(1)
|
||||
|
||||
|
||||
def load_silero_vad(onnx=False, opset_version=16):
|
||||
available_ops = [15, 16]
|
||||
if onnx and opset_version not in available_ops:
|
||||
raise Exception(f'Available ONNX opset_version: {available_ops}')
|
||||
|
||||
if onnx:
|
||||
if opset_version == 16:
|
||||
model_name = 'silero_vad.onnx'
|
||||
else:
|
||||
model_name = f'silero_vad_16k_op{opset_version}.onnx'
|
||||
else:
|
||||
model_name = 'silero_vad.jit'
|
||||
package_path = "silero_vad.data"
|
||||
|
||||
try:
|
||||
import importlib_resources as impresources
|
||||
model_file_path = str(impresources.files(package_path).joinpath(model_name))
|
||||
except:
|
||||
from importlib import resources as impresources
|
||||
try:
|
||||
with impresources.path(package_path, model_name) as f:
|
||||
model_file_path = f
|
||||
except:
|
||||
model_file_path = str(impresources.files(package_path).joinpath(model_name))
|
||||
|
||||
if onnx:
|
||||
model = OnnxWrapper(model_file_path, force_onnx_cpu=True)
|
||||
else:
|
||||
model = init_jit_model(model_file_path)
|
||||
|
||||
return model
|
||||
@@ -0,0 +1,500 @@
|
||||
import torch
|
||||
import torchaudio
|
||||
from typing import Callable, List
|
||||
import warnings
|
||||
|
||||
languages = ['ru', 'en', 'de', 'es']
|
||||
|
||||
|
||||
class OnnxWrapper():
|
||||
|
||||
def __init__(self, path, force_onnx_cpu=False):
|
||||
import numpy as np
|
||||
global np
|
||||
import onnxruntime
|
||||
|
||||
opts = onnxruntime.SessionOptions()
|
||||
opts.inter_op_num_threads = 1
|
||||
opts.intra_op_num_threads = 1
|
||||
|
||||
if force_onnx_cpu and 'CPUExecutionProvider' in onnxruntime.get_available_providers():
|
||||
self.session = onnxruntime.InferenceSession(path, providers=['CPUExecutionProvider'], sess_options=opts)
|
||||
else:
|
||||
self.session = onnxruntime.InferenceSession(path, sess_options=opts)
|
||||
|
||||
self.reset_states()
|
||||
if '16k' in path:
|
||||
warnings.warn('This model support only 16000 sampling rate!')
|
||||
self.sample_rates = [16000]
|
||||
else:
|
||||
self.sample_rates = [8000, 16000]
|
||||
|
||||
def _validate_input(self, x, sr: int):
|
||||
if x.dim() == 1:
|
||||
x = x.unsqueeze(0)
|
||||
if x.dim() > 2:
|
||||
raise ValueError(f"Too many dimensions for input audio chunk {x.dim()}")
|
||||
|
||||
if sr != 16000 and (sr % 16000 == 0):
|
||||
step = sr // 16000
|
||||
x = x[:,::step]
|
||||
sr = 16000
|
||||
|
||||
if sr not in self.sample_rates:
|
||||
raise ValueError(f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)")
|
||||
if sr / x.shape[1] > 31.25:
|
||||
raise ValueError("Input audio chunk is too short")
|
||||
|
||||
return x, sr
|
||||
|
||||
def reset_states(self, batch_size=1):
|
||||
self._state = torch.zeros((2, batch_size, 128)).float()
|
||||
self._context = torch.zeros(0)
|
||||
self._last_sr = 0
|
||||
self._last_batch_size = 0
|
||||
|
||||
def __call__(self, x, sr: int):
|
||||
|
||||
x, sr = self._validate_input(x, sr)
|
||||
num_samples = 512 if sr == 16000 else 256
|
||||
|
||||
if x.shape[-1] != num_samples:
|
||||
raise ValueError(f"Provided number of samples is {x.shape[-1]} (Supported values: 256 for 8000 sample rate, 512 for 16000)")
|
||||
|
||||
batch_size = x.shape[0]
|
||||
context_size = 64 if sr == 16000 else 32
|
||||
|
||||
if not self._last_batch_size:
|
||||
self.reset_states(batch_size)
|
||||
if (self._last_sr) and (self._last_sr != sr):
|
||||
self.reset_states(batch_size)
|
||||
if (self._last_batch_size) and (self._last_batch_size != batch_size):
|
||||
self.reset_states(batch_size)
|
||||
|
||||
if not len(self._context):
|
||||
self._context = torch.zeros(batch_size, context_size)
|
||||
|
||||
x = torch.cat([self._context, x], dim=1)
|
||||
if sr in [8000, 16000]:
|
||||
ort_inputs = {'input': x.numpy(), 'state': self._state.numpy(), 'sr': np.array(sr, dtype='int64')}
|
||||
ort_outs = self.session.run(None, ort_inputs)
|
||||
out, state = ort_outs
|
||||
self._state = torch.from_numpy(state)
|
||||
else:
|
||||
raise ValueError()
|
||||
|
||||
self._context = x[..., -context_size:]
|
||||
self._last_sr = sr
|
||||
self._last_batch_size = batch_size
|
||||
|
||||
out = torch.from_numpy(out)
|
||||
return out
|
||||
|
||||
def audio_forward(self, x, sr: int):
|
||||
outs = []
|
||||
x, sr = self._validate_input(x, sr)
|
||||
self.reset_states()
|
||||
num_samples = 512 if sr == 16000 else 256
|
||||
|
||||
if x.shape[1] % num_samples:
|
||||
pad_num = num_samples - (x.shape[1] % num_samples)
|
||||
x = torch.nn.functional.pad(x, (0, pad_num), 'constant', value=0.0)
|
||||
|
||||
for i in range(0, x.shape[1], num_samples):
|
||||
wavs_batch = x[:, i:i+num_samples]
|
||||
out_chunk = self.__call__(wavs_batch, sr)
|
||||
outs.append(out_chunk)
|
||||
|
||||
stacked = torch.cat(outs, dim=1)
|
||||
return stacked.cpu()
|
||||
|
||||
|
||||
class Validator():
|
||||
def __init__(self, url, force_onnx_cpu):
|
||||
self.onnx = True if url.endswith('.onnx') else False
|
||||
torch.hub.download_url_to_file(url, 'inf.model')
|
||||
if self.onnx:
|
||||
import onnxruntime
|
||||
if force_onnx_cpu and 'CPUExecutionProvider' in onnxruntime.get_available_providers():
|
||||
self.model = onnxruntime.InferenceSession('inf.model', providers=['CPUExecutionProvider'])
|
||||
else:
|
||||
self.model = onnxruntime.InferenceSession('inf.model')
|
||||
else:
|
||||
self.model = init_jit_model(model_path='inf.model')
|
||||
|
||||
def __call__(self, inputs: torch.Tensor):
|
||||
with torch.no_grad():
|
||||
if self.onnx:
|
||||
ort_inputs = {'input': inputs.cpu().numpy()}
|
||||
outs = self.model.run(None, ort_inputs)
|
||||
outs = [torch.Tensor(x) for x in outs]
|
||||
else:
|
||||
outs = self.model(inputs)
|
||||
|
||||
return outs
|
||||
|
||||
|
||||
def read_audio(path: str,
|
||||
sampling_rate: int = 16000):
|
||||
list_backends = torchaudio.list_audio_backends()
|
||||
|
||||
assert len(list_backends) > 0, 'The list of available backends is empty, please install backend manually. \
|
||||
\n Recommendations: \n \tSox (UNIX OS) \n \tSoundfile (Windows OS, UNIX OS) \n \tffmpeg (Windows OS, UNIX OS)'
|
||||
|
||||
try:
|
||||
effects = [
|
||||
['channels', '1'],
|
||||
['rate', str(sampling_rate)]
|
||||
]
|
||||
|
||||
wav, sr = torchaudio.sox_effects.apply_effects_file(path, effects=effects)
|
||||
except:
|
||||
wav, sr = torchaudio.load(path)
|
||||
|
||||
if wav.size(0) > 1:
|
||||
wav = wav.mean(dim=0, keepdim=True)
|
||||
|
||||
if sr != sampling_rate:
|
||||
transform = torchaudio.transforms.Resample(orig_freq=sr,
|
||||
new_freq=sampling_rate)
|
||||
wav = transform(wav)
|
||||
sr = sampling_rate
|
||||
|
||||
assert sr == sampling_rate
|
||||
return wav.squeeze(0)
|
||||
|
||||
|
||||
def save_audio(path: str,
|
||||
tensor: torch.Tensor,
|
||||
sampling_rate: int = 16000):
|
||||
torchaudio.save(path, tensor.unsqueeze(0), sampling_rate, bits_per_sample=16)
|
||||
|
||||
|
||||
def init_jit_model(model_path: str,
|
||||
device=torch.device('cpu')):
|
||||
model = torch.jit.load(model_path, map_location=device)
|
||||
model.eval()
|
||||
return model
|
||||
|
||||
|
||||
def make_visualization(probs, step):
|
||||
import pandas as pd
|
||||
pd.DataFrame({'probs': probs},
|
||||
index=[x * step for x in range(len(probs))]).plot(figsize=(16, 8),
|
||||
kind='area', ylim=[0, 1.05], xlim=[0, len(probs) * step],
|
||||
xlabel='seconds',
|
||||
ylabel='speech probability',
|
||||
colormap='tab20')
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def get_speech_timestamps(audio: torch.Tensor,
|
||||
model,
|
||||
threshold: float = 0.5,
|
||||
sampling_rate: int = 16000,
|
||||
min_speech_duration_ms: int = 250,
|
||||
max_speech_duration_s: float = float('inf'),
|
||||
min_silence_duration_ms: int = 100,
|
||||
speech_pad_ms: int = 30,
|
||||
return_seconds: bool = False,
|
||||
visualize_probs: bool = False,
|
||||
progress_tracking_callback: Callable[[float], None] = None,
|
||||
neg_threshold: float = None,
|
||||
window_size_samples: int = 512,):
|
||||
|
||||
"""
|
||||
This method is used for splitting long audios into speech chunks using silero VAD
|
||||
|
||||
Parameters
|
||||
----------
|
||||
audio: torch.Tensor, one dimensional
|
||||
One dimensional float torch.Tensor, other types are casted to torch if possible
|
||||
|
||||
model: preloaded .jit/.onnx silero VAD model
|
||||
|
||||
threshold: float (default - 0.5)
|
||||
Speech threshold. Silero VAD outputs speech probabilities for each audio chunk, probabilities ABOVE this value are considered as SPEECH.
|
||||
It is better to tune this parameter for each dataset separately, but "lazy" 0.5 is pretty good for most datasets.
|
||||
|
||||
sampling_rate: int (default - 16000)
|
||||
Currently silero VAD models support 8000 and 16000 (or multiply of 16000) sample rates
|
||||
|
||||
min_speech_duration_ms: int (default - 250 milliseconds)
|
||||
Final speech chunks shorter min_speech_duration_ms are thrown out
|
||||
|
||||
max_speech_duration_s: int (default - inf)
|
||||
Maximum duration of speech chunks in seconds
|
||||
Chunks longer than max_speech_duration_s will be split at the timestamp of the last silence that lasts more than 100ms (if any), to prevent agressive cutting.
|
||||
Otherwise, they will be split aggressively just before max_speech_duration_s.
|
||||
|
||||
min_silence_duration_ms: int (default - 100 milliseconds)
|
||||
In the end of each speech chunk wait for min_silence_duration_ms before separating it
|
||||
|
||||
speech_pad_ms: int (default - 30 milliseconds)
|
||||
Final speech chunks are padded by speech_pad_ms each side
|
||||
|
||||
return_seconds: bool (default - False)
|
||||
whether return timestamps in seconds (default - samples)
|
||||
|
||||
visualize_probs: bool (default - False)
|
||||
whether draw prob hist or not
|
||||
|
||||
progress_tracking_callback: Callable[[float], None] (default - None)
|
||||
callback function taking progress in percents as an argument
|
||||
|
||||
neg_threshold: float (default = threshold - 0.15)
|
||||
Negative threshold (noise or exit threshold). If model's current state is SPEECH, values BELOW this value are considered as NON-SPEECH.
|
||||
|
||||
window_size_samples: int (default - 512 samples)
|
||||
!!! DEPRECATED, DOES NOTHING !!!
|
||||
|
||||
Returns
|
||||
----------
|
||||
speeches: list of dicts
|
||||
list containing ends and beginnings of speech chunks (samples or seconds based on return_seconds)
|
||||
"""
|
||||
|
||||
if not torch.is_tensor(audio):
|
||||
try:
|
||||
audio = torch.Tensor(audio)
|
||||
except:
|
||||
raise TypeError("Audio cannot be casted to tensor. Cast it manually")
|
||||
|
||||
if len(audio.shape) > 1:
|
||||
for i in range(len(audio.shape)): # trying to squeeze empty dimensions
|
||||
audio = audio.squeeze(0)
|
||||
if len(audio.shape) > 1:
|
||||
raise ValueError("More than one dimension in audio. Are you trying to process audio with 2 channels?")
|
||||
|
||||
if sampling_rate > 16000 and (sampling_rate % 16000 == 0):
|
||||
step = sampling_rate // 16000
|
||||
sampling_rate = 16000
|
||||
audio = audio[::step]
|
||||
warnings.warn('Sampling rate is a multiply of 16000, casting to 16000 manually!')
|
||||
else:
|
||||
step = 1
|
||||
|
||||
if sampling_rate not in [8000, 16000]:
|
||||
raise ValueError("Currently silero VAD models support 8000 and 16000 (or multiply of 16000) sample rates")
|
||||
|
||||
window_size_samples = 512 if sampling_rate == 16000 else 256
|
||||
|
||||
model.reset_states()
|
||||
min_speech_samples = sampling_rate * min_speech_duration_ms / 1000
|
||||
speech_pad_samples = sampling_rate * speech_pad_ms / 1000
|
||||
max_speech_samples = sampling_rate * max_speech_duration_s - window_size_samples - 2 * speech_pad_samples
|
||||
min_silence_samples = sampling_rate * min_silence_duration_ms / 1000
|
||||
min_silence_samples_at_max_speech = sampling_rate * 98 / 1000
|
||||
|
||||
audio_length_samples = len(audio)
|
||||
|
||||
speech_probs = []
|
||||
for current_start_sample in range(0, audio_length_samples, window_size_samples):
|
||||
chunk = audio[current_start_sample: current_start_sample + window_size_samples]
|
||||
if len(chunk) < window_size_samples:
|
||||
chunk = torch.nn.functional.pad(chunk, (0, int(window_size_samples - len(chunk))))
|
||||
speech_prob = model(chunk, sampling_rate).item()
|
||||
speech_probs.append(speech_prob)
|
||||
# caculate progress and seng it to callback function
|
||||
progress = current_start_sample + window_size_samples
|
||||
if progress > audio_length_samples:
|
||||
progress = audio_length_samples
|
||||
progress_percent = (progress / audio_length_samples) * 100
|
||||
if progress_tracking_callback:
|
||||
progress_tracking_callback(progress_percent)
|
||||
|
||||
triggered = False
|
||||
speeches = []
|
||||
current_speech = {}
|
||||
|
||||
if neg_threshold is None:
|
||||
neg_threshold = max(threshold - 0.15, 0.01)
|
||||
temp_end = 0 # to save potential segment end (and tolerate some silence)
|
||||
prev_end = next_start = 0 # to save potential segment limits in case of maximum segment size reached
|
||||
|
||||
for i, speech_prob in enumerate(speech_probs):
|
||||
if (speech_prob >= threshold) and temp_end:
|
||||
temp_end = 0
|
||||
if next_start < prev_end:
|
||||
next_start = window_size_samples * i
|
||||
|
||||
if (speech_prob >= threshold) and not triggered:
|
||||
triggered = True
|
||||
current_speech['start'] = window_size_samples * i
|
||||
continue
|
||||
|
||||
if triggered and (window_size_samples * i) - current_speech['start'] > max_speech_samples:
|
||||
if prev_end:
|
||||
current_speech['end'] = prev_end
|
||||
speeches.append(current_speech)
|
||||
current_speech = {}
|
||||
if next_start < prev_end: # previously reached silence (< neg_thres) and is still not speech (< thres)
|
||||
triggered = False
|
||||
else:
|
||||
current_speech['start'] = next_start
|
||||
prev_end = next_start = temp_end = 0
|
||||
else:
|
||||
current_speech['end'] = window_size_samples * i
|
||||
speeches.append(current_speech)
|
||||
current_speech = {}
|
||||
prev_end = next_start = temp_end = 0
|
||||
triggered = False
|
||||
continue
|
||||
|
||||
if (speech_prob < neg_threshold) and triggered:
|
||||
if not temp_end:
|
||||
temp_end = window_size_samples * i
|
||||
if ((window_size_samples * i) - temp_end) > min_silence_samples_at_max_speech: # condition to avoid cutting in very short silence
|
||||
prev_end = temp_end
|
||||
if (window_size_samples * i) - temp_end < min_silence_samples:
|
||||
continue
|
||||
else:
|
||||
current_speech['end'] = temp_end
|
||||
if (current_speech['end'] - current_speech['start']) > min_speech_samples:
|
||||
speeches.append(current_speech)
|
||||
current_speech = {}
|
||||
prev_end = next_start = temp_end = 0
|
||||
triggered = False
|
||||
continue
|
||||
|
||||
if current_speech and (audio_length_samples - current_speech['start']) > min_speech_samples:
|
||||
current_speech['end'] = audio_length_samples
|
||||
speeches.append(current_speech)
|
||||
|
||||
for i, speech in enumerate(speeches):
|
||||
if i == 0:
|
||||
speech['start'] = int(max(0, speech['start'] - speech_pad_samples))
|
||||
if i != len(speeches) - 1:
|
||||
silence_duration = speeches[i+1]['start'] - speech['end']
|
||||
if silence_duration < 2 * speech_pad_samples:
|
||||
speech['end'] += int(silence_duration // 2)
|
||||
speeches[i+1]['start'] = int(max(0, speeches[i+1]['start'] - silence_duration // 2))
|
||||
else:
|
||||
speech['end'] = int(min(audio_length_samples, speech['end'] + speech_pad_samples))
|
||||
speeches[i+1]['start'] = int(max(0, speeches[i+1]['start'] - speech_pad_samples))
|
||||
else:
|
||||
speech['end'] = int(min(audio_length_samples, speech['end'] + speech_pad_samples))
|
||||
|
||||
if return_seconds:
|
||||
audio_length_seconds = audio_length_samples / sampling_rate
|
||||
for speech_dict in speeches:
|
||||
speech_dict['start'] = max(round(speech_dict['start'] / sampling_rate, 1), 0)
|
||||
speech_dict['end'] = min(round(speech_dict['end'] / sampling_rate, 1), audio_length_seconds)
|
||||
elif step > 1:
|
||||
for speech_dict in speeches:
|
||||
speech_dict['start'] *= step
|
||||
speech_dict['end'] *= step
|
||||
|
||||
if visualize_probs:
|
||||
make_visualization(speech_probs, window_size_samples / sampling_rate)
|
||||
|
||||
return speeches
|
||||
|
||||
|
||||
class VADIterator:
|
||||
def __init__(self,
|
||||
model,
|
||||
threshold: float = 0.5,
|
||||
sampling_rate: int = 16000,
|
||||
min_silence_duration_ms: int = 100,
|
||||
speech_pad_ms: int = 30
|
||||
):
|
||||
|
||||
"""
|
||||
Class for stream imitation
|
||||
|
||||
Parameters
|
||||
----------
|
||||
model: preloaded .jit/.onnx silero VAD model
|
||||
|
||||
threshold: float (default - 0.5)
|
||||
Speech threshold. Silero VAD outputs speech probabilities for each audio chunk, probabilities ABOVE this value are considered as SPEECH.
|
||||
It is better to tune this parameter for each dataset separately, but "lazy" 0.5 is pretty good for most datasets.
|
||||
|
||||
sampling_rate: int (default - 16000)
|
||||
Currently silero VAD models support 8000 and 16000 sample rates
|
||||
|
||||
min_silence_duration_ms: int (default - 100 milliseconds)
|
||||
In the end of each speech chunk wait for min_silence_duration_ms before separating it
|
||||
|
||||
speech_pad_ms: int (default - 30 milliseconds)
|
||||
Final speech chunks are padded by speech_pad_ms each side
|
||||
"""
|
||||
|
||||
self.model = model
|
||||
self.threshold = threshold
|
||||
self.sampling_rate = sampling_rate
|
||||
|
||||
if sampling_rate not in [8000, 16000]:
|
||||
raise ValueError('VADIterator does not support sampling rates other than [8000, 16000]')
|
||||
|
||||
self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000
|
||||
self.speech_pad_samples = sampling_rate * speech_pad_ms / 1000
|
||||
self.reset_states()
|
||||
|
||||
def reset_states(self):
|
||||
|
||||
self.model.reset_states()
|
||||
self.triggered = False
|
||||
self.temp_end = 0
|
||||
self.current_sample = 0
|
||||
|
||||
@torch.no_grad()
|
||||
def __call__(self, x, return_seconds=False):
|
||||
"""
|
||||
x: torch.Tensor
|
||||
audio chunk (see examples in repo)
|
||||
|
||||
return_seconds: bool (default - False)
|
||||
whether return timestamps in seconds (default - samples)
|
||||
"""
|
||||
|
||||
if not torch.is_tensor(x):
|
||||
try:
|
||||
x = torch.Tensor(x)
|
||||
except:
|
||||
raise TypeError("Audio cannot be casted to tensor. Cast it manually")
|
||||
|
||||
window_size_samples = len(x[0]) if x.dim() == 2 else len(x)
|
||||
self.current_sample += window_size_samples
|
||||
|
||||
speech_prob = self.model(x, self.sampling_rate).item()
|
||||
|
||||
if (speech_prob >= self.threshold) and self.temp_end:
|
||||
self.temp_end = 0
|
||||
|
||||
if (speech_prob >= self.threshold) and not self.triggered:
|
||||
self.triggered = True
|
||||
speech_start = max(0, self.current_sample - self.speech_pad_samples - window_size_samples)
|
||||
return {'start': int(speech_start) if not return_seconds else round(speech_start / self.sampling_rate, 1)}
|
||||
|
||||
if (speech_prob < self.threshold - 0.15) and self.triggered:
|
||||
if not self.temp_end:
|
||||
self.temp_end = self.current_sample
|
||||
if self.current_sample - self.temp_end < self.min_silence_samples:
|
||||
return None
|
||||
else:
|
||||
speech_end = self.temp_end + self.speech_pad_samples - window_size_samples
|
||||
self.temp_end = 0
|
||||
self.triggered = False
|
||||
return {'end': int(speech_end) if not return_seconds else round(speech_end / self.sampling_rate, 1)}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def collect_chunks(tss: List[dict],
|
||||
wav: torch.Tensor):
|
||||
chunks = []
|
||||
for i in tss:
|
||||
chunks.append(wav[i['start']: i['end']])
|
||||
return torch.cat(chunks)
|
||||
|
||||
|
||||
def drop_chunks(tss: List[dict],
|
||||
wav: torch.Tensor):
|
||||
chunks = []
|
||||
cur_start = 0
|
||||
for i in tss:
|
||||
chunks.append((wav[cur_start: i['start']]))
|
||||
cur_start = i['end']
|
||||
return torch.cat(chunks)
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,436 @@
|
||||
import time
|
||||
import aiohttp
|
||||
import asyncio
|
||||
from tabulate import tabulate
|
||||
from typing import Dict, List
|
||||
from core.utils.llm import create_instance as create_llm_instance
|
||||
from core.utils.tts import create_instance as create_tts_instance
|
||||
from core.utils.util import read_config
|
||||
import statistics
|
||||
from config.settings import get_config_file
|
||||
import inspect
|
||||
import os
|
||||
import logging
|
||||
|
||||
# 设置全局日志级别为WARNING，抑制INFO级别日志
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
|
||||
|
||||
class AsyncPerformanceTester:
|
||||
def __init__(self):
|
||||
self.config = read_config(get_config_file())
|
||||
self.test_sentences = self.config.get("module_test", {}).get(
|
||||
"test_sentences",
|
||||
["你好，请介绍一下你自己", "What's the weather like today?",
|
||||
"请用100字概括量子计算的基本原理和应用前景"]
|
||||
)
|
||||
self.results = {
|
||||
"llm": {},
|
||||
"tts": {},
|
||||
"combinations": []
|
||||
}
|
||||
|
||||
async def _check_ollama_service(self, base_url: str, model_name: str) -> bool:
|
||||
"""异步检查Ollama服务状态"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
# 检查服务是否可用
|
||||
async with session.get(f"{base_url}/api/version") as response:
|
||||
if response.status != 200:
|
||||
print(f"🚫 Ollama服务未启动或无法访问: {base_url}")
|
||||
return False
|
||||
|
||||
# 检查模型是否存在
|
||||
async with session.get(f"{base_url}/api/tags") as response:
|
||||
if response.status == 200:
|
||||
data = await response.json()
|
||||
models = data.get("models", [])
|
||||
if not any(model["name"] == model_name for model in models):
|
||||
print(f"🚫 Ollama模型 {model_name} 未找到，请先使用 ollama pull {model_name} 下载")
|
||||
return False
|
||||
else:
|
||||
print(f"🚫 无法获取Ollama模型列表")
|
||||
return False
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"🚫 无法连接到Ollama服务: {str(e)}")
|
||||
return False
|
||||
|
||||
async def _test_tts(self, tts_name: str, config: Dict) -> Dict:
|
||||
"""异步测试单个TTS性能"""
|
||||
try:
|
||||
logging.getLogger("core.providers.tts.base").setLevel(logging.WARNING)
|
||||
|
||||
token_fields = ["access_token", "api_key", "token"]
|
||||
if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in
|
||||
token_fields):
|
||||
print(f"⏭️ TTS {tts_name} 未配置access_token/api_key，已跳过")
|
||||
return {"name": tts_name, "type": "tts", "errors": 1}
|
||||
|
||||
module_type = config.get('type', tts_name)
|
||||
tts = create_tts_instance(
|
||||
module_type,
|
||||
config,
|
||||
delete_audio_file=True
|
||||
)
|
||||
|
||||
print(f"🎵 测试 TTS: {tts_name}")
|
||||
|
||||
tmp_file = tts.generate_filename()
|
||||
await tts.text_to_speak("连接测试", tmp_file)
|
||||
|
||||
if not tmp_file or not os.path.exists(tmp_file):
|
||||
print(f"❌ {tts_name} 连接失败")
|
||||
return {"name": tts_name, "type": "tts", "errors": 1}
|
||||
|
||||
total_time = 0
|
||||
test_count = len(self.test_sentences[:2])
|
||||
|
||||
for i, sentence in enumerate(self.test_sentences[:2], 1):
|
||||
start = time.time()
|
||||
tmp_file = tts.generate_filename()
|
||||
await tts.text_to_speak(sentence, tmp_file)
|
||||
duration = time.time() - start
|
||||
total_time += duration
|
||||
|
||||
if tmp_file and os.path.exists(tmp_file):
|
||||
print(f"✓ {tts_name} [{i}/{test_count}]")
|
||||
else:
|
||||
print(f"✗ {tts_name} [{i}/{test_count}]")
|
||||
return {"name": tts_name, "type": "tts", "errors": 1}
|
||||
|
||||
return {
|
||||
"name": tts_name,
|
||||
"type": "tts",
|
||||
"avg_time": total_time / test_count,
|
||||
"errors": 0
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ {tts_name} 测试失败: {str(e)}")
|
||||
return {"name": tts_name, "type": "tts", "errors": 1}
|
||||
|
||||
async def _test_llm(self, llm_name: str, config: Dict) -> Dict:
|
||||
"""异步测试单个LLM性能"""
|
||||
try:
|
||||
# 对于Ollama，跳过api_key检查并进行特殊处理
|
||||
if llm_name == "Ollama":
|
||||
base_url = config.get('base_url', 'http://localhost:11434')
|
||||
model_name = config.get('model_name')
|
||||
if not model_name:
|
||||
print(f"🚫 Ollama未配置model_name")
|
||||
return {"name": llm_name, "type": "llm", "errors": 1}
|
||||
|
||||
if not await self._check_ollama_service(base_url, model_name):
|
||||
return {"name": llm_name, "type": "llm", "errors": 1}
|
||||
else:
|
||||
if "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]):
|
||||
print(f"🚫 跳过未配置的LLM: {llm_name}")
|
||||
return {"name": llm_name, "type": "llm", "errors": 1}
|
||||
|
||||
# 获取实际类型（兼容旧配置）
|
||||
module_type = config.get('type', llm_name)
|
||||
llm = create_llm_instance(module_type, config)
|
||||
|
||||
# 统一使用UTF-8编码
|
||||
test_sentences = [s.encode('utf-8').decode('utf-8') for s in self.test_sentences]
|
||||
|
||||
# 创建所有句子的测试任务
|
||||
sentence_tasks = []
|
||||
for sentence in test_sentences:
|
||||
sentence_tasks.append(self._test_single_sentence(llm_name, llm, sentence))
|
||||
|
||||
# 并发执行所有句子测试
|
||||
sentence_results = await asyncio.gather(*sentence_tasks)
|
||||
|
||||
# 处理结果
|
||||
valid_results = [r for r in sentence_results if r is not None]
|
||||
if not valid_results:
|
||||
print(f"⚠️ {llm_name} 无有效数据，可能配置错误")
|
||||
return {"name": llm_name, "type": "llm", "errors": 1}
|
||||
|
||||
first_token_times = [r["first_token_time"] for r in valid_results]
|
||||
response_times = [r["response_time"] for r in valid_results]
|
||||
|
||||
# 过滤异常数据
|
||||
mean = statistics.mean(response_times)
|
||||
stdev = statistics.stdev(response_times) if len(response_times) > 1 else 0
|
||||
filtered_times = [t for t in response_times if t <= mean + 3 * stdev]
|
||||
|
||||
if len(filtered_times) < len(test_sentences) * 0.5:
|
||||
print(f"⚠️ {llm_name} 有效数据不足，可能网络不稳定")
|
||||
return {"name": llm_name, "type": "llm", "errors": 1}
|
||||
|
||||
return {
|
||||
"name": llm_name,
|
||||
"type": "llm",
|
||||
"avg_response": sum(response_times) / len(response_times),
|
||||
"avg_first_token": sum(first_token_times) / len(first_token_times),
|
||||
"std_first_token": statistics.stdev(first_token_times) if len(first_token_times) > 1 else 0,
|
||||
"std_response": statistics.stdev(response_times) if len(response_times) > 1 else 0,
|
||||
"errors": 0
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"LLM {llm_name} 测试失败: {str(e)}")
|
||||
return {"name": llm_name, "type": "llm", "errors": 1}
|
||||
|
||||
async def _test_single_sentence(self, llm_name: str, llm, sentence: str) -> Dict:
|
||||
"""测试单个句子的性能"""
|
||||
try:
|
||||
print(f"📝 {llm_name} 开始测试: {sentence[:20]}...")
|
||||
sentence_start = time.time()
|
||||
first_token_received = False
|
||||
first_token_time = None
|
||||
|
||||
async def process_response():
|
||||
nonlocal first_token_received, first_token_time
|
||||
for chunk in llm.response("perf_test", [{"role": "user", "content": sentence}]):
|
||||
if not first_token_received and chunk.strip() != '':
|
||||
first_token_time = time.time() - sentence_start
|
||||
first_token_received = True
|
||||
print(f"✓ {llm_name} 首个Token: {first_token_time:.3f}s")
|
||||
yield chunk
|
||||
|
||||
response_chunks = []
|
||||
async for chunk in process_response():
|
||||
response_chunks.append(chunk)
|
||||
|
||||
response_time = time.time() - sentence_start
|
||||
print(f"✓ {llm_name} 完成响应: {response_time:.3f}s")
|
||||
|
||||
if first_token_time is None:
|
||||
first_token_time = response_time # 如果没有检测到first token，使用总响应时间
|
||||
|
||||
return {
|
||||
"name": llm_name,
|
||||
"type": "llm",
|
||||
"first_token_time": first_token_time,
|
||||
"response_time": response_time
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"⚠️ {llm_name} 句子测试失败: {str(e)}")
|
||||
return None
|
||||
|
||||
def _generate_combinations(self):
|
||||
"""生成最佳组合建议"""
|
||||
valid_llms = [
|
||||
k for k, v in self.results["llm"].items()
|
||||
if v["errors"] == 0 and v["avg_first_token"] >= 0.05
|
||||
]
|
||||
valid_tts = [k for k, v in self.results["tts"].items() if v["errors"] == 0]
|
||||
|
||||
# 找出基准值
|
||||
min_first_token = min([self.results["llm"][llm]["avg_first_token"] for llm in valid_llms]) if valid_llms else 1
|
||||
min_tts_time = min([self.results["tts"][tts]["avg_time"] for tts in valid_tts]) if valid_tts else 1
|
||||
|
||||
for llm in valid_llms:
|
||||
for tts in valid_tts:
|
||||
# 计算相对性能分数（越小越好）
|
||||
llm_score = self.results["llm"][llm]["avg_first_token"] / min_first_token
|
||||
tts_score = self.results["tts"][tts]["avg_time"] / min_tts_time
|
||||
|
||||
# 计算稳定性分数（标准差/平均值，越小越稳定）
|
||||
llm_stability = self.results["llm"][llm]["std_first_token"] / self.results["llm"][llm][
|
||||
"avg_first_token"]
|
||||
|
||||
# 综合得分（考虑性能和稳定性）
|
||||
# 性能权重0.7，稳定性权重0.3
|
||||
llm_final_score = llm_score * 0.7 + llm_stability * 0.3
|
||||
|
||||
# 总分 = LLM得分(70%) + TTS得分(30%)
|
||||
total_score = llm_final_score * 0.7 + tts_score * 0.3
|
||||
|
||||
self.results["combinations"].append({
|
||||
"llm": llm,
|
||||
"tts": tts,
|
||||
"score": total_score,
|
||||
"details": {
|
||||
"llm_first_token": self.results["llm"][llm]["avg_first_token"],
|
||||
"llm_stability": llm_stability,
|
||||
"tts_time": self.results["tts"][tts]["avg_time"]
|
||||
}
|
||||
})
|
||||
|
||||
# 分数越小越好
|
||||
self.results["combinations"].sort(key=lambda x: x["score"])
|
||||
|
||||
def _print_results(self):
|
||||
"""打印测试结果"""
|
||||
llm_table = []
|
||||
for name, data in self.results["llm"].items():
|
||||
if data["errors"] == 0:
|
||||
stability = data["std_first_token"] / data["avg_first_token"]
|
||||
llm_table.append([
|
||||
name, # 不需要固定宽度，让tabulate自己处理对齐
|
||||
f"{data['avg_first_token']:.3f}秒",
|
||||
f"{data['avg_response']:.3f}秒",
|
||||
f"{stability:.3f}"
|
||||
])
|
||||
|
||||
if llm_table:
|
||||
print("\nLLM 性能排行:")
|
||||
print(tabulate(
|
||||
llm_table,
|
||||
headers=["模型名称", "首字耗时", "总耗时", "稳定性"],
|
||||
tablefmt="github",
|
||||
colalign=("left", "right", "right", "right"),
|
||||
disable_numparse=True
|
||||
))
|
||||
else:
|
||||
print("\n⚠️ 没有可用的LLM模块进行测试。")
|
||||
|
||||
tts_table = []
|
||||
for name, data in self.results["tts"].items():
|
||||
if data["errors"] == 0:
|
||||
tts_table.append([
|
||||
name, # 不需要固定宽度
|
||||
f"{data['avg_time']:.3f}秒"
|
||||
])
|
||||
|
||||
if tts_table:
|
||||
print("\nTTS 性能排行:")
|
||||
print(tabulate(
|
||||
tts_table,
|
||||
headers=["模型名称", "合成耗时"],
|
||||
tablefmt="github",
|
||||
colalign=("left", "right"),
|
||||
disable_numparse=True
|
||||
))
|
||||
else:
|
||||
print("\n⚠️ 没有可用的TTS模块进行测试。")
|
||||
|
||||
if self.results["combinations"]:
|
||||
print("\n推荐配置组合 (得分越小越好):")
|
||||
combo_table = []
|
||||
for combo in self.results["combinations"][:5]:
|
||||
combo_table.append([
|
||||
f"{combo['llm']} + {combo['tts']}", # 不需要固定宽度
|
||||
f"{combo['score']:.3f}",
|
||||
f"{combo['details']['llm_first_token']:.3f}秒",
|
||||
f"{combo['details']['llm_stability']:.3f}",
|
||||
f"{combo['details']['tts_time']:.3f}秒"
|
||||
])
|
||||
|
||||
print(tabulate(
|
||||
combo_table,
|
||||
headers=["组合方案", "综合得分", "LLM首字耗时", "稳定性", "TTS合成耗时"],
|
||||
tablefmt="github",
|
||||
colalign=("left", "right", "right", "right", "right"),
|
||||
disable_numparse=True
|
||||
))
|
||||
else:
|
||||
print("\n⚠️ 没有可用的模块组合建议。")
|
||||
|
||||
def _process_results(self, all_results):
|
||||
"""处理测试结果"""
|
||||
for result in all_results:
|
||||
if result["errors"] == 0:
|
||||
if result["type"] == "llm":
|
||||
self.results["llm"][result["name"]] = result
|
||||
else:
|
||||
self.results["tts"][result["name"]] = result
|
||||
|
||||
async def run(self):
|
||||
"""执行全量异步测试"""
|
||||
print("🔍 开始筛选可用模块...")
|
||||
|
||||
# 创建所有测试任务
|
||||
all_tasks = []
|
||||
|
||||
# LLM测试任务
|
||||
for llm_name, config in self.config.get("LLM", {}).items():
|
||||
# 检查配置有效性
|
||||
if llm_name == "CozeLLM":
|
||||
if any(x in config.get("bot_id", "") for x in ["你的"]) \
|
||||
or any(x in config.get("user_id", "") for x in ["你的"]):
|
||||
print(f"⏭️ LLM {llm_name} 未配置bot_id/user_id，已跳过")
|
||||
continue
|
||||
elif "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]):
|
||||
print(f"⏭️ LLM {llm_name} 未配置api_key，已跳过")
|
||||
continue
|
||||
|
||||
# 对于Ollama，先检查服务状态
|
||||
if llm_name == "Ollama":
|
||||
base_url = config.get('base_url', 'http://localhost:11434')
|
||||
model_name = config.get('model_name')
|
||||
if not model_name:
|
||||
print(f"🚫 Ollama未配置model_name")
|
||||
continue
|
||||
|
||||
if not await self._check_ollama_service(base_url, model_name):
|
||||
continue
|
||||
|
||||
print(f"📋 添加LLM测试任务: {llm_name}")
|
||||
module_type = config.get('type', llm_name)
|
||||
llm = create_llm_instance(module_type, config)
|
||||
|
||||
# 为每个句子创建独立任务
|
||||
for sentence in self.test_sentences:
|
||||
sentence = sentence.encode('utf-8').decode('utf-8')
|
||||
all_tasks.append(self._test_single_sentence(llm_name, llm, sentence))
|
||||
|
||||
# TTS测试任务
|
||||
for tts_name, config in self.config.get("TTS", {}).items():
|
||||
token_fields = ["access_token", "api_key", "token"]
|
||||
if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in
|
||||
token_fields):
|
||||
print(f"⏭️ TTS {tts_name} 未配置access_token/api_key，已跳过")
|
||||
continue
|
||||
print(f"🎵 添加TTS测试任务: {tts_name}")
|
||||
all_tasks.append(self._test_tts(tts_name, config))
|
||||
|
||||
print(
|
||||
f"\n✅ 找到 {len([t for t in all_tasks if 'test_single_sentence' in str(t)]) / len(self.test_sentences):.0f} 个可用LLM模块")
|
||||
print(f"✅ 找到 {len([t for t in all_tasks if '_test_tts' in str(t)])} 个可用TTS模块")
|
||||
print("\n⏳ 开始并发测试所有模块...\n")
|
||||
|
||||
# 并发执行所有测试任务
|
||||
all_results = await asyncio.gather(*all_tasks, return_exceptions=True)
|
||||
|
||||
# 处理LLM结果
|
||||
llm_results = {}
|
||||
for result in [r for r in all_results if r and isinstance(r, dict) and r.get("type") == "llm"]:
|
||||
llm_name = result["name"]
|
||||
if llm_name not in llm_results:
|
||||
llm_results[llm_name] = {
|
||||
"name": llm_name,
|
||||
"type": "llm",
|
||||
"first_token_times": [],
|
||||
"response_times": [],
|
||||
"errors": 0
|
||||
}
|
||||
llm_results[llm_name]["first_token_times"].append(result["first_token_time"])
|
||||
llm_results[llm_name]["response_times"].append(result["response_time"])
|
||||
|
||||
# 计算LLM平均值和标准差
|
||||
for llm_name, data in llm_results.items():
|
||||
if len(data["first_token_times"]) >= len(self.test_sentences) * 0.5:
|
||||
self.results["llm"][llm_name] = {
|
||||
"name": llm_name,
|
||||
"type": "llm",
|
||||
"avg_response": sum(data["response_times"]) / len(data["response_times"]),
|
||||
"avg_first_token": sum(data["first_token_times"]) / len(data["first_token_times"]),
|
||||
"std_first_token": statistics.stdev(data["first_token_times"]) if len(
|
||||
data["first_token_times"]) > 1 else 0,
|
||||
"std_response": statistics.stdev(data["response_times"]) if len(data["response_times"]) > 1 else 0,
|
||||
"errors": 0
|
||||
}
|
||||
|
||||
# 处理TTS结果
|
||||
for result in [r for r in all_results if r and isinstance(r, dict) and r.get("type") == "tts"]:
|
||||
if result["errors"] == 0:
|
||||
self.results["tts"][result["name"]] = result
|
||||
|
||||
# 生成组合建议并打印结果
|
||||
print("\n📊 生成测试报告...")
|
||||
self._generate_combinations()
|
||||
self._print_results()
|
||||
|
||||
|
||||
async def main():
|
||||
tester = AsyncPerformanceTester()
|
||||
await tester.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Generated
+4854
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
[tool.poetry]
|
||||
name = "xiaozhi-esp32-server"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["kalicyh <34980061+kaliCYH@users.noreply.github.com>"]
|
||||
readme = "README.md"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.10"
|
||||
pyyml = "0.0.2"
|
||||
torch = "2.2.2"
|
||||
silero-vad = "5.1.2"
|
||||
websockets = "14.2"
|
||||
numpy = "1.26.4"
|
||||
pydub = "0.25.1"
|
||||
funasr = "1.2.3"
|
||||
torchaudio = "2.2.2"
|
||||
openai = "1.61.0"
|
||||
google-generativeai = "0.8.4"
|
||||
edge-tts = "7.0.0"
|
||||
httpx = "0.27.2"
|
||||
aiohttp = "3.9.3"
|
||||
aiohttp-cors = "0.7.0"
|
||||
ormsgpack = "1.7.0"
|
||||
ruamel-yaml = "0.18.10"
|
||||
setuptools = "^75.8.0"
|
||||
loguru = "^0.7.3"
|
||||
opuslib-next = "^1.1.2"
|
||||
fastapi = {extras = ["all"], version = "^0.115.8"}
|
||||
uvicorn = "^0.34.0"
|
||||
pyjwt = "^2.10.1"
|
||||
python-jose = {extras = ["cryptography"], version = "^3.3.0"}
|
||||
bcrypt = "^4.2.1"
|
||||
sqlalchemy = "^2.0.38"
|
||||
pymysql = "^1.1.1"
|
||||
asyncpg = "^0.30.0"
|
||||
onnxruntime = "1.19.2"
|
||||
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
pyyml==0.0.2
|
||||
torch==2.2.2
|
||||
silero_vad==5.1.2
|
||||
websockets==14.2
|
||||
opuslib_next==1.1.2
|
||||
numpy==1.26.4
|
||||
pydub==0.25.1
|
||||
funasr==1.2.3
|
||||
torchaudio==2.2.2
|
||||
openai==1.61.0
|
||||
google-generativeai==0.8.4
|
||||
edge_tts==7.0.0
|
||||
httpx==0.27.2
|
||||
aiohttp==3.9.3
|
||||
aiohttp_cors==0.7.0
|
||||
ormsgpack==1.7.0
|
||||
ruamel.yaml==0.18.10
|
||||
loguru==0.7.3
|
||||
requests==2.32.3
|
||||
cozepy==0.12.0
|
||||
mem0ai==0.1.62
|
||||
Reference in New Issue
Block a user