Mem0ai (#199)

* update:增加mem0ai包依赖

* update:文档增加mem0ai的描述

---------

Co-authored-by: hrz <1710360675@qq.com>

.github/workflows/docker-image.yml

@@ -16,6 +16,14 @@ jobs:
       issues: write
 
     steps:
+      - name: Check Disk Space
+        run: |
+          df -h
+          docker system df
+      - name: Clean up Docker resources
+        run: |
+          docker system prune -af
+          docker builder prune -af
       - name: Check out the repo
         uses: actions/checkout@v4
 
@@ -43,4 +51,4 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:${{ env.VERSION }}
             ghcr.io/${{ github.repository }}:latest
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm64

.gitignore

@@ -1,5 +1,6 @@
 # Byte-compiled / optimized / DLL files
 __pycache__/
+.idea/
 *.py[cod]
 *$py.class
 
@@ -131,6 +132,9 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
+# music directory
+music/
+
 # pytype static type analyzer
 .pytype/
 
@@ -145,3 +149,4 @@ tmp
 .private_config.yaml
 .env.development
 docker-compose.yml
+web/vue/node_modules

Dockerfile

@@ -1,20 +1,4 @@
-# 第一阶段：前端构建
+# 第一阶段：构建 Python 依赖
-
-FROM kalicyh/node:v18-alpine AS frontend-builder
-
-WORKDIR /app/ZhiKongTaiWeb
-
-# RUN corepack enable && yarn config set registry https://registry.npmmirror.com
-
-COPY ZhiKongTaiWeb/package.json ZhiKongTaiWeb/yarn.lock ./
-
-RUN yarn install --frozen-lockfile
-
-COPY ZhiKongTaiWeb . 
-RUN yarn build
-
-# 第二阶段：构建 Python 依赖
-
 FROM kalicyh/poetry:v3.10_xiaozhi AS builder
 
 WORKDIR /app
@@ -24,19 +8,6 @@ COPY . .
 # 检查是否有缺失
 RUN poetry install --no-root
 
-# 使用清华源加速apt安装，该镜像内置所以注释
-# RUN rm -rf /etc/apt/sources.list.d/* && \
-#     echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list && \
-#     echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \
-#     echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-backports main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \
-#     echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bookworm-security main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \
-#     apt-get update && \
-#     apt-get install -y --no-install-recommends libopus0 ffmpeg && \
-#     apt-get clean
-
-# 从构建阶段复制虚拟环境和前端构建产物
-COPY --from=frontend-builder /app/ZhiKongTaiWeb/dist /app/manager/static/webui
-
 # 设置虚拟环境路径
 ENV PATH="/app/.venv/bin:$PATH"
 

Dockerfile-pip

@@ -25,7 +25,7 @@ COPY requirements.txt .
 
 # 优化apt安装
 RUN pip install --no-cache-dir -r requirements.txt \
-    -i https://pypi.tuna.tsinghua.edu.cn/simple
+    -i https://mirrors.aliyun.com/pypi/simple/
 
 # 第三阶段：生产镜像
 FROM python:3.10-slim
@@ -33,8 +33,8 @@ FROM python:3.10-slim
 WORKDIR /opt/xiaozhi-esp32-server
 
 # 优化apt安装
-RUN echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list && \
+RUN echo "deb https://mirrors.aliyun.com/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list && \
-    echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \
+    echo "deb https://mirrors.aliyun.com/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \
     apt-get update && \
     apt-get install -y --no-install-recommends libopus0 ffmpeg && \
     apt-get clean && \

README.md

@@ -1,4 +1,15 @@
-![图片](docs/images/banner.png)
+[![SVG Banners](https://svg-banners.vercel.app/api?type=origin&text1=你好😃，小智📟&text2=开源小智ESP-32后端服务&width=830&height=210)](https://github.com/xinnan-tech/xiaozhi-esp32-server)
+<p align="center">
+  <a href="https://github.com/xinnan-tech/xiaozhi-esp32-server/graphs/contributors">
+    <img alt="GitHub Contributors" src="https://img.shields.io/github/contributors/xinnan-tech/xiaozhi-esp32-server" />
+  </a>
+  <a href="https://github.com/xinnan-tech/xiaozhi-esp32-server/issues">
+    <img alt="Issues" src="https://img.shields.io/github/issues/xinnan-tech/xiaozhi-esp32-server?color=0088ff" />
+  </a>
+  <a href="https://github.com/xinnan-tech/xiaozhi-esp32-server/pulls">
+    <img alt="GitHub pull requests" src="https://img.shields.io/github/issues-pr/xinnan-tech/xiaozhi-esp32-server?color=0088ff" />
+  </a>
+</p>
 
 # 小智 ESP-32 后端服务(xiaozhi-esp32-server)
 
@@ -14,15 +25,65 @@
 本项目需要配合 ESP32 硬件设备使用。如果您已经购买了 ESP32 相关硬件，且成功对接过虾哥部署的后端服务，并希望独立搭建自己的
 `xiaozhi-esp32` 后端服务，那么本项目非常适合您。
 
-跳转到[使用方式](#使用方式-)
+想看使用效果？请猛戳视频 🎥
 
-想看使用效果？请猛戳这个视频 🎥
+<table>
-
+  <tr>
-<a href="https://www.bilibili.com/video/BV1FMFyejExX">
+    <td>
- <picture>
+        <a href="https://www.bilibili.com/video/BV1FMFyejExX" target="_blank">
-   <img alt="小智esp32连接自己的后台模型" src="docs/images/demo.png" />
+         <picture>
- </picture>
+           <img alt="小智esp32连接自己的后台模型" src="docs/images/demo1.png" />
-</a>
+         </picture>
+        </a>
+    </td>
+    <td>
+        <a href="https://www.bilibili.com/video/BV1CDKWemEU6" target="_blank">
+         <picture>
+           <img alt="自定义音色" src="docs/images/demo2.png" />
+         </picture>
+        </a>
+    </td>
+    <td>
+        <a href="https://www.bilibili.com/video/BV12yA2egEaC" target="_blank">
+         <picture>
+           <img alt="使用粤语交流" src="docs/images/demo3.png" />
+         </picture>
+        </a>
+    </td>
+    <td>
+        <a href="https://www.bilibili.com/video/av114036381327149" target="_blank">
+         <picture>
+           <img alt="控制家电开关" src="docs/images/demo5.png" />
+         </picture>
+        </a>
+    </td>
+  </tr>
+  <tr>
+    <td>
+        <a href="https://www.bilibili.com/video/BV1Vy96YCE3R" target="_blank">
+         <picture>
+           <img alt="自定义音色" src="docs/images/demo6.png" />
+         </picture>
+        </a>
+    </td>
+    <td>
+        <a href="https://www.bilibili.com/video/BV1VC96Y5EMH" target="_blank">
+         <picture>
+           <img alt="播放音乐" src="docs/images/demo7.png" />
+         </picture>
+        </a>
+    </td>
+    <td>
+        <a href="https://www.bilibili.com/video/BV1kgA2eYEQ9" target="_blank">
+         <picture>
+           <img alt="成本最低配置" src="docs/images/demo4.png" />
+         </picture>
+        </a>
+    </td>
+    <td>
+    </td>
+  </tr>
+</table>
 
 ---
 
@@ -30,16 +91,24 @@
 
 - **硬件**：一套兼容 `xiaozhi-esp32`
   的硬件设备（具体型号请参考 [此处](https://rcnv1t9vps13.feishu.cn/wiki/DdgIw4BUgivWDPkhMj1cGIYCnRf)）。
-- **服务器**：至少 4 核 CPU、8G 内存的电脑或服务器。
+
-- **固件编译**：请将后端服务的接口地址更新至 `xiaozhi-esp32` 项目中，再重新编译固件并烧录到设备上。
+- **电脑或服务器**：至少 4 核 CPU、8G 内存的电脑。
+- **固件编译**：请将本后端服务的接口地址更新至 `xiaozhi-esp32` 项目中，再重新编译`xiaozhi-esp32`固件并烧录到设备上。
+
+如果你没有esp32相关的硬件设备，但是非常想体验该项目，可以使用以下的项目让你的电脑、手机模拟成esp32设备。
+
+- [小智安卓端](https://github.com/TOM88812/xiaozhi-android-client)
+- [小智电脑端](https://github.com/Huang-junsen/py-xiaozhi)
 
 ---
 
 ## 警告 ⚠️
 
-本项目成立时间较短，还未通过网络安全测评，请勿在生产环境中使用。
+1、本项目为开源软件，本软件与对接的任何第三方API服务商（包括但不限于语音识别、大模型、语音合成等平台）均不存在商业合作关系，不为其服务质量及资金安全提供任何形式的担保。
+建议使用者优先选择持有相关业务牌照的服务商，并仔细阅读其服务协议及隐私政策。本软件不托管任何账户密钥、不参与资金流转、不承担充值资金损失风险。
 
-如果您在公网环境中部署学习本项目，请务必在配置文件 `config.yaml` 中开启防护：
+2、本项目成立时间较短，还未通过网络安全测评，请勿在生产环境中使用。 如果您在公网环境中部署学习本项目，请务必在配置文件
+`config.yaml` 中开启防护：
 
 ```yaml
 server:
@@ -118,9 +187,18 @@ server:
 
 ### ASR
 
-| 类型  |  平台名称  | 使用方式 | 收费模式 | 备注 |
+| 类型  |   平台名称    | 使用方式 | 收费模式 | 备注 |
-|:---:|:------:|:----:|:----:|:--:|
+|:---:|:---------:|:----:|:----:|:--:|
-| ASR | FunASR | 本地使用 |  免费  |    |
+| ASR |  FunASR   | 本地使用 |  免费  |    |
+| ASR | DoubaoASR | 接口调用 |  收费  |    |
+
+---
+
+### Memory
+
+|   类型   |  平台名称  | 使用方式 | 收费模式 | 备注 |
+|:------:|:------:|:----:|:----:|:--:|
+| Memory | mem0ai | 接口调用 |  免费  |    |
 
 ---
 
@@ -128,9 +206,25 @@ server:
 
 ### 一、[部署文档](./docs/Deployment.md)
 
-1.**[本地源码运行](./docs/Deployment.md#%E6%96%B9%E5%BC%8F%E4%B8%89%E6%9C%AC%E5%9C%B0%E6%BA%90%E7%A0%81%E8%BF%90%E8%A1%8C)**
+本项目支持以下三种部署方式，您可根据实际需求选择。
-  
+
-适合熟悉 Conda 环境或希望从零搭建运行环境的用户。  
+本项目的文档主要是`文字版本`的教程，如果你想要`视频版本`
+的教程，您可以学习一下[这个大佬的手把手教程](https://www.bilibili.com/video/BV1gePuejEvT)。
+
+如果你能把`文字版本的教程`和`视频版本的教程`结合起来一起看，可以让你更快上手。
+
+1. [Docker 快速部署](./docs/Deployment.md)
+
+适合快速体验的普通用户，不需过多环境配置。缺点是，拉取镜像有点慢。
+
+2. [借助 Docker 环境运行部署](./docs/Deployment.md#%E6%96%B9%E5%BC%8F%E4%BA%8C%E5%80%9F%E5%8A%A9docker%E7%8E%AF%E5%A2%83%E8%BF%90%E8%A1%8C%E9%83%A8%E7%BD%B2)
+
+适用于已安装 Docker 且希望对代码进行自定义修改的软件工程师。
+
+3. [本地源码运行](./docs/Deployment.md#%E6%96%B9%E5%BC%8F%E4%B8%89%E6%9C%AC%E5%9C%B0%E6%BA%90%E7%A0%81%E8%BF%90%E8%A1%8C)
+
+适合熟悉`Conda` 环境或希望从零搭建运行环境的用户。
+
 对于对响应速度要求较高的场景，推荐使用本地源码运行方式以降低额外开销。
 
 ### 二、[固件编译](./docs/firmware-build.md)
@@ -166,13 +260,15 @@ VAD:
 
 ### 4、为什么我说的话，小智识别出来很多韩文、日文、英文？🇰🇷
 
-建议：检查一下`models/SenseVoiceSmall`是否已经有`model.pt`文件，如果没有就要下载，查看这里[下载语音识别模型文件](docs/Deployment.md#模型文件)
+建议：检查一下`models/SenseVoiceSmall`是否已经有`model.pt`
+文件，如果没有就要下载，查看这里[下载语音识别模型文件](docs/Deployment.md#模型文件)
 
 ### 5、为什么会出现“TTS 任务出错 文件不存在”？📁
 
 建议：检查一下是否正确使用`conda` 安装了`libopus`和`ffmpeg`库。
 
 如果没有安装，就安装
+
 ```
 conda install conda-forge::libopus
 conda install conda-forge::ffmpeg
@@ -238,14 +334,19 @@ TTS 性能排行:
 
 ### 7、更多问题，可联系我们反馈 💬
 
-![图片](docs/images/wechat.jpg)
+我们的联系方式放在[百度网盘中,点击前往](https://pan.baidu.com/s/1x6USjvP1nTRsZ45XlJu65Q)，提取码是`223y`。
+
+网盘里有“硬件烧录QQ群”、“开源服务端交流群”、“产品建议联系人” 三张图片，请根据需要选择加入。
+
+- 硬件烧录QQ群：适用于硬件烧录问题
+- 开源服务端交流群：适用于服务端问题
+- 产品建议联系人：适用于产品功能、产品设计等建议
 
 ---
 
 ## 鸣谢 🙏
 
 - 本项目受 [百聆语音对话机器人](https://github.com/wwbin2017/bailing) 启发，并在其基础上实现。
-- 感谢 [腾讯云](https://cloud.tencent.com/) 提供免费 Docker 镜像空间。
 - 感谢 [十方融海](https://www.tenclass.com/) 对小智通讯协议提供的详尽文档支持。
 
 <a href="https://star-history.com/#xinnan-tech/xiaozhi-esp32-server&Date">

README_en.md

@@ -1,427 +1,291 @@
-![图片](docs/images/banner.png)
 
-# Xiaozhi ESP-32 Back-end Service (xiaozhi-esp32-server)
+[![SVG Banners](https://svg-banners.vercel.app/api?type=origin&text1=Hello😃,%20XiaoZhi📟&text2=Open%20Source%20XiaoZhi%20ESP-32%20Backend%20Service&width=830&height=210)](https://github.com/xinnan-tech/xiaozhi-esp32-server)
+<p align="center">
+  <a href="https://github.com/xinnan-tech/xiaozhi-esp32-server/graphs/contributors">
+    <img alt="GitHub Contributors" src="https://img.shields.io/github/contributors/xinnan-tech/xiaozhi-esp32-server" />
+  </a>
+  <a href="https://github.com/xinnan-tech/xiaozhi-esp32-server/issues">
+    <img alt="Issues" src="https://img.shields.io/github/issues/xinnan-tech/xiaozhi-esp32-server?color=0088ff" />
+  </a>
+  <a href="https://github.com/xinnan-tech/xiaozhi-esp32-server/pulls">
+    <img alt="GitHub pull requests" src="https://img.shields.io/github/issues-pr/xinnan-tech/xiaozhi-esp32-server?color=0088ff" />
+  </a>
+</p>
 
-（[中文](README.md) | English）
+# XiaoZhi ESP-32 Backend Service (xiaozhi-esp32-server)
 
-This project provides backend services for the open-source smart hardware
+([中文](README.md) | English)
-project [xiaozhi-esp32](https://github.com/78/xiaozhi-esp32)。Implemented in Python following
-the[Xiaozhi Communication Protocol](https://ccnphfhqs21z.feishu.cn/wiki/M0XiwldO9iJwHikpXD5cEx71nKh)。
 
-## Target Audience
+This project provides the backend service for the open source smart hardware project [xiaozhi-esp32](https://github.com/78/xiaozhi-esp32). It is implemented in `Python` based on the [XiaoZhi Communication Protocol](https://ccnphfhqs21z.feishu.cn/wiki/M0XiwldO9iJwHikpXD5cEx71nKh).
 
-This project requires compatible esp32 hardware devices. If you have purchased esp32 hardware, successfully connected to
+---
-Xiage's deployed backend, and wish to independently set up the `xiaozhi-esp32` backend service, this project is for
-you.
 
-To see a demo, watch this video:
+## Target Audience 👥
 
-<a href="https://www.bilibili.com/video/BV1FMFyejExX">
+This project is designed to be used in conjunction with ESP32 hardware devices. If you have already purchased an ESP32 device, successfully connected to the backend service deployed by XieGe, and now wish to set up your own `xiaozhi-esp32` backend service, then this project is perfect for you.
- <picture>
-   <img alt="小智esp32连接自己的后台模型" src="docs/images/demo.png" />
- </picture>
-</a>
 
-To fully experience this project, follow these steps:
+Want to see it in action? Check out the videos 🎥
 
-- Prepare hardware compatible with the `xiaozhi-esp32` project. For supported
+<table>
-  models, [click here](https://rcnv1t9vps13.feishu.cn/wiki/DdgIw4BUgivWDPkhMj1cGIYCnRf).
+  <tr>
-- Use a computer/server with at least 4-core CPU and 8GB RAM to run this project. After deployment, you'll see the
+    <td>
-  service endpoint address in the console.
+      <a href="https://www.bilibili.com/video/BV1FMFyejExX" target="_blank">
-- Download the `xiaozhi-esp32` project, replace the default `endpoint address` with your own, compile, and flash the
+        <picture>
-  firmware to your device.
+          <img alt="XiaoZhi ESP32 connecting to a custom backend model" src="docs/images/demo1.png" />
-- Start the device and check your server console logs to verify successful connection.
+        </picture>
+      </a>
+    </td>
+    <td>
+      <a href="https://www.bilibili.com/video/BV1CDKWemEU6" target="_blank">
+        <picture>
+          <img alt="Custom Voice" src="docs/images/demo2.png" />
+        </picture>
+      </a>
+    </td>
+    <td>
+      <a href="https://www.bilibili.com/video/BV12yA2egEaC" target="_blank">
+        <picture>
+          <img alt="Conversing in Cantonese" src="docs/images/demo3.png" />
+        </picture>
+      </a>
+    </td>
+    <td>
+      <a href="https://www.bilibili.com/video/av114036381327149" target="_blank">
+        <picture>
+          <img alt="Control Home Appliances" src="docs/images/demo5.png" />
+        </picture>
+      </a>
+    </td>
+    <td>
+      <a href="https://www.bilibili.com/video/BV1kgA2eYEQ9" target="_blank">
+        <picture>
+          <img alt="Lowest Cost Configuration" src="docs/images/demo4.png" />
+        </picture>
+      </a>
+    </td>
+    <td>
+    </td>
+  </tr>
+</table>
 
-## Warning
+---
 
-This project has been established for a short time and has not passed the network security assessment, so please do not
+## System Requirements and Deployment Prerequisites 🖥️
-use it in the production environment.
 
-## Feature List
+- **Hardware**: A set of devices compatible with `xiaozhi-esp32` (for specific models, please refer to [this link](https://rcnv1t9vps13.feishu.cn/wiki/DdgIw4BUgivWDPkhMj1cGIYCnRf)).
+- **Server**: A computer with at least a 4-core CPU and 8GB of memory.
+- **Firmware Compilation**: Please update the backend service API endpoint in the `xiaozhi-esp32` project, then recompile the firmware and flash it to your device.
 
-## Implemented
+---
 
-- `xiaozhi-esp32` WebSocket communication protocol
+## Warning ⚠️
-- Supports wake-word initiated dialogue, manual dialogue, and real-time interruption of dialogue.
-- Support for 5 languages: Mandarin, Cantonese, English, Japanese, Korean (FunASR - default)
-- Flexible LLM switching (openai:ChatGLM - default, Aliyun, DeepSeek; dify:Dify)
-- Flexible TTS switching (EdgeTTS - default, ByteDance Doubao TTS)
 
-## In Progress
+This project is relatively new and has not yet undergone network security evaluations. **Do not use it in a production environment.**
 
-- Sleep mode after inactivity
+If you deploy this project on a public network for learning purposes, be sure to enable protection in the configuration file `config.yaml`:
-- Dialogue memory
-- Change the mood mode
 
-## Supported Services
+```yaml
-
+server:
-| Type | Service    |  Usage   | Pricing Model	 | Notes                                                                      |
+  auth:
-|:-----|:-----------|:--------:|:---------------|:---------------------------------------------------------------------------|
+    # Enable protection
-| LLM  | Aliyun     | openai API call | Token-based    | [Apply for API Key](https://bailian.console.aliyun.com/?apiKey=1#/api-key) |
+    enabled: true  
-| LLM  | DeepSeek   | openai API call | Token-based    | [Apply for API Key](https://platform.deepseek.com/)                        |
-| LLM  | Bigmodel   | openai API call | Free           | [Create API Key](https://bigmodel.cn/usercenter/proj-mgmt/apikeys)         |
-| LLM  | Dify       | dify API call | Token-based    | Self-hosted                                                                |
-| TTS  | HuoshanTTS | API call | Token-based    | [Create API Key](https://console.volcengine.com/speech/service/8)          |
-| TTS  | EdgeTTS    | API call | Free           |                                                                            |
-| VAD  | SileroVAD  |  Local   | Free           |                                                                            |
-| ASR  | FunASR     |  Local   | Free           |                                                                            |
-
-In fact, any LLM that supports OpenAI API calls can be integrated and used.
-
-# Deployment
-
-This project supports rapid deployment of docker and local source code operation. If you want to have a quick
-experience, it is recommended to use docker to deploy. If you want to have an in-depth understanding of this project, it
-is recommended to run the local source code.
-
-## Method 1: Quick deployment of docker
-
-The docker image has supported the CPU of x86 architecture and arm64 architecture, and supports running on Chinese
-operating systems.
-
-1. Install docker
-
-If your computer has not installed docker, you can follow the tutorial here to install
-it:[Install docker](https://www.runoob.com/docker/ubuntu-docker-install.html)
-
-2. Create a directory
-
-After installation, you need to find a directory for the configuration file for this project. Let's call it the
-`project directory` for the time being. This directory is preferably a newly created empty directory.
-
-3. Download the configuration file
-
-Open with a browser[This link](https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/config.yaml)。
-
-On the right side of the page, find the button named `RAW`, next to the `RAW` button, find the download icon, click the
-Download button, and download the `config.yaml` file. Download the file to your `project directory`.
-
-4. Configure Project
-
-Modify the `config.yaml` file to configure the various parameters required for this project. The default LLM uses
-`ChatGLMLLM`, you need to configure the key to start.
-The default TTS uses `EdgeTTS`. This does not require configuration. If you need to replace it with`Doubao TTS`, you
-need to
-configure the key.
-
-Configuration description: This is the default component of each function, such as LLM default to use the `ChatGLMLLM`
-model. If you need to switch the model, it is the corresponding name.
-
-The default configuration of this project is only the lowest operating cost configuration（`glm-4-flash`and`EdgeTTS`are
-free），If you need to be better and faster, you need to combine the use of the deployment environment to switch the use
-of each component。
-
-```
-selected_module:
-  ASR: FunASR
-  VAD: SileroVAD
-  LLM: ChatGLMLLM
-  TTS: EdgeTTS
 ```
 
-For example, to modify the components used by the `LLM`, it depends on which `LLM` API interfaces are supported by this project. Currently, the supported ones are `openai` and `dify`. We welcome validation and support for more LLM platforms' interfaces.
+Once protection is enabled, you will need to validate the machine's token or MAC address based on your actual situation. Please refer to the configuration documentation for details.
-When using it, change the `selected_module` to the corresponding name of the following LLM configurations:
-
-```
-LLM:
-  AliLLM:
-    type: openai
-    ...
-  DeepSeekLLM:
-    type: openai
-    ...
-  ChatGLMLLM:
-    type: openai
-    ...
-  DifyLLM:
-    type: openai
-    ...
-```
 
-Some services, for example, if you use the TTS` of the `dify` and` bean bags, you need a key, remember to add the
+---
-configuration file!
 
-5. Execute the docker command
+## Feature List ✨
-
-Open the command line tool, `cd` enter your `project directory`, and execute the following command
 
-```
+### Implemented ✅
-#If you are Linux, execute
-ls
-#If you are Windows, execute
-dir
-```
 
-If you can see the `config.yaml` file, you have indeed entered the `project directory`, and then execute the following
+- **Communication Protocol**  
-command:
+  Based on the `xiaozhi-esp32` protocol, data exchange is implemented via WebSocket.
-
+- **Dialogue Interaction**  
-```
+  Supports wake-up dialogues, manual conversations, and real-time interruptions. Automatically enters sleep mode after long periods of inactivity.
-docker run -d --name xiaozhi-esp32-server --restart always --security-opt seccomp:unconfined -p 8000:8000 -v $(pwd)/config.yaml:/opt/xiaozhi-esp32-server/config.yaml ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest
+- **Multilingual Recognition**  
-```
+  Supports Mandarin, Cantonese, English, Japanese, and Korean (default using FunASR).
-
+- **LLM Module**  
-If executed for the first time, it may take several minutes, and you have to be patient to wait for it to complete the
+  Allows flexible switching of LLM modules. The default is ChatGLMLLM, with options to use AliLLM, DeepSeek, Ollama, and others.
-pull. After normal pulling is completed, you can execute the following command on the command line to see if the service
+- **TTS Module**  
-is started successfully.
+  Supports multiple TTS interfaces including EdgeTTS (default) and Volcano Engine Doubao TTS to meet speech synthesis requirements.
-
-```
-docker ps
-```
-
-If you can see `xiaozhi-server`, it means that the service starts successfully. Then you can further execute the
-following command to view the service log
-
-```
-docker logs -f xiaozhi-esp32-server
-```
-
-If you can see, similar to the following logs, it is a sign that the service of this project is successfully launched.
-
-```
-2025-xx-xx xx:51:59,492 - core.server - INFO - Server is running at ws://xx.xx.xx.xxx:8000
-2025-xx-xx xx:51:59,516 - websockets.server - INFO - server listening on 0.0.0.0:8000
-```
-
-Next, you can start `compiling esp32 firmware`. Please go down and turn to the relevant chapter on
-`compiling esp32 firmware`. So since you are deploying with docker, you have to check the IP of your native computer by
-yourself.
-Normally, assuming your ip is `192.168.1.25`, then your interface address is: `ws://192.168.1.25:8000`. This information
-is very useful, and it is required to `compile esp32 firmware` later.
-
-## Method 2 : Local Source Code Deployment
-
-### 1.Install Prerequisites
-
-This project uses 'conda' to manage dependencies, and after installation, start executing the following commands:
-
-```
-conda remove -n xiaozhi-esp32-server --all -y
-conda create -n xiaozhi-esp32-server python=3.10 -y
-conda activate xiaozhi-esp32-server
-```
-
-After executing the above command, if your computer is Windows or Mac, execute the following statement:
-
-```
-conda activate xiaozhi-esp32-server
-conda install conda-forge::libopus
-conda install conda-forge::ffmpeg
-```
-
-If your computer is ubuntu, execute the following statement:
-
-```
-apt-get install libopus0 ffmpeg 
-```
-
-### 2.Install Dependencies
-
-```
-# Clone the project
-cd xiaozhi-esp32-server
-conda activate xiaozhi-esp32-server
-pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
-pip install -r requirements.txt
-```
-
-### 3.Download ASR Model
-
-Download [SenseVoiceSmall](https://modelscope.cn/models/iic/SenseVoiceSmall/resolve/master/model.pt) to
-`model/SenseVoiceSmall`.
-
-By default, the `SenseVoiceSmall` model is used to convert voice to text. Because the model is large, it needs to be
-downloaded independently. After downloading, place the `model.pt` file in the `model/SenseVoiceSmall` directory. Choose
-any of the following two download routes.
-
-- Line 1: Download Ali Magic
-  Tower[SenseVoiceSmall](https://modelscope.cn/models/iic/SenseVoiceSmall/resolve/master/model.pt)
-- Line 2: Baidu Netdisk download[SenseVoiceSmall](https://pan.baidu.com/share/init?surl=QlgM58FHhYv1tFnUT_A8Sg&pwd=qvna)
-  提取码: `qvna`
 
-### 4.Configure Project
+### In Development 🚧
-
-Modify the `config.yaml` file to configure the various parameters required for this project. The default LLM uses
-`ChatGLMLLM`, you need to configure the key to start.
-The default TTS uses `EdgeTTS`. This does not require configuration. If you need to replace it with`Doubao TTS`, you
-need to
-configure the key.
-
-Configuration description: This is the default component of each function, such as LLM default to use the `ChatGLMLLM`
-model. If you need to switch the model, it is the corresponding name.
-
-The default configuration of this project is only the lowest operating cost configuration（`glm-4-flash`and`EdgeTTS`are
-free），If you need to be better and faster, you need to combine the use of the deployment environment to switch the use
-of each component。
 
-```
+- Conversation Memory Feature
-selected_module:
+- Multiple Mood Modes
-  ASR: FunASR
+- Smart Control Panel Web UI
-  VAD: SileroVAD
-  LLM: ChatGLMLLM
-  TTS: EdgeTTS
-```
-
-For example, to modify the components used by the `LLM`, it depends on which `LLM` API interfaces are supported by this project. Currently, the supported ones are `openai` and `dify`. We welcome validation and support for more LLM platforms' interfaces.
-When using it, change the `selected_module` to the corresponding name of the following LLM configurations:
-
-```
-LLM:
-  AliLLM:
-    type: openai
-    ...
-  DeepSeekLLM:
-    type: openai
-    ...
-  ChatGLMLLM:
-    type: openai
-    ...
-  DifyLLM:
-    type: openai
-    ...
-```
-
-Some services, for example, if you use the TTS` of the `dify` and` bean bags, you need a key, remember to add the
-configuration file!
-
-### 5.Run the Project
 
-Run the Project
+![Screenshot](docs/images/webui.png)
 
-```
+---
-# Make sure to execute in the root directory of this project
-conda activate xiaozhi-esp32-server
-python app.py
-```
 
-You'll see the WebSocket endpoint in logs:
+## Supported Platforms/Components 📋
 
-```
+### LLM
-2025-xx-xx xx:51:59,492 - core.server - INFO - Server is running at ws://192.168.1.25:8000
-2025-xx-xx xx:51:59,516 - websockets.server - INFO - server listening on 0.0.0.0:8000
-```
 
-Among them, the `ws://192.168.1.25:8000` is the interface address provided by this project. Of course, your own machine
+| Type |         Platform Name         |         Usage Method          |   Pricing Model   |                                 Remarks                                  |
-is different from mine. Remember to find your own address.
+|:----:|:-----------------------------:|:-----------------------------:|:-----------------:|:-------------------------------------------------------------------------:|
+| LLM  |   AliLLM (阿里百炼)           | OpenAI API call               | Token consumption | [Click to apply for API key](https://bailian.console.aliyun.com/?apiKey=1#/api-key) |
+| LLM  | DeepSeekLLM (深度求索)        | OpenAI API call               | Token consumption |          [Click to apply for API key](https://platform.deepseek.com/)         |
+| LLM  | ChatGLMLLM (智谱)             | OpenAI API call               | Free              | Although free, you still need to [click to apply for an API key](https://bigmodel.cn/usercenter/proj-mgmt/apikeys) |
+| LLM  | OllamaLLM                   | Ollama API call               | Free/Custom       | Requires pre-downloading the model (`ollama pull`); service URL: `http://localhost:11434` |
+| LLM  | DifyLLM                     | Dify API call                 | Token consumption | For local deployment. Note that prompt configuration must be set in the Dify console. |
+| LLM  | GeminiLLM                   | Gemini API call               | Free              | [Click to apply for API key](https://aistudio.google.com/apikey) |
+| LLM  | CozeLLM                     | Coze API call                 | Token consumption | Requires providing bot_id, user_id, and personal token. |
+| LLM  | Home Assistant              | Home Assistant voice assistant API call | Free       | Requires providing a Home Assistant token. |
 
-# Compile ESP32 Firmware
+In fact, any LLM that supports OpenAI API calls can be integrated.
 
-1. Download `xiaozhi-esp32` project, configure the project environment according to this
+---
-   tutorial [" Windows builds ESP IDF 5.3.2 Development Environment and Compiles Xiaozhi "](https://icnynnzcwou8.feishu.cn/wiki/JEYDwTTALi5s2zkGlFGcDiRknXf)
-   Cure
 
-2. Open the `xiaozhi-esp32/main/kconfig.projbuild` file, find the content of the` websocket_url` `default`, change the
+### TTS
-   ` wss: // api.tenclass.net` to your own address, such as
 
-Before modification:
+| Type |             Platform Name              | Usage Method |   Pricing Model   |                                       Remarks                                        |
+|:----:|:--------------------------------------:|:------------:|:-----------------:|:--------------------------------------------------------------------------------------:|
+| TTS  |               EdgeTTS                  | API call     | Free              | Default TTS based on Microsoft's speech synthesis technology.                        |
+| TTS  | DoubaoTTS (火山引擎豆包 TTS)             | API call     | Token consumption | [Click to create an API key](https://console.volcengine.com/speech/service/8); it is recommended to use the paid version for higher concurrency. |
+| TTS  | CosyVoiceSiliconflow                   | API call     | Token consumption | Requires application for the Siliconflow API key; output format is WAV.                |
+| TTS  | CozeCnTTS                              | API call     | Token consumption | Requires providing a Coze API key; output format is WAV.                               |
+| TTS  | FishSpeech                             | API call     | Free/Custom       | Starts a local TTS service; see the configuration file for startup instructions.       |
+| TTS  | GPT_SOVITS_V2                          | API call     | Free/Custom       | Starts a local TTS service, suitable for personalized speech synthesis scenarios.      |
 
-```
+---
-config WEBSOCKET_URL
-    depends on CONNECTION_TYPE_WEBSOCKET
-    string "Websocket URL"
-    default "wss://api.tenclass.net/xiaozhi/v1/"
-    help
-        Communication with the server through websocket after wake up.
-```
 
-After modification (example):
+### VAD
 
-```
+| Type |    Platform Name    | Usage Method | Pricing Model | Remarks |
-config WEBSOCKET_URL
+|:----:|:-------------------:|:------------:|:-------------:|:-------:|
-    depends on CONNECTION_TYPE_WEBSOCKET
+| VAD  | SileroVAD           | Local        | Free          |         |
-    string "Websocket URL"
-    default "ws://192.168.1.25:8000/xiaozhi/v1/"
-    help
-        Communication with the server through websocket after wake up.
-```
 
-3. Configure build settings:
+---
 
-```
+### ASR
-# The terminal command line enters the root directory of xiaozhi-esp32
-cd xiaozhi-esp32
-# For example, the board I use is ESP32S3, so the compile target is ESP32S3. If your board is other models, please replace it with the corresponding model
-idf.py set-target esp32s3
-# Enter the menu configuration
-idf.py menuconfig
-```
 
-![图片](docs/images/build_setting01.png)
+| Type |    Platform Name    | Usage Method | Pricing Model | Remarks |
+|:----:|:-------------------:|:------------:|:-------------:|:-------:|
+| ASR  | FunASR              | Local        | Free          |         |
+| ASR  | DoubaoASR           | API call     | Paid          |         |
 
-After entering the menu configuration, then enter `xiaozhi assistant`, set the` connection_type` to `websocket`
+---
-Go back to the main menu, then enter `xiaozhi assistant`, set the `BOARD_TYPE` of your board
-Save exit and return to the terminal command line.
 
-![图片](docs/images/build_setting02.png)
+## Usage 🚀
 
-4. Build and package:
+### 1. [Deployment Documentation](./docs/Deployment.md)
 
-```
+This project supports three deployment methods. Choose the one that best fits your needs.
-idf.py build
-cd scripts
-python release.py
-```
 
-After the compilation is successful, the firmware file `merged-binary.bin` is generated in the` build` directory in the
+The documentation provided here is a **written tutorial**. If you prefer a **video tutorial**, you can refer to [this expert's hands-on guide](https://www.bilibili.com/video/BV1gePuejEvT).
-project root directory.
-This `merged-binary.bin` is the firmware file that will be recorded on the hardware.
 
-6. Flash
+Combining both the written and video tutorials can help you get started more quickly.
-   Connect the ESP32 device to the computer, use the Chrome browser, and open the following URL
 
-```
+1. [Docker Quick Deployment](./docs/Deployment.md)  
-https://espressif.github.io/esp-launchpad/
+   Suitable for general users who want a quick experience without extensive environment configuration. The only downside is that pulling the image can be a bit slow.
-```
 
-Open this
+2. [Deployment Using Docker Environment](./docs/Deployment.md#%E6%96%B9%E5%BC%8F%E4%BA%8C%E5%80%9F%E5%8A%A9docker%E7%8E%AF%E5%A2%83%E8%BF%90%E8%A1%8C%E9%83%A8%E7%BD%B2)  
-tutorial, [Flash Tools/Web -side Burning Folding Step (No IDF Development Environment)](https://ccnphfhqs21z.feishu.cn/wiki/Zpz4wXBtdimBrLk25WdcXzxcnNS).
+   Ideal for software engineers who already have Docker installed and wish to customize the code.
-Turn to: `Method 2: ESP-LAUNCHPAD browser web-end burning`, start from
-`3. Burning firmware/download to the development board`, follow the tutorial operation.
 
-# FAQ
+3. [Running from Local Source Code](./docs/Deployment.md#%E6%96%B9%E5%BC%8F%E4%B8%89%E6%9C%AC%E5%9C%B0%E6%BA%90%E7%A0%81%E8%BF%90%E8%A1%8C)  
+   Suitable for users familiar with the `Conda` environment or those who wish to build the runtime environment from scratch.
 
-## 1、TTS often fails, often overtime
+For scenarios requiring higher response speeds, running from the local source code is recommended to reduce additional overhead.
 
-Suggestion: If the `Edgetts` is slow or often fails, you can replace it with a bean bag TTS` with a volcanic engine. If
+### 2. [Firmware Compilation](./docs/firmware-build.md)
-both are slow, the network environment may need to be optimized.
 
-## 2、Big model reply is a bit slow
+Click [here](./docs/firmware-build.md) for a detailed guide on firmware compilation.
 
-Suggestions: Both big models and TTS are dependent interfaces. If the network environment is not good, you can consider
+After successful compilation and network connection, wake up XiaoZhi using the wake-up word and monitor the server console for output.
-changing the local model. Or try to switch different interface models.
 
-## 3、Why is my ChatGLMLLM replying to a bit? Obviously it is Xiaozhi, but treats me as Xiaozhi.
+---
 
-Suggestion: You can modify the prompts in the configuration file first. You can also replace the free `GLM-4-FLASH` to
+## Frequently Asked Questions ❓
-the model of other toll versions of `ChatGlm`.
 
-## 4、I want to control the operation of electric lights, air conditioners, remote switching and other operations through Xiaozhi.
+### 1. TTS often fails and times out ⏰
 
-Suggestion: In the configuration file, set the `LLM` to`DifyLLM`, and then arrange the smart application by the
+**Suggestion:**  
-`Dify`.
+If `EdgeTTS` frequently fails, please first check whether you are using a proxy (VPN). If so, try disabling the proxy and try again. If you are using Volcano Engine Doubao TTS and it often fails, it is recommended to use the paid version since the trial only supports 2 concurrent requests.
 
-## 5、I said very slowly, I paused, Xiaozhi always grabbed me, what to do.
+### 2. I want to control lights, air conditioners, remote power on/off, etc. with XiaoZhi 💡
 
-Suggestion: In the configuration file, find this section, change the `min_silence_duration_ms` value, such as change to
+**Suggestion:**  
-` 1000`.
+Set the `LLM` to `HomeAssistant` in the configuration file and use the `HomeAssistant` API to perform the relevant controls.
 
-```
+### 3. I speak slowly, and XiaoZhi always interrupts during pauses 🗣️
+
+**Suggestion:**  
+Locate the following section in the configuration file and increase the value of `min_silence_duration_ms` (for example, change it to `1000`):
+
+```yaml
 VAD:
   SileroVAD:
     threshold: 0.5
     model_dir: models/snakers4_silero-vad
-    min_silence_duration_ms: 700  # 如果说话停顿比较长，可以把这个值设置大一些
+    min_silence_duration_ms: 700  # If your pauses are longer, increase this value
 ```
 
-## 6、For more questions, contact us to feedback
+### 4. Why does XiaoZhi recognize a lot of Korean, Japanese, and English in what I say? 🇰🇷
 
-![图片](docs/images/wechat.jpg)
+**Suggestion:**  
+Check whether the `model.pt` file exists in the `models/SenseVoiceSmall` directory. If it does not, please download it. See [Download ASR Model Files](docs/Deployment.md#模型文件) for details.
 
-# Acknowledgments
+### 5. Why does the error “TTS task error: file does not exist” occur? 📁
 
-- This project is inspired by the [Bailin Voice Dialogue Robot](https://github.com/wwbin2017/bailing) project, and the
+**Suggestion:**  
-  basic idea of the project is completed。
+Verify that you have correctly installed the `libopus` and `ffmpeg` libraries using `conda`. If not, install them using:
-- Thanks to [Tencent Cloud] (https://cloud.tencent.com/) for providing free docker space for this project。
+
-- Thanks to [tenclass](https://www.tenclass.com/)Provide adequate documentation support on Xiaozhi Communication
+```
-  Protocol。
+conda install conda-forge::libopus
+conda install conda-forge::ffmpeg
+```
+
+### 6. How can I improve XiaoZhi's dialogue response speed? ⚡
+
+The default configuration of this project is designed to be cost-effective. It is recommended that beginners first use the default free models to ensure that the system runs smoothly, then optimize for faster response times.  
+To improve response speed, you can try replacing individual components. Below are the response time test results for each component (for reference only, not a guarantee):
+
+**LLM Performance Ranking:**
+
+| Module Name  | Average First Token Time | Average Total Response Time |
+|--------------|--------------------------|-----------------------------|
+| AliLLM       | 0.547s                   | 1.485s                      |
+| ChatGLMLLM   | 0.677s                   | 3.057s                      |
+| OllamaLLM    | 0.003s                   | 0.003s                      |
+
+**TTS Performance Ranking:**
+
+| Module Name                | Average Synthesis Time |
+|----------------------------|------------------------|
+| EdgeTTS                    | 1.019s                 |
+| DoubaoTTS                  | 0.503s                 |
+| CosyVoiceSiliconflow       | 3.732s                 |
+
+**Recommended Configuration Combination (Overall Response Speed):**
+
+| Combination Scheme                | Overall Score | LLM First Token | TTS Synthesis |
+|-----------------------------------|---------------|-----------------|---------------|
+| AliLLM + DoubaoTTS                | 0.539         | 0.547s          | 0.503s        |
+| AliLLM + EdgeTTS                  | 0.642         | 0.547s          | 1.019s        |
+| ChatGLMLLM + DoubaoTTS            | 0.642         | 0.677s          | 0.503s        |
+| ChatGLMLLM + EdgeTTS              | 0.745         | 0.677s          | 1.019s        |
+| AliLLM + CosyVoiceSiliconflow     | 1.184         | 0.547s          | 3.732s        |
+
+**Conclusion 🔍**
+
+_As of February 19, 2025, if my computer were located in Haizhu District, Guangzhou, Guangdong Province, and connected via China Unicom, I would prioritize using:_
+
+- **LLM:** `AliLLM`
+- **TTS:** `DoubaoTTS`
+
+### 7. For more questions, feel free to contact us for feedback 💬
+
+Our contact information is in [Baidu Netdisk](https://pan.baidu.com/s/1x6USjvP1nTRsZ45XlJu65Q)，The extraction code is`223y`。
+
+---
+
+## Acknowledgements 🙏
+
+- This project was inspired by the [Bailing Voice Dialogue Robot](https://github.com/wwbin2017/bailing) and implemented based on it.
+- Many thanks to [Tenclass](https://www.tenclass.com/) for providing detailed documentation support for the XiaoZhi communication protocol.
 
 <a href="https://star-history.com/#xinnan-tech/xiaozhi-esp32-server&Date">
  <picture>

app.py

@@ -1,17 +1,20 @@
 import asyncio
 from config.logger import setup_logging
-from config.settings import load_config
+from config.settings import load_config, check_config_file
 from core.websocket_server import WebSocketServer
 from manager.http_server import WebUI
 from aiohttp import web
-from core.utils.util import get_local_ip
+from core.utils.util import get_local_ip, check_ffmpeg_installed
 
 TAG = __name__
 
+
 async def main():
+    check_config_file()
+    check_ffmpeg_installed()
     logger = setup_logging()
     config = load_config()
-    
+
     # 启动 WebSocket 服务器
     ws_server = WebSocketServer(config)
     ws_task = asyncio.create_task(ws_server.start())
@@ -33,7 +36,7 @@ async def main():
             logger.bind(tag=TAG).info(f"WebUI server is running at http://{local_ip}:{port}")
         except Exception as e:
             logger.bind(tag=TAG).error(f"Failed to start WebUI server: {e}")
-    
+
     try:
         # 等待 WebSocket 服务器运行
         await ws_task
@@ -42,5 +45,6 @@ async def main():
         if webui_runner:
             await webui_runner.cleanup()
 
+
 if __name__ == "__main__":
     asyncio.run(main())

config.yaml

@@ -22,13 +22,28 @@ server:
     # 可选:设备白名单，如果设置了白名单，那么白名单的机器无论是什么token都可以连接。
     #allowed_devices:
     #  - "24:0A:C4:1D:3B:F0"  # MAC地址列表
+log:
+  # 设置控制台输出的日志格式，时间、日志级别、标签、消息
+  log_format: "<green>{time:YY-MM-DD HH:mm:ss}</green>[<light-blue>{extra[tag]}</light-blue>] - <level>{level}</level> - <light-green>{message}</light-green>"
+  # 设置日志文件输出的格式，时间、日志级别、标签、消息
+  log_format_simple: "{time:YYYY-MM-DD HH:mm:ss} - {name} - {level} - {extra[tag]} - {message}"
+  # 设置日志等级：INFO、DEBUG
+  log_level: INFO
+  # 设置日志路径
+  log_dir: tmp
+  # 设置日志文件
+  log_file: "server.log"
+  # 设置数据文件路径
+  data_dir: data
 manager:
   # 是否启用管理后台
   # 目前这个模块还在开发中，建议：不要修改enabled选项
   enabled: false
   ip: 0.0.0.0
   port: 8002
-
+iot:
+  Speaker:
+    volume: 100
 xiaozhi:
   type: hello
   version: 1
@@ -65,12 +80,26 @@ selected_module:
   LLM: ChatGLMLLM
   # TTS将根据配置名称对应的type调用实际的TTS适配器
   TTS: EdgeTTS
+  Memory: mem0ai
 
+Memory:
+  mem0ai:
+    type: mem0ai
+    # https://app.mem0.ai/dashboard/api-keys
+    # 每月有1000次免费调用
+    api_key: 你的mem0ai api key
+    
 ASR:
   FunASR:
+    type: fun_local
     model_dir: models/SenseVoiceSmall
     output_dir: tmp/
-
+  DoubaoASR:
+    type: doubao
+    appid: 你的火山引擎语音合成服务appid
+    access_token: 你的火山引擎语音合成服务access_token
+    cluster: volcengine_input_common
+    output_dir: tmp/
 VAD:
   SileroVAD:
     threshold: 0.5
@@ -126,8 +155,13 @@ LLM:
     type: coze
     bot_id: 你的bot_id
     user_id: 你的user_id
-    base_url: "https://api.coze.cn/open_api/v2/chat" # 服务地址
     personal_access_token: 你的coze个人令牌
+  LMStudioLLM:
+    # 定义LLM API类型
+    type: openai
+    model_name: deepseek-r1-distill-llama-8b@q4_k_m # 使用的模型名称，需要预先在社区下载
+    url: http://localhost:1234/v1 # LM Studio服务地址
+    api_key: lm-studio # LM Studio服务的固定API Key
   HomeAssistant:
     # 定义LLM API类型
     type: homeassistant
@@ -148,8 +182,10 @@ TTS:
     # 山引擎语音一定要购买花钱，起步价30元，就有100并发了。如果用免费的只有2个并发，会经常报tts错误
     # 购买服务后，购买免费的音色后，可能要等半小时左右，才能使用。
     # 地址：https://console.volcengine.com/speech/service/8
+    api_url: https://openspeech.bytedance.com/api/v1/tts
     voice: BV001_streaming
     output_file: tmp/
+    authorization: "Bearer;"
     appid: 你的火山引擎语音合成服务appid
     access_token: 你的火山引擎语音合成服务access_token
     cluster: volcano_tts
@@ -278,10 +314,42 @@ TTS:
     # volume: 50
     # speech_rate: 0
     # pitch_rate: 0
-
+    # 添加 302.ai TTS 配置
+    # token申请地址：https://dash.302.ai/
+  TTS302AI:
+    # 302AI语音合成服务，需要先在302平台创建账户充值，并获取密钥信息
+    # 获取api_keyn路径：https://dash.302.ai/apis/list
+    # 价格，$35/百万字符。火山原版¥450元/万字符
+    type: doubao
+    api_url: https://api.302ai.cn/doubao/tts_hd
+    authorization: "Bearer "
+    voice: "zh_female_wanwanxiaohe_moon_bigtts"
+    output_file: tmp/
+    access_token: "你的302API密钥"
 # 模块测试配置
 module_test:
   test_sentences:  # 自定义测试语句
     - "你好，请介绍一下你自己"
     - "What's the weather like today?"
     - "请用100字概括量子计算的基本原理和应用前景"
+
+# 本地音乐播放配置
+music:
+  music_commands:
+    - "来一首歌"
+    - "唱一首歌"
+    - "播放音乐"
+    - "来点音乐"
+    - "背景音乐"
+    - "放首歌"
+    - "播放歌曲"
+    - "来点背景音乐"
+    - "我想听歌"
+    - "我要听歌"
+    - "放点音乐"
+  music_dir: "./music"  # 音乐文件存放路径，将从该目录及子目录下搜索音乐文件
+  music_ext: # 音乐文件类型，p3格式效率最高
+    - ".mp3"
+    - ".wav"
+    - ".p3"
+  refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒

config/logger.py

@@ -1,27 +1,29 @@
 import os
 import sys
 from loguru import logger
+from config.settings import load_config
+
+def setup_logging():
+    """从配置文件中读取日志配置，并设置日志输出格式和级别"""
+    config = load_config()
+    log_config = config["log"]
+    log_format = log_config.get("log_format", "<green>{time:YY-MM-DD HH:mm:ss}</green>[<light-blue>{extra[tag]}</light-blue>] - <level>{level}</level> - <light-green>{message}</light-green>")
+    log_format_simple = log_config.get("log_format_file", "{time:YYYY-MM-DD HH:mm:ss} - {name} - {level} - {extra[tag]} - {message}")
+    log_level = log_config.get("log_level", "INFO")
+    log_dir = log_config.get("log_dir", "tmp")
+    log_file = log_config.get("log_file", "server.log")
+    data_dir = log_config.get("data_dir", "data")
 
-def setup_logging(log_dir='tmp', data_dir='data'):
-    """配置全局彩色日志（不同区块不同标签）"""
     os.makedirs(log_dir, exist_ok=True)
     os.makedirs(data_dir, exist_ok=True)
 
-    # 设置日志格式，时间、日志级别、标签、消息
-    log_format = (
-        # "<green>{time:YYYY-MM-DD HH:mm:ss}</green>"
-        "[<light-blue>{extra[tag]}</light-blue>]"
-        " - <level>{level}</level> - "
-        "<light-green>{message}</light-green>"
-    )
-
     # 配置日志输出
     logger.remove()
 
     # 输出到控制台
-    logger.add(sys.stdout, format=log_format, level="INFO")
+    logger.add(sys.stdout, format=log_format, level=log_level)
 
     # 输出到文件
-    logger.add(os.path.join(log_dir, "server.log"), format="{time:YYYY-MM-DD HH:mm:ss} - {name} - {level} - {extra[tag]} - {message}", level="INFO")
+    logger.add(os.path.join(log_dir, log_file), format=log_format_simple, level=log_level)
 
     return logger

config/settings.py

@@ -1,22 +1,26 @@
 import os
 import argparse
 from ruamel.yaml import YAML
+from collections.abc import Mapping
 from core.utils.util import read_config, get_project_dir
 
+default_config_file = "config.yaml"
+
 
 def get_config_file():
-    default_config_file = "config.yaml"
+    global default_config_file
     # 判断是否存在私有的配置文件
+    config_file = default_config_file
     if os.path.exists(get_project_dir() + "data/." + default_config_file):
-        default_config_file = "data/." + default_config_file
+        config_file = "data/." + default_config_file
-    return default_config_file
+    return config_file
 
 
 def load_config():
     """加载配置文件"""
     parser = argparse.ArgumentParser(description="Server configuration")
-    default_config_file = get_config_file()
+    config_file = get_config_file()
-    parser.add_argument("--config_path", type=str, default=default_config_file)
+    parser.add_argument("--config_path", type=str, default=config_file)
     args = parser.parse_args()
     return read_config(args.config_path)
 
@@ -27,3 +31,54 @@ def update_config(config):
     """将配置保存到YAML文件"""
     with open(get_config_file(), 'w') as f:
         yaml.dump(config, f)
+
+
+def find_missing_keys(new_config, old_config, parent_key=''):
+    """
+    递归查找缺失的配置项
+    返回格式：[缺失配置路径]
+    """
+    missing_keys = []
+
+    if not isinstance(new_config, Mapping):
+        return missing_keys
+
+    for key, value in new_config.items():
+        # 构建当前配置路径
+        full_path = f"{parent_key}.{key}" if parent_key else key
+
+        # 检查键是否存在
+        if key not in old_config:
+            missing_keys.append(full_path)
+            continue
+
+        # 递归检查嵌套字典
+        if isinstance(value, Mapping):
+            sub_missing = find_missing_keys(
+                value,
+                old_config[key],
+                parent_key=full_path
+            )
+            missing_keys.extend(sub_missing)
+
+    return missing_keys
+
+
+def check_config_file():
+    old_config_file = get_config_file()
+    global default_config_file
+    if not old_config_file.startswith('data'):
+        return
+    old_config = read_config(get_project_dir() + old_config_file)
+    new_config = read_config(get_project_dir() + default_config_file)
+    # 查找缺失的配置项
+    missing_keys = find_missing_keys(new_config, old_config)
+
+    if missing_keys:
+        error_msg = "您的配置文件太旧了，缺少了：\n"
+        error_msg += "\n".join(f"- {key}" for key in missing_keys)
+        error_msg += "\n建议您：\n"
+        error_msg += "1、备份data/.config.yaml文件\n"
+        error_msg += "2、将根目录的config.yaml文件复制到data下，重命名为.config.yaml\n"
+        error_msg += "3、将密钥逐个复制到新的配置文件中\n"
+        raise ValueError(error_msg)

core/connection.py

@@ -4,25 +4,30 @@ import uuid
 import time
 import queue
 import asyncio
+import traceback
 from config.logger import setup_logging
 import threading
 import websockets
 from typing import Dict, Any
-from collections import deque
-from core.utils.util import is_segment
 from core.utils.dialogue import Message, Dialogue
 from core.handle.textHandle import handleTextMessage
 from core.utils.util import get_string_no_punctuation_or_emoji
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
-from core.handle.audioHandle import handleAudioMessage, sendAudioMessage
+from core.handle.sendAudioHandle import sendAudioMessage
+from core.handle.receiveAudioHandle import handleAudioMessage
 from config.private_config import PrivateConfig
 from core.auth import AuthMiddleware, AuthenticationError
-from core.utils.auth_code_gen import AuthCodeGenerator  # 添加导入
+from core.utils.auth_code_gen import AuthCodeGenerator
 
 TAG = __name__
 
+
+class TTSException(RuntimeError):
+    pass
+
+
 class ConnectionHandler:
-    def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts):
+    def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory):
         self.config = config
         self.logger = setup_logging()
         self.auth = AuthMiddleware(config)
@@ -41,15 +46,15 @@ class ConnectionHandler:
         self.loop = asyncio.get_event_loop()
         self.stop_event = threading.Event()
         self.tts_queue = queue.Queue()
+        self.audio_play_queue = queue.Queue()
         self.executor = ThreadPoolExecutor(max_workers=10)
-        self.scheduled_tasks = deque()
 
         # 依赖的组件
         self.vad = _vad
         self.asr = _asr
         self.llm = _llm
         self.tts = _tts
-        self.dialogue = None
+        self.memory = _memory
 
         # vad相关变量
         self.client_audio_buffer = bytes()
@@ -67,33 +72,37 @@ class ConnectionHandler:
         self.dialogue = Dialogue()
 
         # tts相关变量
-        self.tts_first_text = None
+        self.tts_first_text_index = -1
-        self.tts_last_text = None
+        self.tts_last_text_index = -1
-        self.tts_start_speak_time = None
+
-        self.tts_duration = 0
+        # iot相关变量
+        self.iot_descriptors = {}
 
         self.cmd_exit = self.config["CMD_exit"]
         self.max_cmd_length = 0
         for cmd in self.cmd_exit:
             if len(cmd) > self.max_cmd_length:
                 self.max_cmd_length = len(cmd)
-        
+
         self.private_config = None
         self.auth_code_gen = AuthCodeGenerator.get_instance()
         self.is_device_verified = False  # 添加设备验证状态标志
-
+        self.music_handler = _music
 
     async def handle_connection(self, ws):
         try:
             # 获取并验证headers
             self.headers = dict(ws.request.headers)
-            self.logger.bind(tag=TAG).info(f"New connection request - Headers: {self.headers}")
+            # 获取客户端ip地址
+            client_ip = ws.remote_address[0]
+            self.logger.bind(tag=TAG).info(f"{client_ip} conn - Headers: {self.headers}")
 
             # 进行认证
             await self.auth.authenticate(self.headers)
 
             device_id = self.headers.get("device-id", None)
-            
+            self.memory.set_role_id(device_id)
+
             # Load private configuration if device_id is provided
             bUsePrivateConfig = self.config.get("use_private_config", False)
             self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
@@ -104,10 +113,10 @@ class ConnectionHandler:
                     # 判断是否已经绑定
                     owner = self.private_config.get_owner()
                     self.is_device_verified = owner is not None
-                    
+
                     if self.is_device_verified:
-                        await self.private_config.update_last_chat_time() 
+                        await self.private_config.update_last_chat_time()
-                    
+
                     llm, tts = self.private_config.create_private_instances()
                     if all([llm, tts]):
                         self.llm = llm
@@ -131,9 +140,14 @@ class ConnectionHandler:
 
             await self.loop.run_in_executor(None, self._initialize_components)
 
-            tts_priority = threading.Thread(target=self._priority_thread, daemon=True)
+            # tts 消化线程
+            tts_priority = threading.Thread(target=self._tts_priority_thread, daemon=True)
             tts_priority.start()
 
+            # 音频播放 消化线程
+            audio_play_priority = threading.Thread(target=self._audio_play_priority_thread, daemon=True)
+            audio_play_priority.start()
+
             try:
                 async for message in self.websocket:
                     await self._route_message(message)
@@ -146,9 +160,12 @@ class ConnectionHandler:
             await ws.close()
             return
         except Exception as e:
-            self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}")
+            stack_trace = traceback.format_exc()
+            self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}")
             await ws.close()
             return
+        finally:
+            await self.memory.save_memory(self.dialogue.dialogue)
 
     async def _route_message(self, message):
         """消息路由"""
@@ -166,7 +183,7 @@ class ConnectionHandler:
             date_time = time.strftime("%Y-%m-%d %H:%M", time.localtime())
             self.prompt = self.prompt.replace("{date_time}", date_time)
         self.dialogue.put(Message(role="system", content=self.prompt))
-          
+
     async def _check_and_broadcast_auth_code(self):
         """检查设备绑定状态并广播认证码"""
         if not self.private_config.get_owner():
@@ -186,65 +203,86 @@ class ConnectionHandler:
             # 如果不使用私有配置，就不需要验证
             return False
         return not self.is_device_verified
-    
+
+
     def chat(self, query):
-        # 如果设备未验证，就发送验证码
         if self.isNeedAuth():
             self.llm_finish_task = True
-            # 创建一个新的事件循环来运行异步函数
+            future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
-            loop = asyncio.new_event_loop()
+            future.result()
-            asyncio.set_event_loop(loop)
-            try:
-                loop.run_until_complete(self._check_and_broadcast_auth_code())
-            finally:
-                loop.close()
             return True
-        
+
         self.dialogue.put(Message(role="user", content=query))
         response_message = []
-        start = 0
+        processed_chars = 0  # 跟踪已处理的字符位置
-        # 提交 LLM 任务
         try:
-            start_time = time.time()  # 记录开始时间
+            start_time = time.time()
-            llm_responses = self.llm.response(self.session_id, self.dialogue.get_llm_dialogue())
+            # 使用带记忆的对话
+            future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
+            memory_str = future.result()
+            
+            self.logger.bind(tag=TAG).info(f"记忆内容: {memory_str}")
+            llm_responses = self.llm.response(
+                self.session_id, 
+                self.dialogue.get_llm_dialogue_with_memory(memory_str)
+            )
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
             return None
-        # 提交 TTS 任务到线程池
+
         self.llm_finish_task = False
+        text_index = 0
         for content in llm_responses:
             response_message.append(content)
-            # 如果中途被打断，就停止生成
             if self.client_abort:
-                start = len(response_message)
                 break
 
-            end_time = time.time()  # 记录结束时间
+            end_time = time.time()
-            self.logger.bind(tag=TAG).debug(f"大模型返回时间时间: {end_time - start_time} 秒, 生成token={content}")
+            self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
-            if is_segment(response_message):
-                segment_text = "".join(response_message[start:])
-                segment_text = get_string_no_punctuation_or_emoji(segment_text)
-                if len(segment_text) > 0:
-                    self.recode_first_last_text(segment_text)
-                    future = self.executor.submit(self.speak_and_play, segment_text)
-                    self.tts_queue.put(future)
-                    start = len(response_message)
 
-        # 处理剩余的响应
+            # 合并当前全部文本并处理未分割部分
-        if start < len(response_message):
+            full_text = "".join(response_message)
-            segment_text = "".join(response_message[start:])
+            current_text = full_text[processed_chars:]  # 从未处理的位置开始
-            if len(segment_text) > 0:
+
-                self.recode_first_last_text(segment_text)
+            # 查找最后一个有效标点
-                future = self.executor.submit(self.speak_and_play, segment_text)
+            punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：")
+            last_punct_pos = -1
+            for punct in punctuations:
+                pos = current_text.rfind(punct)
+                if pos > last_punct_pos:
+                    last_punct_pos = pos
+
+            # 找到分割点则处理
+            if last_punct_pos != -1:
+                segment_text_raw = current_text[:last_punct_pos + 1]
+                segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
+                if segment_text:
+                    # 强制设置空字符，测试TTS出错返回语音的健壮性
+                    # if text_index % 2 == 0:
+                    #     segment_text = " "
+                    text_index += 1
+                    self.recode_first_last_text(segment_text, text_index)
+                    future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                    self.tts_queue.put(future)
+                    processed_chars += len(segment_text_raw)  # 更新已处理字符位置
+
+        # 处理最后剩余的文本
+        full_text = "".join(response_message)
+        remaining_text = full_text[processed_chars:]
+        if remaining_text:
+            segment_text = get_string_no_punctuation_or_emoji(remaining_text)
+            if segment_text:
+                text_index += 1
+                self.recode_first_last_text(segment_text, text_index)
+                future = self.executor.submit(self.speak_and_play, segment_text, text_index)
                 self.tts_queue.put(future)
 
         self.llm_finish_task = True
-        # 更新对话
         self.dialogue.put(Message(role="assistant", content="".join(response_message)))
         self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
         return True
 
-    def _priority_thread(self):
+    def _tts_priority_thread(self):
         while not self.stop_event.is_set():
             text = None
             try:
@@ -252,33 +290,28 @@ class ConnectionHandler:
                 if future is None:
                     continue
                 text = None
+                opus_datas, text_index, tts_file = [], 0, None
                 try:
                     self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
-                    tts_file, text = future.result(timeout=10)
+                    tts_file, text, text_index = future.result(timeout=10)
                     if text is None or len(text) <= 0:
-                        continue
+                        self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
-                    if tts_file is None:
+                    elif tts_file is None:
-                        self.logger.bind(tag=TAG).error(f"TTS文件生成失败: {text}")
+                        self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}")
-                        continue
-                    self.logger.bind(tag=TAG).debug(f"TTS文件生成完毕，文件路径: {tts_file}")
-                    if os.path.exists(tts_file):
-                        opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
                     else:
-                        self.logger.bind(tag=TAG).error(f"TTS文件不存在: {tts_file}")
+                        self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
-                        opus_datas = []
+                        if os.path.exists(tts_file):
-                        duration = 0
+                            opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
+                        else:
+                            self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
                 except TimeoutError:
-                    self.logger.bind(tag=TAG).error("TTS 任务超时")
+                    self.logger.bind(tag=TAG).error("TTS超时")
-                    continue
                 except Exception as e:
-                    self.logger.bind(tag=TAG).error(f"TTS 任务出错: {e}")
+                    self.logger.bind(tag=TAG).error(f"TTS出错: {e}")
-                    continue
                 if not self.client_abort:
                     # 如果没有中途打断就发送语音
-                    asyncio.run_coroutine_threadsafe(
+                    self.audio_play_queue.put((opus_datas, text, text_index))
-                        sendAudioMessage(self, opus_datas, duration, text), self.loop
+                if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
-                    )
-                if self.tts.delete_audio_file and os.path.exists(tts_file):
                     os.remove(tts_file)
             except Exception as e:
                 self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
@@ -287,35 +320,46 @@ class ConnectionHandler:
                     self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
                     self.loop
                 )
-                self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text}{e}")
+                self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
 
-    def speak_and_play(self, text):
+    def _audio_play_priority_thread(self):
+        while not self.stop_event.is_set():
+            text = None
+            try:
+                opus_datas, text, text_index = self.audio_play_queue.get()
+                future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
+                                                          self.loop)
+                future.result()
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")
+
+    def speak_and_play(self, text, text_index=0):
         if text is None or len(text) <= 0:
             self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
-            return None, text
+            return None, text, text_index
         tts_file = self.tts.to_tts(text)
         if tts_file is None:
             self.logger.bind(tag=TAG).error(f"tts转换失败，{text}")
-            return None, text
+            return None, text, text_index
         self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}")
-        return tts_file, text
+        return tts_file, text, text_index
 
     def clearSpeakStatus(self):
         self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
         self.asr_server_receive = True
-        self.tts_last_text = None
+        self.tts_last_text_index = -1
-        self.tts_first_text = None
+        self.tts_first_text_index = -1
-        self.tts_duration = 0
-        self.tts_start_speak_time = None
 
-    def recode_first_last_text(self, text):
+    def recode_first_last_text(self, text, text_index=0):
-        if not self.tts_first_text:
+        if self.tts_first_text_index == -1:
             self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}")
-            self.tts_first_text = text
+            self.tts_first_text_index = text_index
-        self.tts_last_text = text
+        self.tts_last_text_index = text_index
 
     async def close(self):
         """资源清理方法"""
+
+        # 清理其他资源
         self.stop_event.set()
         self.executor.shutdown(wait=False)
         if self.websocket:
@@ -328,9 +372,3 @@ class ConnectionHandler:
         self.client_have_voice_last_time = 0
         self.client_voice_stop = False
         self.logger.bind(tag=TAG).debug("VAD states reset.")
-
-    def stop_all_tasks(self):
-        while self.scheduled_tasks:
-            task = self.scheduled_tasks.popleft()
-            task.cancel()
-        self.scheduled_tasks.clear()

core/handle/abortHandle.py

@@ -1,4 +1,5 @@
 import json
+import queue
 from config.logger import setup_logging
 
 TAG = __name__
@@ -9,8 +10,6 @@ async def handleAbortMessage(conn):
     logger.bind(tag=TAG).info("Abort message received")
     # 设置成打断状态，会自动打断llm、tts任务
     conn.client_abort = True
-    # 打断屏显任务
-    conn.stop_all_tasks()
     # 打断客户端说话状态
     await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
     conn.clearSpeakStatus()

core/handle/audioHandle.py

@@ -1,164 +0,0 @@
-from config.logger import setup_logging
-import json
-import asyncio
-import time
-from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji
-
-TAG = __name__
-logger = setup_logging()
-
-
-async def handleAudioMessage(conn, audio):
-    if not conn.asr_server_receive:
-        logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
-        return
-    if conn.client_listen_mode == "auto":
-        have_voice = conn.vad.is_vad(conn, audio)
-    else:
-        have_voice = conn.client_have_voice
-
-    # 如果本次没有声音，本段也没声音，就把声音丢弃了
-    if have_voice == False and conn.client_have_voice == False:
-        await no_voice_close_connect(conn)
-        conn.asr_audio.clear()
-        return
-    conn.client_no_voice_last_time = 0.0
-    conn.asr_audio.append(audio)
-    # 如果本段有声音，且已经停止了
-    if conn.client_voice_stop:
-        conn.client_abort = False
-        conn.asr_server_receive = False
-        text, file_path = conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
-        logger.bind(tag=TAG).info(f"识别文本: {text}")
-        text_len, text_without_punctuation = remove_punctuation_and_length(text)
-        if text_len <= conn.max_cmd_length and await handleCMDMessage(conn, text_without_punctuation):
-            return
-        if text_len > 0:
-            await startToChat(conn, text)
-        else:
-            conn.asr_server_receive = True
-        conn.asr_audio.clear()
-        conn.reset_vad_states()
-
-
-async def handleCMDMessage(conn, text):
-    cmd_exit = conn.cmd_exit
-    for cmd in cmd_exit:
-        if text == cmd:
-            logger.bind(tag=TAG).info("识别到明确的退出命令".format(text))
-            await finishToChat(conn)
-            return True
-    return False
-
-
-async def finishToChat(conn):
-    await conn.close()
-
-
-async def isLLMWantToFinish(conn):
-    first_text = conn.tts_first_text
-    last_text = conn.tts_last_text
-    _, last_text_without_punctuation = remove_punctuation_and_length(last_text)
-    if "再见" in last_text_without_punctuation or "拜拜" in last_text_without_punctuation:
-        return True
-    _, first_text_without_punctuation = remove_punctuation_and_length(first_text)
-    if "再见" in first_text_without_punctuation or "拜拜" in first_text_without_punctuation:
-        return True
-    return False
-
-
-async def startToChat(conn, text):
-    # 异步发送 stt 信息
-    stt_task = asyncio.create_task(
-        schedule_with_interrupt(0, send_stt_message(conn, text))
-    )
-    conn.scheduled_tasks.append(stt_task)
-    conn.executor.submit(conn.chat, text)
-
-
-async def sendAudioMessage(conn, audios, duration, text):
-    base_delay = conn.tts_duration
-
-    # 发送 tts.start
-    if text == conn.tts_first_text:
-        logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
-        conn.tts_start_speak_time = time.time()
-
-    # 发送 sentence_start（每个音频文件之前发送一次）
-    sentence_task = asyncio.create_task(
-        schedule_with_interrupt(base_delay, send_tts_message(conn, "sentence_start", text))
-    )
-    conn.scheduled_tasks.append(sentence_task)
-
-    conn.tts_duration += duration
-
-    # 发送音频数据
-    for idx, opus_packet in enumerate(audios):
-        await conn.websocket.send(opus_packet)
-
-    if conn.llm_finish_task and text == conn.tts_last_text:
-        stop_duration = conn.tts_duration - (time.time() - conn.tts_start_speak_time)
-        stop_task = asyncio.create_task(
-            schedule_with_interrupt(stop_duration, send_tts_message(conn, 'stop'))
-        )
-        conn.scheduled_tasks.append(stop_task)
-        if await isLLMWantToFinish(conn):
-            finish_task = asyncio.create_task(
-                schedule_with_interrupt(stop_duration, finishToChat(conn))
-            )
-            conn.scheduled_tasks.append(finish_task)
-
-
-async def send_tts_message(conn, state, text=None):
-    """发送 TTS 状态消息"""
-    message = {
-        "type": "tts",
-        "state": state,
-        "session_id": conn.session_id
-    }
-    if text is not None:
-        message["text"] = text
-
-    await conn.websocket.send(json.dumps(message))
-    if state == "stop":
-        conn.clearSpeakStatus()
-
-
-async def send_stt_message(conn, text):
-    """发送 STT 状态消息"""
-    stt_text = get_string_no_punctuation_or_emoji(text)
-    await conn.websocket.send(json.dumps({
-        "type": "stt",
-        "text": stt_text,
-        "session_id": conn.session_id}
-    ))
-    await conn.websocket.send(
-        json.dumps({
-            "type": "llm",
-            "text": "😊",
-            "emotion": "happy",
-            "session_id": conn.session_id}
-        ))
-    await send_tts_message(conn, "start")
-
-
-async def schedule_with_interrupt(delay, coro):
-    """可中断的延迟调度"""
-    try:
-        await asyncio.sleep(delay)
-        await coro
-    except asyncio.CancelledError:
-        pass
-
-
-async def no_voice_close_connect(conn):
-    if conn.client_no_voice_last_time == 0.0:
-        conn.client_no_voice_last_time = time.time() * 1000
-    else:
-        no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
-        close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
-        if no_voice_time > 1000 * close_connection_no_voice_time:
-            conn.client_abort = False
-            conn.asr_server_receive = False
-            prompt = "时间过得真快，我都好久没说话了。请你用十个字左右话跟我告别，以“再见”或“拜拜拜”为结尾"
-            await startToChat(conn, prompt)

core/handle/iotHandle.py

@@ -0,0 +1,152 @@
+import json
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class IotDescriptor:
+    """
+    A class to represent an IoT descriptor.
+    Attributes:
+    ----------
+    name : str
+        The name of the IoT descriptor.
+    description : str
+        A brief description of the IoT descriptor.
+    properties : dict
+        A dictionary containing properties of the IoT descriptor.
+    methods : dict
+        A dictionary containing methods of the IoT descriptor.
+    -------
+    """
+
+    def __init__(self, name, description, properties, methods):
+        self.name = name
+        self.description = description
+        self.properties = []
+        self.methods = []
+
+        # 根据描述创建属性
+        for key, value in properties.items():
+            # "volume":{"description":"当前音量 值","type":"number"}
+            """
+            等价于
+            {
+                'name': 名字,
+                'description': 描述,
+                'value': 0
+            }
+            """
+            # setattr(self, key, {}) # 创建一个空字典, 名字是属性名
+            property_item = globals()[key] = {}  # 创建一个空字典, 名字是属性名
+            property_item['name'] = key
+            property_item["description"] = value["description"]
+            if value["type"] == "number":
+                property_item["value"] = 0
+            elif value["type"] == "boolean":
+                property_item["value"] = False
+            else:
+                property_item["value"] = ""
+            self.properties.append(property_item)
+
+        # 根据描述创建方法
+        for key, value in methods.items():
+            # "SetVolume": {"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}}
+            """
+            等价于
+            SetVolume = {
+                `description`: 描述,
+                `volume`: {
+                    `description`: 描述,
+                    `value`: 0
+                }
+            }
+            """
+            # setattr(self, key, {}) # 创建一个空字典, 名字是方法名
+            method = globals()[key] = {}  # 创建一个空字典, 名字是方法名
+            method["description"] = value["description"]
+            method['name'] = key
+            for k, v in value["parameters"].items():
+                # 不同的参数解析
+                method[k] = {}
+                method[k]["description"] = v["description"]
+                if v["type"] == "number":
+                    method[k]["value"] = 0
+                elif v["type"] == "boolean":
+                    method[k]["value"] = False
+                else:
+                    method[k]["value"] = ""
+
+            self.methods.append(method)
+
+
+async def handleIotDescriptors(conn, descriptors):
+    """
+    处理物联网描述
+    示例: [{
+        "name":"Speaker",
+        "description":"当前 AI 机器人的扬声器",
+        "properties":{
+            "volume":{"description":"当前音量 值","type":"number"}  可以有boolean, number, string三种类型
+        },
+        "methods":{
+            "SetVolume":{
+                "description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}
+            }
+        }
+    }]
+    descriptors: 描述列表
+    """
+    for descriptor in descriptors:
+        iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
+                                       descriptor["methods"])
+        conn.iot_descriptors[descriptor["name"]] = iot_descriptor
+
+    # 暂时从配置文件中设置音量，后期通过意图识别控制音量
+    default_iot_volume = 100
+    if "iot" in conn.config:
+        default_iot_volume = conn.config["iot"]["Speaker"]["volume"]
+    logger.bind(tag=TAG).info(f"服务端设置音量为{default_iot_volume}")
+    await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": default_iot_volume})
+
+
+async def send_iot_conn(conn, name, method_name, parameters):
+    """
+    发送物联网指令
+    name: 设备名称 "Speaker"
+    method: 方法 "SetVolume"
+    parameters: 参数, 是一个字典 {"volume": 100}
+    发送示例:
+    {
+        "type": "iot",
+        "commands": [
+            {
+                "name" :  "Speaker",
+                "method": "SetVolume",
+                "parameters": {
+                    "volume": 100
+                    }
+            }
+        ]
+    }
+    """
+
+    for key, value in conn.iot_descriptors.items():
+        if key == name:
+            # 找到了设备
+            for method in value.methods:
+                # 找到了方法
+                if method["name"] == method_name:
+                    await conn.websocket.send(json.dumps({
+                        "type": "iot",
+                        "commands": [
+                            {
+                                "name": name,
+                                "method": method_name,
+                                "parameters": parameters
+                            }
+                        ]
+                    }))
+                    return
+    logger.bind(tag=TAG).error(f"未找到方法{method_name}")

core/handle/musicHandler.py

@@ -0,0 +1,153 @@
+from config.logger import setup_logging
+import os
+import random
+import difflib
+import re
+import traceback
+from pathlib import Path
+import time
+from core.handle.sendAudioHandle import send_stt_message
+from core.utils import p3
+
+TAG = __name__
+logger = setup_logging()
+
+
+def _extract_song_name(text):
+    """从用户输入中提取歌名"""
+    for keyword in ["听", "播放", "放", "唱"]:
+        if keyword in text:
+            parts = text.split(keyword)
+            if len(parts) > 1:
+                return parts[1].strip()
+    return None
+
+
+def _find_best_match(potential_song, music_files):
+    """查找最匹配的歌曲"""
+    best_match = None
+    highest_ratio = 0
+
+    for music_file in music_files:
+        song_name = os.path.splitext(music_file)[0]
+        ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
+        if ratio > highest_ratio and ratio > 0.4:
+            highest_ratio = ratio
+            best_match = music_file
+    return best_match
+
+class MusicManager:
+    def __init__(self, music_dir, music_ext):
+        self.music_dir = Path(music_dir)
+        self.music_ext = music_ext
+
+    def get_music_files(self):
+        music_files = []
+        for file in self.music_dir.rglob("*"):
+            # 判断是否是文件
+            if file.is_file():
+                # 获取文件扩展名
+                ext = file.suffix.lower()
+                # 判断扩展名是否在列表中
+                if ext in self.music_ext:
+                    # music_files.append(str(file.resolve()))  # 添加绝对路径
+                    # 添加相对路径
+                    music_files.append(str(file.relative_to(self.music_dir)))
+        return music_files
+
+class MusicHandler:
+    def __init__(self, config):
+        self.config = config
+        self.music_related_keywords = []
+
+        if "music" in self.config:
+            self.music_config = self.config["music"]
+            self.music_dir = os.path.abspath(
+                self.music_config.get("music_dir", "./music")  # 默认路径修改
+            )
+            self.music_related_keywords = self.music_config.get("music_commands", [])
+            self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
+            self.refresh_time = self.music_config.get("refresh_time", 60)
+        else:
+            self.music_dir = os.path.abspath("./music")
+            self.music_related_keywords = ["来一首歌", "唱一首歌", "播放音乐", "来点音乐", "背景音乐", "放首歌",
+                                           "播放歌曲", "来点背景音乐", "我想听歌", "我要听歌", "放点音乐"]
+            self.music_ext = (".mp3", ".wav", ".p3")
+            self.refresh_time = 60
+
+        # 获取音乐文件列表
+        self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
+        self.scan_time = time.time()
+        logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}")
+
+    async def handle_music_command(self, conn, text):
+        """处理音乐播放指令"""
+        clean_text = re.sub(r'[^\w\s]', '', text).strip()
+        logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
+
+        # 尝试匹配具体歌名
+        if os.path.exists(self.music_dir):
+            if time.time() - self.scan_time > self.refresh_time:
+                # 刷新音乐文件列表
+                self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
+                self.scan_time = time.time()
+                logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}")
+
+            potential_song = _extract_song_name(clean_text)
+            if potential_song:
+                best_match = _find_best_match(potential_song, self.music_files)
+                if best_match:
+                    logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
+                    await self.play_local_music(conn, specific_file=best_match)
+                    return True
+
+        # 检查是否是通用播放音乐命令
+        if any(cmd in clean_text for cmd in self.music_related_keywords):
+            await self.play_local_music(conn)
+            return True
+
+        return False
+
+    async def play_local_music(self, conn, specific_file=None):
+        """播放本地音乐文件"""
+        try:
+            if not os.path.exists(self.music_dir):
+                logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}")
+                return
+
+            # 确保路径正确性
+            if specific_file:
+                music_path = os.path.join(self.music_dir, specific_file)
+                if not os.path.exists(music_path):
+                    logger.bind(tag=TAG).error(f"指定的音乐文件不存在: {music_path}")
+                    return
+                selected_music = specific_file
+            else:
+                if time.time() - self.scan_time > self.refresh_time:
+                    # 刷新音乐文件列表
+                    self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
+                    self.scan_time = time.time()
+                    logger.bind(tag=TAG).debug(f"刷新的音乐文件列表: {self.music_files}")
+
+                if not self.music_files:
+                    logger.bind(tag=TAG).error("未找到MP3音乐文件")
+                    return
+                selected_music = random.choice(self.music_files)
+                music_path = os.path.join(self.music_dir, selected_music)
+                if not os.path.exists(music_path):
+                    logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
+                    return
+            text = f"正在播放{selected_music}"
+            await send_stt_message(conn, text)
+            conn.tts_first_text_index = 0
+            conn.tts_last_text_index = 0
+            conn.llm_finish_task = True
+            if music_path.endswith(".p3"):
+                opus_packets, duration = p3.decode_opus_from_file(music_path)
+            else:
+                opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
+            conn.audio_play_queue.put((opus_packets, selected_music, 0))
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
+            logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")

core/handle/receiveAudioHandle.py

@@ -0,0 +1,77 @@
+from config.logger import setup_logging
+import time
+from core.utils.util import remove_punctuation_and_length
+from core.handle.sendAudioHandle import send_stt_message
+
+TAG = __name__
+logger = setup_logging()
+
+
+async def handleAudioMessage(conn, audio):
+    if not conn.asr_server_receive:
+        logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
+        return
+    if conn.client_listen_mode == "auto":
+        have_voice = conn.vad.is_vad(conn, audio)
+    else:
+        have_voice = conn.client_have_voice
+
+    # 如果本次没有声音，本段也没声音，就把声音丢弃了
+    if have_voice == False and conn.client_have_voice == False:
+        await no_voice_close_connect(conn)
+        conn.asr_audio.clear()
+        return
+    conn.client_no_voice_last_time = 0.0
+    conn.asr_audio.append(audio)
+    # 如果本段有声音，且已经停止了
+    if conn.client_voice_stop:
+        conn.client_abort = False
+        conn.asr_server_receive = False
+        # 音频太短了，无法识别
+        if len(conn.asr_audio) < 3:
+            conn.asr_server_receive = True
+        else:
+            text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
+            logger.bind(tag=TAG).info(f"识别文本: {text}")
+            text_len, text_without_punctuation = remove_punctuation_and_length(text)
+            if await conn.music_handler.handle_music_command(conn, text_without_punctuation):
+                conn.asr_server_receive = True
+                conn.asr_audio.clear()
+                return
+            if text_len <= conn.max_cmd_length and await handleCMDMessage(conn, text_without_punctuation):
+                return
+            if text_len > 0:
+                await startToChat(conn, text)
+            else:
+                conn.asr_server_receive = True
+        conn.asr_audio.clear()
+        conn.reset_vad_states()
+
+
+async def handleCMDMessage(conn, text):
+    cmd_exit = conn.cmd_exit
+    for cmd in cmd_exit:
+        if text == cmd:
+            logger.bind(tag=TAG).info("识别到明确的退出命令".format(text))
+            await conn.close()
+            return True
+    return False
+
+
+async def startToChat(conn, text):
+    # 异步发送 stt 信息
+    await send_stt_message(conn, text)
+    conn.executor.submit(conn.chat, text)
+
+
+async def no_voice_close_connect(conn):
+    if conn.client_no_voice_last_time == 0.0:
+        conn.client_no_voice_last_time = time.time() * 1000
+    else:
+        no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
+        close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
+        if no_voice_time > 1000 * close_connection_no_voice_time:
+            conn.client_abort = False
+            conn.asr_server_receive = False
+            prompt = "时间过得真快，我都好久没说话了。请你用十个字左右话跟我告别，以“再见”或“拜拜”为结尾"
+            await startToChat(conn, prompt)

core/handle/sendAudioHandle.py

@@ -0,0 +1,83 @@
+from config.logger import setup_logging
+import json
+import asyncio
+import time
+from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji
+
+TAG = __name__
+logger = setup_logging()
+
+
+async def isLLMWantToFinish(last_text):
+    _, last_text_without_punctuation = remove_punctuation_and_length(last_text)
+    if "再见" in last_text_without_punctuation or "拜拜" in last_text_without_punctuation:
+        return True
+    return False
+
+
+async def sendAudioMessage(conn, audios, text, text_index=0):
+    # 发送句子开始消息
+    if text_index == conn.tts_first_text_index:
+        logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
+    await send_tts_message(conn, "sentence_start", text)
+
+    # 初始化流控参数
+    frame_duration = 60  # 毫秒
+    start_time = time.perf_counter()  # 使用高精度计时器
+    play_position = 0  # 已播放的时长（毫秒）
+
+    for opus_packet in audios:
+        if conn.client_abort:
+            return
+
+        # 计算当前包的预期发送时间
+        expected_time = start_time + (play_position / 1000)
+        current_time = time.perf_counter()
+
+        # 等待直到预期时间
+        delay = expected_time - current_time
+        if delay > 0:
+            await asyncio.sleep(delay)
+
+        # 发送音频包
+        await conn.websocket.send(opus_packet)
+        play_position += frame_duration  # 更新播放位置
+    await send_tts_message(conn, "sentence_end", text)
+    # 发送结束消息（如果是最后一个文本）
+    if conn.llm_finish_task and text_index == conn.tts_last_text_index:
+        await send_tts_message(conn, 'stop', None)
+        if await isLLMWantToFinish(text):
+            await conn.close()
+
+
+async def send_tts_message(conn, state, text=None):
+    """发送 TTS 状态消息"""
+    message = {
+        "type": "tts",
+        "state": state,
+        "session_id": conn.session_id
+    }
+    if text is not None:
+        message["text"] = text
+
+    await conn.websocket.send(json.dumps(message))
+    if state == "stop":
+        conn.clearSpeakStatus()
+
+
+async def send_stt_message(conn, text):
+    """发送 STT 状态消息"""
+    stt_text = get_string_no_punctuation_or_emoji(text)
+    await conn.websocket.send(json.dumps({
+        "type": "stt",
+        "text": stt_text,
+        "session_id": conn.session_id}
+    ))
+    await conn.websocket.send(
+        json.dumps({
+            "type": "llm",
+            "text": "😊",
+            "emotion": "happy",
+            "session_id": conn.session_id}
+        ))
+    await send_tts_message(conn, "start")

core/handle/textHandle.py

@@ -2,7 +2,8 @@ from config.logger import setup_logging
 import json
 from core.handle.abortHandle import handleAbortMessage
 from core.handle.helloHandle import handleHelloMessage
-from core.handle.audioHandle import startToChat
+from core.handle.receiveAudioHandle import startToChat
+from core.handle.iotHandle import handleIotDescriptors
 
 TAG = __name__
 logger = setup_logging()
@@ -36,5 +37,8 @@ async def handleTextMessage(conn, message):
                 conn.asr_audio.clear()
                 if "text" in msg_json:
                     await startToChat(conn, msg_json["text"])
+        elif msg_json["type"] == "iot":
+            if "descriptors" in msg_json:
+                await handleIotDescriptors(conn, msg_json["descriptors"])
     except json.JSONDecodeError:
         await conn.websocket.send(message)

core/providers/asr/base.py

@@ -0,0 +1,19 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Tuple, List
+
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProviderBase(ABC):
+    @abstractmethod
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """解码Opus数据并保存为WAV文件"""
+        pass
+
+    @abstractmethod
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        pass

core/providers/asr/doubao.py

@@ -0,0 +1,286 @@
+import time
+import io
+import wave
+import os
+from typing import Optional, Tuple, List
+import uuid
+import websockets
+import json
+import gzip
+
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+CLIENT_FULL_REQUEST = 0b0001
+CLIENT_AUDIO_ONLY_REQUEST = 0b0010
+
+NO_SEQUENCE = 0b0000
+NEG_SEQUENCE = 0b0010
+
+SERVER_FULL_RESPONSE = 0b1001
+SERVER_ACK = 0b1011
+SERVER_ERROR_RESPONSE = 0b1111
+
+NO_SERIALIZATION = 0b0000
+JSON = 0b0001
+THRIFT = 0b0011
+CUSTOM_TYPE = 0b1111
+NO_COMPRESSION = 0b0000
+GZIP = 0b0001
+CUSTOM_COMPRESSION = 0b1111
+
+
+def parse_response(res):
+    """
+    protocol_version(4 bits), header_size(4 bits),
+    message_type(4 bits), message_type_specific_flags(4 bits)
+    serialization_method(4 bits) message_compression(4 bits)
+    reserved （8bits) 保留字段
+    header_extensions 扩展头(大小等于 8 * 4 * (header_size - 1) )
+    payload 类似与http 请求体
+    """
+    protocol_version = res[0] >> 4
+    header_size = res[0] & 0x0f
+    message_type = res[1] >> 4
+    message_type_specific_flags = res[1] & 0x0f
+    serialization_method = res[2] >> 4
+    message_compression = res[2] & 0x0f
+    reserved = res[3]
+    header_extensions = res[4:header_size * 4]
+    payload = res[header_size * 4:]
+    result = {}
+    payload_msg = None
+    payload_size = 0
+    if message_type == SERVER_FULL_RESPONSE:
+        payload_size = int.from_bytes(payload[:4], "big", signed=True)
+        payload_msg = payload[4:]
+    elif message_type == SERVER_ACK:
+        seq = int.from_bytes(payload[:4], "big", signed=True)
+        result['seq'] = seq
+        if len(payload) >= 8:
+            payload_size = int.from_bytes(payload[4:8], "big", signed=False)
+            payload_msg = payload[8:]
+    elif message_type == SERVER_ERROR_RESPONSE:
+        code = int.from_bytes(payload[:4], "big", signed=False)
+        result['code'] = code
+        payload_size = int.from_bytes(payload[4:8], "big", signed=False)
+        payload_msg = payload[8:]
+    if payload_msg is None:
+        return result
+    if message_compression == GZIP:
+        payload_msg = gzip.decompress(payload_msg)
+    if serialization_method == JSON:
+        payload_msg = json.loads(str(payload_msg, "utf-8"))
+    elif serialization_method != NO_SERIALIZATION:
+        payload_msg = str(payload_msg, "utf-8")
+    result['payload_msg'] = payload_msg
+    result['payload_size'] = payload_size
+    return result
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        self.appid = config.get("appid")
+        self.cluster = config.get("cluster")
+        self.access_token = config.get("access_token")
+        self.output_dir = config.get("output_dir")
+
+        self.host = "openspeech.bytedance.com"
+        self.ws_url = f"wss://{self.host}/api/v2/asr"
+        self.success_code = 1000
+        self.seg_duration = 15000
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """将Opus音频数据解码并保存为WAV文件"""
+        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    @staticmethod
+    def _generate_header(message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE) -> bytearray:
+        """Generate protocol header."""
+        header = bytearray()
+        header_size = 1
+        header.append((0b0001 << 4) | header_size)  # Protocol version
+        header.append((message_type << 4) | message_type_specific_flags)
+        header.append((0b0001 << 4) | 0b0001)  # JSON serialization & GZIP compression
+        header.append(0x00)  # reserved
+        return header
+
+    def _construct_request(self, reqid) -> dict:
+        """Construct the request payload."""
+        return {
+            "app": {
+                "appid": f"{self.appid}",
+                "cluster": self.cluster,
+                "token": self.access_token,
+            },
+            "user": {
+                "uid": str(uuid.uuid4()),
+            },
+            "request": {
+                "reqid": reqid,
+                "show_utterances": False,
+                "sequence": 1
+            },
+            "audio": {
+                "format": "wav",
+                "rate": 16000,
+                "language": "zh-CN",
+                "bits": 16,
+                "channel": 1,
+                "codec": "raw",
+            },
+        }
+
+    async def _send_request(self, audio_data: List[bytes], segment_size: int) -> Optional[str]:
+        """Send request to Volcano ASR service."""
+        try:
+            auth_header = {'Authorization': 'Bearer; {}'.format(self.access_token)}
+            async with websockets.connect(self.ws_url, additional_headers=auth_header) as websocket:
+                # Prepare request data
+                request_params = self._construct_request(str(uuid.uuid4()))
+                print(request_params)
+                payload_bytes = str.encode(json.dumps(request_params))
+                payload_bytes = gzip.compress(payload_bytes)
+                full_client_request = self._generate_header()
+                full_client_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
+                full_client_request.extend(payload_bytes)  # payload
+
+                # Send header and metadata
+                # full_client_request
+                await websocket.send(full_client_request)
+                res = await websocket.recv()
+                result = parse_response(res)
+                if 'payload_msg' in result and result['payload_msg']['code'] != self.success_code:
+                    logger.bind(tag=TAG).error(f"ASR error: {result}")
+                    return None
+
+                for seq, (chunk, last) in enumerate(self.slice_data(audio_data, segment_size), 1):
+                    if last:
+                        audio_only_request = self._generate_header(
+                            message_type=CLIENT_AUDIO_ONLY_REQUEST,
+                            message_type_specific_flags=NEG_SEQUENCE
+                        )
+                    else:
+                        audio_only_request = self._generate_header(
+                            message_type=CLIENT_AUDIO_ONLY_REQUEST
+                        )
+                    payload_bytes = gzip.compress(chunk)
+                    audio_only_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
+                    audio_only_request.extend(payload_bytes)  # payload
+                    # Send audio data
+                    await websocket.send(audio_only_request)
+
+                # Receive response
+                response = await websocket.recv()
+                result = parse_response(response)
+
+                if 'payload_msg' in result and result['payload_msg']['code'] == self.success_code:
+                    if len(result['payload_msg']['result']) > 0:
+                        return result['payload_msg']['result'][0]["text"]
+                    return None
+                else:
+                    logger.bind(tag=TAG).error(f"ASR error: {result}")
+                    return None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
+            return None
+
+    @staticmethod
+    def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        return pcm_data
+
+    @staticmethod
+    def read_wav_info(data: io.BytesIO = None) -> (int, int, int, int, int):
+        with io.BytesIO(data) as _f:
+            wave_fp = wave.open(_f, 'rb')
+            nchannels, sampwidth, framerate, nframes = wave_fp.getparams()[:4]
+            wave_bytes = wave_fp.readframes(nframes)
+        return nchannels, sampwidth, framerate, nframes, len(wave_bytes)
+
+    @staticmethod
+    def slice_data(data: bytes, chunk_size: int) -> (list, bool):
+        """
+        slice data
+        :param data: wav data
+        :param chunk_size: the segment size in one request
+        :return: segment data, last flag
+        """
+        data_len = len(data)
+        offset = 0
+        while offset + chunk_size < data_len:
+            yield data[offset: offset + chunk_size], False
+            offset += chunk_size
+        else:
+            yield data[offset: data_len], True
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        try:
+            # 合并所有opus数据包
+            pcm_data = self.decode_opus(opus_data, session_id)
+            combined_pcm_data = b''.join(pcm_data)
+
+            wav_buffer = io.BytesIO()
+
+            with wave.open(wav_buffer, "wb") as wav_file:
+                wav_file.setnchannels(1)  # 设置声道数
+                wav_file.setsampwidth(2)  # 设置采样宽度
+                wav_file.setframerate(16000)  # 设置采样率
+                wav_file.writeframes(combined_pcm_data)  # 写入 PCM 数据
+
+            # 获取封装后的 WAV 数据
+            wav_data = wav_buffer.getvalue()
+            nchannels, sampwidth, framerate, nframes, wav_len = self.read_wav_info(wav_data)
+            size_per_sec = nchannels * sampwidth * framerate
+            segment_size = int(size_per_sec * self.seg_duration / 1000)
+
+            # 语音识别
+            start_time = time.time()
+            text = await self._send_request(wav_data, segment_size)
+            if text:
+                logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+                return text, None
+            return "", None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None

core/providers/asr/fun_local.py

@@ -0,0 +1,110 @@
+import time
+import wave
+import os
+import sys
+import io
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+import uuid
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+
+from funasr import AutoModel
+from funasr.utils.postprocess_utils import rich_transcription_postprocess
+
+TAG = __name__
+logger = setup_logging()
+
+
+# 捕获标准输出
+class CaptureOutput:
+    def __enter__(self):
+        self._output = io.StringIO()
+        self._original_stdout = sys.stdout
+        sys.stdout = self._output
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        sys.stdout = self._original_stdout
+        self.output = self._output.getvalue()
+        self._output.close()
+
+        # 将捕获到的内容通过 logger 输出
+        if self.output:
+            logger.bind(tag=TAG).info(self.output.strip())
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        self.model_dir = config.get("model_dir")
+        self.output_dir = config.get("output_dir")  # 修正配置键名
+        self.delete_audio_file = delete_audio_file
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+        with CaptureOutput():
+            self.model = AutoModel(
+                model=self.model_dir,
+                vad_kwargs={"max_single_segment_time": 30000},
+                disable_update=True,
+                hub="hf"
+                # device="cuda:0",  # 启用GPU加速
+            )
+
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """将Opus音频数据解码并保存为WAV文件"""
+        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """语音转文本主处理逻辑"""
+        file_path = None
+        try:
+            # 保存音频文件
+            start_time = time.time()
+            file_path = self.save_audio_to_file(opus_data, session_id)
+            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
+
+            # 语音识别
+            start_time = time.time()
+            result = self.model.generate(
+                input=file_path,
+                cache={},
+                language="auto",
+                use_itn=True,
+                batch_size_s=60,
+            )
+            text = rich_transcription_postprocess(result[0]["text"])
+            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+
+            return text, file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None
+
+        finally:
+            # 文件清理逻辑
+            if self.delete_audio_file and file_path and os.path.exists(file_path):
+                try:
+                    os.remove(file_path)
+                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

core/providers/llm/coze/coze.py

@@ -3,94 +3,35 @@ import requests
 import json
 import re
 from core.providers.llm.base import LLMProviderBase
+import os
+# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
+from cozepy import COZE_CN_BASE_URL
+from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType  # noqa
 
 TAG = __name__
 logger = setup_logging()
 
-# 定义用于匹配中文标点符号的正则表达式（包括句号、感叹号、问号、分号）
-punctuation_pattern = re.compile(r'([。！？；])')
-
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.personal_access_token = config.get("personal_access_token")
         self.bot_id = config.get("bot_id")
-        self.user_id = config.get("user_id")  # 默认用户 ID
+        self.user_id = config.get("user_id")
-        self.base_url = config.get("base_url")
 
     def response(self, session_id, dialogue):
-        try:
+        coze_api_token = self.personal_access_token
-            # 从对话中取出最新的用户消息
+        coze_api_base = COZE_CN_BASE_URL
-            last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+
-            data = {
+        last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
-                "conversation_id": session_id,
+
-                "bot_id": self.bot_id,
+        coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
-                "user": self.user_id,
+
-                "query": last_msg["content"],
+        for event in coze.chat.stream(
-                "stream": True
+            bot_id=self.bot_id,
-            }
+            user_id=self.user_id,
-            logger.bind(tag=TAG).info(f"发送到 Coze API 的请求数据: {json.dumps(data, ensure_ascii=False)}")
+            additional_messages=[
-            
+                Message.build_user_question_text(last_msg["content"]),
-            headers = {
+            ],
-                'Authorization': f'Bearer {self.personal_access_token}',
+        ):
-                'Content-Type': 'application/json',
+            if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
-                'Accept': '*/*',
+                print(event.message.content, end="", flush=True)
-                'Host': 'api.coze.cn',
+                yield event.message.content
-                'Connection': 'keep-alive'
-            }
-            
-            response = requests.post(
-                self.base_url,
-                headers=headers,
-                json=data,
-                stream=True
-            )
-            logger.bind(tag=TAG).info(f"请求状态: {response.status_code}")
-            
-            if response.status_code == 200:
-                # 对每一行流数据进行处理，不做跨块累积
-                for line_bytes in response.iter_lines(decode_unicode=False):
-                    if not line_bytes:
-                        continue
-                    try:
-                        # 使用 utf-8 解码，错误部分用替换符
-                        line = line_bytes.decode('utf-8', errors='replace')
-                    except Exception as e:
-                        logger.bind(tag=TAG).error(f"解码失败: {e}")
-                        continue
-                    if line.startswith("data:"):
-                        data_str = line[len("data:"):].strip()
-                        if data_str == "[DONE]":
-                            break
-                        try:
-                            data_chunk = json.loads(data_str)
-                        except json.JSONDecodeError as e:
-                            logger.bind(tag=TAG).error(f"JSON解析失败: {e} 数据: {line}")
-                            continue
-                        msg = data_chunk.get("message", {})
-                        if msg.get("role") == "assistant" and msg.get("type") == "answer":
-                            content = msg.get("content", "")
-                            # 如果返回内容中包含标点符号，则按标点拆分，立即返回每个片段
-                            if punctuation_pattern.search(content):
-                                # 利用 finditer 找到每个标点，并返回以标点结尾的片段
-                                start = 0
-                                for match in punctuation_pattern.finditer(content):
-                                    end = match.end()
-                                    sentence = content[start:end].strip()
-                                    if sentence:
-                                        yield sentence
-                                    start = end
-                                # 如果拆分后剩余内容也返回（不含标点），直接返回
-                                if start < len(content):
-                                    remainder = content[start:].strip()
-                                    if remainder:
-                                        yield remainder
-                            else:
-                                # 如果没有标点，则直接返回这块内容
-                                if content.strip():
-                                    yield content.strip()
-            else:
-                logger.bind(tag=TAG).error(f"请求失败，状态码: {response.status_code}")
-                yield f"【Coze服务响应异常：请求失败，状态码 {response.status_code}】"
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error in Coze response generation: {e}")
-            yield "【Coze服务响应异常】"

core/providers/llm/openai/openai.py

@@ -25,12 +25,25 @@ class LLMProvider(LLMProviderBase):
                 messages=dialogue,
                 stream=True
             )
+            
+            is_active = True
             for chunk in responses:
-                # 检查是否存在有效的choice且content不为空
+                try:
-                if chunk.choices and len(chunk.choices) > 0:
+                    # 检查是否存在有效的choice且content不为空
-                    delta = chunk.choices[0].delta
+                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
-                    content = getattr(delta, 'content', '')
+                    content = delta.content if hasattr(delta, 'content') else ''
-                    if content:  # 仅在content非空时生成
+                except IndexError:
+                    content = ''
+                if content:
+                    # 处理标签跨多个chunk的情况
+                    if '<think>' in content:
+                        is_active = False
+                        content = content.split('<think>')[0]
+                    if '</think>' in content:
+                        is_active = True
+                        content = content.split('</think>')[-1]
+                    if is_active:
                         yield content
+
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")

core/providers/memory/base.py

@@ -0,0 +1,23 @@
+from abc import ABC, abstractmethod
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class MemoryProviderBase(ABC):
+    def __init__(self, config):
+        self.config = config
+        self.role_id = None
+
+    @abstractmethod
+    async def save_memory(self, msgs):
+        """Save a new memory for specific role and return memory ID"""
+        print("this is base func", msgs)
+
+    @abstractmethod
+    async def query_memory(self, query: str) -> str:
+        """Query memories for specific role based on similarity"""
+        return "please implement query method"
+
+    def set_role_id(self, role_id: str):
+        self.role_id = role_id

core/providers/memory/mem0ai/mem0ai.py

@@ -0,0 +1,74 @@
+from ..base import MemoryProviderBase, logger
+from mem0 import MemoryClient
+
+TAG = __name__
+
+class MemoryProvider(MemoryProviderBase):
+    def __init__(self, config):
+        super().__init__(config)
+        self.api_key = config.get("api_key", "")
+        self.api_version = config.get("api_version", "v1.1")
+        if len(self.api_key) == 0 or "你" in self.api_key:
+            logger.bind(tag=TAG).error("你还没配置Mem0ai的密钥，请在配置文件中配置密钥，否则无法提供记忆服务")
+            self.use_mem0 = False
+            return
+        else:
+            self.use_mem0 = True
+        self.client = MemoryClient(api_key=self.api_key)
+
+    async def save_memory(self, msgs):
+        if not self.use_mem0:
+            return None
+        if len(msgs) < 2:
+            return None
+        
+        try:
+            # Format the content as a message list for mem0
+            messages = [
+                {"role": message.role, "content": message.content}
+                for message in msgs if message.role != "system"
+            ]
+            result = self.client.add(messages, user_id=self.role_id, output_format=self.api_version)
+            logger.bind(tag=TAG).debug(f"Save memory result: {result}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
+            return None
+
+    async def query_memory(self, query: str)-> str:
+        if not self.use_mem0:
+            return ""
+        try:
+            results = self.client.search(
+                query,
+                user_id=self.role_id,
+                output_format=self.api_version
+            )
+            if not results or 'results' not in results:
+                return ""
+                
+            # Format each memory entry with its update time up to minutes
+            memories = []
+            for entry in results['results']:
+                timestamp = entry.get('updated_at', '')
+                if timestamp:
+                    try:
+                        # Parse and reformat the timestamp
+                        dt = timestamp.split('.')[0]  # Remove milliseconds
+                        formatted_time = dt.replace('T', ' ')
+                    except:
+                        formatted_time = timestamp
+                memory = entry.get('memory', '')
+                if timestamp and memory:
+                    # Store tuple of (timestamp, formatted_string) for sorting
+                    memories.append((timestamp, f"[{formatted_time}] {memory}"))
+            
+            # Sort by timestamp in descending order (newest first)
+            memories.sort(key=lambda x: x[0], reverse=True)
+            
+            # Extract only the formatted strings
+            memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
+            logger.bind(tag=TAG).debug(f"Query results: {memories_str}")
+            return memories_str
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}")
+            return ""

core/providers/tts/doubao.py

@@ -14,10 +14,9 @@ class TTSProvider(TTSProviderBase):
         self.access_token = config.get("access_token")
         self.cluster = config.get("cluster")
         self.voice = config.get("voice")
-
+        self.api_url = config.get("api_url")
-        self.host = "openspeech.bytedance.com"
+        self.authorization = config.get("authorization")
-        self.api_url = f"https://{self.host}/api/v1/tts"
+        self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
-        self.header = {"Authorization": f"Bearer;{self.access_token}"}
 
     def generate_filename(self, extension=".wav"):
         return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
@@ -25,7 +24,7 @@ class TTSProvider(TTSProviderBase):
     async def text_to_speak(self, text, output_file):
         request_json = {
             "app": {
-                "appid": self.appid,
+                "appid": f"{self.appid}",
                 "token": "access_token",
                 "cluster": self.cluster
             },
@@ -49,8 +48,13 @@ class TTSProvider(TTSProviderBase):
             }
         }
 
-        resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
+        try:
-        if "data" in resp.json():
+            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
-            data = resp.json()["data"]
+            if "data" in resp.json():
-            file_to_save = open(output_file, "wb")
+                data = resp.json()["data"]
-            file_to_save.write(base64.b64decode(data))
+                file_to_save = open(output_file, "wb")
+                file_to_save.write(base64.b64decode(data))
+            else:
+                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+        except Exception as e:
+            raise Exception(f"{__name__} error: {e}")

core/providers/tts/fishspeech.py

@@ -89,7 +89,7 @@ class TTSProvider(TTSProviderBase):
         self.channels = config.get("channels",1)
         self.rate = config.get("rate",44100)
         self.api_key = config.get("api_key","YOUR_API_KEY")
-        if not self.api_key or "你" in self.api_key:
+        if "你" in self.api_key:
             logger.bind(tag=TAG).error("你还没配置FishSpeech TTS的密钥，请在配置文件中配置密钥，否则无法正常工作")
             return
         self.normalize = config.get("normalize",True)

core/utils/asr.py

@@ -1,132 +1,24 @@
-import time
+import importlib
-import wave
+import logging
 import os
 import sys
-import io
+import time
-from abc import ABC, abstractmethod
+import wave
-from config.logger import setup_logging
-from typing import Optional, Tuple, List
 import uuid
-
+from abc import ABC, abstractmethod
-import opuslib_next
+from typing import Optional, Tuple, List
-from funasr import AutoModel
+from core.providers.asr.base import ASRProviderBase
-from funasr.utils.postprocess_utils import rich_transcription_postprocess
+from config.logger import setup_logging
 
 TAG = __name__
 logger = setup_logging()
 
-# 捕获标准输出
+def create_instance(class_name: str, *args, **kwargs) -> ASRProviderBase:
-class CaptureOutput:
-    def __enter__(self):
-        self._output = io.StringIO()
-        self._original_stdout = sys.stdout
-        sys.stdout = self._output
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        sys.stdout = self._original_stdout
-        self.output = self._output.getvalue()
-        self._output.close()
-
-        # 将捕获到的内容通过 logger 输出
-        if self.output:
-            logger.bind(tag=TAG).info(self.output.strip())
-
-class ASR(ABC):
-    @abstractmethod
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """解码Opus数据并保存为WAV文件"""
-        pass
-
-    @abstractmethod
-    def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
-        """将语音数据转换为文本"""
-        pass
-
-
-class FunASR(ASR):
-    def __init__(self, config: dict, delete_audio_file: bool):
-        self.model_dir = config.get("model_dir")
-        self.output_dir = config.get("output_dir")  # 修正配置键名
-        self.delete_audio_file = delete_audio_file
-
-        # 确保输出目录存在
-        os.makedirs(self.output_dir, exist_ok=True)
-        with CaptureOutput():
-            self.model = AutoModel(
-                model=self.model_dir,
-                vad_kwargs={"max_single_segment_time": 30000},
-                disable_update=True,
-                hub="hf"
-                # device="cuda:0",  # 启用GPU加速
-            )
-
-    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
-        """将Opus音频数据解码并保存为WAV文件"""
-        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
-        file_path = os.path.join(self.output_dir, file_name)
-
-        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
-        pcm_data = []
-
-        for opus_packet in opus_data:
-            try:
-                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
-                pcm_data.append(pcm_frame)
-            except opuslib_next.OpusError as e:
-                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
-
-        with wave.open(file_path, "wb") as wf:
-            wf.setnchannels(1)
-            wf.setsampwidth(2)  # 2 bytes = 16-bit
-            wf.setframerate(16000)
-            wf.writeframes(b"".join(pcm_data))
-
-        return file_path
-
-    def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
-        """语音转文本主处理逻辑"""
-        file_path = None
-        try:
-            # 保存音频文件
-            start_time = time.time()
-            file_path = self.save_audio_to_file(opus_data, session_id)
-            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
-
-            # 语音识别
-            start_time = time.time()
-            result = self.model.generate(
-                input=file_path,
-                cache={},
-                language="auto",
-                use_itn=True,
-                batch_size_s=60,
-            )
-            text = rich_transcription_postprocess(result[0]["text"])
-            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
-
-            return text, file_path
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
-            return None, None
-
-        finally:
-            # 文件清理逻辑
-            if self.delete_audio_file and file_path and os.path.exists(file_path):
-                try:
-                    os.remove(file_path)
-                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
-                except Exception as e:
-                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
-
-
-def create_instance(class_name: str, *args, **kwargs) -> ASR:
     """工厂方法创建ASR实例"""
-    cls_map = {
+    if os.path.exists(os.path.join('core', 'providers', 'asr', f'{class_name}.py')):
-        "FunASR": FunASR,
+        lib_name = f'core.providers.asr.{class_name}'
-        # 可扩展其他ASR实现
+        if lib_name not in sys.modules:
-    }
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].ASRProvider(*args, **kwargs)
 
-    if cls := cls_map.get(class_name):
+    raise ValueError(f"不支持的ASR类型: {class_name}，请检查该配置的type是否设置正确")
-        return cls(*args, **kwargs)
-    raise ValueError(f"不支持的ASR类型: {class_name}")

core/utils/dialogue.py

@@ -24,3 +24,27 @@ class Dialogue:
         for m in self.dialogue:
             dialogue.append({"role": m.role, "content": m.content})
         return dialogue
+
+    def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
+        # 构建带记忆的对话
+        dialogue = []
+        
+        # 添加系统提示和记忆
+        system_message = next(
+            (msg for msg in self.dialogue if msg.role == "system"), None
+        )
+
+
+        if system_message:
+            enhanced_system_prompt = (
+                f"{system_message.content}\n\n"
+                f"相关记忆：\n{memory_str}"
+            )
+            dialogue.append({"role": "system", "content": enhanced_system_prompt})
+
+        # 添加用户和助手的对话
+        for msg in self.dialogue:
+            if msg.role != "system":  # 跳过原始的系统消息
+                dialogue.append({"role": msg.role, "content": msg.content})
+
+        return dialogue

core/utils/llm.py

@@ -1,11 +1,13 @@
 import os
 import sys
+
+# 添加项目根目录到Python路径
+current_dir = os.path.dirname(os.path.abspath(__file__))
+project_root = os.path.abspath(os.path.join(current_dir, "..", ".."))
+sys.path.insert(0, project_root)
+
 from config.logger import setup_logging
 import importlib
-from datetime import datetime
-from core.utils.util import is_segment
-from core.utils.util import get_string_no_punctuation_or_emoji
-from core.utils.util import read_config, get_project_dir
 
 logger = setup_logging()
 
@@ -19,41 +21,3 @@ def create_instance(class_name, *args, **kwargs):
         return sys.modules[lib_name].LLMProvider(*args, **kwargs)
 
     raise ValueError(f"不支持的LLM类型: {class_name}，请检查该配置的type是否设置正确")
-
-
-if __name__ == "__main__":
-    """
-      响应速度测试
-    """
-    config = read_config(get_project_dir() + "config.yaml")
-    llm = create_instance(
-        config["selected_module"]["LLM"]
-        if not "type" in config["LLM"][config["selected_module"]["LLM"]]
-        else
-        config["LLM"][config["selected_module"]["LLM"]]["type"],
-        config["LLM"][config["selected_module"]["LLM"]]
-    )
-
-    start_time = datetime.now()
-
-    dialogue = []
-    dialogue.append({"role": "system", "content": config.get("prompt")})
-    dialogue.append({"role": "user", "content": "你好小智"})
-    llm_responses = llm.response("test", dialogue)
-    response_message = []
-    first_text = None
-    start = 0
-
-    for content in llm_responses:
-        response_message.append(content)
-
-        if is_segment(response_message):
-            segment_text = "".join(response_message[start:])
-            segment_text = get_string_no_punctuation_or_emoji(segment_text)
-            if len(segment_text) > 0:
-                if first_text is None:
-                    first_text = segment_text
-                    print("大模型首次返回耗时：" + str(datetime.now() - start_time))
-                start = len(response_message)
-
-    print("大模型返回总耗时：" + str(datetime.now() - start_time))

core/utils/memory.py

@@ -0,0 +1,17 @@
+import os
+import sys
+import importlib
+from config.logger import setup_logging
+from core.utils.util import read_config, get_project_dir
+
+logger = setup_logging()
+
+def create_instance(class_name, *args, **kwargs):
+    if os.path.exists(os.path.join('core', 'providers', 'memory', class_name, f'{class_name}.py')):
+        lib_name = f'core.providers.memory.{class_name}.{class_name}'
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].MemoryProvider(*args, **kwargs)
+
+    raise ValueError(f"不支持的记忆服务类型: {class_name}")
+

core/utils/p3.py

@@ -0,0 +1,33 @@
+import struct
+
+def decode_opus_from_file(input_file):
+    """
+    从p3文件中解码 Opus 数据，并返回一个 Opus 数据包的列表以及总时长。
+    """
+    opus_datas = []
+    total_frames = 0
+    sample_rate = 16000  # 文件采样率
+    frame_duration_ms = 60  # 帧时长
+    frame_size = int(sample_rate * frame_duration_ms / 1000)
+
+    with open(input_file, 'rb') as f:
+        while True:
+            # 读取头部（4字节）：[1字节类型，1字节保留，2字节长度]
+            header = f.read(4)
+            if not header:
+                break
+
+            # 解包头部信息
+            _, _, data_len = struct.unpack('>BBH', header)
+
+            # 根据头部指定的长度读取 Opus 数据
+            opus_data = f.read(data_len)
+            if len(opus_data) != data_len:
+                raise ValueError(f"Data length({len(opus_data)}) mismatch({data_len}) in the file.")
+
+            opus_datas.append(opus_data)
+            total_frames += 1
+
+    # 计算总时长
+    total_duration = (total_frames * frame_duration_ms) / 1000.0
+    return opus_datas, total_duration

core/utils/tts.py

@@ -2,8 +2,6 @@ import os
 import sys
 from config.logger import setup_logging
 import importlib
-from datetime import datetime
-from core.utils.util import read_config, get_project_dir
 
 logger = setup_logging()
 
@@ -16,26 +14,4 @@ def create_instance(class_name, *args, **kwargs):
             sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
         return sys.modules[lib_name].TTSProvider(*args, **kwargs)
 
-    raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")
+    raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")
-
-
-if __name__ == "__main__":
-    """
-      响应速度测试
-    """
-    config = read_config(get_project_dir() + "config.yaml")
-    tts = create_instance(
-        config["selected_module"]["TTS"]
-        if not 'type' in config["TTS"][config["selected_module"]["TTS"]]
-        else
-        config["TTS"][config["selected_module"]["TTS"]]["type"],
-        config["TTS"][config["selected_module"]["TTS"]],
-        config["delete_audio"]
-    )
-    tts.output_file = get_project_dir() + tts.output_file
-    start = datetime.now()
-    file_path = tts.to_tts("你好，测试,我是人工智能小智")
-    print("语音合成耗时：" + str(datetime.now() - start))
-    start = datetime.now()
-    tts.wav_to_opus_data(file_path)
-    print("语音opus耗时：" + str(datetime.now() - start))

core/utils/util.py

@@ -3,6 +3,7 @@ import re
 import json
 import yaml
 import socket
+import subprocess
 
 
 def get_project_dir():
@@ -34,13 +35,6 @@ def write_json_file(file_path, data):
         json.dump(data, file, ensure_ascii=False, indent=4)
 
 
-def is_segment(tokens):
-    if tokens[-1] in (",", ".", "?", "，", "。", "？", "！", "!", ";", "；", ":", "："):
-        return True
-    else:
-        return False
-
-
 def is_punctuation_or_emoji(char):
     """检查字符是否为空格、指定标点或表情符号"""
     # 定义需要去除的中英文标点（包括全角/半角）
@@ -90,7 +84,7 @@ def remove_punctuation_and_length(text):
                       char not in full_width_punctuations and char not in half_width_punctuations and char not in space and char not in full_width_space])
 
     if result == "Yeah":
-        return 0
+        return 0, ""
     return len(result), result
 
 
@@ -120,4 +114,30 @@ def check_password(password):
         return False
 
     # 如果满足所有条件，则返回True
-    return True
+    return True
+
+def check_ffmpeg_installed():
+    ffmpeg_installed = False
+    try:
+        # 执行ffmpeg -version命令，并捕获输出
+        result = subprocess.run(
+            ['ffmpeg', '-version'],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=True  # 如果返回码非零则抛出异常
+        )
+        # 检查输出中是否包含版本信息（可选）
+        output = result.stdout + result.stderr
+        if 'ffmpeg version' in output.lower():
+            ffmpeg_installed = True
+        return False
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        # 命令执行失败或未找到
+        ffmpeg_installed = False
+    if not ffmpeg_installed:
+        error_msg = "您的电脑还没正确安装ffmpeg\n"
+        error_msg += "\n建议您：\n"
+        error_msg += "1、按照项目的安装文档，正确进入conda环境\n"
+        error_msg += "2、查阅安装文档，如何在conda环境中安装ffmpeg\n"
+        raise ValueError(error_msg)

core/websocket_server.py

@@ -2,18 +2,25 @@ import asyncio
 import websockets
 from config.logger import setup_logging
 from core.connection import ConnectionHandler
+from core.handle.musicHandler import MusicHandler
 from core.utils.util import get_local_ip
-from core.utils import asr, vad, llm, tts
+from core.utils import asr, vad, llm, tts, memory
 
 TAG = __name__
 
+
 class WebSocketServer:
     def __init__(self, config: dict):
         self.config = config
         self.logger = setup_logging()
-        self._vad, self._asr, self._llm, self._tts = self._create_processing_instances()
+        self._vad, self._asr, self._llm, self._tts, self._music, self._memory = self._create_processing_instances()
+        self.active_connections = set()  # 添加全局连接记录
 
     def _create_processing_instances(self):
+        memory_cls_name = self.config["selected_module"].get("Memory", "mem0ai") # 默认使用mem0ai
+        has_memory_cfg = self.config.get("Memory") and memory_cls_name in self.config["Memory"]
+        memory_cfg = self.config["Memory"][memory_cls_name] if has_memory_cfg else {}
+
         """创建处理模块实例"""
         return (
             vad.create_instance(
@@ -21,7 +28,10 @@ class WebSocketServer:
                 self.config["VAD"][self.config["selected_module"]["VAD"]]
             ),
             asr.create_instance(
-                self.config["selected_module"]["ASR"],
+                self.config["selected_module"]["ASR"]
+                if not 'type' in self.config["ASR"][self.config["selected_module"]["ASR"]]
+                else
+                self.config["ASR"][self.config["selected_module"]["ASR"]]["type"],
                 self.config["ASR"][self.config["selected_module"]["ASR"]],
                 self.config["delete_audio"]
             ),
@@ -39,7 +49,9 @@ class WebSocketServer:
                 self.config["TTS"][self.config["selected_module"]["TTS"]]["type"],
                 self.config["TTS"][self.config["selected_module"]["TTS"]],
                 self.config["delete_audio"]
-            )
+            ),
+            MusicHandler(self.config),
+            memory.create_instance(memory_cls_name, memory_cfg),
         )
 
     async def start(self):
@@ -58,5 +70,10 @@ class WebSocketServer:
 
     async def _handle_connection(self, websocket):
         """处理新连接，每次创建独立的ConnectionHandler"""
-        handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts)
+        # 创建ConnectionHandler时传入当前server实例
-        await handler.handle_connection(websocket)
+        handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._music, self._memory)
+        self.active_connections.add(handler)
+        try:
+            await handler.handle_connection(websocket)
+        finally:
+            self.active_connections.discard(handler)

docker-compose-example.yml

@@ -1,17 +0,0 @@
-services:
-  xiaozhi-esp32-server:
-    image: ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest
-    container_name: xiaozhi-esp32-server
-    restart: always
-    #security_opt:
-    #  - seccomp:unconfined
-    ports:
-      - "8000:8000" # ws服务端
-      - "8002:8002" # 管理后台
-    volumes:
-      # 配置文件 记得创建文件夹
-      - ./data:/opt/xiaozhi-esp32-server/data
-      # 模型文件 //可以挂载本地下载好的模型
-      #- ./models:/opt/xiaozhi-esp32-server/models
-      # 临时文件 //可以看log和音频文件 记得创建文件夹
-      #- ./tmp:/opt/xiaozhi-esp32-server/tmp

docker-compose.yml

@@ -1,13 +1,18 @@
+version: '3'
 services:
   xiaozhi-esp32-server:
-    image: ghcr.io/kalicyh/xiaozhi-esp32-server:latest
+    image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:latest
     container_name: xiaozhi-esp32-server
     restart: always
-    #security_opt:
+    security_opt:
-    #  - seccomp:unconfined
+      - seccomp:unconfined
     ports:
+      # ws服务端
       - "8000:8000"
+      # 管理后台
       - "8002:8002"
     volumes:
+      # 配置文件目录
       - ./data:/app/data
+      # 模型文件挂接，很重要
       - ./models/SenseVoiceSmall/model.pt:/app/models/SenseVoiceSmall/model.pt

docs/Deployment-silm.md

@@ -1,217 +0,0 @@
-# 部署指南
-
-本项目支持 **x86** 和 **arm64** 架构，可运行于国产操作系统。
-
-## 方式一：快速部署（推荐）
-
-### 1. 安装 Docker
-
-如果未安装 Docker，可参考 [Docker 安装教程](https://www.runoob.com/docker/ubuntu-docker-install.html)。
-
-### 2. 创建项目目录
-
-选择一个空目录作为 **项目目录**。
-
-### 3. 下载配置文件
-
-访问 [config.yaml](https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/config.yaml)，点击 `RAW` 旁的下载按钮，保存到 **项目目录**。
-
-### 4. 运行 Docker
-
-打开终端，`cd` 进入 **项目目录**，执行：
-
-```sh
-docker run -d --name xiaozhi-esp32-server --restart always --security-opt seccomp:unconfined \
-  -p 8000:8000 \
-  -v $(pwd)/config.yaml:/opt/xiaozhi-esp32-server/config.yaml \
-  ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest
-```
-
-### 5. 确认运行状态
-
-```sh
-docker ps  # 查看容器状态
-docker logs -f xiaozhi-esp32-server  # 查看日志
-```
-
-## 方式二：使用 Docker Compose 部署
-
-### 1. 创建 `docker-compose.yml`
-在 `项目目录` 下创建 `docker-compose.yml` 文件，内容如下：
-
-```yaml
-version: '3'
-services:
-  xiaozhi-esp32-server:
-    image: ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest
-    container_name: xiaozhi-esp32-server
-    restart: always
-    security_opt:
-      - seccomp:unconfined
-    ports:
-      - "8000:8000"
-    volumes:
-      - ./config.yaml:/opt/xiaozhi-esp32-server/config.yaml
-```
-
-### 2. 启动服务
-```sh
-docker-compose up -d
-```
-
-### 3. 确认运行状态
-```sh
-docker-compose ps
-docker-compose logs -f
-```
-
-### 4. 停止和删除容器
-```sh
-docker-compose down
-```
-
-### 5. 版本升级
-
-#### 备份配置文件
-```sh
-cp config.yaml config_backup.yaml
-```
-
-#### 更新 Docker 镜像
-```sh
-docker-compose pull
-docker-compose down
-docker-compose up -d
-```
-
-
-## 方式三：源码部署（免环境部署/可修改代码）
-
-### 1. 下载源码
-
-```sh
-git clone https://github.com/xinnan-tech/xiaozhi-esp32-server.git
-cd xiaozhi-esp32-server
-```
-
-或手动下载 [ZIP 包](https://github.com/xinnan-tech/xiaozhi-esp32-server/archive/refs/heads/main.zip)，解压后重命名为 `xiaozhi-esp32-server`。
-
-## 2.运行docker
-
-修改完配置后，打开命令行工具，`cd`进入到你的项目目录下，执行以下命令
-
-```sh
-docker run -it --name xiaozhi-env --restart always --security-opt seccomp:unconfined \
-  -p 8000:8000 \
-  -p 8002:8002 \
-  -v ./:/app \
-  kalicyh/poetry:v3.10_latest
-```
-
-然后就和正常开发一样了
-
-## 3.安装依赖
-
-在刚刚的打开的终端运行
-
-```sh
-poetry install --no-root
-```
-
-```sh
-apt-get update
-apt-get install -y --no-install-recommends libopus0 ffmpeg
-```
-
-速度慢可以尝试使用清华镜像
-
-```sh
-echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list
-echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list
-echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-backports main contrib non-free non-free-firmware" >> /etc/apt/sources.list
-echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bookworm-security main contrib non-free non-free-firmware" >> /etc/apt/sources.list
-apt-get update
-apt-get install -y --no-install-recommends libopus0 ffmpeg
-```
-
-## 4.运行项目
-
-```sh
-poetry run python app.py
-```
-
-## 方式三：本地运行（适用于开发）
-
-### 1. 安装环境
-
-```sh
-conda create -n xiaozhi-esp32-server python=3.10 -y
-conda activate xiaozhi-esp32-server
-```
-
-#### Mac/Windows:
-
-```sh
-conda install conda-forge::libopus conda-forge::ffmpeg
-```
-
-#### Ubuntu:
-
-```sh
-apt-get install libopus0 ffmpeg
-```
-
-### 2. 下载源码 & 安装依赖
-
-```sh
-git clone https://github.com/xinnan-tech/xiaozhi-esp32-server.git
-cd xiaozhi-esp32-server
-pip install -r requirements.txt
-```
-
-### 3. 运行项目
-
-```sh
-python app.py
-```
-
-## 版本升级
-
-```sh
-docker stop xiaozhi-esp32-server
-docker rm xiaozhi-esp32-server
-docker rmi ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest
-```
-
-然后重新按照 **方式一** 或 **方式二** 运行 Docker。
-
-## 重要配置
-
-修改 `config.yaml` 以适配不同 LLM 和 TTS 组件。
-
-示例：
-
-```yaml
-selected_module:
-  ASR: FunASR
-  VAD: SileroVAD
-  LLM: ChatGLMLLM
-  TTS: EdgeTTS
-```
-
-如需使用 `Dify` 或 `DeepSeekLLM`，修改 `LLM` 部分并填写密钥。
-
-## 模型文件下载
-
-语音转文字默认使用 `SenseVoiceSmall`，需下载 `model.pt` 并存放至 `model/SenseVoiceSmall` 目录。
-
-- [阿里魔塔下载](https://modelscope.cn/models/iic/SenseVoiceSmall/resolve/master/model.pt)
-- [百度网盘下载](https://pan.baidu.com/share/init?surl=QlgM58FHhYv1tFnUT_A8Sg&pwd=qvna) 提取码：`qvna`
-
----
-
-如遇问题，可参考 `docker logs -f xiaozhi-esp32-server` 进行排查。
-
-**至此，部署完成！** 🚀
-
-

docs/Deployment.md

@@ -1,29 +1,188 @@
+# 方式一：docker快速部署
 
-# 本地源码运行
+docker镜像已支持x86架构、arm64架构的CPU，支持在国产操作系统上运行。
+
+## 1. 安装docker
+
+如果您的电脑还没安装docker，可以按照这里的教程安装：[docker安装](https://www.runoob.com/docker/ubuntu-docker-install.html)
+
+## 2. 创建目录
+
+安装完后，你需要为这个项目找一个安放配置文件的目录，我们暂且称它为`项目目录`，这个目录最好是一个新建的空的目录。
+
+创建好目录后，你需要在`项目目录`下面创建`data`文件夹和`models`文件夹，`models`下面还要再创建`SenseVoiceSmall`文件夹。
+
+最终目录结构如下所示：
+
+```
+你的项目根目录
+  ├─ data
+  ├─ models
+     ├─ SenseVoiceSmall
+```
+
+## 4. 下载语音识别模型文件
+
+你需要下载语音识别的模型文件，因为本项目的默认语音识别用的是本地离线语音识别方案。可通过这个方式下载
+[跳转到下载语音识别模型文件](#模型文件)
+
+下载完后，回到本教程。
+
+## 3. 下载docker-compose.yaml
+
+用浏览器打开[这个链接](https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docker-compose.yml)。
+
+在页面的右侧找到名称为`RAW`按钮，在`RAW`按钮的旁边，找到下载的图标，点击下载按钮，下载`docker-compose.yml`文件。 把文件下载到你的
+`项目目录`中。
+
+下载完后，回到本教程继续往下。
+
+## 3. 下载配置文件
+
+用浏览器打开[这个链接](https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/config.yaml)。
+
+在页面的右侧找到名称为`RAW`按钮，在`RAW`按钮的旁边，找到下载的图标，点击下载按钮，下载`config.yaml`文件。 把文件下载到你的
+`项目目录`下面的`data`文件夹中，然后把`config.yaml`文件重命名为`.config.yaml`。
+
+下载完配置文件后，我们确认一下整个`项目目录`里面的文件如下所示：
+
+```
+你的项目根目录
+  ├─ docker-compose.yml
+  ├─ data
+    ├─ .config.yaml
+  ├─ models
+     ├─ SenseVoiceSmall
+       ├─ model.pt
+```
+
+如果你的文件目录结构也是上面的，就继续往下。如果不是，你就再仔细看看是不是漏操作了什么。
+
+## 4. 配置项目文件
+
+接下里，程序还不能直接运行，你需要配置一下，你到底使用的是什么模型。你可以看这个教程：
+[跳转到配置项目文件](#配置项目)
+
+配置完项目文件后，回到本教程继续往下。
+
+## 5. 执行docker命令
+
+打开命令行工具，使用`终端`或`命令行`工具 进入到你的`项目目录`，执行以下命令
+
+```
+docker-compose up -d
+```
+
+执行完后，再执行以下命令，查看日志信息。
+
+```
+docker logs -f xiaozhi-esp32-server
+```
+
+这时，你就要留意日志信息，可以根据这个教程，判断是否成功了。[跳转到运行状态确认](#运行状态确认)
+
+## 6.版本升级操作
+
+如果后期想升级版本，可以这么操作
+
+1、备份好`data`文件夹中的`.config.yaml`文件，一些关键的配置到时复制到新的`.config.yaml`文件里。
+请注意是对关键密钥逐个复制，不要直接覆盖。因为新的`.config.yaml`文件可能有一些新的配置项，旧的`.config.yaml`文件不一定有。
+
+2、执行以下命令
+
+```
+docker stop xiaozhi-esp32-server
+docker rm xiaozhi-esp32-server
+docker rmi ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:latest
+```
+
+3、重新按docker方式部署
+
+# 方式二：借助Docker环境运行部署
+
+开发人员如果不想安装`conda`环境，可以使用这种方法管理好依赖。
+
+## 1.克隆项目
+
+## 2.[跳转到下载语音识别模型文件](#模型文件)
+
+## 3.[跳转到配置项目文件](#配置项目)
+
+## 4.运行docker
+
+修改完配置后，打开命令行工具，`cd`进入到你的项目目录下，执行以下命令
+
+```sh
+docker run -it --name xiaozhi-env --restart always --security-opt seccomp:unconfined \
+  -p 8000:8000 \
+  -p 8002:8002 \
+  -v ./:/app \
+  kalicyh/poetry:v3.10_xiaozhi
+```
+
+然后就和正常开发一样了
+
+## 5.安装依赖
+
+在刚刚的打开的终端运行
+
+```sh
+poetry install --no-root
+```
+
+```sh
+apt-get update
+apt-get install -y --no-install-recommends libopus0 ffmpeg
+```
+
+速度慢可以尝试使用清华镜像
+
+```sh
+echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list
+echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list
+echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-backports main contrib non-free non-free-firmware" >> /etc/apt/sources.list
+echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bookworm-security main contrib non-free non-free-firmware" >> /etc/apt/sources.list
+apt-get update
+apt-get install -y --no-install-recommends libopus0 ffmpeg
+```
+
+## 6.运行项目
+
+```sh
+poetry run python app.py
+```
+
+# 方式三：本地源码运行
 
 ## 1.安装基础环境
 
-本项目使用`conda`管理依赖环境，安装好后，开始执行以下命令。
+本项目使用`conda`管理依赖环境。如果不方便安装`conda`，需要根据实际的操作系统安装好`libopus`和`ffmpeg`。
+如果确定使用`conda`，则安装好后，开始执行以下命令。
+
+重要提示！windows 用户，可以通过安装`Anaconda`来管理环境。安装好`Anaconda`后，在`开始`那里搜索`anaconda`相关的关键词，
+找到`Anaconda Prpmpt`，使用管理员身份运行它。如下图。
+
+![conda_prompt](./images/conda_env_1.png)
+
+运行之后，如果你能看到命令行窗口前面有一个(base)字样，说明你成功进入了`conda`环境。那么你就可以执行以下命令了。
+
+![conda_env](./images/conda_env_2.png)
 
 ```
 conda remove -n xiaozhi-esp32-server --all -y
 conda create -n xiaozhi-esp32-server python=3.10 -y
 conda activate xiaozhi-esp32-server
+
+# 添加清华源通道
+conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
+conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free
+conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge
+
+conda install libopus -y
+conda install ffmpeg -y
 ```
 
-执行以上命令后， 如果你的电脑是Windows或Mac，执行下面的语句：
+请注意，以上命令，不是一股脑执行就成功的，你需要一步步执行，每一步执行完后，都检查一下输出的日志，查看是否成功。
-
-```
-conda activate xiaozhi-esp32-server
-conda install conda-forge::libopus
-conda install conda-forge::ffmpeg
-```
-
-如果你的电脑是ubuntu，执行下面的语句：
-
-```
-apt-get install libopus0 ffmpeg 
-```
 
 ## 2.安装本项目依赖
 
@@ -37,15 +196,23 @@ apt-get install libopus0 ffmpeg
 你需要把它重命名成`xiaozhi-esp32-server`，好了请记住这个目录，我们暂且称它为`项目目录`。
 
 ```
-# 使用dos或者终端，进入到你的项目目录，执行以下命令
+# 继续使用conda环境，进入到你的项目目录，执行以下命令
 conda activate xiaozhi-esp32-server
 pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
 pip install -r requirements.txt
 ```
 
-## 3.[跳转到下载语音识别模型文件](#模型文件)
+## 3.下载语音识别模型文件
 
-## 4.[跳转到配置项目文件](#配置项目)
+你需要下载语音识别的模型文件，因为本项目的默认语音识别用的是本地离线语音识别方案。可通过这个方式下载
+[跳转到下载语音识别模型文件](#模型文件)
+
+下载完后，回到本教程。
+
+## 4.配置项目文件
+
+接下里，程序还不能直接运行，你需要配置一下，你到底使用的是什么模型。你可以看这个教程：
+[跳转到配置项目文件](#配置项目)
 
 ## 5.运行项目
 
@@ -55,25 +222,21 @@ conda activate xiaozhi-esp32-server
 python app.py
 ```
 
-## 6.[跳转到运行状态确认](#运行状态确认)
+这时，你就要留意日志信息，可以根据这个教程，判断是否成功了。[跳转到运行状态确认](#运行状态确认)
 
 # 汇总
 
 ## 配置项目
 
-修改`config.yaml`文件，配置本项目所需的各种参数。默认的LLM使用的是`ChatGLMLLM`
+如果你的`项目目录`目录没有`data`，你需要创建`data`目录。
-，你需要配置密钥，因为他们的模型，虽然有免费的，但是仍要去[官网](https://bigmodel.cn/usercenter/proj-mgmt/apikeys)注册密钥，才能启动。
+如果你的`data`下面没有`.config.yaml`文件，你可以把源码目录下的`config.yaml`文件复制一份，重命名为`.config.yaml`
-默认的TTS使用的是`EdgeTTS`，这个无需配置，如果你需要更换成`豆包TTS`，则需要配置密钥。
 
-```
+修改`项目目录`下`data`目录下的`.config.yaml`文件，配置本项目必须的两个配置。
-# 如果您是一名开发者，建议阅读以下内容。如果不是开发者，可以忽略这部分内容。
+
-# 在开发中，在项目根目录创建data目录，将【config.yaml】复制一份，改成【.config.yaml】，放进data目录中
+- 默认的LLM使用的是`ChatGLMLLM`，你需要配置密钥，因为他们的模型，虽然有免费的，但是仍要去[官网](https://bigmodel.cn/usercenter/proj-mgmt/apikeys)注册密钥，才能启动。
-# 系统会优先读取【data/.config.yaml】文件的配置。
+- 默认的记忆层`mem0ai`，你需要配置密钥，因为他们的API，虽然有免费额度，但是仍要去[官网](https://app.mem0.ai/dashboard/api-keys)注册密钥，才能启动。
-# 这样做，可以避免在提交代码的时候，错误地提交密钥信息，保护您的密钥安全。
-```
 
 配置说明：这里是各个功能使用的默认组件，例如LLM默认使用`ChatGLMLLM`模型。如果需要切换模型，就是改对应的名称。
-
 本项目的默认配置仅是成本最低配置（`glm-4-flash`和`EdgeTTS`都是免费的），如果需要更优的更快的搭配，需要自己结合部署环境切换各组件的使用。
 
 ```
@@ -104,35 +267,29 @@ LLM:
 
 ## 模型文件
 
-下载源码后，需要下载模型文件。 默认使用`SenseVoiceSmall`模型，进行语音转文字。因为模型较大，需要独立下载，下载后把`model.pt`
+本项目语音识别模型，默认使用`SenseVoiceSmall`模型，进行语音转文字。因为模型较大，需要独立下载，下载后把`model.pt`
-文件放在`model/SenseVoiceSmall`
+文件放在`models/SenseVoiceSmall`
 目录下。下面两个下载路线任选一个。
 
 - 线路一：阿里魔塔下载[SenseVoiceSmall](https://modelscope.cn/models/iic/SenseVoiceSmall/resolve/master/model.pt)
 - 线路二：百度网盘下载[SenseVoiceSmall](https://pan.baidu.com/share/init?surl=QlgM58FHhYv1tFnUT_A8Sg&pwd=qvna) 提取码:
   `qvna`
 
-huggingface地址：
-
-```
-https://huggingface.co/jinaai/jina-embeddings-v2-base-zh?clone=true
-https://huggingface.co/FunAudioLLM/SenseVoiceSmall/tree/main?clone=true
-```
-
-镜像地址`https://hf-mirror.com/`
-
-
 ## 运行状态确认
 
 如果你能看到，类似以下日志,则是本项目服务启动成功的标志。
 
 ```
-2025-xx-xx xx:51:59,492 - core.server - INFO - Server is running at ws://xx.xx.xx.xxx:8000
+25-02-23 12:01:09[core.websocket_server] - INFO - Server is running at ws://xxx.xx.xx.xx:8000
-2025-xx-xx xx:51:59,516 - websockets.server - INFO - server listening on 0.0.0.0:8000
+25-02-23 12:01:09[core.websocket_server] - INFO - =======上面的地址是websocket协议地址，请勿用浏览器访问=======
 ```
 
-接下来，你就可以开始 `编译esp32固件`了，请往下翻，翻到编译`esp32固件`相关章节。
+正常来说，如果您是通过源码运行本项目，日志会有你的接口地址信息。
-正常来说，假设你的ip是`192.168.1.25`，那么你的接口地址就是：`ws://192.168.1.25:8000`。这个信息很有用的，后面`编译esp32固件`
+但是如果你用docker部署，那么你的日志里给出的接口地址信息就不是真实的接口地址。
-需要用到。
 
-请注意，你的接口地址是`websocket`协议的地址，你可以使用`apifox`等工具调试。但是不能直接用浏览器打开访问，如果用浏览器打开，日志会显示错误，会让你怀疑是否部署成功了。
+最正确的方法，是根据电脑的局域网IP来确定你的接口地址。
+如果你的电脑的局域网IP比如是`192.168.1.25`，那么你的接口地址就是：`ws://192.168.1.25:8000`。
+
+这个信息很有用的，后面`编译esp32固件`需要用到。
+
+接下来，你就可以开始 [编译esp32固件](firmware-build.md)了。

docs/Deployment_old.md

@@ -1,250 +0,0 @@
-# 方式一：docker快速部署
-
-docker镜像已支持x86架构、arm64架构的CPU，支持在国产操作系统上运行。
-
-## 1. 安装docker
-
-如果您的电脑还没安装docker，可以按照这里的教程安装：[docker安装](https://www.runoob.com/docker/ubuntu-docker-install.html)
-
-## 2. 创建目录
-
-安装完后，你需要为这个项目找一个安放配置文件的目录，我们暂且称它为`项目目录`，这个目录最好是一个新建的空的目录。
-
-## 3. 下载配置文件
-
-用浏览器打开[这个链接](https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/config.yaml)。
-
-在页面的右侧找到名称为`RAW`按钮，在`RAW`按钮的旁边，找到下载的图标，点击下载按钮，下载`config.yaml`文件。 把文件下载到你的
-`项目目录`。
-
-## 4.[跳转到配置项目文件](#配置项目)
-
-## 5. 执行docker命令
-
-打开命令行工具，`cd` 进入到你的`项目目录`，执行以下命令
-
-```
-#如果你是linux，执行
-ls
-#如果你是windows，执行
-dir
-```
-
-如果你能看到`config.yaml`文件，确确实实进入到了`项目目录`，接着执行以下命令：
-
-```
-docker run -d --name xiaozhi-esp32-server --restart always --security-opt seccomp:unconfined -p 8000:8000 -v $(pwd)/config.yaml:/opt/xiaozhi-esp32-server/config.yaml ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest
-```
-
-## 6.[跳转到运行状态确认](#运行状态确认)
-
-## [跳转到版本升级操作](#版本升级操作)
-
-# 方式二：借助docker环境运行部署（仅限开发人员/小白勿用）
-
-## 1.克隆项目
-
-## 2.[跳转到下载语音识别模型文件](#模型文件)
-
-## 3.[跳转到配置项目文件](#配置项目)
-
-## 4.运行docker
-
-修改完配置后，打开命令行工具，`cd`进入到你的项目目录下，执行以下命令
-
-```sh
-docker run -it --name xiaozhi-env --restart always --security-opt seccomp:unconfined \
-  -p 8000:8000 \
-  -p 8002:8002 \
-  -v ./:/app \
-  ccr.ccs.tencentyun.com/kalicyh/poetry:v3.10_latest
-```
-
-然后就和正常开发一样了
-
-## 5.安装依赖
-
-在刚刚的打开的终端运行
-
-```sh
-poetry install --no-root
-```
-
-```sh
-apt-get update
-apt-get install -y --no-install-recommends libopus0 ffmpeg
-```
-
-速度慢可以尝试使用清华镜像
-
-```sh
-echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list
-echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list
-echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-backports main contrib non-free non-free-firmware" >> /etc/apt/sources.list
-echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bookworm-security main contrib non-free non-free-firmware" >> /etc/apt/sources.list
-apt-get update
-apt-get install -y --no-install-recommends libopus0 ffmpeg
-```
-
-## 6.运行项目
-
-```sh
-poetry run python app.py
-```
-
-# 方式三：本地源码运行
-
-## 1.安装基础环境
-
-本项目使用`conda`管理依赖环境，安装好后，开始执行以下命令。
-
-```
-conda remove -n xiaozhi-esp32-server --all -y
-conda create -n xiaozhi-esp32-server python=3.10 -y
-conda activate xiaozhi-esp32-server
-```
-
-执行以上命令后， 如果你的电脑是Windows或Mac，执行下面的语句：
-
-```
-conda activate xiaozhi-esp32-server
-conda install conda-forge::libopus
-conda install conda-forge::ffmpeg
-```
-
-如果你的电脑是ubuntu，执行下面的语句：
-
-```
-apt-get install libopus0 ffmpeg 
-```
-
-## 2.安装本项目依赖
-
-你先要下载本项目源码，源码可以通过`git clone`命令下载，如果你不熟悉`git clone`命令。
-
-你可以用浏览器打开这个地址`https://github.com/xinnan-tech/xiaozhi-esp32-server.git`
-
-打开完，找到页面中一个绿色的按钮，写着`Code`的按钮，点开它，然后你就看到`Download ZIP`的按钮。
-
-点击它，下载本项目源码压缩包。下载到你电脑后，解压它，此时它的名字可能叫`xiaozhi-esp32-server-main`
-你需要把它重命名成`xiaozhi-esp32-server`，好了请记住这个目录，我们暂且称它为`项目目录`。
-
-```
-# 使用dos或者终端，进入到你的项目目录，执行以下命令
-conda activate xiaozhi-esp32-server
-pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
-pip install -r requirements.txt
-```
-
-## 3.[跳转到下载语音识别模型文件](#模型文件)
-
-## 4.[跳转到配置项目文件](#配置项目)
-
-## 5.运行项目
-
-```
-# 确保在本项目的根目录下执行
-conda activate xiaozhi-esp32-server
-python app.py
-```
-
-## 6.[跳转到运行状态确认](#运行状态确认)
-
-# 汇总
-
-## 配置项目
-
-修改`config.yaml`文件，配置本项目所需的各种参数。默认的LLM使用的是`ChatGLMLLM`
-，你需要配置密钥，因为他们的模型，虽然有免费的，但是仍要去[官网](https://bigmodel.cn/usercenter/proj-mgmt/apikeys)注册密钥，才能启动。
-默认的TTS使用的是`EdgeTTS`，这个无需配置，如果你需要更换成`豆包TTS`，则需要配置密钥。
-
-```
-# 如果您是一名开发者，建议阅读以下内容。如果不是开发者，可以忽略这部分内容。
-# 在开发中，在项目根目录创建data目录，将【config.yaml】复制一份，改成【.config.yaml】，放进data目录中
-# 系统会优先读取【data/.config.yaml】文件的配置。
-# 这样做，可以避免在提交代码的时候，错误地提交密钥信息，保护您的密钥安全。
-```
-
-配置说明：这里是各个功能使用的默认组件，例如LLM默认使用`ChatGLMLLM`模型。如果需要切换模型，就是改对应的名称。
-
-本项目的默认配置仅是成本最低配置（`glm-4-flash`和`EdgeTTS`都是免费的），如果需要更优的更快的搭配，需要自己结合部署环境切换各组件的使用。
-
-```
-selected_module:
-  ASR: FunASR
-  VAD: SileroVAD
-  LLM: ChatGLMLLM
-  TTS: EdgeTTS
-```
-
-比如修改`LLM`使用的组件，就看本项目支持哪些`LLM` API接口，当前支持的是`openai`、`dify`。欢迎验证和支持更多LLM平台的接口。
-使用时，在`selected_module`修改成对应的如下LLM配置的名称：
-
-```
-LLM:
-  DeepSeekLLM:
-    type: openai
-    ...
-  ChatGLMLLM:
-    type: openai
-    ...
-  DifyLLM:
-    type: dify
-    ...
-```
-
-有些服务，比如如果你使用`Dify`、`豆包的TTS`，是需要密钥的，记得在配置文件加上哦！
-
-## 模型文件
-
-下载源码后，需要下载模型文件。 默认使用`SenseVoiceSmall`模型，进行语音转文字。因为模型较大，需要独立下载，下载后把`model.pt`
-文件放在`model/SenseVoiceSmall`
-目录下。下面两个下载路线任选一个。
-
-- 线路一：阿里魔塔下载[SenseVoiceSmall](https://modelscope.cn/models/iic/SenseVoiceSmall/resolve/master/model.pt)
-- 线路二：百度网盘下载[SenseVoiceSmall](https://pan.baidu.com/share/init?surl=QlgM58FHhYv1tFnUT_A8Sg&pwd=qvna) 提取码:
-  `qvna`
-
-
-## 运行状态确认
-
-如果首次执行，可能需要几分钟时间，你要耐心等待他完成拉取。正常拉取完成后，你可以在命令行执行以下命令查看服务是否启动成功
-
-```
-docker ps
-```
-
-如果你能看到`xiaozhi-server`，说明服务启动成功。那你还可以进一步执行以下命令，查看服务的日志
-
-```
-docker logs -f xiaozhi-esp32-server
-```
-
-如果你能看到，类似以下日志,则是本项目服务启动成功的标志。
-
-```
-2025-xx-xx xx:51:59,492 - core.server - INFO - Server is running at ws://xx.xx.xx.xxx:8000
-2025-xx-xx xx:51:59,516 - websockets.server - INFO - server listening on 0.0.0.0:8000
-```
-
-接下来，你就可以开始 `编译esp32固件`了，请往下翻，翻到编译`esp32固件`相关章节。那么由于你是用docker部署，你要自己查看自己本机电脑的ip是多少。
-正常来说，假设你的ip是`192.168.1.25`，那么你的接口地址就是：`ws://192.168.1.25:8000`。这个信息很有用的，后面`编译esp32固件`
-需要用到。
-
-请注意，你的接口地址是`websocket`协议的地址，你可以使用`apifox`等工具调试。但是不能直接用浏览器打开访问，如果用浏览器打开，日志会显示错误，会让你怀疑是否部署成功了。
-
-## 版本升级操作
-
-如果想升级版本，可以这么操作
-
-1、备份好`config.yaml`文件，一些关键的配置到时复制到新的`config.yaml`文件里。
-
-2、执行以下命令
-
-```
-docker stop xiaozhi-esp32-server
-docker rm xiaozhi-esp32-server
-docker rmi ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest
-```
-
-3、重新开始安装

docs/docker-build.md

@@ -1,4 +1,7 @@
-# 编译docker镜像
+# 本地编译docker镜像方法
+
+现在本项目已经使用github自动编译docker功能，本文档是提供给有本地编译docker镜像需求的朋友准备的。
+
 1、安装docker
 ```
 sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
@@ -6,7 +9,7 @@ sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin
 2、编译docker镜像
 ```
 # 普通编译
-docker build -t xiaozhi-esp32-server:local -f ./Dockerfile .
+docker build -t xiaozhi-esp32-server:local -f ./Dockerfile-pip .
 ```
 3、测试本地镜像
 ```

performance_tester.py

@@ -1,27 +1,27 @@
 import time
+import aiohttp
+import asyncio
 from tabulate import tabulate
-from typing import Dict
+from typing import Dict, List
 from core.utils.llm import create_instance as create_llm_instance
 from core.utils.tts import create_instance as create_tts_instance
 from core.utils.util import read_config
 import statistics
 from config.settings import get_config_file
-from concurrent.futures import ThreadPoolExecutor
 import inspect
 import os
-import requests
 import logging
 
 # 设置全局日志级别为WARNING，抑制INFO级别日志
 logging.basicConfig(level=logging.WARNING)
 
-class PerformanceTester:
+
+class AsyncPerformanceTester:
     def __init__(self):
         self.config = read_config(get_config_file())
-        # 从配置读取测试句子，如果不存在则使用默认
         self.test_sentences = self.config.get("module_test", {}).get(
-            "test_sentences", 
+            "test_sentences",
-            ["你好，请介绍一下你自己", "What's the weather like today?", 
+            ["你好，请介绍一下你自己", "What's the weather like today?",
              "请用100字概括量子计算的基本原理和应用前景"]
         )
         self.results = {
@@ -30,258 +30,407 @@ class PerformanceTester:
             "combinations": []
         }
 
-    def _test_llm(self, llm_name: str, config: Dict) -> Dict:
+    async def _check_ollama_service(self, base_url: str, model_name: str) -> bool:
-        """测试单个LLM性能"""
+        """异步检查Ollama服务状态"""
+        async with aiohttp.ClientSession() as session:
+            try:
+                # 检查服务是否可用
+                async with session.get(f"{base_url}/api/version") as response:
+                    if response.status != 200:
+                        print(f"🚫 Ollama服务未启动或无法访问: {base_url}")
+                        return False
+
+                # 检查模型是否存在
+                async with session.get(f"{base_url}/api/tags") as response:
+                    if response.status == 200:
+                        data = await response.json()
+                        models = data.get("models", [])
+                        if not any(model["name"] == model_name for model in models):
+                            print(f"🚫 Ollama模型 {model_name} 未找到，请先使用 ollama pull {model_name} 下载")
+                            return False
+                    else:
+                        print(f"🚫 无法获取Ollama模型列表")
+                        return False
+                return True
+            except Exception as e:
+                print(f"🚫 无法连接到Ollama服务: {str(e)}")
+                return False
+
+    async def _test_tts(self, tts_name: str, config: Dict) -> Dict:
+        """异步测试单个TTS性能"""
         try:
-            # 跳过未配置密钥的模块
+            logging.getLogger("core.providers.tts.base").setLevel(logging.WARNING)
-            if "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]):
+
-                print(f"🚫 跳过未配置的LLM: {llm_name}")
+            token_fields = ["access_token", "api_key", "token"]
-                return {"errors": 1}
+            if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in
-            
+                   token_fields):
+                print(f"⏭️  TTS {tts_name} 未配置access_token/api_key，已跳过")
+                return {"name": tts_name, "type": "tts", "errors": 1}
+
+            module_type = config.get('type', tts_name)
+            tts = create_tts_instance(
+                module_type,
+                config,
+                delete_audio_file=True
+            )
+
+            print(f"🎵 测试 TTS: {tts_name}")
+
+            tmp_file = tts.generate_filename()
+            await tts.text_to_speak("连接测试", tmp_file)
+
+            if not tmp_file or not os.path.exists(tmp_file):
+                print(f"❌ {tts_name} 连接失败")
+                return {"name": tts_name, "type": "tts", "errors": 1}
+
+            total_time = 0
+            test_count = len(self.test_sentences[:2])
+
+            for i, sentence in enumerate(self.test_sentences[:2], 1):
+                start = time.time()
+                tmp_file = tts.generate_filename()
+                await tts.text_to_speak(sentence, tmp_file)
+                duration = time.time() - start
+                total_time += duration
+
+                if tmp_file and os.path.exists(tmp_file):
+                    print(f"✓ {tts_name} [{i}/{test_count}]")
+                else:
+                    print(f"✗ {tts_name} [{i}/{test_count}]")
+                    return {"name": tts_name, "type": "tts", "errors": 1}
+
+            return {
+                "name": tts_name,
+                "type": "tts",
+                "avg_time": total_time / test_count,
+                "errors": 0
+            }
+
+        except Exception as e:
+            print(f"⚠️ {tts_name} 测试失败: {str(e)}")
+            return {"name": tts_name, "type": "tts", "errors": 1}
+
+    async def _test_llm(self, llm_name: str, config: Dict) -> Dict:
+        """异步测试单个LLM性能"""
+        try:
+            # 对于Ollama，跳过api_key检查并进行特殊处理
+            if llm_name == "Ollama":
+                base_url = config.get('base_url', 'http://localhost:11434')
+                model_name = config.get('model_name')
+                if not model_name:
+                    print(f"🚫 Ollama未配置model_name")
+                    return {"name": llm_name, "type": "llm", "errors": 1}
+
+                if not await self._check_ollama_service(base_url, model_name):
+                    return {"name": llm_name, "type": "llm", "errors": 1}
+            else:
+                if "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]):
+                    print(f"🚫 跳过未配置的LLM: {llm_name}")
+                    return {"name": llm_name, "type": "llm", "errors": 1}
+
             # 获取实际类型（兼容旧配置）
             module_type = config.get('type', llm_name)
             llm = create_llm_instance(module_type, config)
-            
+
             # 统一使用UTF-8编码
             test_sentences = [s.encode('utf-8').decode('utf-8') for s in self.test_sentences]
-            
+
-            total_time = 0
+            # 创建所有句子的测试任务
-            first_token_times = []
+            sentence_tasks = []
-            valid_times = []
-            
             for sentence in test_sentences:
-                sentence_start = time.time()  # 记录整句开始时间
+                sentence_tasks.append(self._test_single_sentence(llm_name, llm, sentence))
-                first_token_received = False
+
-                
+            # 并发执行所有句子测试
-                # 遍历响应流
+            sentence_results = await asyncio.gather(*sentence_tasks)
-                for chunk in llm.response("perf_test", [{"role": "user", "content": sentence}]):
+
-                    if not first_token_received and chunk.strip() != '':
+            # 处理结果
-                        first_token_times.append(time.time() - sentence_start)
+            valid_results = [r for r in sentence_results if r is not None]
-                        first_token_received = True
+            if not valid_results:
-                
-                # 计算整句耗时
-                sentence_duration = time.time() - sentence_start
-                total_time += sentence_duration
-                valid_times.append(sentence_duration)
-            
-            # 新增有效性检查
-            if len(first_token_times) == 0 or len(valid_times) == 0:
                 print(f"⚠️  {llm_name} 无有效数据，可能配置错误")
-                return {"errors": 1}
+                return {"name": llm_name, "type": "llm", "errors": 1}
-                
+
-            # 过滤异常数据（超过3倍标准差）
+            first_token_times = [r["first_token_time"] for r in valid_results]
-            mean = statistics.mean(valid_times)
+            response_times = [r["response_time"] for r in valid_results]
-            stdev = statistics.stdev(valid_times) if len(valid_times) > 1 else 0
+
-            filtered_times = [t for t in valid_times if t <= mean + 3*stdev]
+            # 过滤异常数据
-            
+            mean = statistics.mean(response_times)
-            # 当有效数据不足时标记错误
+            stdev = statistics.stdev(response_times) if len(response_times) > 1 else 0
+            filtered_times = [t for t in response_times if t <= mean + 3 * stdev]
+
             if len(filtered_times) < len(test_sentences) * 0.5:
                 print(f"⚠️  {llm_name} 有效数据不足，可能网络不稳定")
-                return {"errors": 1}
+                return {"name": llm_name, "type": "llm", "errors": 1}
 
             return {
-                "avg_response": total_time / len(test_sentences),
+                "name": llm_name,
-                "avg_first_token": sum(first_token_times)/len(first_token_times),
+                "type": "llm",
+                "avg_response": sum(response_times) / len(response_times),
+                "avg_first_token": sum(first_token_times) / len(first_token_times),
                 "std_first_token": statistics.stdev(first_token_times) if len(first_token_times) > 1 else 0,
-                "std_response": statistics.stdev(valid_times) if len(valid_times) > 1 else 0,
+                "std_response": statistics.stdev(response_times) if len(response_times) > 1 else 0,
                 "errors": 0
             }
         except Exception as e:
             print(f"LLM {llm_name} 测试失败: {str(e)}")
-            return {"errors": 1}
+            return {"name": llm_name, "type": "llm", "errors": 1}
 
-    def _test_tts(self, tts_name: str, config: Dict) -> Dict:
+    async def _test_single_sentence(self, llm_name: str, llm, sentence: str) -> Dict:
-        """测试单个TTS性能"""
+        """测试单个句子的性能"""
         try:
-            # 关闭详细日志
+            print(f"📝 {llm_name} 开始测试: {sentence[:20]}...")
-            logging.getLogger("core.providers.tts.base").setLevel(logging.WARNING)
+            sentence_start = time.time()
-            
+            first_token_received = False
-            # 跳过未配置密钥的模块
+            first_token_time = None
-            token_fields = ["access_token", "api_key", "token"]
-            if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in token_fields):
-                print(f"⏭️  TTS {tts_name} 未配置access_token/api_key，已跳过")
-                return {"errors": 1}
-            
-            # 获取实际类型（兼容旧配置）
-            module_type = config.get('type', tts_name)
-            tts = create_tts_instance(
-                module_type,
-                config, 
-                delete_audio_file=True  # 确保参数名称正确
-            )
-            
-            # 简化后的输出
-            print(f"\n🎵 正在测试 TTS: {tts_name}")
-            print(f"🔊 测试 {tts_name}：", end="", flush=True)
-            
-            # 连接测试
-            test_conn = tts.to_tts("连接测试")
-            if not os.path.exists(test_conn):
-                print("❌ 连接失败")
-                return {"errors": 1}
-            else:
-                print("✅")
-    
-            total_time = 0
-            test_count = len(self.test_sentences[:2])
-            
-            for i, sentence in enumerate(self.test_sentences[:2], 1):
-                start = time.time()
-                file_path = tts.to_tts(sentence)
-                duration = time.time() - start
-                total_time += duration
-                
-                # 显示简单的进度标识
-                if os.path.exists(file_path):
-                    print(f"✓[{i}/{test_count}]", end="", flush=True)
-                else:
-                    print(f"✗[{i}/{test_count}]", end="", flush=True)
-            
-            print()  # 换行
-            return {"avg_time": total_time / test_count, "errors": 0}
-    
-        except requests.exceptions.ConnectionError:
-            print(f"\n⛔ {tts_name} 无法连接服务端")
-            return {"errors": 1}
-        except Exception as e:
-            print(f"\n⚠️ {tts_name} 测试失败: {str(e)}")
-            return {"errors": 1}
 
-    def run(self):
+            async def process_response():
-        """执行全量测试并自动跳过未配置的模块"""
+                nonlocal first_token_received, first_token_time
-        print("🔍 开始自动检测已配置的模块...")
+                for chunk in llm.response("perf_test", [{"role": "user", "content": sentence}]):
-        
+                    if not first_token_received and chunk.strip() != '':
-        # 测试所有LLM
+                        first_token_time = time.time() - sentence_start
-        for llm_name, config in self.config.get("LLM", {}).items():
+                        first_token_received = True
-            # 特殊处理CozeLLM的配置检查
+                        print(f"✓ {llm_name} 首个Token: {first_token_time:.3f}s")
-            if llm_name == "CozeLLM":
+                    yield chunk
-                if any(x in config.get("bot_id", "") for x in ["你的"]) \
+
-                or any(x in config.get("user_id", "") for x in ["你的"]):
+            response_chunks = []
-                    print(f"⏭️  LLM {llm_name} 未配置bot_id/user_id，已跳过")
+            async for chunk in process_response():
-                    continue
+                response_chunks.append(chunk)
-            # 通用的api_key配置检查
+
-            if "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder"]):
+            response_time = time.time() - sentence_start
-                print(f"⏭️  LLM {llm_name} 未配置api_key，已跳过")
+            print(f"✓ {llm_name} 完成响应: {response_time:.3f}s")
-                continue
+
-                
+            if first_token_time is None:
-            print(f"🚀 正在测试 LLM: {llm_name}")
+                first_token_time = response_time  # 如果没有检测到first token，使用总响应时间
-            self.results["llm"][llm_name] = self._test_llm(llm_name, config)
+
-        
+            return {
-        # 测试所有TTS
+                "name": llm_name,
-        for tts_name, config in self.config.get("TTS", {}).items():
+                "type": "llm",
-            # 根据不同服务的token字段检测
+                "first_token_time": first_token_time,
-            token_fields = ["access_token", "api_key", "token"]
+                "response_time": response_time
-            if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in token_fields):
+            }
-                print(f"⏭️  TTS {tts_name} 未配置access_token/api_key，已跳过")
+        except Exception as e:
-                continue
+            print(f"⚠️ {llm_name} 句子测试失败: {str(e)}")
-                
+            return None
-            print(f"🎵 正在测试 TTS: {tts_name}")
-            self.results["tts"][tts_name] = self._test_tts(tts_name, config)
-        
-        # 生成组合建议
-        self._generate_combinations()
-        self._print_results()
 
     def _generate_combinations(self):
         """生成最佳组合建议"""
-        # 调整过滤条件，例如设为 >= 0.05
         valid_llms = [
-            k for k, v in self.results["llm"].items() 
+            k for k, v in self.results["llm"].items()
             if v["errors"] == 0 and v["avg_first_token"] >= 0.05
         ]
         valid_tts = [k for k, v in self.results["tts"].items() if v["errors"] == 0]
 
+        # 找出基准值
+        min_first_token = min([self.results["llm"][llm]["avg_first_token"] for llm in valid_llms]) if valid_llms else 1
+        min_tts_time = min([self.results["tts"][tts]["avg_time"] for tts in valid_tts]) if valid_tts else 1
+
         for llm in valid_llms:
             for tts in valid_tts:
-                llm_weight = 0.8 if self.results["llm"][llm]["avg_first_token"] < 1.0 else 0.6
+                # 计算相对性能分数（越小越好）
-                tts_weight = 1 - llm_weight
+                llm_score = self.results["llm"][llm]["avg_first_token"] / min_first_token
-                score = (
+                tts_score = self.results["tts"][tts]["avg_time"] / min_tts_time
-                    self.results["llm"][llm]["avg_first_token"] * llm_weight +
+
-                    self.results["tts"][tts]["avg_time"] * tts_weight
+                # 计算稳定性分数（标准差/平均值，越小越稳定）
-                )
+                llm_stability = self.results["llm"][llm]["std_first_token"] / self.results["llm"][llm][
+                    "avg_first_token"]
+
+                # 综合得分（考虑性能和稳定性）
+                # 性能权重0.7，稳定性权重0.3
+                llm_final_score = llm_score * 0.7 + llm_stability * 0.3
+
+                # 总分 = LLM得分(70%) + TTS得分(30%)
+                total_score = llm_final_score * 0.7 + tts_score * 0.3
+
                 self.results["combinations"].append({
                     "llm": llm,
                     "tts": tts,
-                    "score": score,
+                    "score": total_score,
                     "details": {
                         "llm_first_token": self.results["llm"][llm]["avg_first_token"],
+                        "llm_stability": llm_stability,
                         "tts_time": self.results["tts"][tts]["avg_time"]
                     }
                 })
 
-        # 按综合得分排序
+        # 分数越小越好
         self.results["combinations"].sort(key=lambda x: x["score"])
 
     def _print_results(self):
-        """控制台输出结果"""
+        """打印测试结果"""
-        # LLM结果表格
         llm_table = []
         for name, data in self.results["llm"].items():
             if data["errors"] == 0:
+                stability = data["std_first_token"] / data["avg_first_token"]
                 llm_table.append([
-                    name,
+                    name,  # 不需要固定宽度，让tabulate自己处理对齐
-                    f"{data['avg_first_token']:.3f}s",
+                    f"{data['avg_first_token']:.3f}秒",
-                    f"{data['avg_response']:.3f}s"
+                    f"{data['avg_response']:.3f}秒",
+                    f"{stability:.3f}"
                 ])
 
         if llm_table:
             print("\nLLM 性能排行:")
             print(tabulate(
                 llm_table,
-                headers=["模块名称", "平均首Token时间", "平均总响应时间"],
+                headers=["模型名称", "首字耗时", "总耗时", "稳定性"],
-                tablefmt="github"
+                tablefmt="github",
+                colalign=("left", "right", "right", "right"),
+                disable_numparse=True
             ))
         else:
             print("\n⚠️ 没有可用的LLM模块进行测试。")
 
-        # TTS结果表格
         tts_table = []
         for name, data in self.results["tts"].items():
             if data["errors"] == 0:
                 tts_table.append([
-                    name,
+                    name,  # 不需要固定宽度
-                    f"{data['avg_time']:.3f}s"
+                    f"{data['avg_time']:.3f}秒"
                 ])
 
         if tts_table:
             print("\nTTS 性能排行:")
             print(tabulate(
                 tts_table,
-                headers=["模块名称", "平均合成时间"],
+                headers=["模型名称", "合成耗时"],
-                tablefmt="github"
+                tablefmt="github",
+                colalign=("left", "right"),
+                disable_numparse=True
             ))
         else:
             print("\n⚠️ 没有可用的TTS模块进行测试。")
 
-        # 最佳组合建议
         if self.results["combinations"]:
-            print("\n推荐配置组合 (综合响应速度):")
+            print("\n推荐配置组合 (得分越小越好):")
             combo_table = []
-            for combo in self.results["combinations"][:5]:  # 显示前5名
+            for combo in self.results["combinations"][:5]:
                 combo_table.append([
-                    f"{combo['llm']} + {combo['tts']}",
+                    f"{combo['llm']} + {combo['tts']}",  # 不需要固定宽度
                     f"{combo['score']:.3f}",
-                    f"{combo['details']['llm_first_token']:.3f}s",
+                    f"{combo['details']['llm_first_token']:.3f}秒",
-                    f"{combo['details']['tts_time']:.3f}s"
+                    f"{combo['details']['llm_stability']:.3f}",
+                    f"{combo['details']['tts_time']:.3f}秒"
                 ])
-            
+
             print(tabulate(
                 combo_table,
-                headers=["组合方案", "综合得分", "LLM首Token", "TTS合成"],
+                headers=["组合方案", "综合得分", "LLM首字耗时", "稳定性", "TTS合成耗时"],
-                tablefmt="github"
+                tablefmt="github",
+                colalign=("left", "right", "right", "right", "right"),
+                disable_numparse=True
             ))
         else:
             print("\n⚠️ 没有可用的模块组合建议。")
 
-    def _execute_with_timeout(self, func, args=(), kwargs={}, timeout=None):
+    def _process_results(self, all_results):
-        with ThreadPoolExecutor(max_workers=1) as executor:
+        """处理测试结果"""
-            future = executor.submit(func, *args, **kwargs)
+        for result in all_results:
-            try:
+            if result["errors"] == 0:
-                result = future.result(timeout)
+                if result["type"] == "llm":
-                return list(result) if inspect.isgenerator(result) else result
+                    self.results["llm"][result["name"]] = result
-            except TimeoutError:
+                else:
-                raise Exception("操作超时")
+                    self.results["tts"][result["name"]] = result
+
+    async def run(self):
+        """执行全量异步测试"""
+        print("🔍 开始筛选可用模块...")
+
+        # 创建所有测试任务
+        all_tasks = []
+
+        # LLM测试任务
+        for llm_name, config in self.config.get("LLM", {}).items():
+            # 检查配置有效性
+            if llm_name == "CozeLLM":
+                if any(x in config.get("bot_id", "") for x in ["你的"]) \
+                        or any(x in config.get("user_id", "") for x in ["你的"]):
+                    print(f"⏭️  LLM {llm_name} 未配置bot_id/user_id，已跳过")
+                    continue
+            elif "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]):
+                print(f"⏭️  LLM {llm_name} 未配置api_key，已跳过")
+                continue
+
+            # 对于Ollama，先检查服务状态
+            if llm_name == "Ollama":
+                base_url = config.get('base_url', 'http://localhost:11434')
+                model_name = config.get('model_name')
+                if not model_name:
+                    print(f"🚫 Ollama未配置model_name")
+                    continue
+
+                if not await self._check_ollama_service(base_url, model_name):
+                    continue
+
+            print(f"📋 添加LLM测试任务: {llm_name}")
+            module_type = config.get('type', llm_name)
+            llm = create_llm_instance(module_type, config)
+
+            # 为每个句子创建独立任务
+            for sentence in self.test_sentences:
+                sentence = sentence.encode('utf-8').decode('utf-8')
+                all_tasks.append(self._test_single_sentence(llm_name, llm, sentence))
+
+        # TTS测试任务
+        for tts_name, config in self.config.get("TTS", {}).items():
+            token_fields = ["access_token", "api_key", "token"]
+            if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in
+                   token_fields):
+                print(f"⏭️  TTS {tts_name} 未配置access_token/api_key，已跳过")
+                continue
+            print(f"🎵 添加TTS测试任务: {tts_name}")
+            all_tasks.append(self._test_tts(tts_name, config))
+
+        print(
+            f"\n✅ 找到 {len([t for t in all_tasks if 'test_single_sentence' in str(t)]) / len(self.test_sentences):.0f} 个可用LLM模块")
+        print(f"✅ 找到 {len([t for t in all_tasks if '_test_tts' in str(t)])} 个可用TTS模块")
+        print("\n⏳ 开始并发测试所有模块...\n")
+
+        # 并发执行所有测试任务
+        all_results = await asyncio.gather(*all_tasks, return_exceptions=True)
+
+        # 处理LLM结果
+        llm_results = {}
+        for result in [r for r in all_results if r and isinstance(r, dict) and r.get("type") == "llm"]:
+            llm_name = result["name"]
+            if llm_name not in llm_results:
+                llm_results[llm_name] = {
+                    "name": llm_name,
+                    "type": "llm",
+                    "first_token_times": [],
+                    "response_times": [],
+                    "errors": 0
+                }
+            llm_results[llm_name]["first_token_times"].append(result["first_token_time"])
+            llm_results[llm_name]["response_times"].append(result["response_time"])
+
+        # 计算LLM平均值和标准差
+        for llm_name, data in llm_results.items():
+            if len(data["first_token_times"]) >= len(self.test_sentences) * 0.5:
+                self.results["llm"][llm_name] = {
+                    "name": llm_name,
+                    "type": "llm",
+                    "avg_response": sum(data["response_times"]) / len(data["response_times"]),
+                    "avg_first_token": sum(data["first_token_times"]) / len(data["first_token_times"]),
+                    "std_first_token": statistics.stdev(data["first_token_times"]) if len(
+                        data["first_token_times"]) > 1 else 0,
+                    "std_response": statistics.stdev(data["response_times"]) if len(data["response_times"]) > 1 else 0,
+                    "errors": 0
+                }
+
+        # 处理TTS结果
+        for result in [r for r in all_results if r and isinstance(r, dict) and r.get("type") == "tts"]:
+            if result["errors"] == 0:
+                self.results["tts"][result["name"]] = result
+
+        # 生成组合建议并打印结果
+        print("\n📊 生成测试报告...")
+        self._generate_combinations()
+        self._print_results()
+
+
+async def main():
+    tester = AsyncPerformanceTester()
+    await tester.run()
+
 
 if __name__ == "__main__":
-    tester = PerformanceTester()
+    asyncio.run(main())
-    tester.run()

requirements.txt

@@ -16,4 +16,7 @@ aiohttp_cors==0.7.0
 ormsgpack==1.7.0
 ruamel.yaml==0.18.10
 loguru==0.7.3
-requests>=2.0.0
+requests==2.32.3
+cozepy==0.12.0
+miniaudio==1.61
+mem0ai==0.1.62