merge main

main/xiaozhi-server/app.py

@@ -0,0 +1,26 @@
+import asyncio
+from config.settings import load_config, check_config_file
+from core.websocket_server import WebSocketServer
+from core.utils.util import check_ffmpeg_installed
+
+TAG = __name__
+
+
+async def main():
+    check_config_file()
+    check_ffmpeg_installed()
+    config = load_config()
+
+    # 启动 WebSocket 服务器
+    ws_server = WebSocketServer(config)
+    ws_task = asyncio.create_task(ws_server.start())
+
+    try:
+        # 等待 WebSocket 服务器运行
+        await ws_task
+    finally:
+        ws_task.cancel()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

main/xiaozhi-server/config.yaml

@@ -0,0 +1,449 @@
+# 如果您是一名开发者，建议阅读以下内容。如果不是开发者，可以忽略这部分内容。
+# 在开发中，在项目根目录创建data目录，将【config.yaml】复制一份，改成【.config.yaml】，放进data目录中
+# 系统会优先读取【data/.config.yaml】文件的配置。
+# 这样做，可以避免在提交代码的时候，错误地提交密钥信息，保护您的密钥安全。
+
+# 服务器基础配置(Basic server configuration)
+server:
+  # 服务器监听地址和端口(Server listening address and port)
+  ip: 0.0.0.0
+  port: 8000
+  # 认证配置
+  auth:
+    # 是否启用认证
+    enabled: false
+    # 设备的token，可以在编译固件的环节，写入你自己定义的token
+    # 固件上的token和以下的token如果能对应，才能连接本服务端
+    tokens:
+      - token: "your-token1" # 设备1的token
+        name: "your-device-name1"  # 设备1标识
+      - token: "your-token2"  # 设备2的token
+        name: "your-device-name2" # 设备2标识
+    # 可选:设备白名单，如果设置了白名单，那么白名单的机器无论是什么token都可以连接。
+    #allowed_devices:
+    #  - "24:0A:C4:1D:3B:F0"  # MAC地址列表
+log:
+  # 设置控制台输出的日志格式，时间、日志级别、标签、消息
+  log_format: "<green>{time:YY-MM-DD HH:mm:ss}</green>[<light-blue>{extra[tag]}</light-blue>] - <level>{level}</level> - <light-green>{message}</light-green>"
+  # 设置日志文件输出的格式，时间、日志级别、标签、消息
+  log_format_simple: "{time:YYYY-MM-DD HH:mm:ss} - {name} - {level} - {extra[tag]} - {message}"
+  # 设置日志等级：INFO、DEBUG
+  log_level: INFO
+  # 设置日志路径
+  log_dir: tmp
+  # 设置日志文件
+  log_file: "server.log"
+  # 设置数据文件路径
+  data_dir: data
+iot:
+  Speaker:
+    # 设置esp32的音量，范围0-100
+    volume: 80
+xiaozhi:
+  type: hello
+  version: 1
+  transport: websocket
+  audio_params:
+    format: opus
+    sample_rate: 16000
+    channels: 1
+    frame_duration: 60
+prompt: |
+  你是一个叫小智/小志的台湾女孩，说话机车，声音好听，习惯简短表达，爱用网络梗。
+  请注意，要像一个人一样说话，请不要回复表情符号、代码、和xml标签。
+  当前时间是:{date_time}，现在我正在和你进行语音聊天，我们开始吧。
+  如果用户希望结束对话，请在最后说“拜拜”或“再见”。
+# 使用完声音文件后删除文件(Delete the sound file when you are done using it)
+delete_audio: true
+
+# 没有语音输入多久后断开连接(秒)，默认2分钟，即120秒
+close_connection_no_voice_time: 120
+
+CMD_exit:
+  - "退出"
+  - "关闭"
+
+# 具体处理时选择的模块(The module selected for specific processing)
+selected_module:
+  # 语音活动检测模块，默认使用SileroVAD模型
+  VAD: SileroVAD
+  # 语音识别模块，默认使用FunASR本地模型
+  ASR: FunASR
+  # 将根据配置名称对应的type调用实际的LLM适配器
+  LLM: ChatGLMLLM
+  # TTS将根据配置名称对应的type调用实际的TTS适配器
+  TTS: EdgeTTS
+  # 记忆模块，默认不开启记忆；如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short
+  Memory: nomem
+  # 意图识别模块，默认不开启。开启后，可以播放音乐、控制音量、识别退出指令
+  # 意图识别使用intent_llm，优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间
+  # 意图识别使用function_call，缺点：需要所选择的LLM支持function_call，优点：按需调用工具、速度快
+  # 如果意图识别设置成 function_call，建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-pro-32k-functioncall-241028
+  Intent: nointent
+
+# 意图识别，是用于理解用户意图的模块，例如：播放音乐
+Intent:
+  # 不使用意图识别
+  nointent:
+    # 不需要动
+    type: nointent
+  intent_llm:
+    # 不需要动
+    type: intent_llm
+  function_call:
+    # 不需要动
+    type: nointent
+
+Memory:
+  mem0ai:
+    type: mem0ai
+    # https://app.mem0.ai/dashboard/api-keys
+    # 每月有1000次免费调用
+    api_key: 你的mem0ai api key
+  nomem:
+    # 不想使用记忆功能，可以使用nomem
+    type: nomem
+  mem_local_short:
+    # 本地记忆功能，通过selected_module的llm总结，数据保存在本地，不会上传到服务器
+    type: mem_local_short
+
+ASR:
+  FunASR:
+    type: fun_local
+    model_dir: models/SenseVoiceSmall
+    output_dir: tmp/
+  DoubaoASR:
+    type: doubao
+    appid: 你的火山引擎语音合成服务appid
+    access_token: 你的火山引擎语音合成服务access_token
+    cluster: volcengine_input_common
+    output_dir: tmp/
+VAD:
+  SileroVAD:
+    threshold: 0.5
+    model_dir: models/snakers4_silero-vad
+    min_silence_duration_ms: 700  # 如果说话停顿比较长，可以把这个值设置大一些
+
+LLM:
+  # 当前支持的type为openai、dify、ollama，可自行适配
+  AliLLM:
+    # 定义LLM API类型
+    type: openai
+    # 可在这里找到你的 api_key https://bailian.console.aliyun.com/?apiKey=1#/api-key
+    base_url: https://dashscope.aliyuncs.com/compatible-mode/v1
+    model_name: qwen-turbo
+    api_key: 你的deepseek web key
+  DoubaoLLM:
+    # 定义LLM API类型
+    type: openai
+    # 先开通服务，打开以下网址，开通的服务搜索Doubao-pro-32k，开通它
+    # 开通改地址：https://console.volcengine.com/ark/region:ark+cn-beijing/openManagement?LLM=%7B%7D&OpenTokenDrawer=false
+    # 免费额度500000token
+    # 开通后，进入这里获取密钥：https://console.volcengine.com/ark/region:ark+cn-beijing/apiKey?apikey=%7B%7D
+    base_url: https://ark.cn-beijing.volces.com/api/v3
+    model_name: doubao-pro-32k-functioncall-241028
+    api_key: 你的doubao web key
+  DeepSeekLLM:
+    # 定义LLM API类型
+    type: openai
+    # 可在这里找到你的api key https://platform.deepseek.com/
+    model_name: deepseek-chat
+    url: https://api.deepseek.com
+    api_key: 你的deepseek web key
+  ChatGLMLLM:
+    # 定义LLM API类型
+    type: openai
+    # glm-4-flash 是免费的，但是还是需要注册填写api_key的
+    # 可在这里找到你的api key https://bigmodel.cn/usercenter/proj-mgmt/apikeys
+    model_name: glm-4-flash
+    url: https://open.bigmodel.cn/api/paas/v4/
+    api_key: 你的chat-glm web key
+  OllamaLLM:
+    # 定义LLM API类型
+    type: ollama
+    model_name: qwen2.5 #  使用的模型名称，需要预先使用ollama pull下载
+    base_url: http://localhost:11434  # Ollama服务地址
+  DifyLLM:
+    # 定义LLM API类型
+    type: dify
+    # 建议使用本地部署的dify接口，国内部分区域访问dify公有云接口可能会受限
+    # 如果使用DifyLLM，配置文件里prompt(提示词)是无效的，需要在dify控制台设置提示词
+    base_url: https://api.dify.cn/v1
+    api_key: 你的DifyLLM web key
+  GeminiLLM:
+    type: gemini
+    # 谷歌Gemini API，需要先在Google Cloud控制台创建API密钥并获取api_key
+    # 若在中国境内使用，请遵守《生成式人工智能服务管理暂行办法》
+    # token申请地址： https://aistudio.google.com/apikey
+    # 若部署地无法访问接口，需要开启科学上网
+    api_key: 你的gemini web key
+    model_name: "gemini-1.5-pro"  #  gemini-1.5-pro 是免费的
+  CozeLLM:
+    # 定义LLM API类型
+    type: coze
+    bot_id: 你的bot_id
+    user_id: 你的user_id
+    personal_access_token: 你的coze个人令牌
+  LMStudioLLM:
+    # 定义LLM API类型
+    type: openai
+    model_name: deepseek-r1-distill-llama-8b@q4_k_m # 使用的模型名称，需要预先在社区下载
+    url: http://localhost:1234/v1 # LM Studio服务地址
+    api_key: lm-studio # LM Studio服务的固定API Key
+  HomeAssistant:
+    # 定义LLM API类型
+    type: homeassistant
+    base_url: http://homeassistant.local:8123
+    agent_id: conversation.chatgpt
+    api_key: 你的home assistant api访问令牌
+  FastgptLLM:
+    # 定义LLM API类型
+    type: fastgpt
+    # 如果使用fastgpt，配置文件里prompt(提示词)是无效的，需要在fastgpt控制台设置提示词
+    base_url: https://host/api/v1
+    api_key: fastgpt-xxx
+    variables:
+      k: "v"
+      k2: "v2"
+
+TTS_SET:
+  TTS_STREAM: false  #是否启动流响应 true/false，默认false
+  MAX_WORKERS: 4    #并发请求tts的数量，这个调太大，本地tts会变慢
+
+TTS:
+  # 当前支持的type为edge、doubao，可自行适配
+  EdgeTTS:
+    # 定义TTS API类型
+    type: edge
+    voice: zh-CN-XiaoxiaoNeural
+    output_file: tmp/
+  DoubaoTTS:
+    # 定义TTS API类型
+    type: doubao
+    # 火山引擎语音合成服务，需要先在火山引擎控制台创建应用并获取appid和access_token
+    # 山引擎语音一定要购买花钱，起步价30元，就有100并发了。如果用免费的只有2个并发，会经常报tts错误
+    # 购买服务后，购买免费的音色后，可能要等半小时左右，才能使用。
+    # 地址：https://console.volcengine.com/speech/service/8
+    api_url: https://openspeech.bytedance.com/api/v1/tts
+    voice: BV001_streaming
+    output_file: tmp/
+    authorization: "Bearer;"
+    appid: 你的火山引擎语音合成服务appid
+    access_token: 你的火山引擎语音合成服务access_token
+    cluster: volcano_tts
+  CosyVoiceSiliconflow:
+    type: siliconflow
+    # 硅基流动TTS
+    # token申请地址 https://cloud.siliconflow.cn/account/ak
+    model: FunAudioLLM/CosyVoice2-0.5B
+    voice: FunAudioLLM/CosyVoice2-0.5B:alex
+    output_file: tmp/
+    access_token: 你的硅基流动API密钥
+    response_format: wav
+  CozeCnTTS:
+    type: cozecn
+    # COZECN TTS
+    # token申请地址 https://www.coze.cn/open/oauth/pats
+    voice: 7426720361733046281
+    output_file: tmp/
+    access_token: 你的coze web key
+    response_format: wav
+  FishSpeech:
+    # 定义TTS API类型
+    #启动tts方法：
+    #python -m tools.api_server
+    #--listen 0.0.0.0:8080
+    #--llama-checkpoint-path "checkpoints/fish-speech-1.5"
+    #--decoder-checkpoint-path "checkpoints/fish-speech-1.5/firefly-gan-vq-fsq-8x1024-21hz-generator.pth"
+    #--decoder-config-name firefly_gan_vq
+    #--compile
+    type: fishspeech
+    output_file: tmp/
+    response_format: wav
+    reference_id: null
+    reference_audio:
+      - "audio_ref/shinchan.wav"
+      - "audio_ref/shinchan2.wav"
+      - "audio_ref/shinchan3.wav"
+      - "audio_ref/shinchan4.wav"
+    reference_text:
+      - "难过的事讨厌的事丢脸的事全都集中起来，在这里，让水冲走所有的烦恼不就好了吗。"
+      - "我还是觉得船到桥头自然直这句话最棒了。"
+      - "我不是小鬼，我是野原新之助，小新这次的目标是成为圆梦之星哦，在努力一下好了，是我的脚自己要走这么快的喔，竞争就是这么激烈。小白，我们走了，成为圆梦之星是不是可以打败怪兽呢。"
+      - "工钱，今天一起算，有磨能使鬼推钱。哎呀，你难倒我了，像我这么乖的小孩，怎么更乖。"
+    normalize: true
+    max_new_tokens: 1024
+    chunk_length: 200
+    top_p: 0.7
+    repetition_penalty: 1.2
+    temperature: 0.7
+    streaming: true
+    use_memory_cache: "on"
+    seed: null
+    channels: 1
+    rate: 44100
+    api_key: "你的api_key"
+    api_url: "http://127.0.0.1:8080/v1/tts"
+  GPT_SOVITS_V2:
+    # 定义TTS API类型
+    #启动tts方法：
+    #python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/caixukun.yaml
+    type: gpt_sovits_v2
+    url: "http://127.0.0.1:9880/tts"
+    output_file: tmp/
+    text_lang: "auto"
+    ref_audio_path: "caixukun.wav"
+    prompt_text: ""
+    prompt_lang: "zh"
+    top_k: 5
+    top_p: 1
+    temperature: 1
+    text_split_method: "cut0"
+    batch_size: 1
+    batch_threshold: 0.75
+    split_bucket: true
+    return_fragment: false
+    speed_factor: 1.0
+    streaming_mode: false
+    seed: -1
+    parallel_infer: true
+    repetition_penalty: 1.35
+    aux_ref_audio_paths: []
+  GPT_SOVITS_V3:
+    type: gpt_sovits_v3
+    url: "http://127.0.0.1:9880/tts"
+    output_file: tmp/
+    text_lang: "auto"
+    ref_audio_path: "caixukun.wav"
+    prompt_lang: "zh"
+    prompt_text: ""
+    top_k: 5
+    top_p: 1
+    temperature: 1
+    sample_steps: 16
+    media_type: "wav"
+    streaming_mode: false
+    threshold: 30
+  MinimaxTTS:
+    # Minimax语音合成服务，需要先在minimax平台创建账户充值，并获取登录信息
+    # 平台地址：https://platform.minimaxi.com/
+    # 充值地址：https://platform.minimaxi.com/user-center/payment/balance
+    # group_id地址：https://platform.minimaxi.com/user-center/basic-information
+    # api_key地址：https://platform.minimaxi.com/user-center/basic-information/interface-key
+    # 定义TTS API类型
+    type: minimax
+    output_file: tmp/
+    group_id: 你的minimax平台groupID
+    api_key: 你的minimax平台接口密钥
+    model: "speech-01-turbo"
+    # 此处设置将优先于voice_setting中voice_id的设置；如都不设置，默认为 female-shaonv
+    voice_id: "female-shaonv"
+    # 以下可不用设置，使用默认设置
+    # voice_setting:
+    #     voice_id: "male-qn-qingse"
+    #     speed: 1
+    #     vol: 1
+    #     pitch: 0
+    #     emotion: "happy"
+    # pronunciation_dict:
+    #     tone:
+    #       - "处理/(chu3)(li3)"
+    #       - "危险/dangerous"
+    # audio_setting:
+    #     sample_rate: 32000
+    #     bitrate: 128000
+    #     format: "mp3"
+    #     channel: 1
+    # timber_weights:
+    #   -
+    #     voice_id: male-qn-qingse
+    #     weight: 1
+    #   -
+    #     voice_id: female-shaonv
+    #     weight: 1
+    # language_boost: auto
+  AliyunTTS:
+    # 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息
+    # 平台地址：https://nls-portal.console.aliyun.com/
+    # appkey地址：https://nls-portal.console.aliyun.com/applist
+    # token地址：https://nls-portal.console.aliyun.com/overview
+    # 定义TTS API类型
+    type: aliyun
+    output_file: tmp/
+    appkey: 你的阿里云智能语音交互服务项目Appkey
+    token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret
+    voice: xiaoyun
+    access_key_id: 你的阿里云账号access_key_id
+    access_key_secret: 你的阿里云账号access_key_secret
+
+    # 以下可不用设置，使用默认设置
+    # format: wav
+    # sample_rate: 16000
+    # volume: 50
+    # speech_rate: 0
+    # pitch_rate: 0
+    # 添加 302.ai TTS 配置
+    # token申请地址：https://dash.302.ai/
+  TTS302AI:
+    # 302AI语音合成服务，需要先在302平台创建账户充值，并获取密钥信息
+    # 获取api_keyn路径：https://dash.302.ai/apis/list
+    # 价格，$35/百万字符。火山原版¥450元/百万字符
+    type: doubao
+    api_url: https://api.302ai.cn/doubao/tts_hd
+    authorization: "Bearer "
+    voice: "zh_female_wanwanxiaohe_moon_bigtts"
+    output_file: tmp/
+    access_token: "你的302API密钥"
+  ACGNTTS:
+    #在线网址：https://acgn.ttson.cn/
+    #token购买：www.ttson.cn
+    #开发相关疑问请提交至3497689533@qq.com
+    #角色id获取地址：ctrl+f快速检索角色——网站管理者不允许发布,可询问网站管理者：1069379506
+    #各参数意义见开发文档：https://www.yuque.com/alexuh/skmti9/wm6taqislegb02gd?singleDoc#
+    type: ttson
+    token: your_token
+    voice_id: 1695
+    speed_factor: 1
+    pitch_factor: 0
+    volume_change_dB: 0
+    to_lang: ZH
+    url: https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=
+    format: mp3
+    output_file: tmp/
+    emotion: 1
+  OpenAITTS:
+    # openai官方文本转语音服务，可支持全球大多数语种
+    type: openai
+    api_key: 你的openai api key
+    # 国内需要使用代理
+    api_url: https://api.openai.com/v1/audio/speech
+    # 可选tts-1或tts-1-hd，tts-1速度更快tts-1-hd质量更好
+    model: tts-1
+    # 演讲者，可选alloy, echo, fable, onyx, nova, shimmer
+    voice: onyx
+    # 语速范围0.25-4.0
+    speed: 1
+    output_file: tmp/
+# 模块测试配置
+module_test:
+  test_sentences:  # 自定义测试语句
+    - "你好，请介绍一下你自己"
+    - "What's the weather like today?"
+    - "请用100字概括量子计算的基本原理和应用前景"
+
+# 本地音乐播放配置
+music:
+  music_dir: "./music"  # 音乐文件存放路径，将从该目录及子目录下搜索音乐文件
+  music_ext: # 音乐文件类型，p3格式效率最高
+    - ".mp3"
+    - ".wav"
+    - ".p3"
+  refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒
+
+# 以下配置在小于等于0.0.9版本中的docker容器中可用
+# 0.0.9以后的新版本源码部署已经无法奏效
+manager:
+  enabled: false
+  ip: 0.0.0.0
+  port: 8002
+use_private_config: false

main/xiaozhi-server/config/functionCallConfig.py

@@ -0,0 +1,36 @@
+FunctionCallConfig = [
+            {
+                "type": "function",
+                "function": {
+                    "name": "handle_exit_intent",
+                    "description": "当用户想结束对话或需要退出系统时调用",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "say_goodbye": {
+                                "type": "string",
+                                "description": "和用户友好结束对话的告别语"
+                            }
+                        },
+                        "required": []
+                    }
+                }
+            },
+            {
+                "type": "function",
+                "function": {
+                    "name": "play_music",
+                    "description": "唱歌、听歌、播放音乐方法。比如用户说播放音乐，参数为：random，比如用户说播放两只老虎，参数为：两只老虎",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "song_name": {
+                                "type": "string",
+                                "description": "歌曲名称，如果没有指定具体歌名则为'random'"
+                            }
+                        },
+                        "required": ["song_name"]
+                    }
+                }
+            }
+        ]

main/xiaozhi-server/config/logger.py

@@ -0,0 +1,29 @@
+import os
+import sys
+from loguru import logger
+from config.settings import load_config
+
+def setup_logging():
+    """从配置文件中读取日志配置，并设置日志输出格式和级别"""
+    config = load_config()
+    log_config = config["log"]
+    log_format = log_config.get("log_format", "<green>{time:YY-MM-DD HH:mm:ss}</green>[<light-blue>{extra[tag]}</light-blue>] - <level>{level}</level> - <light-green>{message}</light-green>")
+    log_format_simple = log_config.get("log_format_file", "{time:YYYY-MM-DD HH:mm:ss} - {name} - {level} - {extra[tag]} - {message}")
+    log_level = log_config.get("log_level", "INFO")
+    log_dir = log_config.get("log_dir", "tmp")
+    log_file = log_config.get("log_file", "server.log")
+    data_dir = log_config.get("data_dir", "data")
+
+    os.makedirs(log_dir, exist_ok=True)
+    os.makedirs(data_dir, exist_ok=True)
+
+    # 配置日志输出
+    logger.remove()
+
+    # 输出到控制台
+    logger.add(sys.stdout, format=log_format, level=log_level)
+
+    # 输出到文件
+    logger.add(os.path.join(log_dir, log_file), format=log_format_simple, level=log_level)
+
+    return logger

main/xiaozhi-server/config/private_config.py

@@ -0,0 +1,241 @@
+import os
+import time
+import yaml
+from config.logger import setup_logging
+from typing import Dict, Any, Optional
+from copy import deepcopy
+from core.utils.util import get_project_dir
+from core.utils import llm, tts
+from core.utils.lock_manager import FileLockManager
+
+TAG = __name__
+
+class PrivateConfig:
+    def __init__(self, device_id: str, default_config: Dict[str, Any], auth_code_gen=None):
+        self.device_id = device_id
+        self.default_config = default_config
+        self.config_path = get_project_dir() + 'data/.private_config.yaml'
+        self.logger = setup_logging()
+        self.private_config = {}
+        self.auth_code_gen = auth_code_gen
+        self.lock_manager = FileLockManager()
+
+    async def load_or_create(self):
+        try:
+            await self.lock_manager.acquire_lock(self.config_path)
+            try:
+                if os.path.exists(self.config_path):
+                    with open(self.config_path, 'r', encoding='utf-8') as f:
+                        all_configs = yaml.safe_load(f) or {}
+                else:
+                    all_configs = {}
+
+                if self.device_id not in all_configs:
+                    # Get selected module names
+                    selected_modules = self.default_config['selected_module']
+                    selected_tts = selected_modules['TTS']
+                    selected_llm = selected_modules['LLM']
+                    selected_asr = selected_modules['ASR']
+                    selected_vad = selected_modules['VAD']
+
+                    # 生成认证码
+                    auth_code = None
+                    if self.auth_code_gen:
+                        auth_code = self.auth_code_gen.generate_code()
+
+                    # Initialize device config with only necessary configurations
+                    device_config = {
+                        'selected_module': deepcopy(selected_modules),
+                        'prompt': self.default_config['prompt'],
+                        'LLM': {
+                            selected_llm: deepcopy(self.default_config['LLM'][selected_llm])
+                        },
+                        'TTS': {
+                            selected_tts: deepcopy(self.default_config['TTS'][selected_tts])
+                        },
+                        'ASR': {
+                            selected_asr: deepcopy(self.default_config['ASR'][selected_asr])
+                        },
+                        'VAD': {
+                            selected_vad: deepcopy(self.default_config['VAD'][selected_vad])
+                        },
+                        'auth_code': auth_code  # 添加认证码字段
+                    }
+                    
+                    all_configs[self.device_id] = device_config
+                    
+                    # Save updated configs
+                    with open(self.config_path, 'w', encoding='utf-8') as f:
+                        yaml.dump(all_configs, f, allow_unicode=True)
+
+                self.private_config = all_configs[self.device_id]
+
+            finally:
+                self.lock_manager.release_lock(self.config_path)
+
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"Error handling private config: {e}")
+            self.private_config = {}
+
+    async def update_config(self, selected_modules: Dict[str, str], prompt: str, nickname: str) -> bool:
+        """更新设备配置
+        Args:
+            selected_modules: 选择的模块配置，格式如 {'LLM': 'AliLLM', 'TTS': 'EdgeTTS',...}
+            prompt: 提示词配置
+        Returns:
+            bool: 更新是否成功
+        """
+        try:
+            await self.lock_manager.acquire_lock(self.config_path)
+            try:
+                # Read main config to get full module configurations
+                main_config = self.default_config
+
+                # Create new device config
+                device_config = {
+                    'selected_module': selected_modules,
+                    'prompt': prompt,
+                    'nickname': nickname,
+                }
+                if self.private_config.get('last_chat_time'):
+                    device_config['last_chat_time'] = self.private_config['last_chat_time']
+                if self.private_config.get('owner'):
+                    device_config['owner'] = self.private_config['owner']
+
+                # Copy full module configurations from main config
+                for module_type, selected_name in selected_modules.items():
+                    if selected_name and selected_name in main_config.get(module_type, {}):
+                        device_config[module_type] = {
+                            selected_name: main_config[module_type][selected_name]
+                        }
+
+                # Read all configs
+                if os.path.exists(self.config_path):
+                    with open(self.config_path, 'r', encoding='utf-8') as f:
+                        all_configs = yaml.safe_load(f) or {}
+                else:
+                    all_configs = {}
+
+                # Update device config
+                all_configs[self.device_id] = device_config
+                self.private_config = device_config
+
+                # Save back to file
+                with open(self.config_path, 'w', encoding='utf-8') as f:
+                    yaml.dump(all_configs, f, allow_unicode=True)
+
+                return True
+            finally:
+                self.lock_manager.release_lock(self.config_path)
+
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"Error updating config: {e}")
+            return False
+
+    async def delete_config(self) -> bool:
+        """删除设备配置
+        Returns:
+            bool: 删除是否成功
+        """
+        try:
+            await self.lock_manager.acquire_lock(self.config_path)
+            try:
+                # 读取所有配置
+                if os.path.exists(self.config_path):
+                    with open(self.config_path, 'r', encoding='utf-8') as f:
+                        all_configs = yaml.safe_load(f) or {}
+                else:
+                    return False
+
+                # 删除设备配置
+                if self.device_id in all_configs:
+                    del all_configs[self.device_id]
+                    
+                    # 保存更新后的配置
+                    with open(self.config_path, 'w', encoding='utf-8') as f:
+                        yaml.dump(all_configs, f, allow_unicode=True)
+                    
+                    self.private_config = {}
+                    return True
+                
+                return False
+            finally:
+                self.lock_manager.release_lock(self.config_path)
+
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"Error deleting config: {e}")
+            return False
+
+    def create_private_instances(self):
+        #  判断存在私有配置，并且self.device_id在私有配置中
+        if not self.private_config:
+            self.logger.bind(tag=TAG).error("Private config not found for device_id: {}", self.device_id)
+            return None, None
+        
+        """创建私有处理模块实例"""
+        config = self.private_config
+        selected_modules = config['selected_module']
+        return (
+            llm.create_instance(
+                selected_modules["LLM"]
+                if not 'type' in config["LLM"][selected_modules["LLM"]]
+                else
+                config["LLM"][selected_modules["LLM"]]['type'],
+                config["LLM"][selected_modules["LLM"]],
+            ),
+            tts.create_instance(
+                selected_modules["TTS"]
+                if not 'type' in config["TTS"][selected_modules["TTS"]]
+                else
+                config["TTS"][selected_modules["TTS"]]["type"],
+                config["TTS"][selected_modules["TTS"]],
+                self.default_config.get("delete_audio", True)  # Using default_config for global settings
+            )
+        )
+
+    async def update_last_chat_time(self, timestamp=None):
+        """更新设备最近一次的聊天时间
+        Args:
+            timestamp: 指定的时间戳,不传则使用当前时间
+        """
+        if not self.private_config:
+            self.logger.bind(tag=TAG).error("Private config not found")
+            return False
+            
+        try:
+            await self.lock_manager.acquire_lock(self.config_path)
+            try:
+                if timestamp is None:
+                    timestamp = int(time.time())
+                    
+                self.private_config['last_chat_time'] = timestamp
+                
+                # 读取所有配置
+                with open(self.config_path, 'r', encoding='utf-8') as f:
+                    all_configs = yaml.safe_load(f) or {}
+                    
+                # 更新当前设备配置    
+                all_configs[self.device_id] = self.private_config
+                
+                # 保存回文件
+                with open(self.config_path, 'w', encoding='utf-8') as f:
+                    yaml.dump(all_configs, f, allow_unicode=True)
+                    
+                return True
+            finally:
+                self.lock_manager.release_lock(self.config_path)
+                
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"Error updating last chat time: {e}")
+            return False
+
+    def get_auth_code(self) -> str:
+        """获取设备的认证码
+        Returns:
+            str: 认证码，如果没有返回空字符串
+        """
+        return self.private_config.get('auth_code', '')
+
+    def get_owner(self) -> Optional[str]:
+        """获取设备当前所有者"""
+        return self.private_config.get('owner')

main/xiaozhi-server/config/settings.py

@@ -0,0 +1,84 @@
+import os
+import argparse
+from ruamel.yaml import YAML
+from collections.abc import Mapping
+from core.utils.util import read_config, get_project_dir
+
+default_config_file = "config.yaml"
+
+
+def get_config_file():
+    global default_config_file
+    # 判断是否存在私有的配置文件
+    config_file = default_config_file
+    if os.path.exists(get_project_dir() + "data/." + default_config_file):
+        config_file = "data/." + default_config_file
+    return config_file
+
+
+def load_config():
+    """加载配置文件"""
+    parser = argparse.ArgumentParser(description="Server configuration")
+    config_file = get_config_file()
+    parser.add_argument("--config_path", type=str, default=config_file)
+    args = parser.parse_args()
+    return read_config(args.config_path)
+
+
+def update_config(config):
+    yaml = YAML()
+    yaml.preserve_quotes = True
+    """将配置保存到YAML文件"""
+    with open(get_config_file(), 'w') as f:
+        yaml.dump(config, f)
+
+
+def find_missing_keys(new_config, old_config, parent_key=''):
+    """
+    递归查找缺失的配置项
+    返回格式：[缺失配置路径]
+    """
+    missing_keys = []
+
+    if not isinstance(new_config, Mapping):
+        return missing_keys
+
+    for key, value in new_config.items():
+        # 构建当前配置路径
+        full_path = f"{parent_key}.{key}" if parent_key else key
+
+        # 检查键是否存在
+        if key not in old_config:
+            missing_keys.append(full_path)
+            continue
+
+        # 递归检查嵌套字典
+        if isinstance(value, Mapping):
+            sub_missing = find_missing_keys(
+                value,
+                old_config[key],
+                parent_key=full_path
+            )
+            missing_keys.extend(sub_missing)
+
+    return missing_keys
+
+
+def check_config_file():
+    old_config_file = get_config_file()
+    global default_config_file
+    if not old_config_file.startswith('data'):
+        return
+    old_config = read_config(get_project_dir() + old_config_file)
+    new_config = read_config(get_project_dir() + default_config_file)
+    # 查找缺失的配置项
+    missing_keys = find_missing_keys(new_config, old_config)
+
+    if missing_keys:
+        error_msg = "您的配置文件太旧了，缺少了：\n"
+        error_msg += "\n".join(f"- {key}" for key in missing_keys)
+        error_msg += "\n建议您：\n"
+        error_msg += "1、备份data/.config.yaml文件\n"
+        error_msg += "2、将根目录的config.yaml文件复制到data下，重命名为.config.yaml\n"
+        error_msg += "3、将密钥逐个复制到新的配置文件中\n"
+        raise ValueError(error_msg)

main/xiaozhi-server/core/auth.py

@@ -0,0 +1,54 @@
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class AuthenticationError(Exception):
+    """认证异常"""
+    pass
+
+
+class AuthMiddleware:
+    def __init__(self, config):
+        self.config = config
+        self.auth_config = config["server"].get("auth", {})
+        # 构建token查找表
+        self.tokens = {
+            item["token"]: item["name"]
+            for item in self.auth_config.get("tokens", [])
+        }
+        # 设备白名单
+        self.allowed_devices = set(
+            self.auth_config.get("allowed_devices", [])
+        )
+
+    async def authenticate(self, headers):
+        """验证连接请求"""
+        # 检查是否启用认证
+        if not self.auth_config.get("enabled", False):
+            return True
+
+        # 检查设备是否在白名单中
+        device_id = headers.get("device-id", "")
+
+        if self.allowed_devices and device_id in self.allowed_devices:
+            return True
+
+        # 验证Authorization header
+        auth_header = headers.get("authorization", "")
+        if not auth_header.startswith("Bearer "):
+            logger.bind(tag=TAG).error("Missing or invalid Authorization header")
+            raise AuthenticationError("Missing or invalid Authorization header")
+
+        token = auth_header.split(" ")[1]
+        if token not in self.tokens:
+            logger.bind(tag=TAG).error(f"Invalid token: {token}")
+            raise AuthenticationError("Invalid token")
+
+        logger.bind(tag=TAG).info(f"Authentication successful - Device: {device_id}, Token: {self.tokens[token]}")
+        return True
+
+    def get_token_name(self, token):
+        """获取token对应的设备名称"""
+        return self.tokens.get(token)

main/xiaozhi-server/core/connection.py

@@ -0,0 +1,576 @@
+import os
+import json
+import uuid
+import time
+import queue
+import asyncio
+import traceback
+from config.logger import setup_logging
+import threading
+import websockets
+from typing import Dict, Any
+from core.utils.dialogue import Message, Dialogue
+from core.handle.textHandle import handleTextMessage
+from core.utils.util import get_string_no_punctuation_or_emoji
+from concurrent.futures import ThreadPoolExecutor, TimeoutError
+from core.handle.sendAudioHandle import sendAudioMessage, sendAudioMessageStream
+from core.handle.receiveAudioHandle import handleAudioMessage
+from core.handle.intentHandler import Action, get_functions, handle_llm_function_call
+from config.private_config import PrivateConfig
+from core.auth import AuthMiddleware, AuthenticationError
+from core.utils.auth_code_gen import AuthCodeGenerator
+
+TAG = __name__
+
+
+class TTSException(RuntimeError):
+    pass
+
+
+class ConnectionHandler:
+    def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory, _intent):
+        self.config = config
+        self.logger = setup_logging()
+        self.auth = AuthMiddleware(config)
+
+        self.tts_stream = self.config.get("TTS_SET", {}).get("TTS_STREAM", False)
+
+        self.websocket = None
+        self.headers = None
+        self.session_id = None
+        self.prompt = None
+        self.welcome_msg = None
+
+        # 客户端状态相关
+        self.client_abort = False
+        self.client_listen_mode = "auto"
+
+        # 线程任务相关
+        self.loop = asyncio.get_event_loop()
+        self.stop_event = threading.Event()
+        self.tts_queue = queue.Queue()
+        self.tts_queue_stream = queue.Queue()
+        self.audio_play_queue = queue.Queue()
+        max_workers = self.config.get("TTS_SET", {}).get("MAX_WORKERS", 10)
+        self.executor = ThreadPoolExecutor(max_workers=max_workers)
+
+        # 依赖的组件
+        self.vad = _vad
+        self.asr = _asr
+        self.llm = _llm
+        self.tts = _tts
+        self.memory = _memory
+        self.intent = _intent
+
+        # vad相关变量
+        self.client_audio_buffer = bytes()
+        self.client_have_voice = False
+        self.client_have_voice_last_time = 0.0
+        self.client_no_voice_last_time = 0.0
+        self.client_voice_stop = False
+
+        # asr相关变量
+        self.asr_audio = []
+        self.asr_server_receive = True
+
+        # llm相关变量
+        self.llm_finish_task = False
+        self.dialogue = Dialogue()
+
+        # tts相关变量
+        self.tts_first_text_index = -1
+        self.tts_last_text_index = -1
+        self.tts_duration = 0
+
+        # iot相关变量
+        self.iot_descriptors = {}
+
+        self.cmd_exit = self.config["CMD_exit"]
+        self.max_cmd_length = 0
+        for cmd in self.cmd_exit:
+            if len(cmd) > self.max_cmd_length:
+                self.max_cmd_length = len(cmd)
+
+        self.private_config = None
+        self.auth_code_gen = AuthCodeGenerator.get_instance()
+        self.is_device_verified = False  # 添加设备验证状态标志
+        self.music_handler = _music
+        self.close_after_chat = False  # 是否在聊天结束后关闭连接
+        self.use_function_call_mode = False
+        if self.config["selected_module"]["Intent"] == 'function_call':
+            self.use_function_call_mode = True
+
+        self.logger.bind(tag=TAG).info(f"use_function_call_mode:{self.use_function_call_mode}")
+
+    async def handle_connection(self, ws):
+        try:
+            # 获取并验证headers
+            self.headers = dict(ws.request.headers)
+            # 获取客户端ip地址
+            client_ip = ws.remote_address[0]
+            self.logger.bind(tag=TAG).info(f"{client_ip} conn - Headers: {self.headers}")
+
+            # 进行认证
+            await self.auth.authenticate(self.headers)
+
+            device_id = self.headers.get("device-id", None)
+            self.memory.init_memory(device_id, self.llm)
+            self.intent.set_llm(self.llm)
+
+            # Load private configuration if device_id is provided
+            bUsePrivateConfig = self.config.get("use_private_config", False)
+            self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
+            if bUsePrivateConfig and device_id:
+                try:
+                    self.private_config = PrivateConfig(device_id, self.config, self.auth_code_gen)
+                    await self.private_config.load_or_create()
+                    # 判断是否已经绑定
+                    owner = self.private_config.get_owner()
+                    self.is_device_verified = owner is not None
+
+                    if self.is_device_verified:
+                        await self.private_config.update_last_chat_time()
+
+                    llm, tts = self.private_config.create_private_instances()
+                    if all([llm, tts]):
+                        self.llm = llm
+                        self.tts = tts
+                        self.logger.bind(tag=TAG).info(f"Loaded private config and instances for device {device_id}")
+                    else:
+                        self.logger.bind(tag=TAG).error(f"Failed to create instances for device {device_id}")
+                        self.private_config = None
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"Error initializing private config: {e}")
+                    self.private_config = None
+                    raise
+
+            # 认证通过,继续处理
+            self.websocket = ws
+            self.session_id = str(uuid.uuid4())
+
+            self.welcome_msg = self.config["xiaozhi"]
+            self.welcome_msg["session_id"] = self.session_id
+            await self.websocket.send(json.dumps(self.welcome_msg))
+
+            await self.loop.run_in_executor(None, self._initialize_components)
+
+            # tts 消化线程
+            tts_priority = threading.Thread(target=self._tts_priority_thread, daemon=True)
+            tts_priority.start()
+
+            # 音频播放 消化线程
+            audio_play_priority = threading.Thread(target=self._audio_play_priority_thread, daemon=True)
+            audio_play_priority.start()
+
+            try:
+                async for message in self.websocket:
+                    await self._route_message(message)
+            except websockets.exceptions.ConnectionClosed:
+                self.logger.bind(tag=TAG).info("客户端断开连接")
+                await self.close()
+
+        except AuthenticationError as e:
+            self.logger.bind(tag=TAG).error(f"Authentication failed: {str(e)}")
+            await ws.close()
+            return
+        except Exception as e:
+            stack_trace = traceback.format_exc()
+            self.logger.bind(tag=TAG).error(f"Connection error: {str(e)}-{stack_trace}")
+            await ws.close()
+            return
+        finally:
+            await self.memory.save_memory(self.dialogue.dialogue)
+
+    async def _route_message(self, message):
+        """消息路由"""
+        if isinstance(message, str):
+            await handleTextMessage(self, message)
+        elif isinstance(message, bytes):
+            await handleAudioMessage(self, message)
+
+    def _initialize_components(self):
+        self.prompt = self.config["prompt"]
+        if self.private_config:
+            self.prompt = self.private_config.private_config.get("prompt", self.prompt)
+        # 赋予LLM时间观念
+        if "{date_time}" in self.prompt:
+            date_time = time.strftime("%Y-%m-%d %H:%M", time.localtime())
+            self.prompt = self.prompt.replace("{date_time}", date_time)
+        self.dialogue.put(Message(role="system", content=self.prompt))
+
+    async def _check_and_broadcast_auth_code(self):
+        """检查设备绑定状态并广播认证码"""
+        if not self.private_config.get_owner():
+            auth_code = self.private_config.get_auth_code()
+            if auth_code:
+                # 发送验证码语音提示
+                text = f"请在后台输入验证码：{' '.join(auth_code)}"
+                self.recode_first_last_text(text)
+                future = self.executor.submit(self.speak_and_play, text)
+                self.tts_queue.put(future)
+            return False
+        return True
+
+    def isNeedAuth(self):
+        bUsePrivateConfig = self.config.get("use_private_config", False)
+        if not bUsePrivateConfig:
+            # 如果不使用私有配置，就不需要验证
+            return False
+        return not self.is_device_verified
+
+    def chat(self, query):
+        if self.isNeedAuth():
+            self.llm_finish_task = True
+            future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
+            future.result()
+            return True
+
+        self.dialogue.put(Message(role="user", content=query))
+
+        response_message = []
+        processed_chars = 0  # 跟踪已处理的字符位置
+        try:
+            start_time = time.time()
+            # 使用带记忆的对话
+            future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
+            memory_str = future.result()
+
+            self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}")
+            llm_responses = self.llm.response(
+                self.session_id,
+                self.dialogue.get_llm_dialogue_with_memory(memory_str)
+            )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
+            return None
+
+        self.llm_finish_task = False
+        text_index = 0
+        for content in llm_responses:
+            response_message.append(content)
+            if self.client_abort:
+                break
+
+            end_time = time.time()
+            self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
+
+            # 合并当前全部文本并处理未分割部分
+            full_text = "".join(response_message)
+            current_text = full_text[processed_chars:]  # 从未处理的位置开始
+
+            # 查找最后一个有效标点
+            punctuations = ("。", "？", "！", "；", "：")
+            last_punct_pos = -1
+            for punct in punctuations:
+                pos = current_text.rfind(punct)
+                if pos > last_punct_pos:
+                    last_punct_pos = pos
+
+            # 找到分割点则处理
+            if last_punct_pos != -1:
+                segment_text_raw = current_text[:last_punct_pos + 1]
+                segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
+                if segment_text:
+                    # 强制设置空字符，测试TTS出错返回语音的健壮性
+                    # if text_index % 2 == 0:
+                    #     segment_text = " "
+                    text_index += 1
+                    self.recode_first_last_text(segment_text, text_index)
+                    if self.tts_stream:
+                        stream_queue = queue.Queue()
+                        self.executor.submit(self.speak_and_play_stream, segment_text, stream_queue)
+                        self.tts_queue_stream.put({
+                            "text": segment_text,
+                            "chunk_queque": stream_queue,
+                            "text_index": text_index
+                        })
+                    else:
+                        future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                        self.tts_queue.put(future)
+                    processed_chars += len(segment_text_raw)  # 更新已处理字符位置
+
+        # 处理最后剩余的文本
+        full_text = "".join(response_message)
+        remaining_text = full_text[processed_chars:]
+        if remaining_text:
+            segment_text = get_string_no_punctuation_or_emoji(remaining_text)
+            if segment_text:
+                text_index += 1
+                self.recode_first_last_text(segment_text, text_index)
+                if self.tts_stream:
+                    stream_queue = queue.Queue()
+                    self.executor.submit(self.speak_and_play_stream, segment_text, stream_queue, text_index)
+                    self.tts_queue_stream.put({
+                        "text": segment_text,
+                        "chunk_queque": stream_queue,
+                        "text_index": text_index
+                    })
+                else:
+                    future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                    self.tts_queue.put(future)
+
+        self.llm_finish_task = True
+        self.dialogue.put(Message(role="assistant", content="".join(response_message)))
+        self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
+        return True
+
+    def chat_with_function_calling(self, query):
+        self.logger.bind(tag=TAG).debug(f"Chat with function calling start: {query}")
+        """Chat with function calling for intent detection using streaming"""
+        if self.isNeedAuth():
+            self.llm_finish_task = True
+            future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
+            future.result()
+            return True
+
+        self.dialogue.put(Message(role="user", content=query))
+
+        # Define intent functions
+        functions = get_functions()
+
+        response_message = []
+        processed_chars = 0  # 跟踪已处理的字符位置
+        function_call_data = None  # 存储function call数据
+
+        try:
+            start_time = time.time()
+
+            # 使用带记忆的对话
+            future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
+            memory_str = future.result()
+
+            # self.logger.bind(tag=TAG).info(f"记忆内容: {memory_str}")
+
+            # 使用支持functions的streaming接口
+            llm_responses = self.llm.response_with_functions(
+                self.session_id,
+                self.dialogue.get_llm_dialogue_with_memory(memory_str),
+                functions=functions
+            )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
+            return None
+
+        self.llm_finish_task = False
+        text_index = 0
+
+        # 处理流式响应
+        for response in llm_responses:
+            if response["type"] == "content":
+                content = response["content"]
+                response_message.append(content)
+
+                if self.client_abort:
+                    break
+
+                end_time = time.time()
+                self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
+
+                # 处理文本分段和TTS逻辑
+                # 合并当前全部文本并处理未分割部分
+                full_text = "".join(response_message)
+                current_text = full_text[processed_chars:]  # 从未处理的位置开始
+
+                # 查找最后一个有效标点
+                punctuations = ("。", "？", "！", "；", "：")
+                last_punct_pos = -1
+                for punct in punctuations:
+                    pos = current_text.rfind(punct)
+                    if pos > last_punct_pos:
+                        last_punct_pos = pos
+
+                # 找到分割点则处理
+                if last_punct_pos != -1:
+                    segment_text_raw = current_text[:last_punct_pos + 1]
+                    segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
+                    if segment_text:
+                        text_index += 1
+                        self.recode_first_last_text(segment_text, text_index)
+                        future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                        self.tts_queue.put(future)
+                        processed_chars += len(segment_text_raw)  # 更新已处理字符位置
+
+            elif response["type"] == "function_call":
+                # Extract function call data
+                function_call_data = {
+                    "name": response["function_call"]["function"]["name"],
+                    "arguments": response["function_call"]["function"]["arguments"]
+                }
+                self.logger.bind(tag=TAG).info(f"Function call detected: {function_call_data}")
+
+        # 处理最后剩余的文本
+        full_text = "".join(response_message)
+        remaining_text = full_text[processed_chars:]
+        if remaining_text:
+            segment_text = get_string_no_punctuation_or_emoji(remaining_text)
+            if segment_text:
+                text_index += 1
+                self.recode_first_last_text(segment_text, text_index)
+                future = self.executor.submit(self.speak_and_play, segment_text, text_index)
+                self.tts_queue.put(future)
+
+        # 存储对话内容
+        self.dialogue.put(Message(role="assistant", content="".join(response_message)))
+
+        # 处理function call
+        if function_call_data:
+            result = handle_llm_function_call(self, function_call_data)
+            if result.action == Action.RESPONSE:
+                text = result.response
+                text_index += 1
+                self.recode_first_last_text(text, text_index)
+                future = self.executor.submit(self.speak_and_play, text, text_index)
+                self.tts_queue.put(future)
+
+        self.llm_finish_task = True
+        self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
+
+        return True
+
+    def _tts_priority_thread(self):
+        if self.tts_stream:
+            self._tts_priority_thread_stream()
+        else:
+            self._tts_priority_thread_no_stream()
+
+    def _tts_priority_thread_no_stream(self):
+        while not self.stop_event.is_set():
+            text = None
+            try:
+                future = self.tts_queue.get()
+                if future is None:
+                    continue
+                text = None
+                opus_datas, text_index, tts_file = [], 0, None
+                try:
+                    self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
+                    tts_file, text, text_index = future.result(timeout=10)
+                    if text is None or len(text) <= 0:
+                        self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
+                    elif tts_file is None:
+                        self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}")
+                    else:
+                        self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
+                        if os.path.exists(tts_file):
+                            opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
+                        else:
+                            self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
+                except TimeoutError:
+                    self.logger.bind(tag=TAG).error("TTS超时")
+                except Exception as e:
+                    self.logger.bind(tag=TAG).error(f"TTS出错: {e}")
+                if not self.client_abort:
+                    # 如果没有中途打断就发送语音
+                    self.audio_play_queue.put((opus_datas, text, text_index))
+                if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
+                    os.remove(tts_file)
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
+                self.clearSpeakStatus()
+                asyncio.run_coroutine_threadsafe(
+                    self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
+                    self.loop
+                )
+                self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
+
+    def _tts_priority_thread_stream(self):
+        while not self.stop_event.is_set():
+            text = None
+            try:
+                tts_stream_queue_msg = self.tts_queue_stream.get()
+                try:
+                    text = tts_stream_queue_msg["text"]
+                    chunk_queque = tts_stream_queue_msg["chunk_queque"]
+                    text_index = tts_stream_queue_msg["text_index"]
+                except TimeoutError:
+                    self.logger.error("TTS 任务超时")
+                    continue
+                except Exception as e:
+                    self.logger.error(f"TTS 任务出错: {e}")
+                    continue
+                if not self.client_abort:
+                    # 如果没有中途打断就发送语音
+                    self.audio_play_queue.put((chunk_queque, text, text_index))
+            except Exception as e:
+                self.clearSpeakStatus()
+                asyncio.run_coroutine_threadsafe(
+                    self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
+                    self.loop
+                )
+                self.logger.error(f"tts_priority priority_thread: {text}{e}")
+
+    def _audio_play_priority_thread(self):
+        while not self.stop_event.is_set():
+            text = None
+            try:
+                if self.tts_stream:
+                    chunk_queque, text, text_index = self.audio_play_queue.get()
+                    future = asyncio.run_coroutine_threadsafe(
+                        sendAudioMessageStream(self, chunk_queque, text, text_index),
+                        self.loop)
+                    future.result()
+                else:
+                    opus_datas, text, text_index = self.audio_play_queue.get()
+                    future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
+                                                              self.loop)
+                    future.result()
+            except Exception as e:
+                self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")
+
+    def speak_and_play(self, text, text_index=0):
+        if text is None or len(text) <= 0:
+            self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
+            return None, text, text_index
+        tts_file = self.tts.to_tts(text)
+        if tts_file is None:
+            self.logger.bind(tag=TAG).error(f"tts转换失败，{text}")
+            return None, text, text_index
+        self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}")
+        return tts_file, text, text_index
+
+    def speak_and_play_stream(self, text, queue: queue.Queue, text_index=0):
+        if text is None or len(text) <= 0:
+            self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
+            return None, text
+        self.tts.to_tts_stream(text, queue, text_index)
+
+    def clearSpeakStatus(self):
+        self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
+        self.asr_server_receive = True
+        self.tts_last_text_index = -1
+        self.tts_first_text_index = -1
+        self.tts_duration = 0
+
+    def recode_first_last_text(self, text, text_index=0):
+        if self.tts_first_text_index == -1:
+            self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}")
+            self.tts_first_text_index = text_index
+        self.tts_last_text_index = text_index
+
+    async def close(self):
+        """资源清理方法"""
+
+        # 清理其他资源
+        self.stop_event.set()
+        self.executor.shutdown(wait=False)
+        if self.websocket:
+            await self.websocket.close()
+        self.logger.bind(tag=TAG).info("连接资源已释放")
+
+    def reset_vad_states(self):
+        self.client_audio_buffer = bytes()
+        self.client_have_voice = False
+        self.client_have_voice_last_time = 0
+        self.client_voice_stop = False
+        self.logger.bind(tag=TAG).debug("VAD states reset.")
+
+    def chat_and_close(self, text):
+        """Chat with the user and then close the connection"""
+        try:
+            # Use the existing chat method
+            self.chat(text)
+
+            # After chat is complete, close the connection
+            self.close_after_chat = True
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"Chat and close error: {str(e)}")

main/xiaozhi-server/core/handle/abortHandle.py

@@ -0,0 +1,16 @@
+import json
+import queue
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+async def handleAbortMessage(conn):
+    logger.bind(tag=TAG).info("Abort message received")
+    # 设置成打断状态，会自动打断llm、tts任务
+    conn.client_abort = True
+    # 打断客户端说话状态
+    await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
+    conn.clearSpeakStatus()
+    logger.bind(tag=TAG).info("Abort message received-end")

main/xiaozhi-server/core/handle/helloHandle.py

@@ -0,0 +1,8 @@
+import json
+from config.logger import setup_logging
+
+logger = setup_logging()
+
+
+async def handleHelloMessage(conn):
+    await conn.websocket.send(json.dumps(conn.welcome_msg))

main/xiaozhi-server/core/handle/intentHandler.py

@@ -0,0 +1,170 @@
+from config.logger import setup_logging
+import json
+from core.handle.sendAudioHandle import send_stt_message
+from core.utils.dialogue import Message
+from config.functionCallConfig import FunctionCallConfig
+import asyncio
+from enum import Enum
+
+TAG = __name__
+logger = setup_logging()
+
+
+class Action(Enum):
+    NOTFOUND = (0, "没有找到函数")
+    NONE = (1, "啥也不干")
+    RESPONSE = (2, "直接回复")
+    REQLLM = (3, "调用函数后再请求llm生成回复")
+
+    def __init__(self, code, message):
+        self.code = code
+        self.message = message
+
+
+class ActionResponse:
+    def __init__(self, action: Action, result, response):
+        self.action = action  # 动作类型
+        self.result = result  # 动作产生的结果
+        self.response = response  # 直接回复的内容
+
+
+def get_functions():
+    """获取功能调用配置"""
+    return FunctionCallConfig
+
+
+def handle_llm_function_call(conn, function_call_data):
+    try:
+        function_name = function_call_data["name"]
+
+        if function_name == "handle_exit_intent":
+            # 处理退出意图
+            try:
+                say_goodbye = json.loads(function_call_data["arguments"]).get("say_goodbye", "再见")
+                conn.close_after_chat = True
+                logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}")
+                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye)
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"处理退出意图错误: {e}")
+
+        elif function_name == "play_music":
+            # 处理音乐播放意图
+            try:
+                song_name = "random"
+                arguments = function_call_data["arguments"]
+                if arguments is not None and len(arguments) > 0:
+                    args = json.loads(arguments)
+                    song_name = args.get("song_name", "random")
+                music_intent = f"播放音乐 {song_name}" if song_name != "random" else "随机播放音乐"
+
+                # 执行音乐播放命令
+                future = asyncio.run_coroutine_threadsafe(
+                    conn.music_handler.handle_music_command(conn, music_intent),
+                    conn.loop
+                )
+                future.result()
+                return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response="还想听什么歌？")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
+        else:
+            return ActionResponse(action=Action.NOTFOUND, result="没有找到对应的函数", response="")
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"处理function call错误: {e}")
+
+    return None
+
+
+async def handle_user_intent(conn, text):
+    """
+    Handle user intent before starting chat
+    
+    Args:
+        conn: Connection object
+        text: User's text input
+    
+    Returns:
+        bool: True if intent was handled, False if should proceed to chat
+    """
+    # 检查是否有明确的退出命令
+    if await check_direct_exit(conn, text):
+        return True
+
+    if conn.use_function_call_mode:
+        # 使用支持function calling的聊天方法,不再进行意图分析
+        return False
+
+    logger.bind(tag=TAG).info(f"分析用户意图: {text}")
+
+    # 使用LLM进行意图分析
+    intent = await analyze_intent_with_llm(conn, text)
+
+    if not intent:
+        return False
+
+    # 处理各种意图
+    return await process_intent_result(conn, intent, text)
+
+
+async def check_direct_exit(conn, text):
+    """检查是否有明确的退出命令"""
+    cmd_exit = conn.cmd_exit
+    for cmd in cmd_exit:
+        if text == cmd:
+            logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
+            await conn.close()
+            return True
+    return False
+
+
+async def analyze_intent_with_llm(conn, text):
+    """使用LLM分析用户意图"""
+    if not hasattr(conn, 'intent') or not conn.intent:
+        logger.bind(tag=TAG).warning("意图识别服务未初始化")
+        return None
+
+    # 创建对话历史记录
+    dialogue = conn.dialogue
+    dialogue.put(Message(role="user", content=text))
+
+    try:
+        intent_result = await conn.intent.detect_intent(dialogue.dialogue)
+        logger.bind(tag=TAG).info(f"意图识别结果: {intent_result}")
+
+        # 尝试解析JSON结果
+        try:
+            intent_data = json.loads(intent_result)
+            if "intent" in intent_data:
+                return intent_data["intent"]
+        except json.JSONDecodeError:
+            # 如果不是JSON格式，尝试直接获取意图文本
+            return intent_result.strip()
+
+    except Exception as e:
+        logger.bind(tag=TAG).error(f"意图识别失败: {str(e)}")
+
+    return None
+
+
+async def process_intent_result(conn, intent, original_text):
+    """处理意图识别结果"""
+    # 处理退出意图
+    if "结束聊天" in intent:
+        logger.bind(tag=TAG).info(f"识别到退出意图: {intent}")
+
+        # 如果正在播放音乐，可以关了 TODO
+
+        # 如果是明确的离别意图，发送告别语并关闭连接
+        await send_stt_message(conn, original_text)
+        conn.executor.submit(conn.chat_and_close, original_text)
+        return True
+
+    # 处理播放音乐意图
+    if "播放音乐" in intent:
+        logger.bind(tag=TAG).info(f"识别到音乐播放意图: {intent}")
+        await conn.music_handler.handle_music_command(conn, intent)
+        return True
+
+    # 其他意图处理可以在这里扩展
+
+    # 默认返回False，表示继续常规聊天流程
+    return False

main/xiaozhi-server/core/handle/iotHandle.py

@@ -0,0 +1,193 @@
+import json
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class IotDescriptor:
+    """
+    A class to represent an IoT descriptor.
+    Attributes:
+    ----------
+    name : str
+        The name of the IoT descriptor.
+    description : str
+        A brief description of the IoT descriptor.
+    properties : dict
+        A dictionary containing properties of the IoT descriptor.
+    methods : dict
+        A dictionary containing methods of the IoT descriptor.
+    -------
+    """
+
+    def __init__(self, name, description, properties, methods):
+        self.name = name
+        self.description = description
+        self.properties = []
+        self.methods = []
+
+        # 根据描述创建属性
+        for key, value in properties.items():
+            # "volume":{"description":"当前音量 值","type":"number"}
+            """
+            等价于
+            {
+                'name': 名字,
+                'description': 描述,
+                'value': 0
+            }
+            """
+            # setattr(self, key, {}) # 创建一个空字典, 名字是属性名
+            property_item = globals()[key] = {}  # 创建一个空字典, 名字是属性名
+            property_item['name'] = key
+            property_item["description"] = value["description"]
+            if value["type"] == "number":
+                property_item["value"] = 0
+            elif value["type"] == "boolean":
+                property_item["value"] = False
+            else:
+                property_item["value"] = ""
+            self.properties.append(property_item)
+
+        # 根据描述创建方法
+        for key, value in methods.items():
+            # "SetVolume": {"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}}
+            """
+            等价于
+            SetVolume = {
+                `description`: 描述,
+                `volume`: {
+                    `description`: 描述,
+                    `value`: 0
+                }
+            }
+            """
+            # setattr(self, key, {}) # 创建一个空字典, 名字是方法名
+            method = globals()[key] = {}  # 创建一个空字典, 名字是方法名
+            method["description"] = value["description"]
+            method['name'] = key
+            for k, v in value["parameters"].items():
+                # 不同的参数解析
+                method[k] = {}
+                method[k]["description"] = v["description"]
+                if v["type"] == "number":
+                    method[k]["value"] = 0
+                elif v["type"] == "boolean":
+                    method[k]["value"] = False
+                else:
+                    method[k]["value"] = ""
+
+            self.methods.append(method)
+
+
+async def handleIotDescriptors(conn, descriptors):
+    """
+    处理物联网描述
+    示例: [{
+        "name":"Speaker",
+        "description":"当前 AI 机器人的扬声器",
+        "properties":{
+            "volume":{"description":"当前音量 值","type":"number"}  可以有boolean, number, string三种类型
+        },
+        "methods":{
+            "SetVolume":{
+                "description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}
+            }
+        }
+    }]
+    descriptors: 描述列表
+    """
+    for descriptor in descriptors:
+        iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
+                                       descriptor["methods"])
+        conn.iot_descriptors[descriptor["name"]] = iot_descriptor
+
+    # 暂时从配置文件中设置音量，后期通过意图识别控制音量
+    default_iot_volume = 100
+    if "iot" in conn.config:
+        default_iot_volume = conn.config["iot"]["Speaker"]["volume"]
+    logger.bind(tag=TAG).info(f"服务端设置音量为{default_iot_volume}")
+    await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": default_iot_volume})
+async def handleIotStatus(conn, states):
+    """
+    处理物联网状态
+    示例: [{
+        "name":"Speaker",
+        "state":{
+            "volume":100
+        }
+    }]
+    states: 状态列表
+    """
+    for state in states:
+        for key, value in conn.iot_descriptors.items():
+            if key == state["name"]:
+                for property_item in value.properties:
+                    # properties为字典列表, 记录各种属性
+                    for k, v in state["state"].items():
+                        # state为字典, 记录各种属性的值, 是需要记录的信息
+                        if property_item["name"] == k:
+                            # 检查一下属性是不是相同的
+                            if type(v) != type(property_item["value"]):
+                                logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
+                                break
+                            else:
+                                property_item["value"] = v
+                                logger.bind(tag=TAG).info(f"物联网状态更新: {key} , {property_item['name']} = {v}")
+                            break
+                break
+
+async def get_iot_status(conn, name, property_name):
+    """
+    获取物联网状态
+    name: 设备名称 "Speaker"
+    property_name: 属性名称 "volume"
+    返回值: 属性值, 实际的属性有int, bool和str三种类型
+    """
+    for key, value in conn.iot_descriptors.items():
+        if key == name:
+            for property_item in value.properties:
+                if property_item["name"] == property_name:
+                    return property_item["value"]
+    return None
+
+async def send_iot_conn(conn, name, method_name, parameters):
+    """
+    发送物联网指令
+    name: 设备名称 "Speaker"
+    method: 方法 "SetVolume"
+    parameters: 参数, 是一个字典 {"volume": 100}
+    发送示例:
+    {
+        "type": "iot",
+        "commands": [
+            {
+                "name" :  "Speaker",
+                "method": "SetVolume",
+                "parameters": {
+                    "volume": 100
+                    }
+            }
+        ]
+    }
+    """
+
+    for key, value in conn.iot_descriptors.items():
+        if key == name:
+            # 找到了设备
+            for method in value.methods:
+                # 找到了方法
+                if method["name"] == method_name:
+                    await conn.websocket.send(json.dumps({
+                        "type": "iot",
+                        "commands": [
+                            {
+                                "name": name,
+                                "method": method_name,
+                                "parameters": parameters
+                            }
+                        ]
+                    }))
+                    return
+    logger.bind(tag=TAG).error(f"未找到方法{method_name}")

main/xiaozhi-server/core/handle/musicHandler.py

@@ -0,0 +1,139 @@
+from config.logger import setup_logging
+import os
+import random
+import difflib
+import re
+import traceback
+from pathlib import Path
+import time
+from core.handle.sendAudioHandle import send_stt_message
+from core.utils import p3
+
+TAG = __name__
+logger = setup_logging()
+
+
+def _extract_song_name(text):
+    """从用户输入中提取歌名"""
+    for keyword in ["播放音乐"]:
+        if keyword in text:
+            parts = text.split(keyword)
+            if len(parts) > 1:
+                return parts[1].strip()
+    return None
+
+
+def _find_best_match(potential_song, music_files):
+    """查找最匹配的歌曲"""
+    best_match = None
+    highest_ratio = 0
+
+    for music_file in music_files:
+        song_name = os.path.splitext(music_file)[0]
+        ratio = difflib.SequenceMatcher(None, potential_song, song_name).ratio()
+        if ratio > highest_ratio and ratio > 0.4:
+            highest_ratio = ratio
+            best_match = music_file
+    return best_match
+
+
+class MusicManager:
+    def __init__(self, music_dir, music_ext):
+        self.music_dir = Path(music_dir)
+        self.music_ext = music_ext
+
+    def get_music_files(self):
+        music_files = []
+        for file in self.music_dir.rglob("*"):
+            # 判断是否是文件
+            if file.is_file():
+                # 获取文件扩展名
+                ext = file.suffix.lower()
+                # 判断扩展名是否在列表中
+                if ext in self.music_ext:
+                    # music_files.append(str(file.resolve()))  # 添加绝对路径
+                    # 添加相对路径
+                    music_files.append(str(file.relative_to(self.music_dir)))
+        return music_files
+
+
+class MusicHandler:
+    def __init__(self, config):
+        self.config = config
+
+        if "music" in self.config:
+            self.music_config = self.config["music"]
+            self.music_dir = os.path.abspath(
+                self.music_config.get("music_dir", "./music")  # 默认路径修改
+            )
+            self.music_ext = self.music_config.get("music_ext", (".mp3", ".wav", ".p3"))
+            self.refresh_time = self.music_config.get("refresh_time", 60)
+        else:
+            self.music_dir = os.path.abspath("./music")
+            self.music_ext = (".mp3", ".wav", ".p3")
+            self.refresh_time = 60
+
+        # 获取音乐文件列表
+        self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
+        self.scan_time = time.time()
+        logger.bind(tag=TAG).debug(f"找到的音乐文件: {self.music_files}")
+
+    async def handle_music_command(self, conn, text):
+        """处理音乐播放指令"""
+        clean_text = re.sub(r'[^\w\s]', '', text).strip()
+        logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
+
+        # 尝试匹配具体歌名
+        if os.path.exists(self.music_dir):
+            if time.time() - self.scan_time > self.refresh_time:
+                # 刷新音乐文件列表
+                self.music_files = MusicManager(self.music_dir, self.music_ext).get_music_files()
+                self.scan_time = time.time()
+                logger.bind(tag=TAG).debug(f"刷新的音乐文件: {self.music_files}")
+
+            potential_song = _extract_song_name(clean_text)
+            if potential_song:
+                best_match = _find_best_match(potential_song, self.music_files)
+                if best_match:
+                    logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
+                    await self.play_local_music(conn, specific_file=best_match)
+                    return True
+        # 检查是否是通用播放音乐命令
+        await self.play_local_music(conn)
+        return True
+
+    async def play_local_music(self, conn, specific_file=None):
+        """播放本地音乐文件"""
+        try:
+            if not os.path.exists(self.music_dir):
+                logger.bind(tag=TAG).error(f"音乐目录不存在: {self.music_dir}")
+                return
+
+            # 确保路径正确性
+            if specific_file:
+                selected_music = specific_file
+                music_path = os.path.join(self.music_dir, specific_file)
+            else:
+                if not self.music_files:
+                    logger.bind(tag=TAG).error("未找到MP3音乐文件")
+                    return
+                selected_music = random.choice(self.music_files)
+                music_path = os.path.join(self.music_dir, selected_music)
+
+            if not os.path.exists(music_path):
+                logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
+                return
+            text = f"正在播放{selected_music}"
+            await send_stt_message(conn, text)
+            conn.tts_first_text_index = 0
+            conn.tts_last_text_index = 0
+            conn.llm_finish_task = True
+            if music_path.endswith(".p3"):
+                opus_packets, duration = p3.decode_opus_from_file(music_path)
+            else:
+                opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
+            conn.audio_play_queue.put((opus_packets, selected_music, 0))
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
+            logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -0,0 +1,74 @@
+from config.logger import setup_logging
+import time
+from core.utils.util import remove_punctuation_and_length
+from core.handle.sendAudioHandle import send_stt_message
+from core.handle.intentHandler import handle_user_intent
+
+TAG = __name__
+logger = setup_logging()
+
+
+async def handleAudioMessage(conn, audio):
+    if not conn.asr_server_receive:
+        logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
+        return
+    if conn.client_listen_mode == "auto":
+        have_voice = conn.vad.is_vad(conn, audio)
+    else:
+        have_voice = conn.client_have_voice
+
+    # 如果本次没有声音，本段也没声音，就把声音丢弃了
+    if have_voice == False and conn.client_have_voice == False:
+        await no_voice_close_connect(conn)
+        conn.asr_audio.clear()
+        return
+    conn.client_no_voice_last_time = 0.0
+    conn.asr_audio.append(audio)
+    # 如果本段有声音，且已经停止了
+    if conn.client_voice_stop:
+        conn.client_abort = False
+        conn.asr_server_receive = False
+        # 音频太短了，无法识别
+        if len(conn.asr_audio) < 3:
+            conn.asr_server_receive = True
+        else:
+            text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
+            logger.bind(tag=TAG).info(f"识别文本: {text}")
+            text_len, _ = remove_punctuation_and_length(text)
+            if text_len > 0:
+                await startToChat(conn, text)
+            else:
+                conn.asr_server_receive = True
+        conn.asr_audio.clear()
+        conn.reset_vad_states()
+
+
+async def startToChat(conn, text):
+    # 首先进行意图分析
+    intent_handled = await handle_user_intent(conn, text)
+    
+    if intent_handled:
+        # 如果意图已被处理，不再进行聊天
+        conn.asr_server_receive = True
+        return
+    
+    # 意图未被处理，继续常规聊天流程
+    await send_stt_message(conn, text)
+    if conn.use_function_call_mode:
+        # 使用支持function calling的聊天方法
+        conn.executor.submit(conn.chat_with_function_calling, text)
+    else:
+        conn.executor.submit(conn.chat, text)
+
+
+async def no_voice_close_connect(conn):
+    if conn.client_no_voice_last_time == 0.0:
+        conn.client_no_voice_last_time = time.time() * 1000
+    else:
+        no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
+        close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
+        if no_voice_time > 1000 * close_connection_no_voice_time:
+            conn.client_abort = False
+            conn.asr_server_receive = False
+            prompt = "时间过得真快，我都好久没说话了。请你用十个字左右话跟我告别，以“再见”或“拜拜”为结尾"
+            await startToChat(conn, prompt)

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -0,0 +1,136 @@
+import traceback
+
+from config.logger import setup_logging
+import json
+import asyncio
+import time
+from core.utils.util import remove_punctuation_and_length, get_string_no_punctuation_or_emoji
+
+TAG = __name__
+logger = setup_logging()
+
+async def sendAudioMessageStream(conn, audios_queue, text, text_index=0):
+    # 发送句子开始消息
+    if text_index == conn.tts_first_text_index:
+        logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
+    await send_tts_message(conn, "sentence_start", text)
+
+    # 初始化流控参数
+    frame_duration = 60  # 毫秒
+    start_time = time.time()  # 使用高精度计时器
+    play_position = 0  # 已播放的时长（毫秒）
+    time_out_stop = False
+    while True:
+        try:
+            start_get_queue = time.time()
+            # 尝试获取数据，如果没有数据，则等待一小段时间再试
+            audio_data_chunke = None
+            try:
+                audio_data_chunke = audios_queue.get(timeout=5)  # 设置超时为1秒
+            except Exception as e:
+                # 如果超时，继续等待
+                logger.bind(tag=TAG).error(f"获取队列超时～{e}")
+
+            audio_opus_datas = audio_data_chunke.get('data') if audio_data_chunke else None
+            duration = audio_data_chunke.get('duration') if audio_data_chunke else 0
+
+            if audio_data_chunke:
+                start_time = time.time()
+            # 检查是否超过 5 秒没有数据
+            if time.time() - start_time > 15:
+                logger.bind(tag=TAG).error("超过15秒没有数据，退出。")
+                break
+
+            if audio_data_chunke and audio_data_chunke.get("end", True):
+                break
+
+            if audio_opus_datas:
+                queue_duration = time.time() - start_get_queue
+                last_duration = conn.tts_duration - queue_duration
+                if last_duration <= 0:
+                    last_duration = 0
+                conn.tts_duration = duration + last_duration
+                for opus_packet in audio_opus_datas:
+                    await conn.websocket.send(opus_packet)
+                start_time = time.time()  # 更新获取数据的时间
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"发生错误: {e}")
+            traceback.print_exc()  # 打印错误堆栈
+    await send_tts_message(conn, "sentence_end", text)
+
+    print(f'{text_index}-{conn.tts_last_text_index}')
+    # 发送结束消息（如果是最后一个文本）
+    if conn.llm_finish_task and text_index == conn.tts_last_text_index:
+        if conn.tts_duration and conn.tts_duration > 0:
+            await asyncio.sleep(conn.tts_duration)
+        await send_tts_message(conn, 'stop', None)
+        if await isLLMWantToFinish(text):
+            await conn.close()
+
+
+
+async def sendAudioMessage(conn, audios, text, text_index=0):
+    # 发送句子开始消息
+    if text_index == conn.tts_first_text_index:
+        logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
+    await send_tts_message(conn, "sentence_start", text)
+
+    # 初始化流控参数
+    frame_duration = 60  # 毫秒
+    start_time = time.perf_counter()  # 使用高精度计时器
+    play_position = 0  # 已播放的时长（毫秒）
+
+    for opus_packet in audios:
+        if conn.client_abort:
+            return
+
+        # 计算当前包的预期发送时间
+        expected_time = start_time + (play_position / 1000)
+        current_time = time.perf_counter()
+
+        # 等待直到预期时间
+        delay = expected_time - current_time
+        if delay > 0:
+            await asyncio.sleep(delay)
+
+        # 发送音频包
+        await conn.websocket.send(opus_packet)
+        play_position += frame_duration  # 更新播放位置
+    await send_tts_message(conn, "sentence_end", text)
+    # 发送结束消息（如果是最后一个文本）
+    if conn.llm_finish_task and text_index == conn.tts_last_text_index:
+        await send_tts_message(conn, 'stop', None)
+        if conn.close_after_chat:
+            await conn.close()
+
+async def send_tts_message(conn, state, text=None):
+    """发送 TTS 状态消息"""
+    message = {
+        "type": "tts",
+        "state": state,
+        "session_id": conn.session_id
+    }
+    if text is not None:
+        message["text"] = text
+
+    await conn.websocket.send(json.dumps(message))
+    if state == "stop":
+        conn.clearSpeakStatus()
+
+
+async def send_stt_message(conn, text):
+    """发送 STT 状态消息"""
+    stt_text = get_string_no_punctuation_or_emoji(text)
+    await conn.websocket.send(json.dumps({
+        "type": "stt",
+        "text": stt_text,
+        "session_id": conn.session_id}
+    ))
+    await conn.websocket.send(
+        json.dumps({
+            "type": "llm",
+            "text": "😊",
+            "emotion": "happy",
+            "session_id": conn.session_id}
+        ))
+    await send_tts_message(conn, "start")

main/xiaozhi-server/core/handle/textHandle.py

@@ -0,0 +1,46 @@
+from config.logger import setup_logging
+import json
+from core.handle.abortHandle import handleAbortMessage
+from core.handle.helloHandle import handleHelloMessage
+from core.handle.receiveAudioHandle import startToChat
+from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
+
+TAG = __name__
+logger = setup_logging()
+
+
+async def handleTextMessage(conn, message):
+    """处理文本消息"""
+    logger.bind(tag=TAG).info(f"收到文本消息：{message}")
+    try:
+        msg_json = json.loads(message)
+        if isinstance(msg_json, int):
+            await conn.websocket.send(message)
+            return
+        if msg_json["type"] == "hello":
+            await handleHelloMessage(conn)
+        elif msg_json["type"] == "abort":
+            await handleAbortMessage(conn)
+        elif msg_json["type"] == "listen":
+            if "mode" in msg_json:
+                conn.client_listen_mode = msg_json["mode"]
+                logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}")
+            if msg_json["state"] == "start":
+                conn.client_have_voice = True
+                conn.client_voice_stop = False
+            elif msg_json["state"] == "stop":
+                conn.client_have_voice = True
+                conn.client_voice_stop = True
+            elif msg_json["state"] == "detect":
+                conn.asr_server_receive = False
+                conn.client_have_voice = False
+                conn.asr_audio.clear()
+                if "text" in msg_json:
+                    await startToChat(conn, msg_json["text"])
+        elif msg_json["type"] == "iot":
+            if "descriptors" in msg_json:
+                await handleIotDescriptors(conn, msg_json["descriptors"])
+            if "states" in msg_json:
+                await handleIotStatus(conn, msg_json["states"])  
+    except json.JSONDecodeError:
+        await conn.websocket.send(message)

main/xiaozhi-server/core/providers/asr/base.py

@@ -0,0 +1,19 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Tuple, List
+
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProviderBase(ABC):
+    @abstractmethod
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """解码Opus数据并保存为WAV文件"""
+        pass
+
+    @abstractmethod
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        pass

main/xiaozhi-server/core/providers/asr/doubao.py

@@ -0,0 +1,286 @@
+import time
+import io
+import wave
+import os
+from typing import Optional, Tuple, List
+import uuid
+import websockets
+import json
+import gzip
+
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+CLIENT_FULL_REQUEST = 0b0001
+CLIENT_AUDIO_ONLY_REQUEST = 0b0010
+
+NO_SEQUENCE = 0b0000
+NEG_SEQUENCE = 0b0010
+
+SERVER_FULL_RESPONSE = 0b1001
+SERVER_ACK = 0b1011
+SERVER_ERROR_RESPONSE = 0b1111
+
+NO_SERIALIZATION = 0b0000
+JSON = 0b0001
+THRIFT = 0b0011
+CUSTOM_TYPE = 0b1111
+NO_COMPRESSION = 0b0000
+GZIP = 0b0001
+CUSTOM_COMPRESSION = 0b1111
+
+
+def parse_response(res):
+    """
+    protocol_version(4 bits), header_size(4 bits),
+    message_type(4 bits), message_type_specific_flags(4 bits)
+    serialization_method(4 bits) message_compression(4 bits)
+    reserved （8bits) 保留字段
+    header_extensions 扩展头(大小等于 8 * 4 * (header_size - 1) )
+    payload 类似与http 请求体
+    """
+    protocol_version = res[0] >> 4
+    header_size = res[0] & 0x0f
+    message_type = res[1] >> 4
+    message_type_specific_flags = res[1] & 0x0f
+    serialization_method = res[2] >> 4
+    message_compression = res[2] & 0x0f
+    reserved = res[3]
+    header_extensions = res[4:header_size * 4]
+    payload = res[header_size * 4:]
+    result = {}
+    payload_msg = None
+    payload_size = 0
+    if message_type == SERVER_FULL_RESPONSE:
+        payload_size = int.from_bytes(payload[:4], "big", signed=True)
+        payload_msg = payload[4:]
+    elif message_type == SERVER_ACK:
+        seq = int.from_bytes(payload[:4], "big", signed=True)
+        result['seq'] = seq
+        if len(payload) >= 8:
+            payload_size = int.from_bytes(payload[4:8], "big", signed=False)
+            payload_msg = payload[8:]
+    elif message_type == SERVER_ERROR_RESPONSE:
+        code = int.from_bytes(payload[:4], "big", signed=False)
+        result['code'] = code
+        payload_size = int.from_bytes(payload[4:8], "big", signed=False)
+        payload_msg = payload[8:]
+    if payload_msg is None:
+        return result
+    if message_compression == GZIP:
+        payload_msg = gzip.decompress(payload_msg)
+    if serialization_method == JSON:
+        payload_msg = json.loads(str(payload_msg, "utf-8"))
+    elif serialization_method != NO_SERIALIZATION:
+        payload_msg = str(payload_msg, "utf-8")
+    result['payload_msg'] = payload_msg
+    result['payload_size'] = payload_size
+    return result
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        self.appid = config.get("appid")
+        self.cluster = config.get("cluster")
+        self.access_token = config.get("access_token")
+        self.output_dir = config.get("output_dir")
+
+        self.host = "openspeech.bytedance.com"
+        self.ws_url = f"wss://{self.host}/api/v2/asr"
+        self.success_code = 1000
+        self.seg_duration = 15000
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """将Opus音频数据解码并保存为WAV文件"""
+        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    @staticmethod
+    def _generate_header(message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE) -> bytearray:
+        """Generate protocol header."""
+        header = bytearray()
+        header_size = 1
+        header.append((0b0001 << 4) | header_size)  # Protocol version
+        header.append((message_type << 4) | message_type_specific_flags)
+        header.append((0b0001 << 4) | 0b0001)  # JSON serialization & GZIP compression
+        header.append(0x00)  # reserved
+        return header
+
+    def _construct_request(self, reqid) -> dict:
+        """Construct the request payload."""
+        return {
+            "app": {
+                "appid": f"{self.appid}",
+                "cluster": self.cluster,
+                "token": self.access_token,
+            },
+            "user": {
+                "uid": str(uuid.uuid4()),
+            },
+            "request": {
+                "reqid": reqid,
+                "show_utterances": False,
+                "sequence": 1
+            },
+            "audio": {
+                "format": "wav",
+                "rate": 16000,
+                "language": "zh-CN",
+                "bits": 16,
+                "channel": 1,
+                "codec": "raw",
+            },
+        }
+
+    async def _send_request(self, audio_data: List[bytes], segment_size: int) -> Optional[str]:
+        """Send request to Volcano ASR service."""
+        try:
+            auth_header = {'Authorization': 'Bearer; {}'.format(self.access_token)}
+            async with websockets.connect(self.ws_url, additional_headers=auth_header) as websocket:
+                # Prepare request data
+                request_params = self._construct_request(str(uuid.uuid4()))
+                print(request_params)
+                payload_bytes = str.encode(json.dumps(request_params))
+                payload_bytes = gzip.compress(payload_bytes)
+                full_client_request = self._generate_header()
+                full_client_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
+                full_client_request.extend(payload_bytes)  # payload
+
+                # Send header and metadata
+                # full_client_request
+                await websocket.send(full_client_request)
+                res = await websocket.recv()
+                result = parse_response(res)
+                if 'payload_msg' in result and result['payload_msg']['code'] != self.success_code:
+                    logger.bind(tag=TAG).error(f"ASR error: {result}")
+                    return None
+
+                for seq, (chunk, last) in enumerate(self.slice_data(audio_data, segment_size), 1):
+                    if last:
+                        audio_only_request = self._generate_header(
+                            message_type=CLIENT_AUDIO_ONLY_REQUEST,
+                            message_type_specific_flags=NEG_SEQUENCE
+                        )
+                    else:
+                        audio_only_request = self._generate_header(
+                            message_type=CLIENT_AUDIO_ONLY_REQUEST
+                        )
+                    payload_bytes = gzip.compress(chunk)
+                    audio_only_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
+                    audio_only_request.extend(payload_bytes)  # payload
+                    # Send audio data
+                    await websocket.send(audio_only_request)
+
+                # Receive response
+                response = await websocket.recv()
+                result = parse_response(response)
+
+                if 'payload_msg' in result and result['payload_msg']['code'] == self.success_code:
+                    if len(result['payload_msg']['result']) > 0:
+                        return result['payload_msg']['result'][0]["text"]
+                    return None
+                else:
+                    logger.bind(tag=TAG).error(f"ASR error: {result}")
+                    return None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
+            return None
+
+    @staticmethod
+    def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        return pcm_data
+
+    @staticmethod
+    def read_wav_info(data: io.BytesIO = None) -> (int, int, int, int, int):
+        with io.BytesIO(data) as _f:
+            wave_fp = wave.open(_f, 'rb')
+            nchannels, sampwidth, framerate, nframes = wave_fp.getparams()[:4]
+            wave_bytes = wave_fp.readframes(nframes)
+        return nchannels, sampwidth, framerate, nframes, len(wave_bytes)
+
+    @staticmethod
+    def slice_data(data: bytes, chunk_size: int) -> (list, bool):
+        """
+        slice data
+        :param data: wav data
+        :param chunk_size: the segment size in one request
+        :return: segment data, last flag
+        """
+        data_len = len(data)
+        offset = 0
+        while offset + chunk_size < data_len:
+            yield data[offset: offset + chunk_size], False
+            offset += chunk_size
+        else:
+            yield data[offset: data_len], True
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """将语音数据转换为文本"""
+        try:
+            # 合并所有opus数据包
+            pcm_data = self.decode_opus(opus_data, session_id)
+            combined_pcm_data = b''.join(pcm_data)
+
+            wav_buffer = io.BytesIO()
+
+            with wave.open(wav_buffer, "wb") as wav_file:
+                wav_file.setnchannels(1)  # 设置声道数
+                wav_file.setsampwidth(2)  # 设置采样宽度
+                wav_file.setframerate(16000)  # 设置采样率
+                wav_file.writeframes(combined_pcm_data)  # 写入 PCM 数据
+
+            # 获取封装后的 WAV 数据
+            wav_data = wav_buffer.getvalue()
+            nchannels, sampwidth, framerate, nframes, wav_len = self.read_wav_info(wav_data)
+            size_per_sec = nchannels * sampwidth * framerate
+            segment_size = int(size_per_sec * self.seg_duration / 1000)
+
+            # 语音识别
+            start_time = time.time()
+            text = await self._send_request(wav_data, segment_size)
+            if text:
+                logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+                return text, None
+            return "", None
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -0,0 +1,110 @@
+import time
+import wave
+import os
+import sys
+import io
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+import uuid
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+
+from funasr import AutoModel
+from funasr.utils.postprocess_utils import rich_transcription_postprocess
+
+TAG = __name__
+logger = setup_logging()
+
+
+# 捕获标准输出
+class CaptureOutput:
+    def __enter__(self):
+        self._output = io.StringIO()
+        self._original_stdout = sys.stdout
+        sys.stdout = self._output
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        sys.stdout = self._original_stdout
+        self.output = self._output.getvalue()
+        self._output.close()
+
+        # 将捕获到的内容通过 logger 输出
+        if self.output:
+            logger.bind(tag=TAG).info(self.output.strip())
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        self.model_dir = config.get("model_dir")
+        self.output_dir = config.get("output_dir")  # 修正配置键名
+        self.delete_audio_file = delete_audio_file
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+        with CaptureOutput():
+            self.model = AutoModel(
+                model=self.model_dir,
+                vad_kwargs={"max_single_segment_time": 30000},
+                disable_update=True,
+                hub="hf"
+                # device="cuda:0",  # 启用GPU加速
+            )
+
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """将Opus音频数据解码并保存为WAV文件"""
+        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """语音转文本主处理逻辑"""
+        file_path = None
+        try:
+            # 保存音频文件
+            start_time = time.time()
+            file_path = self.save_audio_to_file(opus_data, session_id)
+            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
+
+            # 语音识别
+            start_time = time.time()
+            result = self.model.generate(
+                input=file_path,
+                cache={},
+                language="auto",
+                use_itn=True,
+                batch_size_s=60,
+            )
+            text = rich_transcription_postprocess(result[0]["text"])
+            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+
+            return text, file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None
+
+        finally:
+            # 文件清理逻辑
+            if self.delete_audio_file and file_path and os.path.exists(file_path):
+                try:
+                    os.remove(file_path)
+                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/intent/base.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class IntentProviderBase(ABC):
+    def __init__(self, config):
+        self.config = config
+        self.intent_options = config.get("intent_options", {
+            "continue_chat": "继续聊天",
+            "end_chat": "结束聊天",
+            "play_music": "播放音乐"
+        })
+
+    def set_llm(self, llm):
+        self.llm = llm
+        logger.bind(tag=TAG).debug("Set LLM for intent provider")
+        
+    @abstractmethod
+    async def detect_intent(self, dialogue_history: List[Dict]) -> str:
+        """
+        检测用户最后一句话的意图
+        Args:
+            dialogue_history: 对话历史记录列表，每条记录包含role和content
+        Returns:
+            返回识别出的意图，格式为:
+            - "继续聊天"
+            - "结束聊天" 
+            - "播放音乐 歌名" 或 "随机播放音乐"
+        """
+        pass

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -0,0 +1,61 @@
+from typing import List, Dict
+from ..base import IntentProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+
+class IntentProvider(IntentProviderBase):
+    def __init__(self, config):
+        super().__init__(config)
+        self.llm = None
+        self.promot = self.get_intent_system_prompt()
+
+    def get_intent_system_prompt(self) -> str:
+        """
+        根据配置的意图选项动态生成系统提示词
+        Returns:
+            格式化后的系统提示词
+        """
+        intent_list = []
+        for key, value in self.intent_options.items():
+            if key == "play_music":
+                intent_list.append(f"{value} [歌名]")
+            else:
+                intent_list.append(value)
+                
+        prompt = (
+            "你是一个意图识别助手。你需要根据和用户的对话记录，重点分析用户的最后一句话，判断用户意图属于以下哪一类：\n"
+            f"{', '.join(intent_list)}\n"
+            "如果是唱歌、听歌、播放音乐，请指定歌名，格式为'播放音乐 [识别出的歌名]'。\n"
+            "如果听不出具体歌名，可以返回'随机播放音乐'。\n"
+            "只需要返回意图结果的json，不要解释。"
+            "返回格式如下：\n"
+            "{intent: '用户意图'}"
+        )
+        return prompt
+    
+    async def detect_intent(self, dialogue_history: List[Dict]) -> str:
+        if not self.llm:
+            raise ValueError("LLM provider not set")
+
+        # 构建用户最后一句话的提示
+        msgStr = ""
+        for msg in dialogue_history:
+            if msg.role == "user":
+                msgStr += f"User: {msg.content}\n"
+            elif msg.role== "assistant":
+                msgStr += f"Assistant: {msg.content}\n"
+
+        user_prompt = f"请分析用户的意图：\n{msgStr}"
+        
+        # 使用LLM进行意图识别
+        intent = self.llm.response_no_stream(
+            system_prompt=self.promot,
+            user_prompt=user_prompt
+        )
+  
+        logger.bind(tag=TAG).info(f"Detected intent: {intent}")
+        return intent.strip()

main/xiaozhi-server/core/providers/intent/nointent/nointent.py

@@ -0,0 +1,18 @@
+from ..base import IntentProviderBase
+from typing import List, Dict
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class IntentProvider(IntentProviderBase):
+    async def detect_intent(self, dialogue_history: List[Dict]) -> str:
+        """
+        默认的意图识别实现，始终返回继续聊天
+        Args:
+            dialogue_history: 对话历史记录列表
+        Returns:
+            固定返回"继续聊天"
+        """
+        logger.bind(tag=TAG).debug("Using NoIntentProvider, always returning continue chat")
+        return self.intent_options["continue_chat"]

main/xiaozhi-server/core/providers/llm/base.py

@@ -0,0 +1,38 @@
+from abc import ABC, abstractmethod
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class LLMProviderBase(ABC):
+    @abstractmethod
+    def response(self, session_id, dialogue):
+        """LLM response generator"""
+        pass
+
+    def response_no_stream(self, system_prompt, user_prompt):
+        try:
+            # 构造对话格式
+            dialogue = [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt}
+            ]
+            result = ""
+            for part in self.response("", dialogue):
+                result += part
+            return result
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in Ollama response generation: {e}")
+            return "【LLM服务响应异常】"
+    
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        """
+        Default implementation for function calling (streaming)
+        This should be overridden by providers that support function calls
+        
+        Returns: generator that yields either text tokens or a special function call token
+        """
+        # For providers that don't support functions, just return regular response
+        for token in self.response(session_id, dialogue):
+            yield {"type": "content", "content": token}

main/xiaozhi-server/core/providers/llm/coze/coze.py

@@ -0,0 +1,37 @@
+from config.logger import setup_logging
+import requests
+import json
+import re
+from core.providers.llm.base import LLMProviderBase
+import os
+# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
+from cozepy import COZE_CN_BASE_URL
+from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType  # noqa
+
+TAG = __name__
+logger = setup_logging()
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.personal_access_token = config.get("personal_access_token")
+        self.bot_id = config.get("bot_id")
+        self.user_id = config.get("user_id")
+
+    def response(self, session_id, dialogue):
+        coze_api_token = self.personal_access_token
+        coze_api_base = COZE_CN_BASE_URL
+
+        last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+
+        coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
+
+        for event in coze.chat.stream(
+            bot_id=self.bot_id,
+            user_id=self.user_id,
+            additional_messages=[
+                Message.build_user_question_text(last_msg["content"]),
+            ],
+        ):
+            if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
+                print(event.message.content, end="", flush=True)
+                yield event.message.content

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -0,0 +1,39 @@
+import json
+from config.logger import setup_logging
+import requests
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.api_key = config["api_key"]
+        self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
+
+    def response(self, session_id, dialogue):
+        try:
+            # 取最后一条用户消息
+            last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+
+            # 发起流式请求
+            with requests.post(
+                    f"{self.base_url}/chat-messages",
+                    headers={"Authorization": f"Bearer {self.api_key}"},
+                    json={
+                        "query": last_msg["content"],
+                        "response_mode": "streaming",
+                        "user": session_id,
+                        "inputs": {}
+                    },
+                    stream=True
+            ) as r:
+                for line in r.iter_lines():
+                    if line.startswith(b'data: '):
+                        event = json.loads(line[6:])
+                        if event.get('answer'):
+                            yield event['answer']
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
+            yield "【服务响应异常】"

main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py

@@ -0,0 +1,65 @@
+import json
+from config.logger import setup_logging
+import requests
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.api_key = config["api_key"]
+        self.base_url = config.get("base_url")
+        self.detail = config.get("detail", False)
+        self.variables = config.get("variables", {})
+
+    def response(self, session_id, dialogue):
+        try:
+            # 取最后一条用户消息
+            last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+
+            # 发起流式请求
+            with requests.post(
+                    f"{self.base_url}/chat/completions",
+                    headers={"Authorization": f"Bearer {self.api_key}"},
+                    json={
+                        "stream": True,
+                        "chatId": session_id,
+                        "detail": self.detail,
+                        "variables": self.variables,
+                        "messages": [
+                            {
+                                "role": "user",
+                                "content": last_msg["content"]
+                            }
+                        ]
+                    },
+                    stream=True
+            ) as r:
+                for line in r.iter_lines():
+                    if line:
+                        try:
+                            if line.startswith(b'data: '):
+                                if line[6:].decode('utf-8') == '[DONE]':
+                                    break
+
+                                data = json.loads(line[6:])
+                                if 'choices' in data and len(data['choices']) > 0:
+                                    delta = data['choices'][0].get('delta', {})
+                                    if delta and 'content' in delta and delta['content'] is not None:
+                                        content = delta['content']
+                                        if '<think>' in content:
+                                            continue
+                                        if '</think>' in content:
+                                            continue
+                                        yield content
+
+                        except json.JSONDecodeError as e:
+                            continue
+                        except Exception as e:
+                            continue
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
+            yield "【服务响应异常】"

main/xiaozhi-server/core/providers/llm/gemini/gemini.py

@@ -0,0 +1,80 @@
+import google.generativeai as genai
+from core.utils.util import check_model_key
+from core.providers.llm.base import LLMProviderBase
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        """初始化Gemini LLM Provider"""
+        self.model_name = config.get("model_name", "gemini-1.5-pro")
+        self.api_key = config.get("api_key")
+
+        have_key = check_model_key("LLM", self.api_key)
+
+        if not have_key:
+            return
+
+        try:
+            # 初始化Gemini客户端
+            genai.configure(api_key=self.api_key)
+            self.model = genai.GenerativeModel(self.model_name)
+
+            # 设置生成参数
+            self.generation_config = {
+                "temperature": 0.7,
+                "top_p": 0.9,
+                "top_k": 40,
+                "max_output_tokens": 2048,
+            }
+            self.chat = None
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Gemini初始化失败: {e}")
+            self.model = None
+
+    def response(self, session_id, dialogue):
+        """生成Gemini对话响应"""
+        if not self.model:
+            yield "【Gemini服务未正确初始化】"
+            return
+
+        try:
+            # 处理对话历史
+            chat_history = []
+            for msg in dialogue[:-1]:  # 历史对话
+                role = "model" if msg["role"] == "assistant" else "user"
+                content = msg["content"].strip()
+                if content:
+                    chat_history.append({
+                        "role": role,
+                        "parts": [content]
+                    })
+
+            # 获取当前消息
+            current_msg = dialogue[-1]["content"]
+
+            # 创建新的聊天会话
+            chat = self.model.start_chat(history=chat_history)
+
+            # 发送消息并获取流式响应
+            response = chat.send_message(
+                current_msg,
+                stream=True,
+                generation_config=self.generation_config
+            )
+
+            # 处理流式响应
+            for chunk in response:
+                if hasattr(chunk, 'text') and chunk.text:
+                    yield chunk.text
+
+        except Exception as e:
+            error_msg = str(e)
+            logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}")
+
+            # 针对不同错误返回友好提示
+            if "Rate limit" in error_msg:
+                yield "【Gemini服务请求太频繁,请稍后再试】"
+            elif "Invalid API key" in error_msg:
+                yield "【Gemini API key无效】"
+            else:
+                yield f"【Gemini服务响应异常: {error_msg}】"

main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py

@@ -0,0 +1,62 @@
+import requests
+from requests.exceptions import RequestException
+from config.logger import setup_logging
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.agent_id = config.get("agent_id")  # 对应 agent_id
+        self.api_key = config.get("api_key")
+        self.base_url = config.get("base_url", config.get("url"))  # 默认使用 base_url
+        self.api_url = f"{self.base_url}/api/conversation/process"  # 拼接完整的 API URL
+
+    def response(self, session_id, dialogue):
+        print(dialogue)
+        try:
+            # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
+
+            # 提取最后一个 role 为 'user' 的 content
+            input_text = None
+            if isinstance(dialogue, list):  # 确保 dialogue 是一个列表
+                # 逆序遍历，找到最后一个 role 为 'user' 的消息
+                for message in reversed(dialogue):
+                    if message.get("role") == "user":  # 找到 role 为 'user' 的消息
+                        input_text = message.get("content", "")
+                        break  # 找到后立即退出循环
+
+            # 构造请求数据
+            payload = {
+                "text": input_text,
+                "agent_id": self.agent_id,
+                "conversation_id": session_id  # 使用 session_id 作为 conversation_id
+            }
+            # 设置请求头
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json"
+            }
+
+            # 发起 POST 请求
+            response = requests.post(self.api_url, json=payload, headers=headers)
+
+            # 检查请求是否成功
+            response.raise_for_status()
+
+            # 解析返回数据
+            data = response.json()
+            speech = data.get("response", {}).get("speech", {}).get("plain", {}).get("speech", "")
+
+            # 返回生成的内容
+            if speech:
+                yield speech
+            else:
+                logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
+
+        except RequestException as e:
+            logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成响应时出错: {e}")

main/xiaozhi-server/core/providers/llm/ollama/ollama.py

@@ -0,0 +1,81 @@
+from config.logger import setup_logging
+from openai import OpenAI
+import json
+from core.providers.llm.base import LLMProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.model_name = config.get("model_name")
+        self.base_url = config.get("base_url", "http://localhost:11434")
+        # Initialize OpenAI client with Ollama base URL
+        #如果没有v1，增加v1
+        if not self.base_url.endswith("/v1"):
+            self.base_url = f"{self.base_url}/v1"
+            
+        self.client = OpenAI(
+            base_url=self.base_url,
+            api_key="ollama"  # Ollama doesn't need an API key but OpenAI client requires one
+        )
+
+    def response(self, session_id, dialogue):
+        try:
+            responses = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=dialogue,
+                stream=True
+            )
+            
+            for chunk in responses:
+                try:
+                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
+                    content = delta.content if hasattr(delta, 'content') else ''
+                    if content:
+                        yield content
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in Ollama response generation: {e}")
+            yield "【Ollama服务响应异常】"
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        try:
+            stream = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=dialogue,
+                stream=True,
+                tools=functions,
+            )
+            
+            current_function_call = None
+            current_content = ""
+            
+            for chunk in stream:
+                delta = chunk.choices[0].delta
+                
+                if delta.content:
+                    current_content += delta.content
+                    yield {"type": "content", "content": delta.content}
+                
+                if delta.tool_calls:
+                    tool_call = delta.tool_calls[0]
+                    # Handle the function call data using proper attribute access
+                    if not current_function_call:
+                        current_function_call = {
+                            "function": {
+                                "name": tool_call.function.name,
+                                "arguments": tool_call.function.arguments
+                            }
+                        }
+            
+            if current_function_call:
+                logger.bind(tag=TAG).debug(f"ollama Function call detected: {current_function_call}")
+                yield {"type": "function_call", "function_call": current_function_call}
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")
+            yield {"type": "content", "content": f"【Ollama服务响应异常: {str(e)}】"}

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -0,0 +1,83 @@
+import openai
+from core.utils.util import check_model_key
+from core.providers.llm.base import LLMProviderBase
+
+
+class LLMProvider(LLMProviderBase):
+    def __init__(self, config):
+        self.model_name = config.get("model_name")
+        self.api_key = config.get("api_key")
+        if 'base_url' in config:
+            self.base_url = config.get("base_url")
+        else:
+            self.base_url = config.get("url")
+        check_model_key("LLM", self.api_key)
+        self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
+
+    def response(self, session_id, dialogue):
+        try:
+            responses = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=dialogue,
+                stream=True
+            )
+            
+            is_active = True
+            for chunk in responses:
+                try:
+                    # 检查是否存在有效的choice且content不为空
+                    delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
+                    content = delta.content if hasattr(delta, 'content') else ''
+                except IndexError:
+                    content = ''
+                if content:
+                    # 处理标签跨多个chunk的情况
+                    if '<think>' in content:
+                        is_active = False
+                        content = content.split('<think>')[0]
+                    if '</think>' in content:
+                        is_active = True
+                        content = content.split('</think>')[-1]
+                    if is_active:
+                        yield content
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
+
+    def response_with_functions(self, session_id, dialogue, functions=None):
+        try:
+            stream = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=dialogue,
+                stream=True,
+                tools=functions,
+            )
+            
+            current_function_call = None
+            current_content = ""
+            
+            for chunk in stream:
+                delta = chunk.choices[0].delta
+                
+                if delta.content:
+                    current_content += delta.content
+                    yield {"type": "content", "content": delta.content}
+                
+                if delta.tool_calls:
+                    tool_call = delta.tool_calls[0]
+                    # Handle the function call data using proper attribute access
+                    if not current_function_call:
+                        current_function_call = {
+                            "function": {
+                                "name": tool_call.function.name,
+                                "arguments": tool_call.function.arguments
+                            }
+                        }
+            
+            if current_function_call:
+                logger.bind(tag=TAG).debug(f"openai Function call detected: {current_function_call}")
+                yield {"type": "function_call", "function_call": current_function_call}
+                
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
+            yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}

main/xiaozhi-server/core/providers/memory/base.py

@@ -0,0 +1,25 @@
+from abc import ABC, abstractmethod
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class MemoryProviderBase(ABC):
+    def __init__(self, config):
+        self.config = config
+        self.role_id = None
+        self.llm = None
+
+    @abstractmethod
+    async def save_memory(self, msgs):
+        """Save a new memory for specific role and return memory ID"""
+        print("this is base func", msgs)
+
+    @abstractmethod
+    async def query_memory(self, query: str) -> str:
+        """Query memories for specific role based on similarity"""
+        return "please implement query method"
+
+    def init_memory(self, role_id, llm):
+        self.role_id = role_id    
+        self.llm = llm

main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py

@@ -0,0 +1,83 @@
+import traceback
+
+from ..base import MemoryProviderBase, logger
+from mem0 import MemoryClient
+from core.utils.util import check_model_key
+
+TAG = __name__
+
+class MemoryProvider(MemoryProviderBase):
+    def __init__(self, config):
+        super().__init__(config)
+        self.api_key = config.get("api_key", "")
+        self.api_version = config.get("api_version", "v1.1")
+        have_key = check_model_key("Mem0ai", self.api_key)
+        if not have_key :
+            self.use_mem0 = False
+            return
+        else:
+            self.use_mem0 = True
+        try:
+            self.client = MemoryClient(api_key=self.api_key)
+            logger.bind(tag=TAG).info("成功连接到 Mem0ai 服务")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"连接到 Mem0ai 服务时发生错误: {str(e)}")
+            logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
+            self.use_mem0 = False
+
+    async def save_memory(self, msgs):
+        if not self.use_mem0:
+            return None
+        if len(msgs) < 2:
+            return None
+        
+        try:
+            # Format the content as a message list for mem0
+            messages = [
+                {"role": message.role, "content": message.content}
+                for message in msgs if message.role != "system"
+            ]
+            result = self.client.add(messages, user_id=self.role_id, output_format=self.api_version)
+            logger.bind(tag=TAG).debug(f"Save memory result: {result}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
+            return None
+
+    async def query_memory(self, query: str)-> str:
+        if not self.use_mem0:
+            return ""
+        try:
+            results = self.client.search(
+                query,
+                user_id=self.role_id,
+                output_format=self.api_version
+            )
+            if not results or 'results' not in results:
+                return ""
+                
+            # Format each memory entry with its update time up to minutes
+            memories = []
+            for entry in results['results']:
+                timestamp = entry.get('updated_at', '')
+                if timestamp:
+                    try:
+                        # Parse and reformat the timestamp
+                        dt = timestamp.split('.')[0]  # Remove milliseconds
+                        formatted_time = dt.replace('T', ' ')
+                    except:
+                        formatted_time = timestamp
+                memory = entry.get('memory', '')
+                if timestamp and memory:
+                    # Store tuple of (timestamp, formatted_string) for sorting
+                    memories.append((timestamp, f"[{formatted_time}] {memory}"))
+            
+            # Sort by timestamp in descending order (newest first)
+            memories.sort(key=lambda x: x[0], reverse=True)
+            
+            # Extract only the formatted strings
+            memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
+            logger.bind(tag=TAG).debug(f"Query results: {memories_str}")
+            return memories_str
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}")
+            return ""

main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py

@@ -0,0 +1,156 @@
+from ..base import MemoryProviderBase, logger
+import time
+import json
+import os
+import yaml
+from core.utils.util import get_project_dir
+
+short_term_memory_prompt = """
+# 时空记忆编织者
+
+## 核心使命
+构建可生长的动态记忆网络，在有限空间内保留关键信息的同时，智能维护信息演变轨迹
+根据对话记录，总结user的重要信息，以便在未来的对话中提供更个性化的服务
+
+## 记忆法则
+### 1. 三维度记忆评估（每次更新必执行）
+| 维度       | 评估标准                  | 权重分 |
+|------------|---------------------------|--------|
+| 时效性     | 信息新鲜度（按对话轮次） | 40%    |
+| 情感强度   | 含💖标记/重复提及次数     | 35%    |
+| 关联密度   | 与其他信息的连接数量      | 25%    |
+
+### 2. 动态更新机制
+**名字变更处理示例：**
+原始记忆："曾用名": ["张三"], "现用名": "张三丰"
+触发条件：当检测到「我叫X」「称呼我Y」等命名信号时
+操作流程：
+1. 将旧名移入"曾用名"列表
+2. 记录命名时间轴："2024-02-15 14:32:启用张三丰"
+3. 在记忆立方追加：「从张三到张三丰的身份蜕变」
+
+### 3. 空间优化策略
+- **信息压缩术**：用符号体系提升密度
+  - ✅"张三丰[北/软工/🐱]"
+  - ❌"北京软件工程师，养猫"
+- **淘汰预警**：当总字数≥900时触发
+  1. 删除权重分<60且3轮未提及的信息
+  2. 合并相似条目（保留时间戳最近的）
+
+## 记忆结构
+输出格式必须为可解析的json字符串，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
+```json
+{
+  "时空档案": {
+    "身份图谱": {
+      "现用名": "",
+      "特征标记": [] 
+    },
+    "记忆立方": [
+      {
+        "事件": "入职新公司",
+        "时间戳": "2024-03-20",
+        "情感值": 0.9,
+        "关联项": ["下午茶"],
+        "保鲜期": 30 
+      }
+    ]
+  },
+  "关系网络": {
+    "高频话题": {"职场": 12},
+    "暗线联系": [""]
+  },
+  "待响应": {
+    "紧急事项": ["需立即处理的任务"], 
+    "潜在关怀": ["可主动提供的帮助"]
+  },
+  "高光语录": [
+    "最打动人心的瞬间，强烈的情感表达，user的原话"
+  ]
+}
+```
+"""
+
+def extract_json_data(json_code):
+    start = json_code.find("```json")
+    # 从start开始找到下一个```结束
+    end = json_code.find("```", start+1)
+    #print("start:", start, "end:", end)
+    if start == -1 or end == -1:
+        try:
+            jsonData = json.loads(json_code)
+            return json_code
+        except Exception as e:
+            print("Error:", e)
+        return ""
+    jsonData = json_code[start+7:end]
+    return jsonData
+
+TAG = __name__
+
+class MemoryProvider(MemoryProviderBase):
+    def __init__(self, config):
+        super().__init__(config)
+        self.short_momery = ""
+        self.memory_path = get_project_dir() + 'data/.memory.yaml'
+        self.load_memory()
+
+    def init_memory(self, role_id, llm):
+        super().init_memory(role_id, llm)
+        self.load_memory()
+    
+    def load_memory(self):
+        all_memory = {}
+        if os.path.exists(self.memory_path):
+            with open(self.memory_path, 'r', encoding='utf-8') as f:
+                all_memory = yaml.safe_load(f) or {}
+        if self.role_id in all_memory:
+            self.short_momery = all_memory[self.role_id]
+    
+    def save_memory_to_file(self):
+        all_memory = {}
+        if os.path.exists(self.memory_path):
+              with open(self.memory_path, 'r', encoding='utf-8') as f:
+                  all_memory = yaml.safe_load(f) or {}
+        all_memory[self.role_id] = self.short_momery
+        with open(self.memory_path, 'w', encoding='utf-8') as f:
+            yaml.dump(all_memory, f, allow_unicode=True)
+        
+    async def save_memory(self, msgs):
+        if self.llm is None:
+            logger.bind(tag=TAG).error("LLM is not set for memory provider")
+            return None
+        
+        if len(msgs) < 2:
+            return None
+        
+        msgStr = ""
+        for msg in msgs:
+            if msg.role == "user":
+                msgStr += f"User: {msg.content}\n"
+            elif msg.role== "assistant":
+                msgStr += f"Assistant: {msg.content}\n"
+        if len(self.short_momery) > 0:
+            msgStr+="历史记忆：\n"
+            msgStr+=self.short_momery
+        
+        #当前时间
+        time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        msgStr += f"当前时间：{time_str}"
+
+        result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
+ 
+        json_str = extract_json_data(result)
+        try:
+            json_data = json.loads(json_str) # 检查json格式是否正确
+            self.short_momery = json_str
+        except Exception as e:
+            print("Error:", e)
+        
+        self.save_memory_to_file()
+        logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
+
+        return self.short_momery
+    
+    async def query_memory(self, query: str)-> str:
+        return self.short_momery

main/xiaozhi-server/core/providers/memory/nomem/nomem.py

@@ -0,0 +1,18 @@
+'''
+不使用记忆，可以选择此模块
+'''
+from ..base import MemoryProviderBase, logger
+
+TAG = __name__
+
+class MemoryProvider(MemoryProviderBase):
+    def __init__(self, config):
+        super().__init__(config)
+      
+    async def save_memory(self, msgs):
+        logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
+        return None
+
+    async def query_memory(self, query: str)-> str:
+        logger.bind(tag=TAG).debug("nomem mode: No memory query is performed.")
+        return ""

main/xiaozhi-server/core/providers/tts/aliyun.py

@@ -0,0 +1,132 @@
+import os
+import uuid
+import json
+import hmac
+import hashlib
+import base64
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+import http.client
+import urllib.parse
+import time
+import uuid
+from urllib import parse
+class AccessToken:
+    @staticmethod
+    def _encode_text(text):
+        encoded_text = parse.quote_plus(text)
+        return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
+    
+    @staticmethod
+    def _encode_dict(dic):
+        keys = dic.keys()
+        dic_sorted = [(key, dic[key]) for key in sorted(keys)]
+        encoded_text = parse.urlencode(dic_sorted)
+        return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
+    
+    @staticmethod
+    def create_token(access_key_id, access_key_secret):
+        parameters = {'AccessKeyId': access_key_id,
+                      'Action': 'CreateToken',
+                      'Format': 'JSON',
+                      'RegionId': 'cn-shanghai',
+                      'SignatureMethod': 'HMAC-SHA1',
+                      'SignatureNonce': str(uuid.uuid1()),
+                      'SignatureVersion': '1.0',
+                      'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                      'Version': '2019-02-28'}
+        # 构造规范化的请求字符串
+        query_string = AccessToken._encode_dict(parameters)
+        print('规范化的请求字符串: %s' % query_string)
+        # 构造待签名字符串
+        string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string)
+        print('待签名的字符串: %s' % string_to_sign)
+        # 计算签名
+        secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'),
+                                   bytes(string_to_sign, encoding='utf-8'),
+                                   hashlib.sha1).digest()
+        signature = base64.b64encode(secreted_string)
+        print('签名: %s' % signature)
+        # 进行URL编码
+        signature = AccessToken._encode_text(signature)
+        print('URL编码后的签名: %s' % signature)
+        # 调用服务
+        full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string)
+        print('url: %s' % full_url)
+        # 提交HTTP GET请求
+        response = requests.get(full_url)
+        if response.ok:
+            root_obj = response.json()
+            key = 'Token'
+            if key in root_obj:
+                token = root_obj[key]['Id']
+                expire_time = root_obj[key]['ExpireTime']
+                return token, expire_time
+        print(response.text)
+        return None, None
+
+
+class TTSProvider(TTSProviderBase):
+
+
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        
+        # 新增空值判断逻辑
+        access_key_id = config.get("access_key_id")
+        access_key_secret = config.get("access_key_secret")
+        if access_key_id and access_key_secret:
+            # 使用密钥对生成临时token
+            token, expire_time = AccessToken.create_token(access_key_id, access_key_secret)
+        else:
+            # 直接使用预生成的长期token
+            token = config.get("token")
+            expire_time = None
+        
+        print('token: %s, expire time(s): %s' % (token, expire_time))
+
+        self.appkey = config.get("appkey")
+        self.token = token
+        self.format = config.get("format", "wav")
+        self.sample_rate = config.get("sample_rate", 16000)
+        self.voice = config.get("voice", "xiaoyun")
+        self.volume = config.get("volume", 50)
+        self.speech_rate = config.get("speech_rate", 0)
+        self.pitch_rate = config.get("pitch_rate", 0)
+
+        self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
+        self.api_url = f"https://{self.host}/stream/v1/tts"
+        self.header = {
+            "Content-Type": "application/json"
+        }
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "appkey": self.appkey,
+            "token": self.token,
+            "text": text,
+            "format": self.format,
+            "sample_rate": self.sample_rate,
+            "voice": self.voice,
+            "volume": self.volume,
+            "speech_rate": self.speech_rate,
+            "pitch_rate": self.pitch_rate
+        }
+
+        print(self.api_url, json.dumps(request_json, ensure_ascii=False))
+        try:
+            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
+            # 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的
+            if resp.headers['Content-Type'].startswith('audio/'):
+                with open(output_file, 'wb') as f:
+                    f.write(resp.content)
+                return output_file
+            else:
+                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+        except Exception as e:
+            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/base.py

@@ -0,0 +1,123 @@
+import asyncio
+from config.logger import setup_logging
+import os
+import numpy as np
+import opuslib_next
+from pydub import AudioSegment
+from abc import ABC, abstractmethod
+import queue
+
+TAG = __name__
+logger = setup_logging()
+
+
+class TTSProviderBase(ABC):
+    def __init__(self, config, delete_audio_file):
+        self.delete_audio_file = delete_audio_file
+        self.output_file = config.get("output_file")
+
+    @abstractmethod
+    def generate_filename(self):
+        pass
+
+    def to_tts(self, text):
+        tmp_file = self.generate_filename()
+        try:
+            max_repeat_time = 5
+            while not os.path.exists(tmp_file) and max_repeat_time > 0:
+                asyncio.run(self.text_to_speak(text, tmp_file))
+                if not os.path.exists(tmp_file):
+                    max_repeat_time = max_repeat_time - 1
+                    logger.bind(tag=TAG).error(f"语音生成失败: {text}:{tmp_file}，再试{max_repeat_time}次")
+
+            if max_repeat_time > 0:
+                logger.bind(tag=TAG).info(f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次")
+
+            return tmp_file
+        except Exception as e:
+            logger.bind(tag=TAG).info(f": {e}")
+            return None
+
+    def to_tts_stream(self, text, queue: queue.Queue, text_index=0):
+        try:
+            asyncio.run(self.text_to_speak_stream(text, queue, text_index))
+        except Exception as e:
+            logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
+            return None
+
+    @abstractmethod
+    async def text_to_speak(self, text, output_file):
+        pass
+
+    async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
+        raise Exception("该TTS还没有实现stream模式")
+
+    def wav_to_opus_data(self, wav_file_path):
+        # 使用pydub加载PCM文件
+        # 获取文件后缀名
+        file_type = os.path.splitext(wav_file_path)[1]
+        if file_type:
+            file_type = file_type.lstrip('.')
+        audio = AudioSegment.from_file(wav_file_path, format=file_type)
+
+        duration = len(audio) / 1000.0
+
+        # 转换为单声道和16kHz采样率（确保与编码器匹配）
+        audio = audio.set_channels(1).set_frame_rate(16000)
+
+        # 获取原始PCM数据（16位小端）
+        raw_data = audio.raw_data
+
+        # 初始化Opus编码器
+        encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+
+        # 编码参数
+        frame_duration = 60  # 60ms per frame
+        frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+
+        opus_datas = []
+        # 按帧处理所有音频数据（包括最后一帧可能补零）
+        for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
+            # 获取当前帧的二进制数据
+            chunk = raw_data[i:i + frame_size * 2]
+
+            # 如果最后一帧不足，补零
+            if len(chunk) < frame_size * 2:
+                chunk += b'\x00' * (frame_size * 2 - len(chunk))
+
+            # 转换为numpy数组处理
+            np_frame = np.frombuffer(chunk, dtype=np.int16)
+
+            # 编码Opus数据
+            opus_data = encoder.encode(np_frame.tobytes(), frame_size)
+            opus_datas.append(opus_data)
+
+        return opus_datas, duration
+
+    def wav_to_opus_data_audio_raw(self, raw_data):
+        # 初始化Opus编码器
+        encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
+
+        # 编码参数
+        frame_duration = 60  # 60ms per frame
+        frame_size = int(16000 * frame_duration / 1000)  # 960 samples/frame
+
+        opus_datas = []
+        # 按帧处理所有音频数据（包括最后一帧可能补零）
+        for i in range(0, len(raw_data), frame_size * 2):  # 16bit=2bytes/sample
+            # 获取当前帧的二进制数据
+            chunk = raw_data[i:i + frame_size * 2]
+
+            # 如果最后一帧不足，补零
+            if len(chunk) < frame_size * 2:
+                # logger.bind(tag=TAG).info("开始补0")
+                chunk += b'\x00' * (frame_size * 2 - len(chunk))
+
+            # 转换为numpy数组处理
+            np_frame = np.frombuffer(chunk, dtype=np.int16)
+
+            # 编码Opus数据
+            opus_data = encoder.encode(np_frame.tobytes(), frame_size)
+            opus_datas.append(opus_data)
+
+        return opus_datas

main/xiaozhi-server/core/providers/tts/cozecn.py

@@ -0,0 +1,38 @@
+import os
+import uuid
+import json
+import base64
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.model = config.get("model")
+        self.access_token = config.get("access_token")
+        self.voice = config.get("voice")
+        self.response_format = config.get("response_format")
+
+        self.host = "api.coze.cn"
+        self.api_url = f"https://{self.host}/v1/audio/speech"
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "model": self.model,
+            "input": text,
+            "voice_id": self.voice,
+            "response_format": self.response_format,
+        }
+        headers = {
+            "Authorization": f"Bearer {self.access_token}",
+            "Content-Type": "application/json"
+        }
+        response = requests.request("POST", self.api_url, json=request_json, headers=headers)
+        data = response.content
+        file_to_save = open(output_file, "wb")
+        file_to_save.write(data)

main/xiaozhi-server/core/providers/tts/doubao.py

@@ -0,0 +1,62 @@
+import os
+import uuid
+import json
+import base64
+import requests
+from datetime import datetime
+from core.utils.util import check_model_key
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.appid = config.get("appid")
+        self.access_token = config.get("access_token")
+        self.cluster = config.get("cluster")
+        self.voice = config.get("voice")
+        self.api_url = config.get("api_url")
+        self.authorization = config.get("authorization")
+        self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
+        check_model_key("TTS", self.access_token)
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "app": {
+                "appid": f"{self.appid}",
+                "token": "access_token",
+                "cluster": self.cluster
+            },
+            "user": {
+                "uid": "1"
+            },
+            "audio": {
+                "voice_type": self.voice,
+                "encoding": "wav",
+                "speed_ratio": 1.0,
+                "volume_ratio": 1.0,
+                "pitch_ratio": 1.0,
+            },
+            "request": {
+                "reqid": str(uuid.uuid4()),
+                "text": text,
+                "text_type": "plain",
+                "operation": "query",
+                "with_frontend": 1,
+                "frontend_type": "unitTson"
+            }
+        }
+
+        try:
+            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
+            if "data" in resp.json():
+                data = resp.json()["data"]
+                file_to_save = open(output_file, "wb")
+                file_to_save.write(base64.b64decode(data))
+            else:
+                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+        except Exception as e:
+            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/edge.py

@@ -0,0 +1,18 @@
+import os
+import uuid
+import edge_tts
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.voice = config.get("voice")
+
+    def generate_filename(self, extension=".mp3"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        communicate = edge_tts.Communicate(text, voice=self.voice)  # Use your preferred voice
+        await communicate.save(output_file)

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -0,0 +1,268 @@
+import base64
+import os
+import traceback
+import uuid
+import queue
+import io
+
+import numpy as np
+import requests
+import ormsgpack
+from pathlib import Path
+
+import torch
+import torchaudio
+from pydantic import BaseModel, Field, conint, model_validator
+from pydub import AudioSegment
+from typing_extensions import Annotated
+from datetime import datetime
+from typing import Literal
+from core.utils.util import check_model_key
+from core.providers.tts.base import TTSProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ServeReferenceAudio(BaseModel):
+    audio: bytes
+    text: str
+
+    @model_validator(mode="before")
+    def decode_audio(cls, values):
+        audio = values.get("audio")
+        if (
+                isinstance(audio, str) and len(audio) > 255
+        ):  # Check if audio is a string (Base64)
+            try:
+                values["audio"] = base64.b64decode(audio)
+            except Exception as e:
+                # If the audio is not a valid base64 string, we will just ignore it and let the server handle it
+                pass
+        return values
+
+    def __repr__(self) -> str:
+        return f"ServeReferenceAudio(text={self.text!r}, audio_size={len(self.audio)})"
+
+
+class ServeTTSRequest(BaseModel):
+    text: str
+    chunk_length: Annotated[int, conint(ge=100, le=300, strict=True)] = 200
+    # Audio format
+    format: Literal["wav", "pcm", "mp3"] = "wav"
+    # References audios for in-context learning
+    references: list[ServeReferenceAudio] = []
+    # Reference id
+    # For example, if you want use https://fish.audio/m/7f92f8afb8ec43bf81429cc1c9199cb1/
+    # Just pass 7f92f8afb8ec43bf81429cc1c9199cb1
+    reference_id: str | None = None
+    seed: int | None = None
+    use_memory_cache: Literal["on", "off"] = "off"
+    # Normalize text for en & zh, this increase stability for numbers
+    normalize: bool = True
+    # not usually used below
+    streaming: bool = False
+    max_new_tokens: int = 1024
+    top_p: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
+    repetition_penalty: Annotated[float, Field(ge=0.9, le=2.0, strict=True)] = 1.2
+    temperature: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
+
+    class Config:
+        # Allow arbitrary types for pytorch related types
+        arbitrary_types_allowed = True
+
+
+def audio_to_bytes(file_path):
+    if not file_path or not Path(file_path).exists():
+        return None
+    with open(file_path, "rb") as wav_file:
+        wav = wav_file.read()
+    return wav
+
+
+def read_ref_text(ref_text):
+    path = Path(ref_text)
+    if path.exists() and path.is_file():
+        with path.open("r", encoding="utf-8") as file:
+            return file.read()
+    return ref_text
+
+
+class TTSProvider(TTSProviderBase):
+
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+
+        self.reference_id = config.get("reference_id")
+        self.reference_audio = config.get("reference_audio", [])
+        self.reference_text = config.get("reference_text", [])
+        self.format = config.get("format", "wav")
+        self.channels = config.get("channels", 1)
+        self.rate = config.get("rate", 44100)
+        self.api_key = config.get("api_key", "YOUR_API_KEY")
+        have_key = check_model_key("FishSpeech TTS", self.api_key)
+        if not have_key:
+            return
+        self.normalize = config.get("normalize", True)
+        self.max_new_tokens = config.get("max_new_tokens", 1024)
+        self.chunk_length = config.get("chunk_length", 200)
+        self.top_p = config.get("top_p", 0.7)
+        self.repetition_penalty = config.get("repetition_penalty", 1.2)
+        self.temperature = config.get("temperature", 0.7)
+        self.streaming = config.get("streaming", False)
+        self.use_memory_cache = config.get("use_memory_cache", "on")
+        self.seed = config.get("seed")
+        self.api_url = config.get("api_url", "http://127.0.0.1:8080/v1/tts")
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        # Prepare reference data
+        byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
+        ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
+
+        data = {
+            "text": text,
+            "references": [
+                ServeReferenceAudio(
+                    audio=audio if audio else b"", text=text
+                )
+                for text, audio in zip(ref_texts, byte_audios)
+            ],
+            "reference_id": self.reference_id,
+            "normalize": self.normalize,
+            "format": self.format,
+            "max_new_tokens": self.max_new_tokens,
+            "chunk_length": self.chunk_length,
+            "top_p": self.top_p,
+            "repetition_penalty": self.repetition_penalty,
+            "temperature": self.temperature,
+            "streaming": self.streaming,
+            "use_memory_cache": self.use_memory_cache,
+            "seed": self.seed,
+        }
+
+        pydantic_data = ServeTTSRequest(**data)
+
+        response = requests.post(
+            self.api_url,
+            data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
+            headers={
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/msgpack",
+            },
+        )
+
+        if response.status_code == 200:
+            audio_content = response.content
+
+            with open(output_file, "wb") as audio_file:
+                audio_file.write(audio_content)
+
+
+
+        else:
+            print(f"Request failed with status code {response.status_code}")
+            print(response.json())
+
+    def _get_audio_from_tts(self, data_bytes):
+        tts_speech = torch.from_numpy(np.array(np.frombuffer(data_bytes, dtype=np.int16))).unsqueeze(dim=0)
+        with io.BytesIO() as bf:
+            torchaudio.save(bf, tts_speech, 44100, format="wav")
+            audio = AudioSegment.from_file(bf, format="wav")
+        audio = audio.set_channels(1).set_frame_rate(16000)
+        return audio
+
+    async def text_to_speak_stream(self, text, queue: queue.Queue, text_index=0):
+        try:
+            # Prepare reference data
+            byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
+            ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
+
+            data = {
+                "text": text,
+                "references": [
+                    ServeReferenceAudio(
+                        audio=audio if audio else b"", text=text
+                    )
+                    for text, audio in zip(ref_texts, byte_audios)
+                ],
+                "reference_id": self.reference_id,
+                "normalize": self.normalize,
+                "format": self.format,
+                "max_new_tokens": self.max_new_tokens,
+                "chunk_length": self.chunk_length,
+                "top_p": self.top_p,
+                "repetition_penalty": self.repetition_penalty,
+                "temperature": self.temperature,
+                "streaming": self.streaming,
+                "use_memory_cache": self.use_memory_cache,
+                "seed": self.seed,
+            }
+
+            pydantic_data = ServeTTSRequest(**data)
+            audio_buff = None
+            chunk_total = b''
+            last_raw = b''
+            audio_raw = b''
+            print("请求tts")
+            with requests.post(
+                    self.api_url,
+                    data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
+                    headers={
+                        "Authorization": f"Bearer {self.api_key}",
+                        "Content-Type": "application/msgpack",
+                    },
+            ) as response:
+                if response.status_code == 200:
+                    for chunk in response.iter_content():
+                        # 拼接当前块和上一块数据
+                        chunk_total += chunk
+                        # 最后一个是静音，说明是一个完整的音频
+                        if len(chunk_total) % 2 == 0 and chunk_total[-2:] == b'\x00\x00':
+                            audio = self._get_audio_from_tts(chunk_total)
+                            audio_raw = audio_raw + audio.raw_data
+                            #长度凑够2贞开始发送，60ms*4=240ms
+                            if len(audio_raw) >= 7680:
+                                duration = 60 * len(audio_raw) // 1920
+                                if (len(audio_raw) % 1920) > 0:
+                                    duration += 60
+                                duration = duration / 1000.0
+                                logger.bind(tag=TAG).info(f'发送数据长度：{len(audio_raw)}')
+                                opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
+                                queue.put({
+                                    "data": opus_datas,
+                                    "duration": duration,
+                                    "end": False,
+                                    "text_index": text_index
+                                })
+                                audio_raw = b''
+                            chunk_total = b''
+                    if len(chunk_total) > 0:
+                        audio = self._get_audio_from_tts(chunk_total)
+                        audio_raw = audio_raw + audio.raw_data
+                        duration = 60 * len(audio_raw) // 1920
+                        if (len(audio_raw) % 1920) > 0:
+                            duration += 60
+                        duration = duration / 1000.0
+                        # 把 audio 转成 opus
+                        logger.bind(tag=TAG).info(f'发送数据长度：{len(audio_raw)}')
+                        opus_datas = self.wav_to_opus_data_audio_raw(audio_raw)
+                        queue.put({
+                            "data": opus_datas,
+                            "duration": duration,
+                            "end": False
+                        })
+
+                else:
+                    print('请求失败:', response.status_code, response.text)
+                queue.put({
+                    "data": None,
+                    "end": True
+                })
+        except Exception as e:
+            logger.bind(tag=TAG).error("tts发生错误")
+            traceback.print_exc()
+            raise e

main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py

@@ -0,0 +1,67 @@
+import os
+import uuid
+import json
+import base64
+import requests
+from config.logger import setup_logging
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.url = config.get("url")
+        self.text_lang = config.get("text_lang", "zh")
+        self.ref_audio_path = config.get("ref_audio_path")
+        self.prompt_text = config.get("prompt_text")
+        self.prompt_lang = config.get("prompt_lang", "zh")
+        self.top_k = config.get("top_k", 5)
+        self.top_p = config.get("top_p", 1)
+        self.temperature = config.get("temperature", 1)
+        self.text_split_method = config.get("text_split_method", "cut0")
+        self.batch_size = config.get("batch_size", 1)
+        self.batch_threshold = config.get("batch_threshold", 0.75)
+        self.split_bucket = config.get("split_bucket", True)
+        self.return_fragment = config.get("return_fragment", False)
+        self.speed_factor = config.get("speed_factor", 1.0)
+        self.streaming_mode = config.get("streaming_mode", False)
+        self.seed = config.get("seed", -1)
+        self.parallel_infer = config.get("parallel_infer", True)
+        self.repetition_penalty = config.get("repetition_penalty", 1.35)
+        self.aux_ref_audio_paths = config.get("aux_ref_audio_paths", [])
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "text": text,
+            "text_lang": self.text_lang,
+            "ref_audio_path": self.ref_audio_path,
+            "aux_ref_audio_paths": self.aux_ref_audio_paths,
+            "prompt_text": self.prompt_text,
+            "prompt_lang": self.prompt_lang,
+            "top_k": self.top_k,
+            "top_p": self.top_p,
+            "temperature": self.temperature,
+            "text_split_method": self.text_split_method,
+            "batch_size": self.batch_size,
+            "batch_threshold": self.batch_threshold,
+            "split_bucket": self.split_bucket,
+            "return_fragment": self.return_fragment,
+            "speed_factor": self.speed_factor,
+            "streaming_mode": self.streaming_mode,
+            "seed": self.seed,
+            "parallel_infer": self.parallel_infer,
+            "repetition_penalty": self.repetition_penalty
+        }
+
+        resp = requests.post(self.url, json=request_json)
+        if resp.status_code == 200:
+            with open(output_file, "wb") as file:
+                file.write(resp.content)
+        else:
+            logger.bind(tag=TAG).error(f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}")

main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py

@@ -0,0 +1,52 @@
+import os
+import uuid
+import requests
+from config.logger import setup_logging
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.url = config.get("url")
+        self.text_lang = config.get("text_lang", "audo")
+        self.ref_audio_path = config.get("ref_audio_path")
+        self.prompt_lang = config.get("prompt_lang")
+        self.prompt_text = config.get("prompt_text")
+        self.top_k = config.get("top_k", 5)
+        self.top_p = config.get("top_p", 1)
+        self.temperature = config.get("temperature", 1)
+        self.sample_steps = config.get("sample_steps", 16)
+        self.media_type = config.get("media_type", "wav")
+        self.streaming_mode = config.get("streaming_mode", False)
+        self.threshold = config.get("threshold", 30)
+
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_params = {
+            "text": text,
+            "text_lang": self.text_lang,
+            "ref_audio_path": self.ref_audio_path,
+            "prompt_lang": self.prompt_lang,
+            "prompt_text": self.prompt_text,
+            "top_k": self.top_k,
+            "top_p": self.top_p,
+            "temperature": self.temperature,
+            "sample_steps": self.sample_steps,
+            "media_type": self.media_type,
+            "streaming_mode": self.streaming_mode,
+            "threshold": self.threshold,
+        }
+
+        resp = requests.get(self.url, params=request_params)
+        if resp.status_code == 200:
+            with open(output_file, "wb") as file:
+                file.write(resp.content)
+        else:
+            logger.bind(tag=TAG).error(f"GPT_SoVITS_V3 TTS请求失败: {resp.status_code} - {resp.text}")

main/xiaozhi-server/core/providers/tts/minimax.py

@@ -0,0 +1,77 @@
+import os
+import uuid
+import json
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.group_id = config.get("group_id")
+        self.api_key = config.get("api_key")
+        self.model = config.get("model")
+        self.voice_id = config.get("voice_id")
+
+        default_voice_setting = {
+            "voice_id": "female-shaonv",
+            "speed": 1,
+            "vol": 1,
+            "pitch": 0,
+            "emotion": "happy"
+        }
+        default_pronunciation_dict = {
+            "tone": [
+                "处理/(chu3)(li3)", "危险/dangerous"
+            ]
+        }
+        defult_audio_setting = {
+            "sample_rate": 32000,
+            "bitrate": 128000,
+            "format": "mp3",
+            "channel": 1
+        }
+        self.voice_setting = {**default_voice_setting, **config.get("voice_setting", {})}
+        self.pronunciation_dict = {**default_pronunciation_dict, **config.get("pronunciation_dict", {})}
+        self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
+        self.timber_weights = config.get("timber_weights", [])
+
+        if self.voice_id:
+            self.voice_setting["voice_id"] = self.voice_id
+
+        self.host = "api.minimax.chat"
+        self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
+        self.header = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}"
+        }
+
+    def generate_filename(self, extension=".mp3"):
+        return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "model": self.model,
+            "text": text,
+            "stream": False,
+            "voice_setting": self.voice_setting,
+            "pronunciation_dict": self.pronunciation_dict,
+            "audio_setting": self.audio_setting,
+        }
+
+        if type(self.timber_weights) is list and len(self.timber_weights) > 0:
+            request_json["timber_weights"] = self.timber_weights
+            request_json["voice_setting"]["voice_id"] = ""
+
+        try:
+            resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
+            # 检查返回请求数据的status_code是否为0
+            if resp.json()["base_resp"]["status_code"] == 0:
+                data = resp.json()['data']['audio']
+                file_to_save = open(output_file, "wb")
+                file_to_save.write(bytes.fromhex(data))
+            else:
+                raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
+        except Exception as e:
+            raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/openai.py

@@ -0,0 +1,40 @@
+import os
+import uuid
+import requests
+from datetime import datetime
+from core.utils.util import check_model_key
+from core.providers.tts.base import TTSProviderBase
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.api_key = config.get("api_key")
+        self.api_url = config.get("api_url", "https://api.openai.com/v1/audio/speech")
+        self.model = config.get("model", "tts-1")
+        self.voice = config.get("voice", "alloy")
+        self.response_format = "wav"
+        self.speed = config.get("speed", 1.0)
+        self.output_file = config.get("output_file", "tmp/")
+        check_model_key("TTS", self.api_key)
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json"
+        }
+        data = {
+            "model": self.model,
+            "input": text,
+            "voice": self.voice,
+            "response_format": "wav",
+            "speed": self.speed
+        }
+        response = requests.post(self.api_url, json=data, headers=headers)
+        if response.status_code == 200:
+            with open(output_file, "wb") as audio_file:
+                audio_file.write(response.content)
+        else:
+            raise Exception(f"OpenAI TTS请求失败: {response.status_code} - {response.text}")

main/xiaozhi-server/core/providers/tts/siliconflow.py

@@ -0,0 +1,39 @@
+import os
+import uuid
+import requests
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.model = config.get("model")
+        self.access_token = config.get("access_token")
+        self.voice = config.get("voice")
+        self.response_format = config.get("response_format")
+        self.sample_rate = config.get("sample_rate")
+        self.speed = config.get("speed")
+        self.gain = config.get("gain")
+
+        self.host = "api.siliconflow.cn"
+        self.api_url = f"https://{self.host}/v1/audio/speech"
+
+    def generate_filename(self, extension=".wav"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        request_json = {
+            "model": self.model,
+            "input": text,
+            "voice": self.voice,
+            "response_format": self.response_format,
+        }
+        headers = {
+            "Authorization": f"Bearer {self.access_token}",
+            "Content-Type": "application/json"
+        }
+        response = requests.request("POST", self.api_url, json=request_json, headers=headers)
+        data = response.content
+        file_to_save = open(output_file, "wb")
+        file_to_save.write(data)

main/xiaozhi-server/core/providers/tts/ttson.py

@@ -0,0 +1,64 @@
+import os
+import uuid
+import json
+import requests
+import shutil
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.url = config.get("url", "https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=")
+        self.voice_id = config.get("voice_id", 1695)
+        self.token = config.get("token")
+        self.to_lang = config.get("to_lang")
+        self.volume_change_dB = config.get("volume_change_dB", 0)
+        self.speed_factor = config.get("speed_factor", 1)
+        self.stream = config.get("stream", False)
+        self.output_file = config.get("output_file")
+        self.pitch_factor = config.get("pitch_factor", 0)
+        self.format = config.get("format", "mp3")
+        self.emotion = config.get("emotion", 1)
+        self.header = {
+            "Content-Type": "application/json"
+        }
+
+    def generate_filename(self, extension=".mp3"):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
+
+    async def text_to_speak(self, text, output_file):
+        url = f'{self.url}{self.token}'
+        result = "firefly"
+        payload = json.dumps({
+            "to_lang": self.to_lang,
+            "text": text,
+            "emotion": self.emotion,
+            "format": self.format,
+            "volume_change_dB": self.volume_change_dB,
+            "voice_id": self.voice_id,
+            "pitch_factor": self.pitch_factor,
+            "speed_factor": self.speed_factor,
+            "token": self.token
+        })
+
+        resp = requests.request("POST", url, data=payload)
+        if resp.status_code != 200:
+            return None
+        resp_json = resp.json()
+        try:
+            result = resp_json['url'] + ':' + str(
+                resp_json[
+                    'port']) + '/flashsummary/retrieveFileData?stream=True&token=' + self.token + '&voice_audio_path=' + \
+                     resp_json['voice_path']
+        except Exception as e:
+            print("error:", e)
+
+        audio_content = requests.get(result)
+        with open(output_file, "wb") as f:
+            f.write(audio_content.content)
+            return True
+        voice_path = resp_json.get("voice_path")
+        des_path = output_file
+        shutil.move(voice_path, des_path)

main/xiaozhi-server/core/utils/asr.py

@@ -0,0 +1,24 @@
+import importlib
+import logging
+import os
+import sys
+import time
+import wave
+import uuid
+from abc import ABC, abstractmethod
+from typing import Optional, Tuple, List
+from core.providers.asr.base import ASRProviderBase
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+def create_instance(class_name: str, *args, **kwargs) -> ASRProviderBase:
+    """工厂方法创建ASR实例"""
+    if os.path.exists(os.path.join('core', 'providers', 'asr', f'{class_name}.py')):
+        lib_name = f'core.providers.asr.{class_name}'
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].ASRProvider(*args, **kwargs)
+
+    raise ValueError(f"不支持的ASR类型: {class_name}，请检查该配置的type是否设置正确")

main/xiaozhi-server/core/utils/auth_code_gen.py

@@ -0,0 +1,97 @@
+import random
+import threading
+import time
+from typing import Set
+
+class AuthCodeGenerator:
+    _instance = None
+    _instance_lock = threading.Lock()
+    
+    def __new__(cls):
+        if not cls._instance:
+            with cls._instance_lock:
+                if not cls._instance:
+                    cls._instance = super(AuthCodeGenerator, cls).__new__(cls)
+                    # 初始化随机种子
+                    random.seed(time.time())
+        return cls._instance
+
+    def __init__(self):
+        # 确保 __init__ 只被调用一次
+        if not hasattr(self, '_initialized'):
+            self._used_codes: Set[str] = set()
+            self._code_timestamps = {}
+            self._lock = threading.Lock()
+            self._code_timeout = 3 * 24 * 60 * 60
+            self._initialized = True
+    
+    @classmethod
+    def get_instance(cls):
+        """获取AuthCodeGenerator的单例实例"""
+        return cls()
+
+    def generate_code(self) -> str:
+        """
+        生成6位数字认证码，确保不重复
+        返回: 6位数字字符串
+        """
+        with self._lock:
+            self._clean_expired_codes()  # 清理过期code
+            while True:
+                # 使用时间戳和已用码数量作为种子，确保每次生成不同的随机数
+                seed = int(time.time() * 1000) + len(self._used_codes)
+                random.seed(seed)
+                
+                # 生成6位随机数字
+                code = ''.join(str(random.randint(0, 9)) for _ in range(6))
+                
+                # 检查是否已存在
+                if code not in self._used_codes:
+                    self._used_codes.add(code)
+                    self._code_timestamps[code] = time.time()
+                    return code
+    
+    def remove_code(self, code: str) -> bool:
+        """
+        删除已使用的认证码
+        参数:
+            code: 要删除的认证码
+        返回:
+            bool: 删除成功返回True，码不存在返回False
+        """
+        print('remove_code', code)
+        with self._lock:
+            if code in self._used_codes:
+                self._used_codes.remove(code)
+                if code in self._code_timestamps:
+                    del self._code_timestamps[code]
+                return True
+            return False
+    
+    def is_code_used(self, code: str) -> bool:
+        """
+        检查认证码是否已被使用
+        参数:
+            code: 要检查的认证码
+        返回:
+            bool: 如果码存在返回True，否则返回False
+        """
+        with self._lock:
+            return code in self._used_codes
+    
+    def clear_codes(self):
+        """清空所有已使用的认证码"""
+        with self._lock:
+            self._used_codes.clear()
+            self._code_timestamps.clear()
+    
+    def _clean_expired_codes(self):
+        """清理过期的认证码"""
+        current_time = time.time()
+        expired_codes = [
+            code for code, timestamp in self._code_timestamps.items()
+            if (current_time - timestamp) > self._code_timeout
+        ]
+        for code in expired_codes:
+            self._used_codes.remove(code)
+            del self._code_timestamps[code]

main/xiaozhi-server/core/utils/dialogue.py

@@ -0,0 +1,53 @@
+import uuid
+from typing import List, Dict
+from datetime import datetime
+
+
+class Message:
+    def __init__(self, role: str, content: str = None, uniq_id: str = None):
+        self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4())
+        self.role = role
+        self.content = content
+
+
+class Dialogue:
+    def __init__(self):
+        self.dialogue: List[Message] = []
+        # 获取当前时间
+        self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+    def put(self, message: Message):
+        self.dialogue.append(message)
+
+    def get_llm_dialogue(self) -> List[Dict[str, str]]:
+        dialogue = []
+        for m in self.dialogue:
+            dialogue.append({"role": m.role, "content": m.content})
+        return dialogue
+
+    def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
+        if memory_str is None or len(memory_str) == 0:
+            return self.get_llm_dialogue()
+        
+        # 构建带记忆的对话
+        dialogue = []
+        
+        # 添加系统提示和记忆
+        system_message = next(
+            (msg for msg in self.dialogue if msg.role == "system"), None
+        )
+
+
+        if system_message:
+            enhanced_system_prompt = (
+                f"{system_message.content}\n\n"
+                f"相关记忆：\n{memory_str}"
+            )
+            dialogue.append({"role": "system", "content": enhanced_system_prompt})
+
+        # 添加用户和助手的对话
+        for msg in self.dialogue:
+            if msg.role != "system":  # 跳过原始的系统消息
+                dialogue.append({"role": msg.role, "content": msg.content})
+
+        return dialogue

main/xiaozhi-server/core/utils/intent.py

@@ -0,0 +1,17 @@
+import os
+import sys
+from config.logger import setup_logging
+import importlib
+
+logger = setup_logging()
+
+
+def create_instance(class_name, *args, **kwargs):
+    # 创建intent实例
+    if os.path.exists(os.path.join('core', 'providers', 'intent', class_name, f'{class_name}.py')):
+        lib_name = f'core.providers.intent.{class_name}.{class_name}'
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].IntentProvider(*args, **kwargs)
+
+    raise ValueError(f"不支持的intent类型: {class_name}，请检查该配置的type是否设置正确")

main/xiaozhi-server/core/utils/llm.py

@@ -0,0 +1,23 @@
+import os
+import sys
+
+# 添加项目根目录到Python路径
+current_dir = os.path.dirname(os.path.abspath(__file__))
+project_root = os.path.abspath(os.path.join(current_dir, "..", ".."))
+sys.path.insert(0, project_root)
+
+from config.logger import setup_logging
+import importlib
+
+logger = setup_logging()
+
+
+def create_instance(class_name, *args, **kwargs):
+    # 创建LLM实例
+    if os.path.exists(os.path.join('core', 'providers', 'llm', class_name, f'{class_name}.py')):
+        lib_name = f'core.providers.llm.{class_name}.{class_name}'
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].LLMProvider(*args, **kwargs)
+
+    raise ValueError(f"不支持的LLM类型: {class_name}，请检查该配置的type是否设置正确")

main/xiaozhi-server/core/utils/lock_manager.py

@@ -0,0 +1,39 @@
+import asyncio
+from typing import Dict
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+class FileLockManager:
+    _instance = None
+    _locks: Dict[str, asyncio.Lock] = {}
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(FileLockManager, cls).__new__(cls)
+        return cls._instance
+
+    @classmethod
+    def get_lock(cls, file_path: str) -> asyncio.Lock:
+        """获取指定文件的锁"""
+        if file_path not in cls._locks:
+            cls._locks[file_path] = asyncio.Lock()
+        return cls._locks[file_path]
+
+    @classmethod
+    async def acquire_lock(cls, file_path: str):
+        """获取锁"""
+        lock = cls.get_lock(file_path)
+        await lock.acquire()
+        logger.bind(tag=TAG).debug(f"Acquired lock for {file_path}")
+
+    @classmethod
+    def release_lock(cls, file_path: str):
+        """释放锁"""
+        if file_path in cls._locks:
+            try:
+                cls._locks[file_path].release()
+                logger.bind(tag=TAG).debug(f"Released lock for {file_path}")
+            except RuntimeError as e:
+                logger.bind(tag=TAG).warning(f"Failed to release lock for {file_path}: {e}")

main/xiaozhi-server/core/utils/memory.py

@@ -0,0 +1,17 @@
+import os
+import sys
+import importlib
+from config.logger import setup_logging
+from core.utils.util import read_config, get_project_dir
+
+logger = setup_logging()
+
+def create_instance(class_name, *args, **kwargs):
+    if os.path.exists(os.path.join('core', 'providers', 'memory', class_name, f'{class_name}.py')):
+        lib_name = f'core.providers.memory.{class_name}.{class_name}'
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].MemoryProvider(*args, **kwargs)
+
+    raise ValueError(f"不支持的记忆服务类型: {class_name}")
+

main/xiaozhi-server/core/utils/p3.py

@@ -0,0 +1,33 @@
+import struct
+
+def decode_opus_from_file(input_file):
+    """
+    从p3文件中解码 Opus 数据，并返回一个 Opus 数据包的列表以及总时长。
+    """
+    opus_datas = []
+    total_frames = 0
+    sample_rate = 16000  # 文件采样率
+    frame_duration_ms = 60  # 帧时长
+    frame_size = int(sample_rate * frame_duration_ms / 1000)
+
+    with open(input_file, 'rb') as f:
+        while True:
+            # 读取头部（4字节）：[1字节类型，1字节保留，2字节长度]
+            header = f.read(4)
+            if not header:
+                break
+
+            # 解包头部信息
+            _, _, data_len = struct.unpack('>BBH', header)
+
+            # 根据头部指定的长度读取 Opus 数据
+            opus_data = f.read(data_len)
+            if len(opus_data) != data_len:
+                raise ValueError(f"Data length({len(opus_data)}) mismatch({data_len}) in the file.")
+
+            opus_datas.append(opus_data)
+            total_frames += 1
+
+    # 计算总时长
+    total_duration = (total_frames * frame_duration_ms) / 1000.0
+    return opus_datas, total_duration

main/xiaozhi-server/core/utils/tts.py

@@ -0,0 +1,17 @@
+import os
+import sys
+from config.logger import setup_logging
+import importlib
+
+logger = setup_logging()
+
+
+def create_instance(class_name, *args, **kwargs):
+    # 创建TTS实例
+    if os.path.exists(os.path.join('core', 'providers', 'tts', f'{class_name}.py')):
+        lib_name = f'core.providers.tts.{class_name}'
+        if lib_name not in sys.modules:
+            sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
+        return sys.modules[lib_name].TTSProvider(*args, **kwargs)
+
+    raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")

main/xiaozhi-server/core/utils/util.py

@@ -0,0 +1,121 @@
+import os
+import json
+import yaml
+import socket
+import subprocess
+import logging
+
+
+def get_project_dir():
+    """获取项目根目录"""
+    return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + '/'
+
+
+def get_local_ip():
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        # Connect to Google's DNS servers
+        s.connect(("8.8.8.8", 80))
+        local_ip = s.getsockname()[0]
+        s.close()
+        return local_ip
+    except Exception as e:
+        return "127.0.0.1"
+
+
+def read_config(config_path):
+    with open(config_path, "r", encoding="utf-8") as file:
+        config = yaml.safe_load(file)
+    return config
+
+
+def write_json_file(file_path, data):
+    """将数据写入 JSON 文件"""
+    with open(file_path, 'w', encoding='utf-8') as file:
+        json.dump(data, file, ensure_ascii=False, indent=4)
+
+
+def is_punctuation_or_emoji(char):
+    """检查字符是否为空格、指定标点或表情符号"""
+    # 定义需要去除的中英文标点（包括全角/半角）
+    punctuation_set = {
+        '，', ',',  # 中文逗号 + 英文逗号
+        '。', '.',  # 中文句号 + 英文句号
+        '！', '!',  # 中文感叹号 + 英文感叹号
+        '-', '－',  # 英文连字符 + 中文全角横线
+        '、'  # 中文顿号
+    }
+    if char.isspace() or char in punctuation_set:
+        return True
+    # 检查表情符号（保留原有逻辑）
+    code_point = ord(char)
+    emoji_ranges = [
+        (0x1F600, 0x1F64F), (0x1F300, 0x1F5FF),
+        (0x1F680, 0x1F6FF), (0x1F900, 0x1F9FF),
+        (0x1FA70, 0x1FAFF), (0x2600, 0x26FF),
+        (0x2700, 0x27BF)
+    ]
+    return any(start <= code_point <= end for start, end in emoji_ranges)
+
+
+def get_string_no_punctuation_or_emoji(s):
+    """去除字符串首尾的空格、标点符号和表情符号"""
+    chars = list(s)
+    # 处理开头的字符
+    start = 0
+    while start < len(chars) and is_punctuation_or_emoji(chars[start]):
+        start += 1
+    # 处理结尾的字符
+    end = len(chars) - 1
+    while end >= start and is_punctuation_or_emoji(chars[end]):
+        end -= 1
+    return ''.join(chars[start:end + 1])
+
+
+def remove_punctuation_and_length(text):
+    # 全角符号和半角符号的Unicode范围
+    full_width_punctuations = '！＂＃＄％＆＇（）＊＋，－。／：；＜＝＞？＠［＼］＾＿｀｛｜｝～'
+    half_width_punctuations = r'!"#$%&\'()*+,-./:;<=>?@[\]^_`{|}~'
+    space = ' '  # 半角空格
+    full_width_space = '　'  # 全角空格
+
+    # 去除全角和半角符号以及空格
+    result = ''.join([char for char in text if
+                      char not in full_width_punctuations and char not in half_width_punctuations and char not in space and char not in full_width_space])
+
+    if result == "Yeah":
+        return 0, ""
+    return len(result), result
+
+def check_model_key(modelType, modelKey):
+    if "你" in modelKey:
+        logging.error("你还没配置" + modelType + "的密钥，请在配置文件中配置密钥，否则无法正常工作")
+        return False
+    return True
+
+
+def check_ffmpeg_installed():
+    ffmpeg_installed = False
+    try:
+        # 执行ffmpeg -version命令，并捕获输出
+        result = subprocess.run(
+            ['ffmpeg', '-version'],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=True  # 如果返回码非零则抛出异常
+        )
+        # 检查输出中是否包含版本信息（可选）
+        output = result.stdout + result.stderr
+        if 'ffmpeg version' in output.lower():
+            ffmpeg_installed = True
+        return False
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        # 命令执行失败或未找到
+        ffmpeg_installed = False
+    if not ffmpeg_installed:
+        error_msg = "您的电脑还没正确安装ffmpeg\n"
+        error_msg += "\n建议您：\n"
+        error_msg += "1、按照项目的安装文档，正确进入conda环境\n"
+        error_msg += "2、查阅安装文档，如何在conda环境中安装ffmpeg\n"
+        raise ValueError(error_msg)

main/xiaozhi-server/core/utils/vad.py

@@ -0,0 +1,77 @@
+from abc import ABC, abstractmethod
+from config.logger import setup_logging
+import opuslib_next
+import time
+import numpy as np
+import torch
+
+TAG = __name__
+logger = setup_logging()
+
+class VAD(ABC):
+    @abstractmethod
+    def is_vad(self, conn, data):
+        """检测音频数据中的语音活动"""
+        pass
+
+
+class SileroVAD(VAD):
+    def __init__(self, config):
+        logger.bind(tag=TAG).info("SileroVAD", config)
+        self.model, self.utils = torch.hub.load(repo_or_dir=config["model_dir"],
+                                                source='local',
+                                                model='silero_vad',
+                                                force_reload=False)
+        (get_speech_timestamps, _, _, _, _) = self.utils
+
+        self.decoder = opuslib_next.Decoder(16000, 1)
+        self.vad_threshold = config.get("threshold")
+        self.silence_threshold_ms = config.get("min_silence_duration_ms")
+
+    def is_vad(self, conn, opus_packet):
+        try:
+            pcm_frame = self.decoder.decode(opus_packet, 960)
+            conn.client_audio_buffer += pcm_frame  # 将新数据加入缓冲区
+
+            # 处理缓冲区中的完整帧（每次处理512采样点）
+            client_have_voice = False
+            while len(conn.client_audio_buffer) >= 512 * 2:
+                # 提取前512个采样点（1024字节）
+                chunk = conn.client_audio_buffer[:512 * 2]
+                conn.client_audio_buffer = conn.client_audio_buffer[512 * 2:]
+
+                # 转换为模型需要的张量格式
+                audio_int16 = np.frombuffer(chunk, dtype=np.int16)
+                audio_float32 = audio_int16.astype(np.float32) / 32768.0
+                audio_tensor = torch.from_numpy(audio_float32)
+
+                # 检测语音活动
+                speech_prob = self.model(audio_tensor, 16000).item()
+                client_have_voice = speech_prob >= self.vad_threshold
+
+                # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
+                if conn.client_have_voice and not client_have_voice:
+                    stop_duration = time.time() * 1000 - conn.client_have_voice_last_time
+                    if stop_duration >= self.silence_threshold_ms:
+                        conn.client_voice_stop = True
+                if client_have_voice:
+                    conn.client_have_voice = True
+                    conn.client_have_voice_last_time = time.time() * 1000
+
+            return client_have_voice
+        except opuslib_next.OpusError as e:
+            logger.bind(tag=TAG).info(f"解码错误: {e}")
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")
+
+
+def create_instance(class_name, *args, **kwargs) -> VAD:
+    # 获取类对象
+    cls_map = {
+        "SileroVAD": SileroVAD,
+        # 可扩展其他SileroVAD实现
+    }
+
+    if cls := cls_map.get(class_name):
+        return cls(*args, **kwargs)
+    raise ValueError(f"不支持的SileroVAD类型: {class_name}")

main/xiaozhi-server/core/websocket_server.py

@@ -0,0 +1,86 @@
+import asyncio
+import websockets
+from config.logger import setup_logging
+from core.connection import ConnectionHandler
+from core.handle.musicHandler import MusicHandler
+from core.utils.util import get_local_ip
+from core.utils import asr, vad, llm, tts, memory, intent
+
+TAG = __name__
+
+
+class WebSocketServer:
+    def __init__(self, config: dict):
+        self.config = config
+        self.logger = setup_logging()
+        self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent = self._create_processing_instances()
+        self.active_connections = set()  # 添加全局连接记录
+
+    def _create_processing_instances(self):
+        memory_cls_name = self.config["selected_module"].get("Memory", "nomem") # 默认使用nomem
+        has_memory_cfg = self.config.get("Memory") and memory_cls_name in self.config["Memory"]
+        memory_cfg = self.config["Memory"][memory_cls_name] if has_memory_cfg else {}
+
+        """创建处理模块实例"""
+        return (
+            vad.create_instance(
+                self.config["selected_module"]["VAD"],
+                self.config["VAD"][self.config["selected_module"]["VAD"]]
+            ),
+            asr.create_instance(
+                self.config["selected_module"]["ASR"]
+                if not 'type' in self.config["ASR"][self.config["selected_module"]["ASR"]]
+                else
+                self.config["ASR"][self.config["selected_module"]["ASR"]]["type"],
+                self.config["ASR"][self.config["selected_module"]["ASR"]],
+                self.config["delete_audio"]
+            ),
+            llm.create_instance(
+                self.config["selected_module"]["LLM"]
+                if not 'type' in self.config["LLM"][self.config["selected_module"]["LLM"]]
+                else
+                self.config["LLM"][self.config["selected_module"]["LLM"]]['type'],
+                self.config["LLM"][self.config["selected_module"]["LLM"]],
+            ),
+            tts.create_instance(
+                self.config["selected_module"]["TTS"]
+                if not 'type' in self.config["TTS"][self.config["selected_module"]["TTS"]]
+                else
+                self.config["TTS"][self.config["selected_module"]["TTS"]]["type"],
+                self.config["TTS"][self.config["selected_module"]["TTS"]],
+                self.config["delete_audio"]
+            ),
+            MusicHandler(self.config),
+            memory.create_instance(memory_cls_name, memory_cfg),
+            intent.create_instance(
+                self.config["selected_module"]["Intent"]
+                if not 'type' in self.config["Intent"][self.config["selected_module"]["Intent"]]
+                else
+                self.config["Intent"][self.config["selected_module"]["Intent"]]["type"],
+                self.config["Intent"][self.config["selected_module"]["Intent"]]
+            ),
+        )
+
+    async def start(self):
+        server_config = self.config["server"]
+        host = server_config["ip"]
+        port = server_config["port"]
+
+        self.logger.bind(tag=TAG).info("Server is running at ws://{}:{}", get_local_ip(), port)
+        self.logger.bind(tag=TAG).info("=======上面的地址是websocket协议地址，请勿用浏览器访问=======")
+        async with websockets.serve(
+                self._handle_connection,
+                host,
+                port
+        ):
+            await asyncio.Future()
+
+    async def _handle_connection(self, websocket):
+        """处理新连接，每次创建独立的ConnectionHandler"""
+        # 创建ConnectionHandler时传入当前server实例
+        handler = ConnectionHandler(self.config, self._vad, self._asr, self._llm, self._tts, self._music, self._memory, self.intent)
+        self.active_connections.add(handler)
+        try:
+            await handler.handle_connection(websocket)
+        finally:
+            self.active_connections.discard(handler)

main/xiaozhi-server/docker-compose.yml

@@ -0,0 +1,38 @@
+# 如果本机已经安装了MySQL，可以直接在数据库中创建名为`xiaozhi_esp32_server`的数据库。
+# 如果还没有MySQL，你可以通过docker安装mysql,执行以下一句话
+# docker run --name xiaozhi-esp32-server-db -e MYSQL_ROOT_PASSWORD=123456 -p 3306:3306 -e MYSQL_DATABASE=xiaozhi_esp32_server -e MYSQL_INITDB_ARGS="--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci" -d mysql:latest
+# 如果你的mysql账号和密码有修改过，记得修改下方数据库的账号和密码
+# 记得修改下方数据库的IP，ip不能写127.0.0.1或localhost，否则容器无法访问，要写你电脑局域网ip
+version: '3'
+services:
+  xiaozhi-esp32-server:
+    image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:server_latest
+    container_name: xiaozhi-esp32-server
+    restart: always
+    security_opt:
+      - seccomp:unconfined
+    environment:
+      - TZ=Asia/Shanghai
+    ports:
+      # ws服务端
+      - "8000:8000"
+    volumes:
+      # 配置文件目录
+      - ./data:/opt/xiaozhi-esp32-server/data
+      # 模型文件挂接，很重要
+      - ./models/SenseVoiceSmall/model.pt:/opt/xiaozhi-esp32-server/models/SenseVoiceSmall/model.pt
+#  #智控台还没开发好，还不能完全使用，会报很多错误，如果是非技术人员，请不要启用智控台服务
+#  xiaozhi-esp32-server-web:
+#    image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:web_latest
+#    container_name: xiaozhi-esp32-server-web
+#    restart: always
+#    ports:
+#      - "8002:8002"
+#    environment:
+#      - TZ=Asia/Shanghai
+#      ##记得改mysql和redis IP 密码
+#      - SPRING_DATASOURCE_DRUID_URL=jdbc:mysql://192.168.1.20:3306/xiaozhi_esp32_server?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai
+#      - SPRING_DATASOURCE_DRUID_USERNAME=root
+#      - SPRING_DATASOURCE_DRUID_PASSWORD=123456
+#      - SPRING_DATA_REDIS_HOST=192.168.1.20
+#      - SPRING_DATA_REDIS_PORT=6379

main/xiaozhi-server/models/SenseVoiceSmall/config.yaml

@@ -0,0 +1,97 @@
+encoder: SenseVoiceEncoderSmall
+encoder_conf:
+    output_size: 512
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 50
+    tp_blocks: 20
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: pe
+    pos_enc_class: SinusoidalPositionEncoder
+    normalize_before: true
+    kernel_size: 11
+    sanm_shfit: 0
+    selfattention_layer_type: sanm
+
+
+model: SenseVoiceSmall
+model_conf:
+    length_normalized_loss: true
+    sos: 1
+    eos: 2
+    ignore_id: -1
+
+tokenizer: SentencepiecesTokenizer
+tokenizer_conf:
+  bpemodel: null
+  unk_symbol: <unk>
+  split_with_space: true
+
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    lfr_m: 7
+    lfr_n: 6
+    cmvn_file: null
+
+
+dataset: SenseVoiceCTCDataset
+dataset_conf:
+  index_ds: IndexDSJsonl
+  batch_sampler: EspnetStyleBatchSampler
+  data_split_num: 32
+  batch_type: token
+  batch_size: 14000
+  max_token_length: 2000
+  min_token_length: 60
+  max_source_length: 2000
+  min_source_length: 60
+  max_target_length: 200
+  min_target_length: 0
+  shuffle: true
+  num_workers: 4
+  sos: ${model_conf.sos}
+  eos: ${model_conf.eos}
+  IndexDSJsonl: IndexDSJsonl
+  retry: 20
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 20
+  keep_nbest_models: 10
+  avg_nbest_model: 10
+  log_interval: 100
+  resume: true
+  validate_interval: 10000
+  save_checkpoint_interval: 10000
+
+optim: adamw
+optim_conf:
+  lr: 0.00002
+scheduler: warmuplr
+scheduler_conf:
+  warmup_steps: 25000
+
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 6
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1

main/xiaozhi-server/models/SenseVoiceSmall/configuration.json

@@ -0,0 +1,14 @@
+{
+  "framework": "pytorch",
+  "task" : "auto-speech-recognition",
+  "model": {"type" : "funasr"},
+  "pipeline": {"type":"funasr-pipeline"},
+  "model_name_in_hub": {
+    "ms":"", 
+    "hf":""},
+  "file_path_metas": {
+    "init_param":"model.pt", 
+    "config":"config.yaml",
+    "tokenizer_conf": {"bpemodel": "chn_jpn_yue_eng_ko_spectok.bpe.model"},
+    "frontend_conf":{"cmvn_file": "am.mvn"}}
+}

main/xiaozhi-server/models/SenseVoiceSmall/demo.py

@@ -0,0 +1,27 @@
+from funasr import AutoModel
+from funasr.utils.postprocess_utils import rich_transcription_postprocess
+
+model_dir = "./"
+
+
+model = AutoModel(
+    model=model_dir,
+    vad_model="fsmn-vad",
+    vad_kwargs={"max_single_segment_time": 30000},
+    # device="cuda:0",
+    hub="hf",
+)
+
+# en
+res = model.generate(
+    input=f"{model.model_path}/example/en.mp3",
+    cache={},
+    language="auto",  # "zn", "en", "yue", "ja", "ko", "nospeech"
+    use_itn=True,
+    batch_size_s=60,
+    merge_vad=True,  #
+    merge_length_s=15,
+)
+text = rich_transcription_postprocess(res[0]["text"])
+print(text)
+

main/xiaozhi-server/models/snakers4_silero-vad/hubconf.py

@@ -0,0 +1,56 @@
+dependencies = ['torch', 'torchaudio']
+import torch
+import os
+import sys
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
+from silero_vad.utils_vad import (init_jit_model,
+                                  get_speech_timestamps,
+                                  save_audio,
+                                  read_audio,
+                                  VADIterator,
+                                  collect_chunks,
+                                  OnnxWrapper)
+
+
+def versiontuple(v):
+    splitted = v.split('+')[0].split(".")
+    version_list = []
+    for i in splitted:
+        try:
+            version_list.append(int(i))
+        except:
+            version_list.append(0)
+    return tuple(version_list)
+
+
+def silero_vad(onnx=False, force_onnx_cpu=False, opset_version=16):
+    """Silero Voice Activity Detector
+    Returns a model with a set of utils
+    Please see https://github.com/snakers4/silero-vad for usage examples
+    """
+    available_ops = [15, 16]
+    if onnx and opset_version not in available_ops:
+        raise Exception(f'Available ONNX opset_version: {available_ops}')
+
+    if not onnx:
+        installed_version = torch.__version__
+        supported_version = '1.12.0'
+        if versiontuple(installed_version) < versiontuple(supported_version):
+            raise Exception(f'Please install torch {supported_version} or greater ({installed_version} installed)')
+
+    model_dir = os.path.join(os.path.dirname(__file__), 'src', 'silero_vad', 'data')
+    if onnx:
+        if opset_version == 16:
+            model_name = 'silero_vad.onnx'
+        else:
+            model_name = f'silero_vad_16k_op{opset_version}.onnx'
+        model = OnnxWrapper(os.path.join(model_dir, model_name), force_onnx_cpu)
+    else:
+        model = init_jit_model(os.path.join(model_dir, 'silero_vad.jit'))
+    utils = (get_speech_timestamps,
+             save_audio,
+             read_audio,
+             VADIterator,
+             collect_chunks)
+
+    return model, utils

main/xiaozhi-server/models/snakers4_silero-vad/src/silero_vad/__init__.py

@@ -0,0 +1,12 @@
+from importlib.metadata import version
+try:
+    __version__ = version(__name__)
+except:
+    pass
+
+from silero_vad.model import load_silero_vad
+from silero_vad.utils_vad import (get_speech_timestamps,
+                                  save_audio,
+                                  read_audio,
+                                  VADIterator,
+                                  collect_chunks)

main/xiaozhi-server/models/snakers4_silero-vad/src/silero_vad/model.py

@@ -0,0 +1,36 @@
+from .utils_vad import init_jit_model, OnnxWrapper
+import torch
+torch.set_num_threads(1)
+
+
+def load_silero_vad(onnx=False, opset_version=16):
+    available_ops = [15, 16]
+    if onnx and opset_version not in available_ops:
+        raise Exception(f'Available ONNX opset_version: {available_ops}')
+
+    if onnx:
+        if opset_version == 16:
+            model_name = 'silero_vad.onnx'
+        else:
+            model_name = f'silero_vad_16k_op{opset_version}.onnx'
+    else:
+        model_name = 'silero_vad.jit'
+    package_path = "silero_vad.data"
+
+    try:
+        import importlib_resources as impresources
+        model_file_path = str(impresources.files(package_path).joinpath(model_name))
+    except:
+        from importlib import resources as impresources
+        try:
+            with impresources.path(package_path, model_name) as f:
+                model_file_path = f
+        except:
+            model_file_path = str(impresources.files(package_path).joinpath(model_name))
+
+    if onnx:
+        model = OnnxWrapper(model_file_path, force_onnx_cpu=True)
+    else:
+        model = init_jit_model(model_file_path)
+
+    return model

main/xiaozhi-server/models/snakers4_silero-vad/src/silero_vad/utils_vad.py

@@ -0,0 +1,500 @@
+import torch
+import torchaudio
+from typing import Callable, List
+import warnings
+
+languages = ['ru', 'en', 'de', 'es']
+
+
+class OnnxWrapper():
+
+    def __init__(self, path, force_onnx_cpu=False):
+        import numpy as np
+        global np
+        import onnxruntime
+
+        opts = onnxruntime.SessionOptions()
+        opts.inter_op_num_threads = 1
+        opts.intra_op_num_threads = 1
+
+        if force_onnx_cpu and 'CPUExecutionProvider' in onnxruntime.get_available_providers():
+            self.session = onnxruntime.InferenceSession(path, providers=['CPUExecutionProvider'], sess_options=opts)
+        else:
+            self.session = onnxruntime.InferenceSession(path, sess_options=opts)
+
+        self.reset_states()
+        if '16k' in path:
+            warnings.warn('This model support only 16000 sampling rate!')
+            self.sample_rates = [16000]
+        else:
+            self.sample_rates = [8000, 16000]
+
+    def _validate_input(self, x, sr: int):
+        if x.dim() == 1:
+            x = x.unsqueeze(0)
+        if x.dim() > 2:
+            raise ValueError(f"Too many dimensions for input audio chunk {x.dim()}")
+
+        if sr != 16000 and (sr % 16000 == 0):
+            step = sr // 16000
+            x = x[:,::step]
+            sr = 16000
+
+        if sr not in self.sample_rates:
+            raise ValueError(f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)")
+        if sr / x.shape[1] > 31.25:
+            raise ValueError("Input audio chunk is too short")
+
+        return x, sr
+
+    def reset_states(self, batch_size=1):
+        self._state = torch.zeros((2, batch_size, 128)).float()
+        self._context = torch.zeros(0)
+        self._last_sr = 0
+        self._last_batch_size = 0
+
+    def __call__(self, x, sr: int):
+
+        x, sr = self._validate_input(x, sr)
+        num_samples = 512 if sr == 16000 else 256
+
+        if x.shape[-1] != num_samples:
+            raise ValueError(f"Provided number of samples is {x.shape[-1]} (Supported values: 256 for 8000 sample rate, 512 for 16000)")
+
+        batch_size = x.shape[0]
+        context_size = 64 if sr == 16000 else 32
+
+        if not self._last_batch_size:
+            self.reset_states(batch_size)
+        if (self._last_sr) and (self._last_sr != sr):
+            self.reset_states(batch_size)
+        if (self._last_batch_size) and (self._last_batch_size != batch_size):
+            self.reset_states(batch_size)
+
+        if not len(self._context):
+            self._context = torch.zeros(batch_size, context_size)
+
+        x = torch.cat([self._context, x], dim=1)
+        if sr in [8000, 16000]:
+            ort_inputs = {'input': x.numpy(), 'state': self._state.numpy(), 'sr': np.array(sr, dtype='int64')}
+            ort_outs = self.session.run(None, ort_inputs)
+            out, state = ort_outs
+            self._state = torch.from_numpy(state)
+        else:
+            raise ValueError()
+
+        self._context = x[..., -context_size:]
+        self._last_sr = sr
+        self._last_batch_size = batch_size
+
+        out = torch.from_numpy(out)
+        return out
+
+    def audio_forward(self, x, sr: int):
+        outs = []
+        x, sr = self._validate_input(x, sr)
+        self.reset_states()
+        num_samples = 512 if sr == 16000 else 256
+
+        if x.shape[1] % num_samples:
+            pad_num = num_samples - (x.shape[1] % num_samples)
+            x = torch.nn.functional.pad(x, (0, pad_num), 'constant', value=0.0)
+
+        for i in range(0, x.shape[1], num_samples):
+            wavs_batch = x[:, i:i+num_samples]
+            out_chunk = self.__call__(wavs_batch, sr)
+            outs.append(out_chunk)
+
+        stacked = torch.cat(outs, dim=1)
+        return stacked.cpu()
+
+
+class Validator():
+    def __init__(self, url, force_onnx_cpu):
+        self.onnx = True if url.endswith('.onnx') else False
+        torch.hub.download_url_to_file(url, 'inf.model')
+        if self.onnx:
+            import onnxruntime
+            if force_onnx_cpu and 'CPUExecutionProvider' in onnxruntime.get_available_providers():
+                self.model = onnxruntime.InferenceSession('inf.model', providers=['CPUExecutionProvider'])
+            else:
+                self.model = onnxruntime.InferenceSession('inf.model')
+        else:
+            self.model = init_jit_model(model_path='inf.model')
+
+    def __call__(self, inputs: torch.Tensor):
+        with torch.no_grad():
+            if self.onnx:
+                ort_inputs = {'input': inputs.cpu().numpy()}
+                outs = self.model.run(None, ort_inputs)
+                outs = [torch.Tensor(x) for x in outs]
+            else:
+                outs = self.model(inputs)
+
+        return outs
+
+
+def read_audio(path: str,
+               sampling_rate: int = 16000):
+    list_backends = torchaudio.list_audio_backends()
+
+    assert len(list_backends) > 0, 'The list of available backends is empty, please install backend manually. \
+                                    \n Recommendations: \n \tSox (UNIX OS) \n \tSoundfile (Windows OS, UNIX OS) \n \tffmpeg (Windows OS, UNIX OS)'
+
+    try:
+        effects = [
+            ['channels', '1'],
+            ['rate', str(sampling_rate)]
+        ]
+
+        wav, sr = torchaudio.sox_effects.apply_effects_file(path, effects=effects)
+    except:
+        wav, sr = torchaudio.load(path)
+
+        if wav.size(0) > 1:
+            wav = wav.mean(dim=0, keepdim=True)
+
+        if sr != sampling_rate:
+            transform = torchaudio.transforms.Resample(orig_freq=sr,
+                                                       new_freq=sampling_rate)
+            wav = transform(wav)
+            sr = sampling_rate
+
+    assert sr == sampling_rate
+    return wav.squeeze(0)
+
+
+def save_audio(path: str,
+               tensor: torch.Tensor,
+               sampling_rate: int = 16000):
+    torchaudio.save(path, tensor.unsqueeze(0), sampling_rate, bits_per_sample=16)
+
+
+def init_jit_model(model_path: str,
+                   device=torch.device('cpu')):
+    model = torch.jit.load(model_path, map_location=device)
+    model.eval()
+    return model
+
+
+def make_visualization(probs, step):
+    import pandas as pd
+    pd.DataFrame({'probs': probs},
+                 index=[x * step for x in range(len(probs))]).plot(figsize=(16, 8),
+                 kind='area', ylim=[0, 1.05], xlim=[0, len(probs) * step],
+                 xlabel='seconds',
+                 ylabel='speech probability',
+                 colormap='tab20')
+
+
+@torch.no_grad()
+def get_speech_timestamps(audio: torch.Tensor,
+                          model,
+                          threshold: float = 0.5,
+                          sampling_rate: int = 16000,
+                          min_speech_duration_ms: int = 250,
+                          max_speech_duration_s: float = float('inf'),
+                          min_silence_duration_ms: int = 100,
+                          speech_pad_ms: int = 30,
+                          return_seconds: bool = False,
+                          visualize_probs: bool = False,
+                          progress_tracking_callback: Callable[[float], None] = None,
+                          neg_threshold: float = None,
+                          window_size_samples: int = 512,):
+
+    """
+    This method is used for splitting long audios into speech chunks using silero VAD
+
+    Parameters
+    ----------
+    audio: torch.Tensor, one dimensional
+        One dimensional float torch.Tensor, other types are casted to torch if possible
+
+    model: preloaded .jit/.onnx silero VAD model
+
+    threshold: float (default - 0.5)
+        Speech threshold. Silero VAD outputs speech probabilities for each audio chunk, probabilities ABOVE this value are considered as SPEECH.
+        It is better to tune this parameter for each dataset separately, but "lazy" 0.5 is pretty good for most datasets.
+
+    sampling_rate: int (default - 16000)
+        Currently silero VAD models support 8000 and 16000 (or multiply of 16000) sample rates
+
+    min_speech_duration_ms: int (default - 250 milliseconds)
+        Final speech chunks shorter min_speech_duration_ms are thrown out
+
+    max_speech_duration_s: int (default -  inf)
+        Maximum duration of speech chunks in seconds
+        Chunks longer than max_speech_duration_s will be split at the timestamp of the last silence that lasts more than 100ms (if any), to prevent agressive cutting.
+        Otherwise, they will be split aggressively just before max_speech_duration_s.
+
+    min_silence_duration_ms: int (default - 100 milliseconds)
+        In the end of each speech chunk wait for min_silence_duration_ms before separating it
+
+    speech_pad_ms: int (default - 30 milliseconds)
+        Final speech chunks are padded by speech_pad_ms each side
+
+    return_seconds: bool (default - False)
+        whether return timestamps in seconds (default - samples)
+
+    visualize_probs: bool (default - False)
+        whether draw prob hist or not
+
+    progress_tracking_callback: Callable[[float], None] (default - None)
+        callback function taking progress in percents as an argument
+
+    neg_threshold: float (default = threshold - 0.15)
+        Negative threshold (noise or exit threshold). If model's current state is SPEECH, values BELOW this value are considered as NON-SPEECH.
+
+    window_size_samples: int (default - 512 samples)
+        !!! DEPRECATED, DOES NOTHING !!!
+
+    Returns
+    ----------
+    speeches: list of dicts
+        list containing ends and beginnings of speech chunks (samples or seconds based on return_seconds)
+    """
+
+    if not torch.is_tensor(audio):
+        try:
+            audio = torch.Tensor(audio)
+        except:
+            raise TypeError("Audio cannot be casted to tensor. Cast it manually")
+
+    if len(audio.shape) > 1:
+        for i in range(len(audio.shape)):  # trying to squeeze empty dimensions
+            audio = audio.squeeze(0)
+        if len(audio.shape) > 1:
+            raise ValueError("More than one dimension in audio. Are you trying to process audio with 2 channels?")
+
+    if sampling_rate > 16000 and (sampling_rate % 16000 == 0):
+        step = sampling_rate // 16000
+        sampling_rate = 16000
+        audio = audio[::step]
+        warnings.warn('Sampling rate is a multiply of 16000, casting to 16000 manually!')
+    else:
+        step = 1
+
+    if sampling_rate not in [8000, 16000]:
+        raise ValueError("Currently silero VAD models support 8000 and 16000 (or multiply of 16000) sample rates")
+
+    window_size_samples = 512 if sampling_rate == 16000 else 256
+
+    model.reset_states()
+    min_speech_samples = sampling_rate * min_speech_duration_ms / 1000
+    speech_pad_samples = sampling_rate * speech_pad_ms / 1000
+    max_speech_samples = sampling_rate * max_speech_duration_s - window_size_samples - 2 * speech_pad_samples
+    min_silence_samples = sampling_rate * min_silence_duration_ms / 1000
+    min_silence_samples_at_max_speech = sampling_rate * 98 / 1000
+
+    audio_length_samples = len(audio)
+
+    speech_probs = []
+    for current_start_sample in range(0, audio_length_samples, window_size_samples):
+        chunk = audio[current_start_sample: current_start_sample + window_size_samples]
+        if len(chunk) < window_size_samples:
+            chunk = torch.nn.functional.pad(chunk, (0, int(window_size_samples - len(chunk))))
+        speech_prob = model(chunk, sampling_rate).item()
+        speech_probs.append(speech_prob)
+        # caculate progress and seng it to callback function
+        progress = current_start_sample + window_size_samples
+        if progress > audio_length_samples:
+            progress = audio_length_samples
+        progress_percent = (progress / audio_length_samples) * 100
+        if progress_tracking_callback:
+            progress_tracking_callback(progress_percent)
+
+    triggered = False
+    speeches = []
+    current_speech = {}
+
+    if neg_threshold is None:
+        neg_threshold = max(threshold - 0.15, 0.01)
+    temp_end = 0  # to save potential segment end (and tolerate some silence)
+    prev_end = next_start = 0  # to save potential segment limits in case of maximum segment size reached
+
+    for i, speech_prob in enumerate(speech_probs):
+        if (speech_prob >= threshold) and temp_end:
+            temp_end = 0
+            if next_start < prev_end:
+                next_start = window_size_samples * i
+
+        if (speech_prob >= threshold) and not triggered:
+            triggered = True
+            current_speech['start'] = window_size_samples * i
+            continue
+
+        if triggered and (window_size_samples * i) - current_speech['start'] > max_speech_samples:
+            if prev_end:
+                current_speech['end'] = prev_end
+                speeches.append(current_speech)
+                current_speech = {}
+                if next_start < prev_end:  # previously reached silence (< neg_thres) and is still not speech (< thres)
+                    triggered = False
+                else:
+                    current_speech['start'] = next_start
+                prev_end = next_start = temp_end = 0
+            else:
+                current_speech['end'] = window_size_samples * i
+                speeches.append(current_speech)
+                current_speech = {}
+                prev_end = next_start = temp_end = 0
+                triggered = False
+                continue
+
+        if (speech_prob < neg_threshold) and triggered:
+            if not temp_end:
+                temp_end = window_size_samples * i
+            if ((window_size_samples * i) - temp_end) > min_silence_samples_at_max_speech:  # condition to avoid cutting in very short silence
+                prev_end = temp_end
+            if (window_size_samples * i) - temp_end < min_silence_samples:
+                continue
+            else:
+                current_speech['end'] = temp_end
+                if (current_speech['end'] - current_speech['start']) > min_speech_samples:
+                    speeches.append(current_speech)
+                current_speech = {}
+                prev_end = next_start = temp_end = 0
+                triggered = False
+                continue
+
+    if current_speech and (audio_length_samples - current_speech['start']) > min_speech_samples:
+        current_speech['end'] = audio_length_samples
+        speeches.append(current_speech)
+
+    for i, speech in enumerate(speeches):
+        if i == 0:
+            speech['start'] = int(max(0, speech['start'] - speech_pad_samples))
+        if i != len(speeches) - 1:
+            silence_duration = speeches[i+1]['start'] - speech['end']
+            if silence_duration < 2 * speech_pad_samples:
+                speech['end'] += int(silence_duration // 2)
+                speeches[i+1]['start'] = int(max(0, speeches[i+1]['start'] - silence_duration // 2))
+            else:
+                speech['end'] = int(min(audio_length_samples, speech['end'] + speech_pad_samples))
+                speeches[i+1]['start'] = int(max(0, speeches[i+1]['start'] - speech_pad_samples))
+        else:
+            speech['end'] = int(min(audio_length_samples, speech['end'] + speech_pad_samples))
+
+    if return_seconds:
+        audio_length_seconds = audio_length_samples / sampling_rate
+        for speech_dict in speeches:
+            speech_dict['start'] = max(round(speech_dict['start'] / sampling_rate, 1), 0)
+            speech_dict['end'] = min(round(speech_dict['end'] / sampling_rate, 1), audio_length_seconds)
+    elif step > 1:
+        for speech_dict in speeches:
+            speech_dict['start'] *= step
+            speech_dict['end'] *= step
+
+    if visualize_probs:
+        make_visualization(speech_probs, window_size_samples / sampling_rate)
+
+    return speeches
+
+
+class VADIterator:
+    def __init__(self,
+                 model,
+                 threshold: float = 0.5,
+                 sampling_rate: int = 16000,
+                 min_silence_duration_ms: int = 100,
+                 speech_pad_ms: int = 30
+                 ):
+
+        """
+        Class for stream imitation
+
+        Parameters
+        ----------
+        model: preloaded .jit/.onnx silero VAD model
+
+        threshold: float (default - 0.5)
+            Speech threshold. Silero VAD outputs speech probabilities for each audio chunk, probabilities ABOVE this value are considered as SPEECH.
+            It is better to tune this parameter for each dataset separately, but "lazy" 0.5 is pretty good for most datasets.
+
+        sampling_rate: int (default - 16000)
+            Currently silero VAD models support 8000 and 16000 sample rates
+
+        min_silence_duration_ms: int (default - 100 milliseconds)
+            In the end of each speech chunk wait for min_silence_duration_ms before separating it
+
+        speech_pad_ms: int (default - 30 milliseconds)
+            Final speech chunks are padded by speech_pad_ms each side
+        """
+
+        self.model = model
+        self.threshold = threshold
+        self.sampling_rate = sampling_rate
+
+        if sampling_rate not in [8000, 16000]:
+            raise ValueError('VADIterator does not support sampling rates other than [8000, 16000]')
+
+        self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000
+        self.speech_pad_samples = sampling_rate * speech_pad_ms / 1000
+        self.reset_states()
+
+    def reset_states(self):
+
+        self.model.reset_states()
+        self.triggered = False
+        self.temp_end = 0
+        self.current_sample = 0
+
+    @torch.no_grad()
+    def __call__(self, x, return_seconds=False):
+        """
+        x: torch.Tensor
+            audio chunk (see examples in repo)
+
+        return_seconds: bool (default - False)
+            whether return timestamps in seconds (default - samples)
+        """
+
+        if not torch.is_tensor(x):
+            try:
+                x = torch.Tensor(x)
+            except:
+                raise TypeError("Audio cannot be casted to tensor. Cast it manually")
+
+        window_size_samples = len(x[0]) if x.dim() == 2 else len(x)
+        self.current_sample += window_size_samples
+
+        speech_prob = self.model(x, self.sampling_rate).item()
+
+        if (speech_prob >= self.threshold) and self.temp_end:
+            self.temp_end = 0
+
+        if (speech_prob >= self.threshold) and not self.triggered:
+            self.triggered = True
+            speech_start = max(0, self.current_sample - self.speech_pad_samples - window_size_samples)
+            return {'start': int(speech_start) if not return_seconds else round(speech_start / self.sampling_rate, 1)}
+
+        if (speech_prob < self.threshold - 0.15) and self.triggered:
+            if not self.temp_end:
+                self.temp_end = self.current_sample
+            if self.current_sample - self.temp_end < self.min_silence_samples:
+                return None
+            else:
+                speech_end = self.temp_end + self.speech_pad_samples - window_size_samples
+                self.temp_end = 0
+                self.triggered = False
+                return {'end': int(speech_end) if not return_seconds else round(speech_end / self.sampling_rate, 1)}
+
+        return None
+
+
+def collect_chunks(tss: List[dict],
+                   wav: torch.Tensor):
+    chunks = []
+    for i in tss:
+        chunks.append(wav[i['start']: i['end']])
+    return torch.cat(chunks)
+
+
+def drop_chunks(tss: List[dict],
+                wav: torch.Tensor):
+    chunks = []
+    cur_start = 0
+    for i in tss:
+        chunks.append((wav[cur_start: i['start']]))
+        cur_start = i['end']
+    return torch.cat(chunks)

main/xiaozhi-server/performance_tester.py

@@ -0,0 +1,436 @@
+import time
+import aiohttp
+import asyncio
+from tabulate import tabulate
+from typing import Dict, List
+from core.utils.llm import create_instance as create_llm_instance
+from core.utils.tts import create_instance as create_tts_instance
+from core.utils.util import read_config
+import statistics
+from config.settings import get_config_file
+import inspect
+import os
+import logging
+
+# 设置全局日志级别为WARNING，抑制INFO级别日志
+logging.basicConfig(level=logging.WARNING)
+
+
+class AsyncPerformanceTester:
+    def __init__(self):
+        self.config = read_config(get_config_file())
+        self.test_sentences = self.config.get("module_test", {}).get(
+            "test_sentences",
+            ["你好，请介绍一下你自己", "What's the weather like today?",
+             "请用100字概括量子计算的基本原理和应用前景"]
+        )
+        self.results = {
+            "llm": {},
+            "tts": {},
+            "combinations": []
+        }
+
+    async def _check_ollama_service(self, base_url: str, model_name: str) -> bool:
+        """异步检查Ollama服务状态"""
+        async with aiohttp.ClientSession() as session:
+            try:
+                # 检查服务是否可用
+                async with session.get(f"{base_url}/api/version") as response:
+                    if response.status != 200:
+                        print(f"🚫 Ollama服务未启动或无法访问: {base_url}")
+                        return False
+
+                # 检查模型是否存在
+                async with session.get(f"{base_url}/api/tags") as response:
+                    if response.status == 200:
+                        data = await response.json()
+                        models = data.get("models", [])
+                        if not any(model["name"] == model_name for model in models):
+                            print(f"🚫 Ollama模型 {model_name} 未找到，请先使用 ollama pull {model_name} 下载")
+                            return False
+                    else:
+                        print(f"🚫 无法获取Ollama模型列表")
+                        return False
+                return True
+            except Exception as e:
+                print(f"🚫 无法连接到Ollama服务: {str(e)}")
+                return False
+
+    async def _test_tts(self, tts_name: str, config: Dict) -> Dict:
+        """异步测试单个TTS性能"""
+        try:
+            logging.getLogger("core.providers.tts.base").setLevel(logging.WARNING)
+
+            token_fields = ["access_token", "api_key", "token"]
+            if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in
+                   token_fields):
+                print(f"⏭️  TTS {tts_name} 未配置access_token/api_key，已跳过")
+                return {"name": tts_name, "type": "tts", "errors": 1}
+
+            module_type = config.get('type', tts_name)
+            tts = create_tts_instance(
+                module_type,
+                config,
+                delete_audio_file=True
+            )
+
+            print(f"🎵 测试 TTS: {tts_name}")
+
+            tmp_file = tts.generate_filename()
+            await tts.text_to_speak("连接测试", tmp_file)
+
+            if not tmp_file or not os.path.exists(tmp_file):
+                print(f"❌ {tts_name} 连接失败")
+                return {"name": tts_name, "type": "tts", "errors": 1}
+
+            total_time = 0
+            test_count = len(self.test_sentences[:2])
+
+            for i, sentence in enumerate(self.test_sentences[:2], 1):
+                start = time.time()
+                tmp_file = tts.generate_filename()
+                await tts.text_to_speak(sentence, tmp_file)
+                duration = time.time() - start
+                total_time += duration
+
+                if tmp_file and os.path.exists(tmp_file):
+                    print(f"✓ {tts_name} [{i}/{test_count}]")
+                else:
+                    print(f"✗ {tts_name} [{i}/{test_count}]")
+                    return {"name": tts_name, "type": "tts", "errors": 1}
+
+            return {
+                "name": tts_name,
+                "type": "tts",
+                "avg_time": total_time / test_count,
+                "errors": 0
+            }
+
+        except Exception as e:
+            print(f"⚠️ {tts_name} 测试失败: {str(e)}")
+            return {"name": tts_name, "type": "tts", "errors": 1}
+
+    async def _test_llm(self, llm_name: str, config: Dict) -> Dict:
+        """异步测试单个LLM性能"""
+        try:
+            # 对于Ollama，跳过api_key检查并进行特殊处理
+            if llm_name == "Ollama":
+                base_url = config.get('base_url', 'http://localhost:11434')
+                model_name = config.get('model_name')
+                if not model_name:
+                    print(f"🚫 Ollama未配置model_name")
+                    return {"name": llm_name, "type": "llm", "errors": 1}
+
+                if not await self._check_ollama_service(base_url, model_name):
+                    return {"name": llm_name, "type": "llm", "errors": 1}
+            else:
+                if "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]):
+                    print(f"🚫 跳过未配置的LLM: {llm_name}")
+                    return {"name": llm_name, "type": "llm", "errors": 1}
+
+            # 获取实际类型（兼容旧配置）
+            module_type = config.get('type', llm_name)
+            llm = create_llm_instance(module_type, config)
+
+            # 统一使用UTF-8编码
+            test_sentences = [s.encode('utf-8').decode('utf-8') for s in self.test_sentences]
+
+            # 创建所有句子的测试任务
+            sentence_tasks = []
+            for sentence in test_sentences:
+                sentence_tasks.append(self._test_single_sentence(llm_name, llm, sentence))
+
+            # 并发执行所有句子测试
+            sentence_results = await asyncio.gather(*sentence_tasks)
+
+            # 处理结果
+            valid_results = [r for r in sentence_results if r is not None]
+            if not valid_results:
+                print(f"⚠️  {llm_name} 无有效数据，可能配置错误")
+                return {"name": llm_name, "type": "llm", "errors": 1}
+
+            first_token_times = [r["first_token_time"] for r in valid_results]
+            response_times = [r["response_time"] for r in valid_results]
+
+            # 过滤异常数据
+            mean = statistics.mean(response_times)
+            stdev = statistics.stdev(response_times) if len(response_times) > 1 else 0
+            filtered_times = [t for t in response_times if t <= mean + 3 * stdev]
+
+            if len(filtered_times) < len(test_sentences) * 0.5:
+                print(f"⚠️  {llm_name} 有效数据不足，可能网络不稳定")
+                return {"name": llm_name, "type": "llm", "errors": 1}
+
+            return {
+                "name": llm_name,
+                "type": "llm",
+                "avg_response": sum(response_times) / len(response_times),
+                "avg_first_token": sum(first_token_times) / len(first_token_times),
+                "std_first_token": statistics.stdev(first_token_times) if len(first_token_times) > 1 else 0,
+                "std_response": statistics.stdev(response_times) if len(response_times) > 1 else 0,
+                "errors": 0
+            }
+        except Exception as e:
+            print(f"LLM {llm_name} 测试失败: {str(e)}")
+            return {"name": llm_name, "type": "llm", "errors": 1}
+
+    async def _test_single_sentence(self, llm_name: str, llm, sentence: str) -> Dict:
+        """测试单个句子的性能"""
+        try:
+            print(f"📝 {llm_name} 开始测试: {sentence[:20]}...")
+            sentence_start = time.time()
+            first_token_received = False
+            first_token_time = None
+
+            async def process_response():
+                nonlocal first_token_received, first_token_time
+                for chunk in llm.response("perf_test", [{"role": "user", "content": sentence}]):
+                    if not first_token_received and chunk.strip() != '':
+                        first_token_time = time.time() - sentence_start
+                        first_token_received = True
+                        print(f"✓ {llm_name} 首个Token: {first_token_time:.3f}s")
+                    yield chunk
+
+            response_chunks = []
+            async for chunk in process_response():
+                response_chunks.append(chunk)
+
+            response_time = time.time() - sentence_start
+            print(f"✓ {llm_name} 完成响应: {response_time:.3f}s")
+
+            if first_token_time is None:
+                first_token_time = response_time  # 如果没有检测到first token，使用总响应时间
+
+            return {
+                "name": llm_name,
+                "type": "llm",
+                "first_token_time": first_token_time,
+                "response_time": response_time
+            }
+        except Exception as e:
+            print(f"⚠️ {llm_name} 句子测试失败: {str(e)}")
+            return None
+
+    def _generate_combinations(self):
+        """生成最佳组合建议"""
+        valid_llms = [
+            k for k, v in self.results["llm"].items()
+            if v["errors"] == 0 and v["avg_first_token"] >= 0.05
+        ]
+        valid_tts = [k for k, v in self.results["tts"].items() if v["errors"] == 0]
+
+        # 找出基准值
+        min_first_token = min([self.results["llm"][llm]["avg_first_token"] for llm in valid_llms]) if valid_llms else 1
+        min_tts_time = min([self.results["tts"][tts]["avg_time"] for tts in valid_tts]) if valid_tts else 1
+
+        for llm in valid_llms:
+            for tts in valid_tts:
+                # 计算相对性能分数（越小越好）
+                llm_score = self.results["llm"][llm]["avg_first_token"] / min_first_token
+                tts_score = self.results["tts"][tts]["avg_time"] / min_tts_time
+
+                # 计算稳定性分数（标准差/平均值，越小越稳定）
+                llm_stability = self.results["llm"][llm]["std_first_token"] / self.results["llm"][llm][
+                    "avg_first_token"]
+
+                # 综合得分（考虑性能和稳定性）
+                # 性能权重0.7，稳定性权重0.3
+                llm_final_score = llm_score * 0.7 + llm_stability * 0.3
+
+                # 总分 = LLM得分(70%) + TTS得分(30%)
+                total_score = llm_final_score * 0.7 + tts_score * 0.3
+
+                self.results["combinations"].append({
+                    "llm": llm,
+                    "tts": tts,
+                    "score": total_score,
+                    "details": {
+                        "llm_first_token": self.results["llm"][llm]["avg_first_token"],
+                        "llm_stability": llm_stability,
+                        "tts_time": self.results["tts"][tts]["avg_time"]
+                    }
+                })
+
+        # 分数越小越好
+        self.results["combinations"].sort(key=lambda x: x["score"])
+
+    def _print_results(self):
+        """打印测试结果"""
+        llm_table = []
+        for name, data in self.results["llm"].items():
+            if data["errors"] == 0:
+                stability = data["std_first_token"] / data["avg_first_token"]
+                llm_table.append([
+                    name,  # 不需要固定宽度，让tabulate自己处理对齐
+                    f"{data['avg_first_token']:.3f}秒",
+                    f"{data['avg_response']:.3f}秒",
+                    f"{stability:.3f}"
+                ])
+
+        if llm_table:
+            print("\nLLM 性能排行:")
+            print(tabulate(
+                llm_table,
+                headers=["模型名称", "首字耗时", "总耗时", "稳定性"],
+                tablefmt="github",
+                colalign=("left", "right", "right", "right"),
+                disable_numparse=True
+            ))
+        else:
+            print("\n⚠️ 没有可用的LLM模块进行测试。")
+
+        tts_table = []
+        for name, data in self.results["tts"].items():
+            if data["errors"] == 0:
+                tts_table.append([
+                    name,  # 不需要固定宽度
+                    f"{data['avg_time']:.3f}秒"
+                ])
+
+        if tts_table:
+            print("\nTTS 性能排行:")
+            print(tabulate(
+                tts_table,
+                headers=["模型名称", "合成耗时"],
+                tablefmt="github",
+                colalign=("left", "right"),
+                disable_numparse=True
+            ))
+        else:
+            print("\n⚠️ 没有可用的TTS模块进行测试。")
+
+        if self.results["combinations"]:
+            print("\n推荐配置组合 (得分越小越好):")
+            combo_table = []
+            for combo in self.results["combinations"][:5]:
+                combo_table.append([
+                    f"{combo['llm']} + {combo['tts']}",  # 不需要固定宽度
+                    f"{combo['score']:.3f}",
+                    f"{combo['details']['llm_first_token']:.3f}秒",
+                    f"{combo['details']['llm_stability']:.3f}",
+                    f"{combo['details']['tts_time']:.3f}秒"
+                ])
+
+            print(tabulate(
+                combo_table,
+                headers=["组合方案", "综合得分", "LLM首字耗时", "稳定性", "TTS合成耗时"],
+                tablefmt="github",
+                colalign=("left", "right", "right", "right", "right"),
+                disable_numparse=True
+            ))
+        else:
+            print("\n⚠️ 没有可用的模块组合建议。")
+
+    def _process_results(self, all_results):
+        """处理测试结果"""
+        for result in all_results:
+            if result["errors"] == 0:
+                if result["type"] == "llm":
+                    self.results["llm"][result["name"]] = result
+                else:
+                    self.results["tts"][result["name"]] = result
+
+    async def run(self):
+        """执行全量异步测试"""
+        print("🔍 开始筛选可用模块...")
+
+        # 创建所有测试任务
+        all_tasks = []
+
+        # LLM测试任务
+        for llm_name, config in self.config.get("LLM", {}).items():
+            # 检查配置有效性
+            if llm_name == "CozeLLM":
+                if any(x in config.get("bot_id", "") for x in ["你的"]) \
+                        or any(x in config.get("user_id", "") for x in ["你的"]):
+                    print(f"⏭️  LLM {llm_name} 未配置bot_id/user_id，已跳过")
+                    continue
+            elif "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]):
+                print(f"⏭️  LLM {llm_name} 未配置api_key，已跳过")
+                continue
+
+            # 对于Ollama，先检查服务状态
+            if llm_name == "Ollama":
+                base_url = config.get('base_url', 'http://localhost:11434')
+                model_name = config.get('model_name')
+                if not model_name:
+                    print(f"🚫 Ollama未配置model_name")
+                    continue
+
+                if not await self._check_ollama_service(base_url, model_name):
+                    continue
+
+            print(f"📋 添加LLM测试任务: {llm_name}")
+            module_type = config.get('type', llm_name)
+            llm = create_llm_instance(module_type, config)
+
+            # 为每个句子创建独立任务
+            for sentence in self.test_sentences:
+                sentence = sentence.encode('utf-8').decode('utf-8')
+                all_tasks.append(self._test_single_sentence(llm_name, llm, sentence))
+
+        # TTS测试任务
+        for tts_name, config in self.config.get("TTS", {}).items():
+            token_fields = ["access_token", "api_key", "token"]
+            if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in
+                   token_fields):
+                print(f"⏭️  TTS {tts_name} 未配置access_token/api_key，已跳过")
+                continue
+            print(f"🎵 添加TTS测试任务: {tts_name}")
+            all_tasks.append(self._test_tts(tts_name, config))
+
+        print(
+            f"\n✅ 找到 {len([t for t in all_tasks if 'test_single_sentence' in str(t)]) / len(self.test_sentences):.0f} 个可用LLM模块")
+        print(f"✅ 找到 {len([t for t in all_tasks if '_test_tts' in str(t)])} 个可用TTS模块")
+        print("\n⏳ 开始并发测试所有模块...\n")
+
+        # 并发执行所有测试任务
+        all_results = await asyncio.gather(*all_tasks, return_exceptions=True)
+
+        # 处理LLM结果
+        llm_results = {}
+        for result in [r for r in all_results if r and isinstance(r, dict) and r.get("type") == "llm"]:
+            llm_name = result["name"]
+            if llm_name not in llm_results:
+                llm_results[llm_name] = {
+                    "name": llm_name,
+                    "type": "llm",
+                    "first_token_times": [],
+                    "response_times": [],
+                    "errors": 0
+                }
+            llm_results[llm_name]["first_token_times"].append(result["first_token_time"])
+            llm_results[llm_name]["response_times"].append(result["response_time"])
+
+        # 计算LLM平均值和标准差
+        for llm_name, data in llm_results.items():
+            if len(data["first_token_times"]) >= len(self.test_sentences) * 0.5:
+                self.results["llm"][llm_name] = {
+                    "name": llm_name,
+                    "type": "llm",
+                    "avg_response": sum(data["response_times"]) / len(data["response_times"]),
+                    "avg_first_token": sum(data["first_token_times"]) / len(data["first_token_times"]),
+                    "std_first_token": statistics.stdev(data["first_token_times"]) if len(
+                        data["first_token_times"]) > 1 else 0,
+                    "std_response": statistics.stdev(data["response_times"]) if len(data["response_times"]) > 1 else 0,
+                    "errors": 0
+                }
+
+        # 处理TTS结果
+        for result in [r for r in all_results if r and isinstance(r, dict) and r.get("type") == "tts"]:
+            if result["errors"] == 0:
+                self.results["tts"][result["name"]] = result
+
+        # 生成组合建议并打印结果
+        print("\n📊 生成测试报告...")
+        self._generate_combinations()
+        self._print_results()
+
+
+async def main():
+    tester = AsyncPerformanceTester()
+    await tester.run()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

main/xiaozhi-server/requirements.txt

@@ -0,0 +1,21 @@
+pyyml==0.0.2
+torch==2.2.2
+silero_vad==5.1.2
+websockets==14.2
+opuslib_next==1.1.2
+numpy==1.26.4
+pydub==0.25.1
+funasr==1.2.3
+torchaudio==2.2.2
+openai==1.61.0
+google-generativeai==0.8.4
+edge_tts==7.0.0
+httpx==0.27.2
+aiohttp==3.9.3
+aiohttp_cors==0.7.0
+ormsgpack==1.7.0
+ruamel.yaml==0.18.10
+loguru==0.7.3
+requests==2.32.3
+cozepy==0.12.0
+mem0ai==0.1.62