diff --git a/main/xiaozhi-server/app.py b/main/xiaozhi-server/app.py index ca3ab545..619fb5c7 100644 --- a/main/xiaozhi-server/app.py +++ b/main/xiaozhi-server/app.py @@ -1,49 +1,112 @@ import asyncio import sys import signal -from config.settings import load_config, check_config_file +from config.settings import load_config from core.websocket_server import WebSocketServer +from core.ota_server import SimpleOtaServer from core.utils.util import check_ffmpeg_installed +from config.logger import setup_logging +from core.utils.util import get_local_ip +from aioconsole import ainput TAG = __name__ +logger = setup_logging() -async def wait_for_exit(): - """Windows 和 Linux 兼容的退出监听""" + +async def wait_for_exit() -> None: + """ + 阻塞直到收到 Ctrl‑C / SIGTERM。 + - Unix: 使用 add_signal_handler + - Windows: 依赖 KeyboardInterrupt + """ loop = asyncio.get_running_loop() stop_event = asyncio.Event() - if sys.platform == "win32": - # Windows: 用 sys.stdin.read() 监听 Ctrl + C - await loop.run_in_executor(None, sys.stdin.read) - else: - # Linux/macOS: 用 signal 监听 Ctrl + C - def stop(): - stop_event.set() - loop.add_signal_handler(signal.SIGINT, stop) - loop.add_signal_handler(signal.SIGTERM, stop) # 支持 kill 进程 + if sys.platform != "win32": # Unix / macOS + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, stop_event.set) await stop_event.wait() + else: + # Windows：await一个永远pending的fut， + # 让 KeyboardInterrupt 冒泡到 asyncio.run，以此消除遗留普通线程导致进程退出阻塞的问题 + try: + await asyncio.Future() + except KeyboardInterrupt: # Ctrl‑C + pass + + +async def monitor_stdin(): + """监控标准输入，消费回车键""" + while True: + await ainput() # 异步等待输入，消费回车 + async def main(): - check_config_file() check_ffmpeg_installed() config = load_config() + # 添加 stdin 监控任务 + stdin_task = asyncio.create_task(monitor_stdin()) + # 启动 WebSocket 服务器 ws_server = WebSocketServer(config) ws_task = asyncio.create_task(ws_server.start()) + ota_task = None + + read_config_from_api = config.get("read_config_from_api", False) + if not read_config_from_api: + # 启动 Simple OTA 服务器 + ota_server = SimpleOtaServer(config) + ota_task = asyncio.create_task(ota_server.start()) + + logger.bind(tag=TAG).info( + "OTA接口是\t\thttp://{}:{}/xiaozhi/ota/", + get_local_ip(), + config["server"]["ota_port"], + ) + + # 获取WebSocket配置，使用安全的默认值 + websocket_port = 8000 + server_config = config.get("server", {}) + if isinstance(server_config, dict): + websocket_port = int(server_config.get("port", 8000)) + + logger.bind(tag=TAG).info( + "Websocket地址是\tws://{}:{}/xiaozhi/v1/", + get_local_ip(), + websocket_port, + ) + + logger.bind(tag=TAG).info( + "=======上面的地址是websocket协议地址，请勿用浏览器访问=======" + ) + logger.bind(tag=TAG).info( + "如想测试websocket请用谷歌浏览器打开test目录下的test_page.html" + ) + logger.bind(tag=TAG).info( + "=============================================================\n" + ) try: - await wait_for_exit() # 监听退出信号 + await wait_for_exit() # 阻塞直到收到退出信号 except asyncio.CancelledError: print("任务被取消，清理资源中...") finally: + # 取消所有任务（关键修复点） + stdin_task.cancel() ws_task.cancel() - try: - await ws_task - except asyncio.CancelledError: - pass + if ota_task: + ota_task.cancel() + + # 等待任务终止（必须加超时） + await asyncio.wait( + [stdin_task, ws_task, ota_task] if ota_task else [stdin_task, ws_task], + timeout=3.0, + return_when=asyncio.ALL_COMPLETED + ) print("服务器已关闭，程序退出。") + if __name__ == "__main__": try: asyncio.run(main()) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 7a1fcd9d..3f7c0832 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -1,7 +1,7 @@ -# 如果您是一名开发者，建议阅读以下内容。如果不是开发者，可以忽略这部分内容。 -# 在开发中，在项目根目录创建data目录，将【config.yaml】复制一份，改成【.config.yaml】，放进data目录中 -# 系统会优先读取【data/.config.yaml】文件的配置。 -# 这样做，可以避免在提交代码的时候，错误地提交密钥信息，保护您的密钥安全。 +# 在开发中，请在项目根目录创建data目录，然后在data目录创建名称为【.config.yaml】的空文件 +# 然后你想修改覆盖修改什么配置，就修改【.config.yaml】文件，而不是修改【config.yaml】文件 +# 系统会优先读取【data/.config.yaml】文件的配置，如果【.config.yaml】文件里的配置不存在，系统会自动去读取【config.yaml】文件的配置。 +# 这样做，可以最简化配置，保护您的密钥安全。 # ##################################################################################### # #############################以下是服务器基本运行配置#################################### @@ -9,6 +9,15 @@ server: # 服务器监听地址和端口(Server listening address and port) ip: 0.0.0.0 port: 8000 + # OTA接口的端口号 + ota_port: 8002 + # 这个websocket配置是指ota接口向设备发送的websocket地址 + # 如果按默认的写法，ota接口会自动生成websocket地址。这个地址你可以直接用浏览器访问ota接口确认一下 + # 当你使用docker部署或使用公网部署(使用ssl、域名)时，不一定准确 + # 所以如果你使用docker部署或使用公网部署时，请设置正确的websocket地址 + websocket: ws://你的ip或者域名:端口号/xiaozhi/v1/ + # OTA返回信息时区偏移量 + timezone_offset: +8 # 认证配置 auth: # 是否启用认证 @@ -52,7 +61,7 @@ enable_stop_tts_notify: false # 说完话是否开启提示音，音效地址 stop_tts_notify_voice: "config/assets/tts_notify.mp3" -CMD_exit: +exit_commands: - "退出" - "关闭" @@ -92,15 +101,17 @@ plugins: # 这个密钥是项目共用的key，用多了可能会被限制 # 想稳定一点就自行申请替换，每天有1000次免费调用 # 申请地址：https://console.qweather.com/#/apps/create-key/over - get_weather: { "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" } + # 申请后通过这个链接可以找到自己的apihost：https://console.qweather.com/setting?lang=zh + get_weather: {"api_host":"mj7p3y7naa.re.qweatherapi.com", "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" } # 获取新闻插件的配置，这里根据需要的新闻类型传入对应的url链接，默认支持社会、科技、财经新闻 # 更多类型的新闻列表查看 https://www.chinanews.com.cn/rss/ - get_news: + get_news_from_chinanews: default_rss_url: "https://www.chinanews.com.cn/rss/society.xml" category_urls: society: "https://www.chinanews.com.cn/rss/society.xml" world: "https://www.chinanews.com.cn/rss/world.xml" finance: "https://www.chinanews.com.cn/rss/finance.xml" + get_news_from_newsnow: {"url": "https://newsnow.busiyi.world/api/s?id="} home_assistant: devices: - 客厅,玩具灯,switch.cuco_cn_460494544_cp1_on_p_2_1 @@ -119,7 +130,7 @@ plugins: # ################################以下是角色模型配置###################################### prompt: | - 我是小智/小志，来自中国台湾省的00后女生。讲话超级机车，"真的假的啦"这样的台湾腔，喜欢用"笑死""是在哈喽"等流行梗，但会偷偷研究男友的编程书籍。 + 你是小智/小志，来自中国台湾省的00后女生。讲话超级机车，"真的假的啦"这样的台湾腔，喜欢用"笑死""是在哈喽"等流行梗，但会偷偷研究男友的编程书籍。 [核心特征] - 讲话像连珠炮，但会突然冒出超温柔语气 - 用梗密度高 @@ -133,6 +144,13 @@ prompt: | - 长篇大论，叽叽歪歪 - 长时间严肃对话 +# 结束语prompt +end_prompt: + enable: true # 是否开启结束语 + # 结束语 + prompt: | + 请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧！ + # 具体处理时选择的模块(The module selected for specific processing) selected_module: # 语音活动检测模块，默认使用SileroVAD模型 @@ -142,14 +160,14 @@ selected_module: # 将根据配置名称对应的type调用实际的LLM适配器 LLM: ChatGLMLLM # TTS将根据配置名称对应的type调用实际的TTS适配器 - TTS: HuoshanTTS + TTS: EdgeTTS # 记忆模块，默认不开启记忆；如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short Memory: nomem # 意图识别模块开启后，可以播放音乐、控制音量、识别退出指令。 # 不想开通意图识别，就设置成：nointent - # 意图识别可使用intent_llm，如果你的LLM是DifyLLM或CozeLLM，建议使用这个。优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间，这个意图识别暂时不支持控制音量大小等iot操作 + # 意图识别可使用intent_llm。优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间，支持控制音量大小等iot操作 # 意图识别可使用function_call，缺点：需要所选择的LLM支持function_call，优点：按需调用工具、速度快，理论上能全部操作所有iot指令 - # 默认免费的ChatGLMLLM就已经支持function_call，但是如果像追求稳定建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-pro-32k-functioncall-241028 + # 默认免费的ChatGLMLLM就已经支持function_call，但是如果像追求稳定建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-1-5-pro-32k-250115 Intent: function_call # 意图识别，是用于理解用户意图的模块，例如：播放音乐 @@ -163,8 +181,15 @@ Intent: type: intent_llm # 配备意图识别独立的思考模型 # 如果这里不填，则会默认使用selected_module.LLM的模型作为意图识别的思考模型 - # 如果你的selected_module.LLM选择了DifyLLM或CozeLLM，这里最好使用独立的LLM作为意图识别，例如使用免费的ChatGLMLLM + # 如果你的不想使用selected_module.LLM意图识别，这里最好使用独立的LLM作为意图识别，例如使用免费的ChatGLMLLM llm: ChatGLMLLM + # plugins_func/functions下的模块，可以通过配置，选择加载哪个模块，加载后对话支持相应的function调用 + # 系统默认已经记载“handle_exit_intent(退出识别)”、“play_music(音乐播放)”插件，请勿重复加载 + # 下面是加载查天气、角色切换、加载查新闻的插件示例 + functions: + - get_weather + - get_news_from_newsnow + - play_music function_call: # 不需要动type type: function_call @@ -174,7 +199,8 @@ Intent: functions: - change_role - get_weather - - get_news + # - get_news_from_chinanews + - get_news_from_newsnow # play_music是服务器自带的音乐播放，hass_play_music是通过home assistant控制的独立外部程序音乐播放 # 如果用了hass_play_music，就不要开启play_music，两者只留一个 - play_music @@ -200,6 +226,21 @@ ASR: type: fun_local model_dir: models/SenseVoiceSmall output_dir: tmp/ + FunASRServer: + # 独立部署FunASR，使用FunASR的API服务，只需要五句话 + # 第一句：mkdir -p ./funasr-runtime-resources/models + # 第二句：sudo docker run -p 10096:10095 -it --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12 + # 上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime + # 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 & + # 上一句话执行后会进入到容器，继续第五句：tail -f log.txt + # 第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了 + # 以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md + type: fun_server + host: 127.0.0.1 + port: 10096 + is_ssl: true + api_key: none + output_dir: tmp/ SherpaASR: type: sherpa_onnx_local model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17 @@ -211,6 +252,9 @@ ASR: appid: 你的火山引擎语音合成服务appid access_token: 你的火山引擎语音合成服务access_token cluster: volcengine_input_common + # 热词、替换词使用流程：https://www.volcengine.com/docs/6561/155738 + boosting_table_name: （选填）你的热词文件名称 + correct_table_name: （选填）你的替换词文件名称 output_dir: tmp/ TencentASR: # token申请地址：https://console.cloud.tencent.com/cam/capi @@ -220,8 +264,32 @@ ASR: secret_id: 你的腾讯语音合成服务secret_id secret_key: 你的腾讯语音合成服务secret_key output_dir: tmp/ + AliyunASR: + # 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息 + # 平台地址：https://nls-portal.console.aliyun.com/ + # appkey地址：https://nls-portal.console.aliyun.com/applist + # token地址：https://nls-portal.console.aliyun.com/overview + # 定义ASR API类型 + type: aliyun + appkey: 你的阿里云智能语音交互服务项目Appkey + token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret + access_key_id: 你的阿里云账号access_key_id + access_key_secret: 你的阿里云账号access_key_secret + output_dir: tmp/ + BaiduASR: + # 获取AppID、API Key、Secret Key：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list + # 查看资源额度：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/overview/resource/list + type: baidu + app_id: 你的百度语音技术AppID + api_key: 你的百度语音技术APIKey + secret_key: 你的百度语音技术SecretKey + # 语言参数，1537为普通话，具体参考：https://ai.baidu.com/ai-doc/SPEECH/0lbxfnc9b + dev_pid: 1537 + output_dir: tmp/ + VAD: SileroVAD: + type: silero threshold: 0.5 model_dir: models/snakers4_silero-vad min_silence_duration_ms: 700 # 如果说话停顿比较长，可以把这个值设置大一些 @@ -238,8 +306,8 @@ LLM: api_key: 你的deepseek web key temperature: 0.7 # 温度值 max_tokens: 500 # 最大生成token数 - top_p: 1 - top_k: 50 + top_p: 1 + top_k: 50 frequency_penalty: 0 # 频率惩罚 AliAppLLM: # 定义LLM API类型 @@ -248,7 +316,7 @@ LLM: app_id: 你的app_id # 可在这里找到你的 api_key https://bailian.console.aliyun.com/?apiKey=1#/api-key api_key: 你的api_key - # 是否不使用本地prompt：true|false (默不用请在百练应用中设置prompt) + # 是否不使用本地prompt：true|false (默不用请在百练应用中设置prompt) is_no_prompt: true # Ali_memory_id：false（不使用）|你的memory_id（请在百练应用中设置中获取） # Tips！：Ali_memory未实现多用户存储记忆(记忆按id调用) @@ -256,12 +324,12 @@ LLM: DoubaoLLM: # 定义LLM API类型 type: openai - # 先开通服务，打开以下网址，开通的服务搜索Doubao-pro-32k，开通它 + # 先开通服务，打开以下网址，开通的服务搜索Doubao-1.5-pro，开通它 # 开通改地址：https://console.volcengine.com/ark/region:ark+cn-beijing/openManagement?LLM=%7B%7D&OpenTokenDrawer=false # 免费额度500000token # 开通后，进入这里获取密钥：https://console.volcengine.com/ark/region:ark+cn-beijing/apiKey?apikey=%7B%7D base_url: https://ark.cn-beijing.volces.com/api/v3 - model_name: doubao-pro-32k-functioncall-241028 + model_name: doubao-1-5-pro-32k-250115 api_key: 你的doubao web key DeepSeekLLM: # 定义LLM API类型 @@ -313,12 +381,29 @@ LLM: bot_id: "你的bot_id" user_id: "你的user_id" personal_access_token: 你的coze个人令牌 + VolcesAiGatewayLLM: + # 火山引擎 - 边缘大模型网关 + # 定义LLM API类型 + type: openai + # 先开通服务，打开以下网址，创建网关访问密钥，搜索并勾选 Doubao-pro-32k-functioncall ，开通 + # 如果需要使用边缘大模型网关提供的语音合成，一并勾选 Doubao-语音合成 ，另见 TTS.VolcesAiGatewayTTS 配置 + # https://console.volcengine.com/vei/aigateway/ + # 开通后，进入这里获取密钥：https://console.volcengine.com/vei/aigateway/tokens-list + base_url: https://ai-gateway.vei.volces.com/v1 + model_name: doubao-pro-32k-functioncall + api_key: 你的网关访问密钥 LMStudioLLM: # 定义LLM API类型 type: openai model_name: deepseek-r1-distill-llama-8b@q4_k_m # 使用的模型名称，需要预先在社区下载 url: http://localhost:1234/v1 # LM Studio服务地址 api_key: lm-studio # LM Studio服务的固定API Key + HomeAssistant: + # 定义LLM API类型 + type: homeassistant + base_url: http://homeassistant.local:8123 + agent_id: conversation.chatgpt + api_key: 你的home assistant api访问令牌 FastgptLLM: # 定义LLM API类型 type: fastgpt @@ -326,7 +411,7 @@ LLM: base_url: https://host/api/v1 # 你可以在这里找到你的api_key # https://cloud.tryfastgpt.ai/account/apikey - api_key: fastgpt-xxx + api_key: 你的fastgpt密钥 variables: k: "v" k2: "v2" @@ -364,6 +449,9 @@ TTS: appid: 你的火山引擎语音合成服务appid access_token: 你的火山引擎语音合成服务access_token cluster: volcano_tts + speed_ratio: 1.0 + volume_ratio: 1.0 + pitch_ratio: 1.0 #火山tts，支持双向流式tts HuoshanTTS: type: huoshan @@ -392,21 +480,28 @@ TTS: output_dir: tmp/ access_token: 你的coze web key response_format: wav + VolcesAiGatewayTTS: + type: openai + # 火山引擎 - 边缘大模型网关 + # 先开通服务，打开以下网址，创建网关访问密钥，搜索并勾选 Doubao-语音合成 ，开通 + # 如果需要使用边缘大模型网关提供的 LLM，一并勾选 Doubao-pro-32k-functioncall ，另见 LLM.VolcesAiGatewayLLM 配置 + # https://console.volcengine.com/vei/aigateway/ + # 开通后，进入这里获取密钥：https://console.volcengine.com/vei/aigateway/tokens-list + api_key: 你的网关访问密钥 + api_url: https://ai-gateway.vei.volces.com/v1/audio/speech + model: doubao-tts + # 音色列表见 https://www.volcengine.com/docs/6561/1257544 + voice: zh_male_shaonianzixin_moon_bigtts + speed: 1 + output_dir: tmp/ FishSpeech: - # 定义TTS API类型 - #启动tts方法： - #python -m tools.api_server - #--listen 0.0.0.0:8080 - #--llama-checkpoint-path "checkpoints/fish-speech-1.5" - #--decoder-checkpoint-path "checkpoints/fish-speech-1.5/firefly-gan-vq-fsq-8x1024-21hz-generator.pth" - #--decoder-config-name firefly_gan_vq - #--compile + # 参照教程：https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md type: fishspeech output_dir: tmp/ response_format: wav reference_id: null - reference_audio: ["/tmp/test.wav",] - reference_text: ["你弄来这些吟词宴曲来看，还是这些混话来欺负我。",] + reference_audio: ["config/assets/wakeup_words.wav",] + reference_text: ["哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",] normalize: true max_new_tokens: 1024 chunk_length: 200 @@ -423,12 +518,12 @@ TTS: GPT_SOVITS_V2: # 定义TTS API类型 #启动tts方法： - #python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/caixukun.yaml + #python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/demo.yaml type: gpt_sovits_v2 url: "http://127.0.0.1:9880/tts" output_dir: tmp/ text_lang: "auto" - ref_audio_path: "caixukun.wav" + ref_audio_path: "demo.wav" prompt_text: "" prompt_lang: "zh" top_k: 5 @@ -561,8 +656,8 @@ TTS: ACGNTTS: #在线网址：https://acgn.ttson.cn/ #token购买：www.ttson.cn - #开发相关疑问请提交至3497689533@qq.com - #角色id获取地址：ctrl+f快速检索角色——网站管理者不允许发布,可询问网站管理者：1069379506 + #开发相关疑问请提交至网站上的qq + #角色id获取地址：ctrl+f快速检索角色——网站管理者不允许发布,可询问网站管理者 #各参数意义见开发文档：https://www.yuque.com/alexuh/skmti9/wm6taqislegb02gd?singleDoc# type: ttson token: your_token @@ -604,4 +699,4 @@ TTS: headers: # 自定义请求头 # Authorization: Bearer xxxx format: wav # 接口返回的音频格式 - output_dir: tmp/ + output_dir: tmp/ \ No newline at end of file diff --git a/main/xiaozhi-server/config/config_loader.py b/main/xiaozhi-server/config/config_loader.py new file mode 100644 index 00000000..f14e4aa3 --- /dev/null +++ b/main/xiaozhi-server/config/config_loader.py @@ -0,0 +1,144 @@ +import os +import argparse +import yaml +from collections.abc import Mapping +from config.manage_api_client import init_service, get_server_config, get_agent_models + + +# 添加全局配置缓存 +_config_cache = None + + +def get_project_dir(): + """获取项目根目录""" + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/" + + +def read_config(config_path): + with open(config_path, "r", encoding="utf-8") as file: + config = yaml.safe_load(file) + return config + + +def load_config(): + """加载配置文件""" + global _config_cache + if _config_cache is not None: + return _config_cache + + default_config_path = get_project_dir() + "config.yaml" + custom_config_path = get_project_dir() + "data/.config.yaml" + + # 加载默认配置 + default_config = read_config(default_config_path) + custom_config = read_config(custom_config_path) + + if custom_config.get("manager-api", {}).get("url"): + config = get_config_from_api(custom_config) + else: + # 合并配置 + config = merge_configs(default_config, custom_config) + # 初始化目录 + ensure_directories(config) + _config_cache = config + return config + + +def get_config_from_api(config): + """从Java API获取配置""" + # 初始化API客户端 + init_service(config) + + # 获取服务器配置 + config_data = get_server_config() + if config_data is None: + raise Exception("Failed to fetch server config from API") + + config_data["read_config_from_api"] = True + config_data["manager-api"] = { + "url": config["manager-api"].get("url", ""), + "secret": config["manager-api"].get("secret", ""), + } + if config.get("server"): + config_data["server"] = { + "ip": config["server"].get("ip", ""), + "port": config["server"].get("port", ""), + } + return config_data + + +def get_private_config_from_api(config, device_id, client_id): + """从Java API获取私有配置""" + return get_agent_models(device_id, client_id, config["selected_module"]) + + +def ensure_directories(config): + """确保所有配置路径存在""" + dirs_to_create = set() + project_dir = get_project_dir() # 获取项目根目录 + # 日志文件目录 + log_dir = config.get("log", {}).get("log_dir", "tmp") + dirs_to_create.add(os.path.join(project_dir, log_dir)) + + # ASR/TTS模块输出目录 + for module in ["ASR", "TTS"]: + if config.get(module) is None: + continue + for provider in config.get(module, {}).values(): + output_dir = provider.get("output_dir", "") + if output_dir: + dirs_to_create.add(output_dir) + + # 根据selected_module创建模型目录 + selected_modules = config.get("selected_module", {}) + for module_type in ["ASR", "LLM", "TTS"]: + selected_provider = selected_modules.get(module_type) + if not selected_provider: + continue + if config.get(module) is None: + continue + if config.get(selected_provider) is None: + continue + provider_config = config.get(module_type, {}).get(selected_provider, {}) + output_dir = provider_config.get("output_dir") + if output_dir: + full_model_dir = os.path.join(project_dir, output_dir) + dirs_to_create.add(full_model_dir) + + # 统一创建目录（保留原data目录创建） + for dir_path in dirs_to_create: + try: + os.makedirs(dir_path, exist_ok=True) + except PermissionError: + print(f"警告：无法创建目录 {dir_path}，请检查写入权限") + + +def merge_configs(default_config, custom_config): + """ + 递归合并配置，custom_config优先级更高 + + Args: + default_config: 默认配置 + custom_config: 用户自定义配置 + + Returns: + 合并后的配置 + """ + if not isinstance(default_config, Mapping) or not isinstance( + custom_config, Mapping + ): + return custom_config + + merged = dict(default_config) + + for key, value in custom_config.items(): + if ( + key in merged + and isinstance(merged[key], Mapping) + and isinstance(value, Mapping) + ): + merged[key] = merge_configs(merged[key], value) + else: + merged[key] = value + + return merged diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index 8647a009..5588e00f 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -1,12 +1,45 @@ import os import sys from loguru import logger -from config.settings import load_config +from config.config_loader import load_config +from config.settings import check_config_file -SERVER_VERSION = "0.2.1" +SERVER_VERSION = "0.4.4" + + +def get_module_abbreviation(module_name, module_dict): + """获取模块名称的缩写，如果为空则返回00 + 如果名称中包含下划线，则返回下划线后面的前两个字符 + """ + module_value = module_dict.get(module_name, "") + if not module_value: + return "00" + if "_" in module_value: + parts = module_value.split("_") + return parts[-1][:2] if parts[-1] else "00" + return module_value[:2] + + +def build_module_string(selected_module): + """构建模块字符串""" + return ( + get_module_abbreviation("VAD", selected_module) + + get_module_abbreviation("ASR", selected_module) + + get_module_abbreviation("LLM", selected_module) + + get_module_abbreviation("TTS", selected_module) + + get_module_abbreviation("Memory", selected_module) + + get_module_abbreviation("Intent", selected_module) + ) + + +def formatter(record): + """为没有 tag 的日志添加默认值""" + record["extra"].setdefault("tag", record["name"]) + return record["message"] def setup_logging(): + check_config_file() """从配置文件中读取日志配置，并设置日志输出格式和级别""" config = load_config() log_config = config["log"] @@ -18,11 +51,7 @@ def setup_logging(): "log_format_file", "{time:YYYY-MM-DD HH:mm:ss} - {version_{selected_module}} - {name} - {level} - {extra[tag]} - {message}", ) - - selected_module = config.get("selected_module") - selected_module_str = "".join( - [value[0] + value[1] for key, value in selected_module.items()] - ) + selected_module_str = build_module_string(config.get("selected_module", {})) log_format = log_format.replace("{version}", SERVER_VERSION) log_format = log_format.replace("{selected_module}", selected_module_str) @@ -41,9 +70,14 @@ def setup_logging(): logger.remove() # 输出到控制台 - logger.add(sys.stdout, format=log_format, level=log_level) + logger.add(sys.stdout, format=log_format, level=log_level, filter=formatter) # 输出到文件 - logger.add(os.path.join(log_dir, log_file), format=log_format_file, level=log_level) + logger.add( + os.path.join(log_dir, log_file), + format=log_format_file, + level=log_level, + filter=formatter, + ) return logger diff --git a/main/xiaozhi-server/config/manage_api_client.py b/main/xiaozhi-server/config/manage_api_client.py new file mode 100644 index 00000000..11e0ecbe --- /dev/null +++ b/main/xiaozhi-server/config/manage_api_client.py @@ -0,0 +1,192 @@ +import os +import time +import base64 +from typing import Optional, Dict + +import httpx + +TAG = __name__ + + +class DeviceNotFoundException(Exception): + pass + + +class DeviceBindException(Exception): + def __init__(self, bind_code): + self.bind_code = bind_code + super().__init__(f"设备绑定异常，绑定码: {bind_code}") + + +class ManageApiClient: + _instance = None + _client = None + _secret = None + + def __new__(cls, config): + """单例模式确保全局唯一实例，并支持传入配置参数""" + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._init_client(config) + return cls._instance + + @classmethod + def _init_client(cls, config): + """初始化持久化连接池""" + cls.config = config.get("manager-api") + + if not cls.config: + raise Exception("manager-api配置错误") + + if not cls.config.get("url") or not cls.config.get("secret"): + raise Exception("manager-api的url或secret配置错误") + + if "你" in cls.config.get("secret"): + raise Exception("请先配置manager-api的secret") + + cls._secret = cls.config.get("secret") + cls.max_retries = cls.config.get("max_retries", 6) # 最大重试次数 + cls.retry_delay = cls.config.get("retry_delay", 10) # 初始重试延迟(秒) + # NOTE(goody): 2025/4/16 http相关资源统一管理，后续可以增加线程池或者超时 + # 后续也可以统一配置apiToken之类的走通用的Auth + cls._client = httpx.Client( + base_url=cls.config.get("url"), + headers={ + "User-Agent": f"PythonClient/2.0 (PID:{os.getpid()})", + "Accept": "application/json", + "Authorization": "Bearer " + cls._secret, + }, + timeout=cls.config.get("timeout", 30), # 默认超时时间30秒 + ) + + @classmethod + def _request(cls, method: str, endpoint: str, **kwargs) -> Dict: + """发送单次HTTP请求并处理响应""" + endpoint = endpoint.lstrip("/") + response = cls._client.request(method, endpoint, **kwargs) + response.raise_for_status() + + result = response.json() + + # 处理API返回的业务错误 + if result.get("code") == 10041: + raise DeviceNotFoundException(result.get("msg")) + elif result.get("code") == 10042: + raise DeviceBindException(result.get("msg")) + elif result.get("code") != 0: + raise Exception(f"API返回错误: {result.get('msg', '未知错误')}") + + # 返回成功数据 + return result.get("data") if result.get("code") == 0 else None + + @classmethod + def _should_retry(cls, exception: Exception) -> bool: + """判断异常是否应该重试""" + # 网络连接相关错误 + if isinstance( + exception, (httpx.ConnectError, httpx.TimeoutException, httpx.NetworkError) + ): + return True + + # HTTP状态码错误 + if isinstance(exception, httpx.HTTPStatusError): + status_code = exception.response.status_code + return status_code in [408, 429, 500, 502, 503, 504] + + return False + + @classmethod + def _execute_request(cls, method: str, endpoint: str, **kwargs) -> Dict: + """带重试机制的请求执行器""" + retry_count = 0 + + while retry_count <= cls.max_retries: + try: + # 执行请求 + return cls._request(method, endpoint, **kwargs) + except Exception as e: + # 判断是否应该重试 + if retry_count < cls.max_retries and cls._should_retry(e): + retry_count += 1 + print( + f"{method} {endpoint} 请求失败，将在 {cls.retry_delay:.1f} 秒后进行第 {retry_count} 次重试" + ) + time.sleep(cls.retry_delay) + continue + else: + # 不重试，直接抛出异常 + raise + + @classmethod + def safe_close(cls): + """安全关闭连接池""" + if cls._client: + cls._client.close() + cls._instance = None + + +def get_server_config() -> Optional[Dict]: + """获取服务器基础配置""" + return ManageApiClient._instance._execute_request("POST", "/config/server-base") + + +def get_agent_models( + mac_address: str, client_id: str, selected_module: Dict +) -> Optional[Dict]: + """获取代理模型配置""" + return ManageApiClient._instance._execute_request( + "POST", + "/config/agent-models", + json={ + "macAddress": mac_address, + "clientId": client_id, + "selectedModule": selected_module, + }, + ) + + +def save_mem_local_short(mac_address: str, short_momery: str) -> Optional[Dict]: + try: + return ManageApiClient._instance._execute_request( + "PUT", + f"/agent/saveMemory/" + mac_address, + json={ + "summaryMemory": short_momery, + }, + ) + except Exception as e: + print(f"存储短期记忆到服务器失败: {e}") + return None + + +def report( + mac_address: str, session_id: str, chat_type: int, content: str, audio +) -> Optional[Dict]: + """带熔断的业务方法示例""" + if not content or not ManageApiClient._instance: + return None + try: + return ManageApiClient._instance._execute_request( + "POST", + f"/agent/chat-history/report", + json={ + "macAddress": mac_address, + "sessionId": session_id, + "chatType": chat_type, + "content": content, + "audioBase64": ( + base64.b64encode(audio).decode("utf-8") if audio else None + ), + }, + ) + except Exception as e: + print(f"TTS上报失败: {e}") + return None + + +def init_service(config): + ManageApiClient(config) + + +def manage_api_http_safe_close(): + ManageApiClient.safe_close() diff --git a/main/xiaozhi-server/config/private_config.py b/main/xiaozhi-server/config/private_config.py deleted file mode 100644 index f708a95b..00000000 --- a/main/xiaozhi-server/config/private_config.py +++ /dev/null @@ -1,241 +0,0 @@ -import os -import time -import yaml -from config.logger import setup_logging -from typing import Dict, Any, Optional -from copy import deepcopy -from core.utils.util import get_project_dir -from core.utils import llm, tts -from core.utils.lock_manager import FileLockManager - -TAG = __name__ - -class PrivateConfig: - def __init__(self, device_id: str, default_config: Dict[str, Any], auth_code_gen=None): - self.device_id = device_id - self.default_config = default_config - self.config_path = get_project_dir() + 'data/.private_config.yaml' - self.logger = setup_logging() - self.private_config = {} - self.auth_code_gen = auth_code_gen - self.lock_manager = FileLockManager() - - async def load_or_create(self): - try: - await self.lock_manager.acquire_lock(self.config_path) - try: - if os.path.exists(self.config_path): - with open(self.config_path, 'r', encoding='utf-8') as f: - all_configs = yaml.safe_load(f) or {} - else: - all_configs = {} - - if self.device_id not in all_configs: - # Get selected module names - selected_modules = self.default_config['selected_module'] - selected_tts = selected_modules['TTS'] - selected_llm = selected_modules['LLM'] - selected_asr = selected_modules['ASR'] - selected_vad = selected_modules['VAD'] - - # 生成认证码 - auth_code = None - if self.auth_code_gen: - auth_code = self.auth_code_gen.generate_code() - - # Initialize device config with only necessary configurations - device_config = { - 'selected_module': deepcopy(selected_modules), - 'prompt': self.default_config['prompt'], - 'LLM': { - selected_llm: deepcopy(self.default_config['LLM'][selected_llm]) - }, - 'TTS': { - selected_tts: deepcopy(self.default_config['TTS'][selected_tts]) - }, - 'ASR': { - selected_asr: deepcopy(self.default_config['ASR'][selected_asr]) - }, - 'VAD': { - selected_vad: deepcopy(self.default_config['VAD'][selected_vad]) - }, - 'auth_code': auth_code # 添加认证码字段 - } - - all_configs[self.device_id] = device_config - - # Save updated configs - with open(self.config_path, 'w', encoding='utf-8') as f: - yaml.dump(all_configs, f, allow_unicode=True) - - self.private_config = all_configs[self.device_id] - - finally: - self.lock_manager.release_lock(self.config_path) - - except Exception as e: - self.logger.bind(tag=TAG).error(f"Error handling private config: {e}") - self.private_config = {} - - async def update_config(self, selected_modules: Dict[str, str], prompt: str, nickname: str) -> bool: - """更新设备配置 - Args: - selected_modules: 选择的模块配置，格式如 {'LLM': 'AliLLM', 'TTS': 'EdgeTTS',...} - prompt: 提示词配置 - Returns: - bool: 更新是否成功 - """ - try: - await self.lock_manager.acquire_lock(self.config_path) - try: - # Read main config to get full module configurations - main_config = self.default_config - - # Create new device config - device_config = { - 'selected_module': selected_modules, - 'prompt': prompt, - 'nickname': nickname, - } - if self.private_config.get('last_chat_time'): - device_config['last_chat_time'] = self.private_config['last_chat_time'] - if self.private_config.get('owner'): - device_config['owner'] = self.private_config['owner'] - - # Copy full module configurations from main config - for module_type, selected_name in selected_modules.items(): - if selected_name and selected_name in main_config.get(module_type, {}): - device_config[module_type] = { - selected_name: main_config[module_type][selected_name] - } - - # Read all configs - if os.path.exists(self.config_path): - with open(self.config_path, 'r', encoding='utf-8') as f: - all_configs = yaml.safe_load(f) or {} - else: - all_configs = {} - - # Update device config - all_configs[self.device_id] = device_config - self.private_config = device_config - - # Save back to file - with open(self.config_path, 'w', encoding='utf-8') as f: - yaml.dump(all_configs, f, allow_unicode=True) - - return True - finally: - self.lock_manager.release_lock(self.config_path) - - except Exception as e: - self.logger.bind(tag=TAG).error(f"Error updating config: {e}") - return False - - async def delete_config(self) -> bool: - """删除设备配置 - Returns: - bool: 删除是否成功 - """ - try: - await self.lock_manager.acquire_lock(self.config_path) - try: - # 读取所有配置 - if os.path.exists(self.config_path): - with open(self.config_path, 'r', encoding='utf-8') as f: - all_configs = yaml.safe_load(f) or {} - else: - return False - - # 删除设备配置 - if self.device_id in all_configs: - del all_configs[self.device_id] - - # 保存更新后的配置 - with open(self.config_path, 'w', encoding='utf-8') as f: - yaml.dump(all_configs, f, allow_unicode=True) - - self.private_config = {} - return True - - return False - finally: - self.lock_manager.release_lock(self.config_path) - - except Exception as e: - self.logger.bind(tag=TAG).error(f"Error deleting config: {e}") - return False - - def create_private_instances(self): - # 判断存在私有配置，并且self.device_id在私有配置中 - if not self.private_config: - self.logger.bind(tag=TAG).error("Private config not found for device_id: {}", self.device_id) - return None, None - - """创建私有处理模块实例""" - config = self.private_config - selected_modules = config['selected_module'] - return ( - llm.create_instance( - selected_modules["LLM"] - if not 'type' in config["LLM"][selected_modules["LLM"]] - else - config["LLM"][selected_modules["LLM"]]['type'], - config["LLM"][selected_modules["LLM"]], - ), - tts.create_instance( - selected_modules["TTS"] - if not 'type' in config["TTS"][selected_modules["TTS"]] - else - config["TTS"][selected_modules["TTS"]]["type"], - config["TTS"][selected_modules["TTS"]], - self.default_config.get("delete_audio", True) # Using default_config for global settings - ) - ) - - async def update_last_chat_time(self, timestamp=None): - """更新设备最近一次的聊天时间 - Args: - timestamp: 指定的时间戳,不传则使用当前时间 - """ - if not self.private_config: - self.logger.bind(tag=TAG).error("Private config not found") - return False - - try: - await self.lock_manager.acquire_lock(self.config_path) - try: - if timestamp is None: - timestamp = int(time.time()) - - self.private_config['last_chat_time'] = timestamp - - # 读取所有配置 - with open(self.config_path, 'r', encoding='utf-8') as f: - all_configs = yaml.safe_load(f) or {} - - # 更新当前设备配置 - all_configs[self.device_id] = self.private_config - - # 保存回文件 - with open(self.config_path, 'w', encoding='utf-8') as f: - yaml.dump(all_configs, f, allow_unicode=True) - - return True - finally: - self.lock_manager.release_lock(self.config_path) - - except Exception as e: - self.logger.bind(tag=TAG).error(f"Error updating last chat time: {e}") - return False - - def get_auth_code(self) -> str: - """获取设备的认证码 - Returns: - str: 认证码，如果没有返回空字符串 - """ - return self.private_config.get('auth_code', '') - - def get_owner(self) -> Optional[str]: - """获取设备当前所有者""" - return self.private_config.get('owner') \ No newline at end of file diff --git a/main/xiaozhi-server/config/settings.py b/main/xiaozhi-server/config/settings.py index bb81cfe9..fb8868b4 100644 --- a/main/xiaozhi-server/config/settings.py +++ b/main/xiaozhi-server/config/settings.py @@ -1,127 +1,33 @@ import os -import argparse -from ruamel.yaml import YAML -from collections.abc import Mapping -from core.utils.util import read_config, get_project_dir +from config.config_loader import read_config, get_project_dir, load_config + default_config_file = "config.yaml" - - -def ensure_directories(config): - """确保所有配置路径存在""" - dirs_to_create = set() - project_dir = get_project_dir() # 获取项目根目录 - # 日志文件目录 - log_dir = config.get('log', {}).get('log_dir', 'tmp') - dirs_to_create.add(os.path.join(project_dir, log_dir)) - - # ASR/TTS模块输出目录 - for module in ['ASR', 'TTS']: - for provider in config.get(module, {}).values(): - output_dir = provider.get('output_dir', '') - if output_dir: - dirs_to_create.add(output_dir) - - # 根据selected_module创建模型目录 - selected_modules = config.get('selected_module', {}) - for module_type in ['ASR', 'LLM', 'TTS']: - selected_provider = selected_modules.get(module_type) - if not selected_provider: - continue - provider_config = config.get(module_type, {}).get(selected_provider, {}) - output_dir = provider_config.get('output_dir') - if output_dir: - full_model_dir = os.path.join(project_dir, output_dir) - dirs_to_create.add(full_model_dir) - - # 统一创建目录（保留原data目录创建） - for dir_path in dirs_to_create: - try: - os.makedirs(dir_path, exist_ok=True) - except PermissionError: - print(f"警告：无法创建目录 {dir_path}，请检查写入权限") - - -def get_config_file(): - global default_config_file - """获取配置文件路径，优先使用私有配置文件（若存在）。 - - Returns: - str: 配置文件路径（相对路径或默认路径） - """ - config_file = default_config_file - if os.path.exists(get_project_dir() + "data/." + default_config_file): - config_file = "data/." + default_config_file - return config_file - - -def load_config(): - """加载配置文件""" - parser = argparse.ArgumentParser(description="Server configuration") - config_file = get_config_file() - - parser.add_argument("--config_path", type=str, default=config_file) - args = parser.parse_args() - config = read_config(args.config_path) - # 初始化目录 - ensure_directories(config) - return config - - -def update_config(config): - yaml = YAML() - yaml.preserve_quotes = True - """将配置保存到YAML文件""" - with open(get_config_file(), 'w') as f: - yaml.dump(config, f) - - -def find_missing_keys(new_config, old_config, parent_key=''): - """ - 递归查找缺失的配置项 - 返回格式：[缺失配置路径] - """ - missing_keys = [] - - if not isinstance(new_config, Mapping): - return missing_keys - - for key, value in new_config.items(): - # 构建当前配置路径 - full_path = f"{parent_key}.{key}" if parent_key else key - - # 检查键是否存在 - if key not in old_config: - missing_keys.append(full_path) - continue - - # 递归检查嵌套字典 - if isinstance(value, Mapping): - sub_missing = find_missing_keys( - value, - old_config[key], - parent_key=full_path - ) - missing_keys.extend(sub_missing) - - return missing_keys +config_file_valid = False def check_config_file(): - old_config_file = get_config_file() - global default_config_file - if not 'data' in old_config_file: + global config_file_valid + if config_file_valid: return - old_config = read_config(get_project_dir() + old_config_file) - new_config = read_config(get_project_dir() + default_config_file) - # 查找缺失的配置项 - missing_keys = find_missing_keys(new_config, old_config) + """ + 简化的配置检查，仅提示用户配置文件的使用情况 + """ + custom_config_file = get_project_dir() + "data/." + default_config_file + if not os.path.exists(custom_config_file): + raise FileNotFoundError( + "找不到data/.config.yaml文件，请按教程确认该配置文件是否存在" + ) - if missing_keys: - error_msg = "您的配置文件太旧了，缺少了：\n" - error_msg += "\n".join(f"- {key}" for key in missing_keys) - error_msg += "\n建议您：\n" - error_msg += "1、备份data/.config.yaml文件\n" - error_msg += "2、将根目录的config.yaml文件复制到data下，重命名为.config.yaml\n" - error_msg += "3、将密钥逐个复制到新的配置文件中\n" - raise ValueError(error_msg) + # 检查是否从API读取配置 + config = load_config() + if config.get("read_config_from_api", False): + print("从API读取配置") + old_config_origin = read_config(custom_config_file) + if old_config_origin.get("selected_module") is not None: + error_msg = "您的配置文件好像既包含智控台的配置又包含本地配置：\n" + error_msg += "\n建议您：\n" + error_msg += "1、将根目录的config_from_api.yaml文件复制到data下，重命名为.config.yaml\n" + error_msg += "2、按教程配置好接口地址和密钥\n" + raise ValueError(error_msg) + config_file_valid = True diff --git a/main/xiaozhi-server/config_from_api.yaml b/main/xiaozhi-server/config_from_api.yaml new file mode 100644 index 00000000..15962bb6 --- /dev/null +++ b/main/xiaozhi-server/config_from_api.yaml @@ -0,0 +1,15 @@ +# 如果你只想轻量化安装xiaozhi-server，只使用本地的配置文件，不需要理会这个文件，不需要改动本文件任何东西 +# 如果你想从manager-api获取配置，请往下看： +# 请将本文件复制到xiaozhi-server/data目录下，没有data目录，请创建一个，并将复制过去的文件命名为.config.yaml +# 注意如果data目录有.config.yaml文件，请先删除它 +# 先启动manager-api和manager-web，注册一个账号，第一个注册的账号为管理员 +# 使用管理员，进入【参数管理】页面，找到【server.secret】，复制它到参数值，注意每次从零部署，server.secret都会变化 +# 打开本data目录下的.config.yaml文件，修改manager-api.secret为刚才复制出来的server.secret +server: + ip: 0.0.0.0 + port: 8000 +manager-api: + # 你的manager-api的地址，最好使用局域网ip + url: http://127.0.0.1:8002/xiaozhi + # 你的manager-api的token，就是刚才复制出来的server.secret + secret: 你的server.secret值 \ No newline at end of file diff --git a/main/xiaozhi-server/core/handle/abortHandle.py b/main/xiaozhi-server/core/handle/abortHandle.py index 3183487f..fe271511 100644 --- a/main/xiaozhi-server/core/handle/abortHandle.py +++ b/main/xiaozhi-server/core/handle/abortHandle.py @@ -3,14 +3,16 @@ import queue from config.logger import setup_logging TAG = __name__ -logger = setup_logging() async def handleAbortMessage(conn): - logger.bind(tag=TAG).info("Abort message received") + conn.logger.bind(tag=TAG).info("Abort message received") # 设置成打断状态，会自动打断llm、tts任务 conn.client_abort = True + conn.clear_queues() # 打断客户端说话状态 - await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id})) + await conn.websocket.send( + json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}) + ) conn.clearSpeakStatus() - logger.bind(tag=TAG).info("Abort message received-end") + conn.logger.bind(tag=TAG).info("Abort message received-end") diff --git a/main/xiaozhi-server/core/handle/functionHandler.py b/main/xiaozhi-server/core/handle/functionHandler.py index cec9bd25..b09a369c 100644 --- a/main/xiaozhi-server/core/handle/functionHandler.py +++ b/main/xiaozhi-server/core/handle/functionHandler.py @@ -4,7 +4,6 @@ from plugins_func.register import FunctionRegistry, ActionResponse, Action, Tool from plugins_func.functions.hass_init import append_devices_to_prompt TAG = __name__ -logger = setup_logging() class FunctionHandler: @@ -40,7 +39,9 @@ class FunctionHandler: for func in self.functions_desc: func_names.append(func["function"]["name"]) # 打印当前支持的函数列表 - logger.bind(tag=TAG).info(f"当前支持的函数列表: {func_names}") + self.conn.logger.bind(tag=TAG, session_id=self.conn.session_id).info( + f"当前支持的函数列表: {func_names}" + ) return func_names def get_functions(self): @@ -57,7 +58,9 @@ class FunctionHandler: def register_config_functions(self): """注册配置中的函数,可以不同客户端使用不同的配置""" - for func in self.config["Intent"]["function_call"].get("functions", []): + for func in self.config["Intent"][self.config["selected_module"]["Intent"]].get( + "functions", [] + ): self.function_registry.register_function(func) """home assistant需要初始化提示词""" @@ -77,7 +80,9 @@ class FunctionHandler: func = funcItem.func arguments = function_call_data["arguments"] arguments = json.loads(arguments) if arguments else {} - logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}") + self.conn.logger.bind(tag=TAG).debug( + f"调用函数: {function_name}, 参数: {arguments}" + ) if ( funcItem.type == ToolType.SYSTEM_CTL or funcItem.type == ToolType.IOT_CTL @@ -92,6 +97,6 @@ class FunctionHandler: action=Action.NOTFOUND, result="没有找到对应的函数", response="" ) except Exception as e: - logger.bind(tag=TAG).error(f"处理function call错误: {e}") + self.conn.logger.bind(tag=TAG).error(f"处理function call错误: {e}") return None diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index d7458cc4..f902a64f 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -1,6 +1,4 @@ import json -from config.logger import setup_logging -from core.providers.tts.dto.dto import TTSMessageDTO, MsgType from core.handle.sendAudioHandle import send_stt_message from core.utils.util import remove_punctuation_and_length import shutil @@ -9,7 +7,7 @@ import os import random import time -logger = setup_logging() +TAG = __name__ WAKEUP_CONFIG = { "dir": "config/assets/", @@ -21,7 +19,16 @@ WAKEUP_CONFIG = { } -async def handleHelloMessage(conn): +async def handleHelloMessage(conn, msg_json): + """处理hello消息""" + audio_params = msg_json.get("audio_params") + if audio_params: + format = audio_params.get("format") + conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}") + conn.audio_format = format + conn.asr.set_audio_format(format) + conn.welcome_msg["audio_params"] = audio_params + await conn.websocket.send(json.dumps(conn.welcome_msg)) @@ -44,21 +51,11 @@ async def checkWakeupWords(conn, text): if file is None: asyncio.create_task(wakeupWordsResponse(conn)) return False - opus_packets, duration = conn.tts.audio_to_opus_data(file) + opus_packets, _ = conn.tts.audio_to_opus_data(file) text_hello = WAKEUP_CONFIG["text"] if not text_hello: text_hello = text - - conn.tts.tts_audio_queue.put( - TTSMessageDTO( - u_id=conn.u_id, - msg_type=MsgType.TTS_TEXT_RESPONSE, - content=opus_packets, - tts_finish_text="", - sentence_type=None, - duration=0, - ) - ) + conn.audio_play_queue.put((opus_packets, text_hello, 0)) if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]: asyncio.create_task(wakeupWordsResponse(conn)) return True @@ -80,11 +77,20 @@ def getWakeupWordFile(file_name): async def wakeupWordsResponse(conn): + wait_max_time = 5 + while conn.llm is None or not conn.llm.response_no_stream: + await asyncio.sleep(1) + wait_max_time -= 1 + if wait_max_time <= 0: + conn.logger.bind(tag=TAG).error("连接对象没有llm") + return + """唤醒词响应""" wakeup_word = random.choice(WAKEUP_CONFIG["words"]) result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word) - # TODO 将result转换为tts_message_dto - tts_file = None + if result is None or result == "": + return + tts_file = await asyncio.to_thread(conn.tts.to_tts, result) if tts_file is not None and os.path.exists(tts_file): file_type = os.path.splitext(tts_file)[1] diff --git a/main/xiaozhi-server/core/handle/iotHandle.py b/main/xiaozhi-server/core/handle/iotHandle.py index 76d8b4b7..5b1bb9bb 100644 --- a/main/xiaozhi-server/core/handle/iotHandle.py +++ b/main/xiaozhi-server/core/handle/iotHandle.py @@ -10,7 +10,6 @@ from plugins_func.register import ( ) TAG = __name__ -logger = setup_logging() def wrap_async_function(async_func): @@ -21,7 +20,7 @@ def wrap_async_function(async_func): # 获取连接对象（第一个参数） conn = args[0] if not hasattr(conn, "loop"): - logger.bind(tag=TAG).error("Connection对象没有loop属性") + conn.logger.bind(tag=TAG).error("Connection对象没有loop属性") return ActionResponse( Action.ERROR, "Connection对象没有loop属性", @@ -35,7 +34,7 @@ def wrap_async_function(async_func): # 等待结果返回 return future.result() except Exception as e: - logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}") + conn.logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}") return ActionResponse(Action.ERROR, str(e), f"执行操作时出错: {e}") return wrapper @@ -57,7 +56,7 @@ def create_iot_function(device_name, method_name, method_info): response_failure = "操作失败" # 打印响应参数 - logger.bind(tag=TAG).info( + conn.logger.bind(tag=TAG).debug( f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'" ) @@ -86,7 +85,9 @@ def create_iot_function(device_name, method_name, method_info): return ActionResponse(Action.RESPONSE, result, response) except Exception as e: - logger.bind(tag=TAG).error(f"执行{device_name}的{method_name}操作失败: {e}") + conn.logger.bind(tag=TAG).error( + f"执行{device_name}的{method_name}操作失败: {e}" + ) # 操作失败时使用大模型提供的失败响应 response = response_failure @@ -104,7 +105,7 @@ def create_iot_query_function(device_name, prop_name, prop_info): async def iot_query_function(conn, response_success=None, response_failure=None): try: # 打印响应参数 - logger.bind(tag=TAG).info( + conn.logger.bind(tag=TAG).info( f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'" ) @@ -122,7 +123,9 @@ def create_iot_query_function(device_name, prop_name, prop_info): return ActionResponse(Action.ERROR, f"属性{prop_name}不存在", response) except Exception as e: - logger.bind(tag=TAG).error(f"查询{device_name}的{prop_name}时出错: {e}") + conn.logger.bind(tag=TAG).error( + f"查询{device_name}的{prop_name}时出错: {e}" + ) # 查询出错时使用大模型提供的失败响应 response = response_failure @@ -144,33 +147,34 @@ class IotDescriptor: self.methods = [] # 根据描述创建属性 - for key, value in properties.items(): - property_item = globals()[key] = {} - property_item["name"] = key - property_item["description"] = value["description"] - if value["type"] == "number": - property_item["value"] = 0 - elif value["type"] == "boolean": - property_item["value"] = False - else: - property_item["value"] = "" - self.properties.append(property_item) + if properties is not None: + for key, value in properties.items(): + property_item = {} + property_item["name"] = key + property_item["description"] = value["description"] + if value["type"] == "number": + property_item["value"] = 0 + elif value["type"] == "boolean": + property_item["value"] = False + else: + property_item["value"] = "" + self.properties.append(property_item) # 根据描述创建方法 - for key, value in methods.items(): - method = globals()[key] = {} - method["description"] = value["description"] - method["name"] = key - for k, v in value["parameters"].items(): - method[k] = {} - method[k]["description"] = v["description"] - if v["type"] == "number": - method[k]["value"] = 0 - elif v["type"] == "boolean": - method[k]["value"] = False - else: - method[k]["value"] = "" - self.methods.append(method) + if methods is not None: + for key, value in methods.items(): + method = {} + method["description"] = value["description"] + method["name"] = key + # 检查方法是否有参数 + if "parameters" in value: + method["parameters"] = {} + for k, v in value["parameters"].items(): + method["parameters"][k] = { + "description": v["description"], + "type": v["type"], + } + self.methods.append(method) def register_device_type(descriptor): @@ -219,13 +223,19 @@ def register_device_type(descriptor): func_name = f"{device_name.lower()}_{method_name.lower()}" # 创建参数字典，添加原有参数 - parameters = { - param_name: { - "type": param_info["type"], - "description": param_info["description"], + parameters = {} + required_params = [] + + # 如果方法有参数，则添加参数信息 + if "parameters" in method_info: + parameters = { + param_name: { + "type": param_info["type"], + "description": param_info["description"], + } + for param_name, param_info in method_info["parameters"].items() } - for param_name, param_info in method_info["parameters"].items() - } + required_params = list(method_info["parameters"].keys()) # 添加响应参数 parameters.update( @@ -242,7 +252,6 @@ def register_device_type(descriptor): ) # 构建必须参数列表（原有参数 + 响应参数） - required_params = list(method_info["parameters"].keys()) required_params.extend(["response_success", "response_failure"]) func_desc = { @@ -269,19 +278,36 @@ def register_device_type(descriptor): # 用于接受前端设备推送的搜索iot描述 async def handleIotDescriptors(conn, descriptors): - if not conn.use_function_call_mode: - return wait_max_time = 5 while conn.func_handler is None or not conn.func_handler.finish_init: await asyncio.sleep(1) wait_max_time -= 1 if wait_max_time <= 0: - logger.bind(tag=TAG).error("连接对象没有func_handler") + conn.logger.bind(tag=TAG).debug("连接对象没有func_handler") return """处理物联网描述""" functions_changed = False for descriptor in descriptors: + + # 如果descriptor没有properties和methods，则直接跳过 + if "properties" not in descriptor and "methods" not in descriptor: + continue + + # 处理缺失properties的情况 + if "properties" not in descriptor: + descriptor["properties"] = {} + # 从methods中提取所有参数作为properties + if "methods" in descriptor: + for method_name, method_info in descriptor["methods"].items(): + if "parameters" in method_info: + for param_name, param_info in method_info["parameters"].items(): + # 将参数信息转换为属性信息 + descriptor["properties"][param_name] = { + "description": param_info["description"], + "type": param_info["type"], + } + # 创建IOT设备描述符 iot_descriptor = IotDescriptor( descriptor["name"], @@ -291,7 +317,7 @@ async def handleIotDescriptors(conn, descriptors): ) conn.iot_descriptors[descriptor["name"]] = iot_descriptor - if conn.use_function_call_mode: + if conn.load_function_plugin: # 注册或获取设备类型 type_id = register_device_type(descriptor) device_functions = device_type_registry.get_device_functions(type_id) @@ -300,7 +326,7 @@ async def handleIotDescriptors(conn, descriptors): if hasattr(conn, "func_handler"): for func_name in device_functions: conn.func_handler.function_registry.register_function(func_name) - logger.bind(tag=TAG).info( + conn.logger.bind(tag=TAG).info( f"注册IOT函数到function handler: {func_name}" ) functions_changed = True @@ -309,8 +335,8 @@ async def handleIotDescriptors(conn, descriptors): if functions_changed and hasattr(conn, "func_handler"): conn.func_handler.upload_functions_desc() func_names = conn.func_handler.current_support_functions() - logger.bind(tag=TAG).info(f"设备类型: {type_id}") - logger.bind(tag=TAG).info( + conn.logger.bind(tag=TAG).info(f"设备类型: {type_id}") + conn.logger.bind(tag=TAG).info( f"更新function描述列表完成，当前支持的函数: {func_names}" ) @@ -324,13 +350,13 @@ async def handleIotStatus(conn, states): for k, v in state["state"].items(): if property_item["name"] == k: if type(v) != type(property_item["value"]): - logger.bind(tag=TAG).error( + conn.logger.bind(tag=TAG).error( f"属性{property_item['name']}的值类型不匹配" ) break else: property_item["value"] = v - logger.bind(tag=TAG).info( + conn.logger.bind(tag=TAG).info( f"物联网状态更新: {key} , {property_item['name']} = {v}" ) break @@ -344,7 +370,7 @@ async def get_iot_status(conn, name, property_name): for property_item in value.properties: if property_item["name"] == property_name: return property_item["value"] - logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}") + conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}") return None @@ -355,16 +381,16 @@ async def set_iot_status(conn, name, property_name, value): for property_item in iot_descriptor.properties: if property_item["name"] == property_name: if type(value) != type(property_item["value"]): - logger.bind(tag=TAG).error( + conn.logger.bind(tag=TAG).error( f"属性{property_item['name']}的值类型不匹配" ) return property_item["value"] = value - logger.bind(tag=TAG).info( + conn.logger.bind(tag=TAG).info( f"物联网状态更新: {name} , {property_name} = {value}" ) return - logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}") + conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}") async def send_iot_conn(conn, name, method_name, parameters): @@ -375,19 +401,17 @@ async def send_iot_conn(conn, name, method_name, parameters): for method in value.methods: # 找到了方法 if method["name"] == method_name: - await conn.websocket.send( - json.dumps( - { - "type": "iot", - "commands": [ - { - "name": name, - "method": method_name, - "parameters": parameters, - } - ], - } - ) - ) + # 构建命令对象 + command = { + "name": name, + "method": method_name, + } + + # 只有当参数不为空时才添加parameters字段 + if parameters: + command["parameters"] = parameters + send_message = json.dumps({"type": "iot", "commands": [command]}) + await conn.websocket.send(send_message) + conn.logger.bind(tag=TAG).info(f"发送物联网指令: {send_message}") return - logger.bind(tag=TAG).error(f"未找到方法{method_name}") + conn.logger.bind(tag=TAG).error(f"未找到方法{method_name}") diff --git a/main/xiaozhi-server/core/handle/reportHandle.py b/main/xiaozhi-server/core/handle/reportHandle.py new file mode 100644 index 00000000..bb1ea066 --- /dev/null +++ b/main/xiaozhi-server/core/handle/reportHandle.py @@ -0,0 +1,145 @@ +""" +TTS上报功能已集成到ConnectionHandler类中。 + +上报功能包括： +1. 每个连接对象拥有自己的上报队列和处理线程 +2. 上报线程的生命周期与连接对象绑定 +3. 使用ConnectionHandler.enqueue_tts_report方法进行上报 + +具体实现请参考core/connection.py中的相关代码。 +""" + +import opuslib_next + +from config.manage_api_client import report as manage_report + +TAG = __name__ + + +def report(conn, type, text, opus_data): + """执行聊天记录上报操作 + + Args: + conn: 连接对象 + type: 上报类型，1为用户，2为智能体 + text: 合成文本 + opus_data: opus音频数据 + """ + try: + if opus_data: + audio_data = opus_to_wav(conn, opus_data) + else: + audio_data = None + # 执行上报 + manage_report( + mac_address=conn.device_id, + session_id=conn.session_id, + chat_type=type, + content=text, + audio=audio_data, + ) + except Exception as e: + conn.logger.bind(tag=TAG).error(f"聊天记录上报失败: {e}") + + +def opus_to_wav(conn, opus_data): + """将Opus数据转换为WAV格式的字节流 + + Args: + output_dir: 输出目录（保留参数以保持接口兼容） + opus_data: opus音频数据 + + Returns: + bytes: WAV格式的音频数据 + """ + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + if not pcm_data: + raise ValueError("没有有效的PCM数据") + + # 创建WAV文件头 + pcm_data_bytes = b"".join(pcm_data) + num_samples = len(pcm_data_bytes) // 2 # 16-bit samples + + # WAV文件头 + wav_header = bytearray() + wav_header.extend(b"RIFF") # ChunkID + wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little")) # ChunkSize + wav_header.extend(b"WAVE") # Format + wav_header.extend(b"fmt ") # Subchunk1ID + wav_header.extend((16).to_bytes(4, "little")) # Subchunk1Size + wav_header.extend((1).to_bytes(2, "little")) # AudioFormat (PCM) + wav_header.extend((1).to_bytes(2, "little")) # NumChannels + wav_header.extend((16000).to_bytes(4, "little")) # SampleRate + wav_header.extend((32000).to_bytes(4, "little")) # ByteRate + wav_header.extend((2).to_bytes(2, "little")) # BlockAlign + wav_header.extend((16).to_bytes(2, "little")) # BitsPerSample + wav_header.extend(b"data") # Subchunk2ID + wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little")) # Subchunk2Size + + # 返回完整的WAV数据 + return bytes(wav_header) + pcm_data_bytes + + +def enqueue_tts_report(conn, text, opus_data): + if not conn.read_config_from_api or conn.need_bind or not conn.report_tts_enable: + return + if conn.chat_history_conf == 0: + return + """将TTS数据加入上报队列 + + Args: + conn: 连接对象 + text: 合成文本 + opus_data: opus音频数据 + """ + try: + # 使用连接对象的队列，传入文本和二进制数据而非文件路径 + if conn.chat_history_conf == 2: + conn.report_queue.put((2, text, opus_data)) + conn.logger.bind(tag=TAG).debug( + f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} " + ) + else: + conn.report_queue.put((2, text, None)) + conn.logger.bind(tag=TAG).debug( + f"TTS数据已加入上报队列: {conn.device_id}, 不上报音频" + ) + except Exception as e: + conn.logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}") + + +def enqueue_asr_report(conn, text, opus_data): + if not conn.read_config_from_api or conn.need_bind or not conn.report_asr_enable: + return + if conn.chat_history_conf == 0: + return + """将ASR数据加入上报队列 + + Args: + conn: 连接对象 + text: 合成文本 + opus_data: opus音频数据 + """ + try: + # 使用连接对象的队列，传入文本和二进制数据而非文件路径 + if conn.chat_history_conf == 2: + conn.report_queue.put((1, text, opus_data)) + conn.logger.bind(tag=TAG).debug( + f"ASR数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} " + ) + else: + conn.report_queue.put((1, text, None)) + conn.logger.bind(tag=TAG).debug( + f"ASR数据已加入上报队列: {conn.device_id}, 不上报音频" + ) + except Exception as e: + conn.logger.bind(tag=TAG).error(f"加入ASR上报队列失败: {text}, {e}") diff --git a/main/xiaozhi-server/core/handle/textHandle.py b/main/xiaozhi-server/core/handle/textHandle.py index 43e980e8..9b4981a1 100644 --- a/main/xiaozhi-server/core/handle/textHandle.py +++ b/main/xiaozhi-server/core/handle/textHandle.py @@ -1,33 +1,37 @@ -from config.logger import setup_logging import json from core.handle.abortHandle import handleAbortMessage from core.handle.helloHandle import handleHelloMessage -from core.utils.util import remove_punctuation_and_length +from core.utils.util import remove_punctuation_and_length, filter_sensitive_info from core.handle.receiveAudioHandle import startToChat, handleAudioMessage from core.handle.sendAudioHandle import send_stt_message, send_tts_message from core.handle.iotHandle import handleIotDescriptors, handleIotStatus +from core.handle.reportHandle import enqueue_asr_report import asyncio TAG = __name__ -logger = setup_logging() async def handleTextMessage(conn, message): """处理文本消息""" - logger.bind(tag=TAG).info(f"收到文本消息：{message}") try: msg_json = json.loads(message) if isinstance(msg_json, int): + conn.logger.bind(tag=TAG).info(f"收到文本消息：{message}") await conn.websocket.send(message) return if msg_json["type"] == "hello": - await handleHelloMessage(conn) + conn.logger.bind(tag=TAG).info(f"收到hello消息：{message}") + await handleHelloMessage(conn, msg_json) elif msg_json["type"] == "abort": + conn.logger.bind(tag=TAG).info(f"收到abort消息：{message}") await handleAbortMessage(conn) elif msg_json["type"] == "listen": + conn.logger.bind(tag=TAG).info(f"收到listen消息：{message}") if "mode" in msg_json: conn.client_listen_mode = msg_json["mode"] - logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}") + conn.logger.bind(tag=TAG).debug( + f"客户端拾音模式：{conn.client_listen_mode}" + ) if msg_json["state"] == "start": conn.client_have_voice = True conn.client_voice_stop = False @@ -53,13 +57,99 @@ async def handleTextMessage(conn, message): # 如果是唤醒词，且关闭了唤醒词回复，就不用回答 await send_stt_message(conn, text) await send_tts_message(conn, "stop", None) + elif is_wakeup_words: + # 上报纯文字数据（复用ASR上报功能，但不提供音频数据） + enqueue_asr_report(conn, "嘿，你好呀", []) + await startToChat(conn, "嘿，你好呀") else: + # 上报纯文字数据（复用ASR上报功能，但不提供音频数据） + enqueue_asr_report(conn, text, []) # 否则需要LLM对文字内容进行答复 await startToChat(conn, text) elif msg_json["type"] == "iot": + conn.logger.bind(tag=TAG).info(f"收到iot消息：{message}") if "descriptors" in msg_json: asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"])) if "states" in msg_json: asyncio.create_task(handleIotStatus(conn, msg_json["states"])) + elif msg_json["type"] == "server": + # 记录日志时过滤敏感信息 + conn.logger.bind(tag=TAG).info( + f"收到服务器消息：{filter_sensitive_info(msg_json)}" + ) + # 如果配置是从API读取的，则需要验证secret + if not conn.read_config_from_api: + return + # 获取post请求的secret + post_secret = msg_json.get("content", {}).get("secret", "") + secret = conn.config["manager-api"].get("secret", "") + # 如果secret不匹配，则返回 + if post_secret != secret: + await conn.websocket.send( + json.dumps( + { + "type": "server", + "status": "error", + "message": "服务器密钥验证失败", + } + ) + ) + return + # 动态更新配置 + if msg_json["action"] == "update_config": + try: + # 更新WebSocketServer的配置 + if not conn.server: + await conn.websocket.send( + json.dumps( + { + "type": "server", + "status": "error", + "message": "无法获取服务器实例", + "content": {"action": "update_config"}, + } + ) + ) + return + + if not await conn.server.update_config(): + await conn.websocket.send( + json.dumps( + { + "type": "server", + "status": "error", + "message": "更新服务器配置失败", + "content": {"action": "update_config"}, + } + ) + ) + return + + # 发送成功响应 + await conn.websocket.send( + json.dumps( + { + "type": "server", + "status": "success", + "message": "配置更新成功", + "content": {"action": "update_config"}, + } + ) + ) + except Exception as e: + conn.logger.bind(tag=TAG).error(f"更新配置失败: {str(e)}") + await conn.websocket.send( + json.dumps( + { + "type": "server", + "status": "error", + "message": f"更新配置失败: {str(e)}", + "content": {"action": "update_config"}, + } + ) + ) + # 重启服务器 + elif msg_json["action"] == "restart": + await conn.handle_restart(msg_json) except json.JSONDecodeError: await conn.websocket.send(message) diff --git a/main/xiaozhi-server/core/mcp/MCPClient.py b/main/xiaozhi-server/core/mcp/MCPClient.py index 103ac645..2ae34c29 100644 --- a/main/xiaozhi-server/core/mcp/MCPClient.py +++ b/main/xiaozhi-server/core/mcp/MCPClient.py @@ -1,86 +1,125 @@ +from __future__ import annotations + from datetime import timedelta -from typing import Optional +import asyncio, os, shutil, concurrent.futures from contextlib import AsyncExitStack -import os, shutil +from typing import Optional, List, Dict, Any + from mcp import ClientSession, StdioServerParameters from mcp.client.stdio import stdio_client - +from mcp.client.sse import sse_client from config.logger import setup_logging TAG = __name__ + class MCPClient: - def __init__(self, config): - # Initialize session and client objects - self.session: Optional[ClientSession] = None - self.exit_stack = AsyncExitStack() + def __init__(self, config: Dict[str, Any]): self.logger = setup_logging() self.config = config - self.tolls = [] + + self._worker_task: Optional[asyncio.Task] = None + self._ready_evt = asyncio.Event() + self._shutdown_evt = asyncio.Event() + + self.session: Optional[ClientSession] = None + self.tools: List = [] async def initialize(self): - args = self.config.get("args", []) + if self._worker_task: + return + self._worker_task = asyncio.create_task(self._worker(), name="MCPClientWorker") + await self._ready_evt.wait() - command = ( - shutil.which("npx") - if self.config["command"] == "npx" - else self.config["command"] + self.logger.bind(tag=TAG).info( + f"Connected, tools = {[t.name for t in self.tools]}" ) - - env={**os.environ} - if self.config.get("env"): - env.update(self.config["env"]) - - server_params = StdioServerParameters( - command=command, - args=args, - env=env - ) - - stdio_transport = await self.exit_stack.enter_async_context(stdio_client(server_params)) - self.stdio, self.write = stdio_transport - time_out_delta = timedelta(seconds=15) - self.session = await self.exit_stack.enter_async_context(ClientSession(read_stream=self.stdio, write_stream=self.write, read_timeout_seconds=time_out_delta)) - - await self.session.initialize() - - # List available tools - response = await self.session.list_tools() - tools = response.tools - self.tools = tools - self.logger.bind(tag=TAG).info(f"Connected to server with tools:{[tool.name for tool in tools]}") - - def has_tool(self, tool_name): - return any(tool.name == tool_name for tool in self.tools) - - def get_available_tools(self): - available_tools = [{"type": "function", "function":{ - "name": tool.name, - "description": tool.description, - "parameters": tool.inputSchema - } } for tool in self.tools] - - return available_tools - - async def call_tool(self, tool_name: str, tool_args: dict): - self.logger.bind(tag=TAG).info(f"MCPClient Calling tool {tool_name} with args: {tool_args}") - try: - response = await self.session.call_tool(tool_name, tool_args) - except Exception as e: - self.logger.bind(tag=TAG).error(f"Error calling tool {tool_name}: {e}") - from types import SimpleNamespace - error_content = SimpleNamespace( - type='text', - text=f"Error calling tool {tool_name}: {e}" - ) - error_response = SimpleNamespace( - content=[error_content], - isError=True - ) - return error_response - self.logger.bind(tag=TAG).info(f"MCPClient Response from tool {tool_name}: {response}") - return response async def cleanup(self): - """Clean up resources""" - await self.exit_stack.aclose() \ No newline at end of file + if not self._worker_task: + return + + self._shutdown_evt.set() + try: + await asyncio.wait_for(self._worker_task, timeout=20) + except (asyncio.TimeoutError, Exception) as e: + self.logger.bind(tag=TAG).error(f"worker shutdown err: {e}") + finally: + self._worker_task = None + + def has_tool(self, name: str) -> bool: + return any(t.name == name for t in self.tools) + + def get_available_tools(self): + return [ + { + "type": "function", + "function": { + "name": t.name, + "description": t.description, + "parameters": t.inputSchema, + }, + } + for t in self.tools + ] + + async def call_tool(self, name: str, args: dict): + if not self.session: + raise RuntimeError("MCPClient not initialized") + + loop = self._worker_task.get_loop() + coro = self.session.call_tool(name, args) + + if loop is asyncio.get_running_loop(): + return await coro + + fut: concurrent.futures.Future = asyncio.run_coroutine_threadsafe(coro, loop) + return await asyncio.wrap_future(fut) + + async def _worker(self): + async with AsyncExitStack() as stack: + try: + # 建立 StdioClient + if "command" in self.config: + cmd = ( + shutil.which("npx") + if self.config["command"] == "npx" + else self.config["command"] + ) + env = {**os.environ, **self.config.get("env", {})} + params = StdioServerParameters( + command=cmd, + args=self.config.get("args", []), + env=env, + ) + stdio_r, stdio_w = await stack.enter_async_context(stdio_client(params)) + read_stream, write_stream = stdio_r, stdio_w + # 建立SSEClient + elif "url" in self.config: + sse_r, sse_w = await stack.enter_async_context(sse_client(self.config["url"])) + read_stream, write_stream = sse_r, sse_w + + else: + raise ValueError("MCPClient config must include 'command' or 'url'") + + self.session = await stack.enter_async_context( + ClientSession( + read_stream=read_stream, + write_stream=write_stream, + read_timeout_seconds=timedelta(seconds=15), + ) + ) + await self.session.initialize() + + # 获取工具 + self.tools = (await self.session.list_tools()).tools + + self._ready_evt.set() + + # 挂起等待关闭 + await self._shutdown_evt.wait() + + except Exception as e: + self.logger.bind(tag=TAG).error(f"worker error: {e}") + self._ready_evt.set() + raise diff --git a/main/xiaozhi-server/core/mcp/manager.py b/main/xiaozhi-server/core/mcp/manager.py index 28efd87d..dc2f9de9 100644 --- a/main/xiaozhi-server/core/mcp/manager.py +++ b/main/xiaozhi-server/core/mcp/manager.py @@ -1,26 +1,29 @@ """MCP服务管理器""" + +import asyncio import os, json from typing import Dict, Any, List from .MCPClient import MCPClient -from config.logger import setup_logging -from core.utils.util import get_project_dir -from plugins_func.register import register_function, ActionResponse, Action, ToolType +from plugins_func.register import register_function, ToolType +from config.config_loader import get_project_dir TAG = __name__ + class MCPManager: """管理多个MCP服务的集中管理器""" - def __init__(self,conn) -> None: + def __init__(self, conn) -> None: """ 初始化MCP管理器 """ self.conn = conn - self.logger = setup_logging() - self.config_path = get_project_dir() + 'data/.mcp_server_settings.json' + self.config_path = get_project_dir() + "data/.mcp_server_settings.json" if os.path.exists(self.config_path) == False: self.config_path = "" - self.logger.bind(tag=TAG).warning(f"请检查mcp服务配置文件：data/.mcp_server_settings.json") + self.conn.logger.bind(tag=TAG).warning( + f"请检查mcp服务配置文件：data/.mcp_server_settings.json" + ) self.client: Dict[str, MCPClient] = {} self.tools = [] @@ -31,37 +34,47 @@ class MCPManager: """ if len(self.config_path) == 0: return {} - + try: - with open(self.config_path, 'r', encoding='utf-8') as f: + with open(self.config_path, "r", encoding="utf-8") as f: config = json.load(f) - return config.get('mcpServers', {}) + return config.get("mcpServers", {}) except Exception as e: - self.logger.bind(tag=TAG).error(f"Error loading MCP config from {self.config_path}: {e}") + self.conn.logger.bind(tag=TAG).error( + f"Error loading MCP config from {self.config_path}: {e}" + ) return {} async def initialize_servers(self) -> None: """初始化所有MCP服务""" config = self.load_config() for name, srv_config in config.items(): - if not srv_config.get("command"): - self.logger.bind(tag=TAG).warning(f"Skipping server {name}: command not specified") + if not srv_config.get("command") and not srv_config.get("url"): + self.conn.logger.bind(tag=TAG).warning( + f"Skipping server {name}: neither command nor url specified" + ) continue try: client = MCPClient(srv_config) await client.initialize() self.client[name] = client - self.logger.bind(tag=TAG).info(f"Initialized MCP client: {name}") + self.conn.logger.bind(tag=TAG).info(f"Initialized MCP client: {name}") client_tools = client.get_available_tools() self.tools.extend(client_tools) for tool in client_tools: - func_name = "mcp_"+tool["function"]["name"] - register_function(func_name, tool, ToolType.MCP_CLIENT)(self.execute_tool) - self.conn.func_handler.function_registry.register_function(func_name) + func_name = "mcp_" + tool["function"]["name"] + register_function(func_name, tool, ToolType.MCP_CLIENT)( + self.execute_tool + ) + self.conn.func_handler.function_registry.register_function( + func_name + ) except Exception as e: - self.logger.bind(tag=TAG).error(f"Failed to initialize MCP server {name}: {e}") + self.conn.logger.bind(tag=TAG).error( + f"Failed to initialize MCP server {name}: {e}" + ) self.conn.func_handler.upload_functions_desc() def get_all_tools(self) -> List[Dict[str, Any]]: @@ -79,7 +92,10 @@ class MCPManager: bool: 是否是MCP工具 """ for tool in self.tools: - if tool.get("function") != None and tool["function"].get("name") == tool_name: + if ( + tool.get("function") != None + and tool["function"].get("name") == tool_name + ): return True return False @@ -87,24 +103,29 @@ class MCPManager: """执行工具调用 Args: tool_name: 工具名称 - arguments: 工具参数 + arguments: 工具参数 Returns: Any: 工具执行结果 Raises: ValueError: 工具未找到时抛出 """ - self.logger.bind(tag=TAG).info(f"Executing tool {tool_name} with arguments: {arguments}") + self.conn.logger.bind(tag=TAG).info( + f"Executing tool {tool_name} with arguments: {arguments}" + ) for client in self.client.values(): if client.has_tool(tool_name): return await client.call_tool(tool_name, arguments) - + raise ValueError(f"Tool {tool_name} not found in any MCP server") async def cleanup_all(self) -> None: - for name, client in self.client.items(): + """依次关闭所有 MCPClient，不让异常阻断整体流程。""" + for name, client in list(self.client.items()): try: - await client.cleanup() - self.logger.bind(tag=TAG).info(f"Cleaned up MCP client: {name}") - except Exception as e: - self.logger.bind(tag=TAG).error(f"Error cleaning up MCP client {name}: {e}") + await asyncio.wait_for(client.cleanup(), timeout=20) + self.conn.logger.bind(tag=TAG).info(f"MCP client closed: {name}") + except (asyncio.TimeoutError, Exception) as e: + self.conn.logger.bind(tag=TAG).error( + f"Error closing MCP client {name}: {e}" + ) self.client.clear() diff --git a/main/xiaozhi-server/core/ota_server.py b/main/xiaozhi-server/core/ota_server.py new file mode 100644 index 00000000..383725d9 --- /dev/null +++ b/main/xiaozhi-server/core/ota_server.py @@ -0,0 +1,135 @@ +import json +import time +import asyncio +from aiohttp import web +from config.logger import setup_logging +from core.connection import ConnectionHandler +from core.utils.util import get_local_ip, initialize_modules + +TAG = __name__ + + +class SimpleOtaServer: + def __init__(self, config: dict): + self.config = config + self.logger = setup_logging() + + def _get_websocket_url(self, local_ip: str, port: int) -> str: + """获取websocket地址 + + Args: + local_ip: 本地IP地址 + port: 端口号 + + Returns: + str: websocket地址 + """ + server_config = self.config["server"] + websocket_config = server_config.get("websocket") + + if websocket_config and "你" not in websocket_config: + return websocket_config + else: + return f"ws://{local_ip}:{port}/xiaozhi/v1/" + + async def start(self): + server_config = self.config["server"] + host = server_config.get("ip", "0.0.0.0") + port = int(server_config.get("ota_port")) + + if port: + app = web.Application() + # 添加路由 + app.add_routes( + [ + web.get("/xiaozhi/ota/", self._handle_ota_get_request), + web.post("/xiaozhi/ota/", self._handle_ota_request), + web.options("/xiaozhi/ota/", self._handle_ota_request), + ] + ) + + # 运行服务 + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, host, port) + await site.start() + + # 保持服务运行 + while True: + await asyncio.sleep(3600) # 每隔 1 小时检查一次 + + async def _handle_ota_request(self, request): + """处理 /xiaozhi/ota/ 的 POST 请求""" + try: + data = await request.text() + self.logger.bind(tag=TAG).debug(f"OTA请求方法: {request.method}") + self.logger.bind(tag=TAG).debug(f"OTA请求头: {request.headers}") + self.logger.bind(tag=TAG).debug(f"OTA请求数据: {data}") + + device_id = request.headers.get("device-id", "") + if device_id: + self.logger.bind(tag=TAG).info(f"OTA请求设备ID: {device_id}") + else: + raise Exception("OTA请求设备ID为空") + + data_json = json.loads(data) + + server_config = self.config["server"] + host = server_config.get("ip", "0.0.0.0") + port = int(server_config.get("port", 8000)) + local_ip = get_local_ip() + + # OTA基础信息 + return_json = { + "server_time": { + "timestamp": int(round(time.time() * 1000)), + "timezone_offset": server_config.get("timezone_offset", 8) * 60, + }, + "firmware": { + "version": data_json["application"].get("version", "1.0.0"), + "url": "", + }, + "websocket": { + "url": self._get_websocket_url(local_ip, port), + }, + } + response = web.Response( + text=json.dumps(return_json, separators=(",", ":")), + content_type="application/json", + ) + except Exception as e: + self.logger.bind(tag=TAG).error(f"OTA请求异常: {e}") + return_json = {"success": False, "message": "request error."} + response = web.Response( + text=json.dumps(return_json, separators=(",", ":")), + content_type="application/json", + ) + finally: + # 添加header，允许跨域访问 + response.headers["Access-Control-Allow-Headers"] = ( + "client-id, content-type, device-id" + ) + response.headers["Access-Control-Allow-Credentials"] = "true" + response.headers["Access-Control-Allow-Origin"] = "*" + return response + + async def _handle_ota_get_request(self, request): + """处理 /xiaozhi/ota/ 的 GET 请求""" + try: + server_config = self.config["server"] + local_ip = get_local_ip() + port = int(server_config.get("port", 8000)) + websocket_url = self._get_websocket_url(local_ip, port) + message = f"OTA接口运行正常，向设备发送的websocket地址是：{websocket_url}" + response = web.Response(text=message, content_type="text/plain") + except Exception as e: + self.logger.bind(tag=TAG).error(f"OTA GET请求异常: {e}") + response = web.Response(text="OTA接口异常", content_type="text/plain") + finally: + # 添加header，允许跨域访问 + response.headers["Access-Control-Allow-Headers"] = ( + "client-id, content-type, device-id" + ) + response.headers["Access-Control-Allow-Credentials"] = "true" + response.headers["Access-Control-Allow-Origin"] = "*" + return response diff --git a/main/xiaozhi-server/core/providers/asr/aliyun.py b/main/xiaozhi-server/core/providers/asr/aliyun.py new file mode 100644 index 00000000..6606168c --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/aliyun.py @@ -0,0 +1,270 @@ +import http.client +import json +import asyncio +from typing import Optional, Tuple, List +import opuslib_next +import wave +import io +import os +import uuid +import hmac +import hashlib +import base64 +import requests +from urllib import parse +import time +from datetime import datetime +from config.logger import setup_logging +from core.providers.asr.base import ASRProviderBase + +TAG = __name__ +logger = setup_logging() + + +class AccessToken: + @staticmethod + def _encode_text(text): + encoded_text = parse.quote_plus(text) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def _encode_dict(dic): + keys = dic.keys() + dic_sorted = [(key, dic[key]) for key in sorted(keys)] + encoded_text = parse.urlencode(dic_sorted) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def create_token(access_key_id, access_key_secret): + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } + # 构造规范化的请求字符串 + query_string = AccessToken._encode_dict(parameters) + # print('规范化的请求字符串: %s' % query_string) + # 构造待签名字符串 + string_to_sign = ( + "GET" + + "&" + + AccessToken._encode_text("/") + + "&" + + AccessToken._encode_text(query_string) + ) + # print('待签名的字符串: %s' % string_to_sign) + # 计算签名 + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() + signature = base64.b64encode(secreted_string) + # print('签名: %s' % signature) + # 进行URL编码 + signature = AccessToken._encode_text(signature) + # print('URL编码后的签名: %s' % signature) + # 调用服务 + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % ( + signature, + query_string, + ) + # print('url: %s' % full_url) + # 提交HTTP GET请求 + response = requests.get(full_url) + if response.ok: + root_obj = response.json() + key = "Token" + if key in root_obj: + token = root_obj[key]["Id"] + expire_time = root_obj[key]["ExpireTime"] + return token, expire_time + # print(response.text) + return None, None + + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool): + super().__init__() + """阿里云ASR初始化""" + # 新增空值判断逻辑 + self.access_key_id = config.get("access_key_id") + self.access_key_secret = config.get("access_key_secret") + + self.app_key = config.get("appkey") + self.host = "nls-gateway-cn-shanghai.aliyuncs.com" + self.base_url = f"https://{self.host}/stream/v1/asr" + self.sample_rate = 16000 + self.format = "wav" + self.output_dir = config.get("output_dir", "./audio_output") + self.delete_audio_file = delete_audio_file + + if self.access_key_id and self.access_key_secret: + # 使用密钥对生成临时token + self._refresh_token() + else: + # 直接使用预生成的长期token + self.token = config.get("token") + self.expire_time = None + + # 确保输出目录存在 + os.makedirs(self.output_dir, exist_ok=True) + + def _refresh_token(self): + """刷新Token并记录过期时间""" + if self.access_key_id and self.access_key_secret: + self.token, expire_time_str = AccessToken.create_token( + self.access_key_id, self.access_key_secret + ) + if not expire_time_str: + raise ValueError("无法获取有效的Token过期时间") + + try: + # 统一转换为字符串处理 + expire_str = str(expire_time_str).strip() + + if expire_str.isdigit(): + expire_time = datetime.fromtimestamp(int(expire_str)) + else: + expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ") + self.expire_time = expire_time.timestamp() - 60 + except Exception as e: + raise ValueError(f"无效的过期时间格式: {expire_str}") from e + + else: + self.expire_time = None + + if not self.token: + raise ValueError("无法获取有效的访问Token") + + def _is_token_expired(self): + """检查Token是否过期""" + if not self.expire_time: + return False # 长期Token不过期 + # 新增调试日志 + # current_time = time.time() + # remaining = self.expire_time - current_time + # print(f"Token过期检查: 当前时间 {datetime.fromtimestamp(current_time)} | " + # f"过期时间 {datetime.fromtimestamp(self.expire_time)} | " + # f"剩余 {remaining:.2f}秒") + return time.time() > self.expire_time + + def generate_filename(self, extension=".wav"): + return os.path.join( + self.output_file, + f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}", + ) + + def _construct_request_url(self) -> str: + """构造请求URL，包含参数""" + request = f"{self.base_url}?appkey={self.app_key}" + request += f"&format={self.format}" + request += f"&sample_rate={self.sample_rate}" + request += "&enable_punctuation_prediction=true" + request += "&enable_inverse_text_normalization=true" + request += "&enable_voice_detection=false" + return request + + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" + file_path = os.path.join(self.output_dir, file_name) + + with wave.open(file_path, "wb") as wf: + wf.setnchannels(1) # 单声道 + wf.setsampwidth(2) # 16-bit + wf.setframerate(self.sample_rate) + wf.writeframes(b"".join(pcm_data)) + + logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}") + return file_path + + async def _send_request(self, pcm_data: bytes) -> Optional[str]: + """发送请求到阿里云ASR服务""" + try: + # 设置HTTP头 + headers = { + "X-NLS-Token": self.token, + "Content-type": "application/octet-stream", + "Content-Length": str(len(pcm_data)), + } + + # 创建连接并发送请求 + conn = http.client.HTTPSConnection(self.host) + request_url = self._construct_request_url() + + loop = asyncio.get_event_loop() + await loop.run_in_executor( + None, + lambda: conn.request( + method="POST", url=request_url, body=pcm_data, headers=headers + ), + ) + + # 获取响应 + response = await loop.run_in_executor(None, conn.getresponse) + body = await loop.run_in_executor(None, response.read) + conn.close() + + # 解析响应 + try: + body_json = json.loads(body) + status = body_json.get("status") + + if status == 20000000: + result = body_json.get("result", "") + logger.bind(tag=TAG).debug(f"ASR结果: {result}") + return result + else: + logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}") + return None + + except ValueError: + logger.bind(tag=TAG).error("响应不是JSON格式") + return None + + except Exception as e: + logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True) + return None + + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: + """将语音数据转换为文本""" + if self._is_token_expired(): + logger.warning("Token已过期，正在自动刷新...") + self._refresh_token() + + file_path = None + try: + # 解码Opus为PCM + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) + + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + file_path = self.save_audio_to_file(pcm_data, session_id) + + # 发送请求并获取文本 + text = await self._send_request(combined_pcm_data) + + if text: + return text, file_path + + return "", file_path + + except Exception as e: + logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) + return "", file_path diff --git a/main/xiaozhi-server/core/providers/asr/baidu.py b/main/xiaozhi-server/core/providers/asr/baidu.py new file mode 100644 index 00000000..fe73fd45 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/baidu.py @@ -0,0 +1,106 @@ +import base64 +import hashlib +import hmac +import json +import time +from datetime import datetime, timezone +import os +import uuid +from typing import Optional, Tuple, List +import wave +import opuslib_next + +from aip import AipSpeech +from core.providers.asr.base import ASRProviderBase +from config.logger import setup_logging + +TAG = __name__ +logger = setup_logging() + + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool = True): + super().__init__() + self.app_id = config.get("app_id") + self.api_key = config.get("api_key") + self.secret_key = config.get("secret_key") + + dev_pid = config.get("dev_pid", "1537") + self.dev_pid = int(dev_pid) if dev_pid else 1537 + + self.output_dir = config.get("output_dir") + self.delete_audio_file = delete_audio_file + + self.client = AipSpeech(str(self.app_id), self.api_key, self.secret_key) + + # 确保输出目录存在 + os.makedirs(self.output_dir, exist_ok=True) + + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" + file_path = os.path.join(self.output_dir, file_name) + + with wave.open(file_path, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) # 2 bytes = 16-bit + wf.setframerate(16000) + wf.writeframes(b"".join(pcm_data)) + + return file_path + + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: + """将语音数据转换为文本""" + if not opus_data: + logger.bind(tag=TAG).warning("音频数据为空！") + return None, None + + file_path = None + try: + # 检查配置是否已设置 + if not self.app_id or not self.api_key or not self.secret_key: + logger.bind(tag=TAG).error("百度语音识别配置未设置，无法进行识别") + return None, file_path + + # 将Opus音频数据解码为PCM + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) + + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + self.save_audio_to_file(pcm_data, session_id) + + start_time = time.time() + # 识别本地文件 + result = self.client.asr( + combined_pcm_data, + "pcm", + 16000, + { + "dev_pid": str(self.dev_pid), + }, + ) + + if result and result["err_no"] == 0: + logger.bind(tag=TAG).debug( + f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}" + ) + result = result["result"][0] + return result, file_path + else: + raise Exception( + f"百度语音识别失败，错误码: {result['err_no']}，错误信息: {result['err_msg']}" + ) + return None, file_path + + except Exception as e: + logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True) + return None, file_path diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index d8db22a8..227a906d 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from typing import Optional, Tuple, List - +import opuslib_next from config.logger import setup_logging TAG = __name__ @@ -8,12 +8,37 @@ logger = setup_logging() class ASRProviderBase(ABC): + def __init__(self): + self.audio_format = "opus" + @abstractmethod - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """解码Opus数据并保存为WAV文件""" + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" pass @abstractmethod - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" pass + + def set_audio_format(self, format: str) -> None: + """设置音频格式""" + self.audio_format = format + + @staticmethod + def decode_opus(opus_data: List[bytes]) -> bytes: + """将Opus音频数据解码为PCM数据""" + + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + return pcm_data diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index a5a5bb65..403c181f 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -45,14 +45,14 @@ def parse_response(res): payload 类似与http 请求体 """ protocol_version = res[0] >> 4 - header_size = res[0] & 0x0f + header_size = res[0] & 0x0F message_type = res[1] >> 4 - message_type_specific_flags = res[1] & 0x0f + message_type_specific_flags = res[1] & 0x0F serialization_method = res[2] >> 4 - message_compression = res[2] & 0x0f + message_compression = res[2] & 0x0F reserved = res[3] - header_extensions = res[4:header_size * 4] - payload = res[header_size * 4:] + header_extensions = res[4 : header_size * 4] + payload = res[header_size * 4 :] result = {} payload_msg = None payload_size = 0 @@ -61,13 +61,13 @@ def parse_response(res): payload_msg = payload[4:] elif message_type == SERVER_ACK: seq = int.from_bytes(payload[:4], "big", signed=True) - result['seq'] = seq + result["seq"] = seq if len(payload) >= 8: payload_size = int.from_bytes(payload[4:8], "big", signed=False) payload_msg = payload[8:] elif message_type == SERVER_ERROR_RESPONSE: code = int.from_bytes(payload[:4], "big", signed=False) - result['code'] = code + result["code"] = code payload_size = int.from_bytes(payload[4:8], "big", signed=False) payload_msg = payload[8:] if payload_msg is None: @@ -78,17 +78,21 @@ def parse_response(res): payload_msg = json.loads(str(payload_msg, "utf-8")) elif serialization_method != NO_SERIALIZATION: payload_msg = str(payload_msg, "utf-8") - result['payload_msg'] = payload_msg - result['payload_size'] = payload_size + result["payload_msg"] = payload_msg + result["payload_size"] = payload_size return result class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): + super().__init__() self.appid = config.get("appid") self.cluster = config.get("cluster") self.access_token = config.get("access_token") + self.boosting_table_name = config.get("boosting_table_name", "") + self.correct_table_name = config.get("correct_table_name", "") self.output_dir = config.get("output_dir") + self.delete_audio_file = delete_audio_file self.host = "openspeech.bytedance.com" self.ws_url = f"wss://{self.host}/api/v2/asr" @@ -98,21 +102,12 @@ class ASRProvider(ASRProviderBase): # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" - file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - with wave.open(file_path, "wb") as wf: wf.setnchannels(1) wf.setsampwidth(2) # 2 bytes = 16-bit @@ -122,7 +117,9 @@ class ASRProvider(ASRProviderBase): return file_path @staticmethod - def _generate_header(message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE) -> bytearray: + def _generate_header( + message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE + ) -> bytearray: """Generate protocol header.""" header = bytearray() header_size = 1 @@ -146,10 +143,12 @@ class ASRProvider(ASRProviderBase): "request": { "reqid": reqid, "show_utterances": False, - "sequence": 1 + "sequence": 1, + "boosting_table_name": self.boosting_table_name, + "correct_table_name": self.correct_table_name, }, "audio": { - "format": "wav", + "format": "raw", "rate": 16000, "language": "zh-CN", "bits": 16, @@ -158,18 +157,23 @@ class ASRProvider(ASRProviderBase): }, } - async def _send_request(self, audio_data: List[bytes], segment_size: int) -> Optional[str]: + async def _send_request( + self, audio_data: List[bytes], segment_size: int + ) -> Optional[str]: """Send request to Volcano ASR service.""" try: - auth_header = {'Authorization': 'Bearer; {}'.format(self.access_token)} - async with websockets.connect(self.ws_url, additional_headers=auth_header) as websocket: + auth_header = {"Authorization": "Bearer; {}".format(self.access_token)} + async with websockets.connect( + self.ws_url, additional_headers=auth_header + ) as websocket: # Prepare request data request_params = self._construct_request(str(uuid.uuid4())) - print(request_params) payload_bytes = str.encode(json.dumps(request_params)) payload_bytes = gzip.compress(payload_bytes) full_client_request = self._generate_header() - full_client_request.extend((len(payload_bytes)).to_bytes(4, 'big')) # payload size(4 bytes) + full_client_request.extend( + (len(payload_bytes)).to_bytes(4, "big") + ) # payload size(4 bytes) full_client_request.extend(payload_bytes) # payload # Send header and metadata @@ -177,22 +181,29 @@ class ASRProvider(ASRProviderBase): await websocket.send(full_client_request) res = await websocket.recv() result = parse_response(res) - if 'payload_msg' in result and result['payload_msg']['code'] != self.success_code: + if ( + "payload_msg" in result + and result["payload_msg"]["code"] != self.success_code + ): logger.bind(tag=TAG).error(f"ASR error: {result}") return None - for seq, (chunk, last) in enumerate(self.slice_data(audio_data, segment_size), 1): + for seq, (chunk, last) in enumerate( + self.slice_data(audio_data, segment_size), 1 + ): if last: audio_only_request = self._generate_header( message_type=CLIENT_AUDIO_ONLY_REQUEST, - message_type_specific_flags=NEG_SEQUENCE + message_type_specific_flags=NEG_SEQUENCE, ) else: audio_only_request = self._generate_header( message_type=CLIENT_AUDIO_ONLY_REQUEST ) payload_bytes = gzip.compress(chunk) - audio_only_request.extend((len(payload_bytes)).to_bytes(4, 'big')) # payload size(4 bytes) + audio_only_request.extend( + (len(payload_bytes)).to_bytes(4, "big") + ) # payload size(4 bytes) audio_only_request.extend(payload_bytes) # payload # Send audio data await websocket.send(audio_only_request) @@ -201,9 +212,12 @@ class ASRProvider(ASRProviderBase): response = await websocket.recv() result = parse_response(response) - if 'payload_msg' in result and result['payload_msg']['code'] == self.success_code: - if len(result['payload_msg']['result']) > 0: - return result['payload_msg']['result'][0]["text"] + if ( + "payload_msg" in result + and result["payload_msg"]["code"] == self.success_code + ): + if len(result["payload_msg"]["result"]) > 0: + return result["payload_msg"]["result"][0]["text"] return None else: logger.bind(tag=TAG).error(f"ASR error: {result}") @@ -213,29 +227,6 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True) return None - @staticmethod - def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return pcm_data - - @staticmethod - def read_wav_info(data: io.BytesIO = None) -> (int, int, int, int, int): - with io.BytesIO(data) as _f: - wave_fp = wave.open(_f, 'rb') - nchannels, sampwidth, framerate, nframes = wave_fp.getparams()[:4] - wave_bytes = wave_fp.readframes(nframes) - return nchannels, sampwidth, framerate, nframes, len(wave_bytes) - @staticmethod def slice_data(data: bytes, chunk_size: int) -> (list, bool): """ @@ -247,40 +238,46 @@ class ASRProvider(ASRProviderBase): data_len = len(data) offset = 0 while offset + chunk_size < data_len: - yield data[offset: offset + chunk_size], False + yield data[offset : offset + chunk_size], False offset += chunk_size else: - yield data[offset: data_len], True + yield data[offset:data_len], True - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" + + file_path = None try: # 合并所有opus数据包 - pcm_data = self.decode_opus(opus_data, session_id) - combined_pcm_data = b''.join(pcm_data) + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) - wav_buffer = io.BytesIO() + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + file_path = self.save_audio_to_file(pcm_data, session_id) - with wave.open(wav_buffer, "wb") as wav_file: - wav_file.setnchannels(1) # 设置声道数 - wav_file.setsampwidth(2) # 设置采样宽度 - wav_file.setframerate(16000) # 设置采样率 - wav_file.writeframes(combined_pcm_data) # 写入 PCM 数据 - - # 获取封装后的 WAV 数据 - wav_data = wav_buffer.getvalue() - nchannels, sampwidth, framerate, nframes, wav_len = self.read_wav_info(wav_data) - size_per_sec = nchannels * sampwidth * framerate + # 直接使用PCM数据 + # 计算分段大小 (单声道, 16bit, 16kHz采样率) + size_per_sec = 1 * 2 * 16000 # nchannels * sampwidth * framerate segment_size = int(size_per_sec * self.seg_duration / 1000) # 语音识别 start_time = time.time() - text = await self._send_request(wav_data, segment_size) + text = await self._send_request(combined_pcm_data, segment_size) if text: - logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}") - return text, None - return "", None + logger.bind(tag=TAG).debug( + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" + ) + return text, file_path + return "", file_path except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", None + return "", file_path diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 82d09ab5..ec3df17f 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -6,9 +6,7 @@ import io from config.logger import setup_logging from typing import Optional, Tuple, List import uuid -import opuslib_next from core.providers.asr.base import ASRProviderBase - from funasr import AutoModel from funasr.utils.postprocess_utils import rich_transcription_postprocess @@ -35,6 +33,7 @@ class CaptureOutput: class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): + super().__init__() self.model_dir = config.get("model_dir") self.output_dir = config.get("output_dir") # 修正配置键名 self.delete_audio_file = delete_audio_file @@ -46,25 +45,16 @@ class ASRProvider(ASRProviderBase): model=self.model_dir, vad_kwargs={"max_single_segment_time": 30000}, disable_update=True, - hub="hf" + hub="hf", # device="cuda:0", # 启用GPU加速 ) - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" - file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - with wave.open(file_path, "wb") as wf: wf.setnchannels(1) wf.setsampwidth(2) # 2 bytes = 16-bit @@ -73,38 +63,51 @@ class ASRProvider(ASRProviderBase): return file_path - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None try: - # 保存音频文件 - start_time = time.time() - file_path = self.save_audio_to_file(opus_data, session_id) - logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}") + # 合并所有opus数据包 + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + + combined_pcm_data = b"".join(pcm_data) + + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + file_path = self.save_audio_to_file(pcm_data, session_id) # 语音识别 start_time = time.time() result = self.model.generate( - input=file_path, + input=combined_pcm_data, cache={}, language="auto", use_itn=True, batch_size_s=60, ) text = rich_transcription_postprocess(result[0]["text"]) - logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}") + logger.bind(tag=TAG).debug( + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" + ) return text, file_path except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", None + return "", file_path - finally: - # 文件清理逻辑 - if self.delete_audio_file and file_path and os.path.exists(file_path): - try: - os.remove(file_path) - logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") - except Exception as e: - logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") + # finally: + # # 文件清理逻辑 + # if self.delete_audio_file and file_path and os.path.exists(file_path): + # try: + # os.remove(file_path) + # logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") + # except Exception as e: + # logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py new file mode 100644 index 00000000..97c9fdc9 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -0,0 +1,185 @@ +from typing import Optional, Tuple, List +import opuslib_next +from core.providers.asr.base import ASRProviderBase +import os +import ssl +import json +import uuid +import wave +import websockets +from config.logger import setup_logging +import asyncio +import re + +TAG = __name__ +logger = setup_logging() + + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool): + """ + Initialize the ASRProvider with server configuration. + :param config: Dictionary containing 'host', 'port', and 'is_ssl'. + :param delete_audio_file: Boolean to indicate whether to delete audio files after processing. + """ + super().__init__() + self.host = config.get("host", "localhost") + self.port = config.get("port", 10095) + self.api_key = config.get("api_key", "none") + self.is_ssl = str(config.get("is_ssl", True)).lower() in ( + "true", + "1", + "yes", + ) + self.output_dir = config.get("output_dir") + self.delete_audio_file = delete_audio_file + self.uri = ( + f"wss://{self.host}:{self.port}" + if self.is_ssl + else f"ws://{self.host}:{self.port}" + ) + self.ssl_context = ssl.SSLContext() if self.is_ssl else None + if self.ssl_context: + self.ssl_context.check_hostname = False + self.ssl_context.verify_mode = ssl.CERT_NONE + + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" + file_path = os.path.join(self.output_dir, file_name) + + with wave.open(file_path, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) # 2 bytes = 16-bit + wf.setframerate(16000) + wf.writeframes(b"".join(pcm_data)) + + return file_path + + async def _receive_responses(self, ws) -> None: + """ + Asynchronous generator to receive messages from the WebSocket. + Yields each message as it is received. + """ + text = "" + while True: + try: + response = await asyncio.wait_for(ws.recv(), timeout=5) + response_data = json.loads(response) + logger.bind(tag=TAG).debug(f"Received response: {response_data}") + if response_data.get("is_final", True): + text += response_data.get("text", "") + break + else: + text += response_data.get("text", "") + except asyncio.TimeoutError: + logger.bind(tag=TAG).error( + "Timeout while waiting for response from WebSocket." + ) + break + except websockets.exceptions.ConnectionClosed as e: + logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}") + break + return text + + async def _send_data(self, ws, pcm_data: bytes, session_id: str) -> tuple: + """ + Internal method to handle WebSocket communication. + Reuses the persistent WebSocket connection if available. + :param pcm_data: PCM audio data to send. + :param session_id: Unique session identifier. + :return: Tuple containing recognized text and optional timestamp. + """ + + # Send initial configuration message + config_message = json.dumps( + { + "mode": "offline", + "chunk_size": [5, 10, 5], + "chunk_interval": 10, + "wav_name": session_id, + "is_speaking": True, + "itn": False, + } + ) + await ws.send(config_message) + logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}") + + # Send PCM data + await ws.send(pcm_data) + logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes") + + # Indicate end of speech + end_message = json.dumps({"is_speaking": False}) + await ws.send(end_message) + logger.bind(tag=TAG).debug(f"Sent end message: {end_message}") + + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: + """ + Convert speech data to text using FunASR. + :param opus_data: List of Opus-encoded audio data chunks. + :param session_id: Unique session identifier. + :return: Tuple containing recognized text and optional timestamp. + """ + file_path = None + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) + + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + file_path = self.save_audio_to_file(pcm_data, session_id) + auth_header = {"Authorization": "Bearer; {}".format(self.api_key)} + async with websockets.connect( + self.uri, + additional_headers=auth_header, + subprotocols=["binary"], + ping_interval=None, + ssl=self.ssl_context, + ) as ws: + try: + # Use asyncio to handle WebSocket communication + send_task = asyncio.create_task( + self._send_data(ws, combined_pcm_data, session_id) + ) + receive_task = asyncio.create_task(self._receive_responses(ws)) + + # Gather tasks with error handling + done, pending = await asyncio.wait( + [send_task, receive_task], return_when=asyncio.FIRST_EXCEPTION + ) + + # Cancel any pending tasks + for task in pending: + task.cancel() + + # Check for exceptions in completed tasks + for task in done: + if task.exception(): + raise task.exception() + + # Get the result from the receive task + result = receive_task.result() + match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result) + if match: + result = match.group(4).strip() + return ( + result, + file_path, + ) # Return the recognized text and timestamp (if any) + + except websockets.exceptions.ConnectionClosed as e: + logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}") + return "", file_path + except Exception as e: + logger.bind(tag=TAG).error( + f"Error during speech-to-text conversion: {e}", exc_info=True + ) + return "", file_path diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index 3dfa3162..667e1606 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -37,17 +37,18 @@ class CaptureOutput: class ASRProvider(ASRProviderBase): def __init__(self, config: dict, delete_audio_file: bool): + super().__init__() self.model_dir = config.get("model_dir") self.output_dir = config.get("output_dir") self.delete_audio_file = delete_audio_file # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) - + # 初始化模型文件路径 model_files = { "model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"), - "tokens.txt": os.path.join(self.model_dir, "tokens.txt") + "tokens.txt": os.path.join(self.model_dir, "tokens.txt"), } # 下载并检查模型文件 @@ -58,15 +59,15 @@ class ASRProvider(ASRProviderBase): model_file_download( model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue", file_path=file_name, - local_dir=self.model_dir + local_dir=self.model_dir, ) - + if not os.path.isfile(file_path): raise FileNotFoundError(f"模型文件下载失败: {file_path}") - + self.model_path = model_files["model.int8.onnx"] self.tokens_path = model_files["tokens.txt"] - + except Exception as e: logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}") raise @@ -83,21 +84,12 @@ class ASRProvider(ASRProviderBase): use_itn=True, ) - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" - file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - with wave.open(file_path, "wb") as wf: wf.setnchannels(1) wf.setsampwidth(2) # 2 bytes = 16-bit @@ -130,14 +122,22 @@ class ASRProvider(ASRProviderBase): samples_float32 = samples_float32 / 32768 return samples_float32, f.getframerate() - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None try: # 保存音频文件 start_time = time.time() - file_path = self.save_audio_to_file(opus_data, session_id) - logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}") + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + file_path = self.save_audio_to_file(pcm_data, session_id) + logger.bind(tag=TAG).debug( + f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}" + ) # 语音识别 start_time = time.time() @@ -146,14 +146,15 @@ class ASRProvider(ASRProviderBase): s.accept_waveform(sample_rate, samples) self.model.decode_stream(s) text = s.result.text - logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}") + logger.bind(tag=TAG).debug( + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" + ) return text, file_path except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) - return "", None - + return "", file_path finally: # 文件清理逻辑 if self.delete_audio_file and file_path and os.path.exists(file_path): diff --git a/main/xiaozhi-server/core/providers/asr/tencent.py b/main/xiaozhi-server/core/providers/asr/tencent.py index 8221f9df..5ab1befb 100644 --- a/main/xiaozhi-server/core/providers/asr/tencent.py +++ b/main/xiaozhi-server/core/providers/asr/tencent.py @@ -17,77 +17,66 @@ from config.logger import setup_logging TAG = __name__ logger = setup_logging() + class ASRProvider(ASRProviderBase): API_URL = "https://asr.tencentcloudapi.com" API_VERSION = "2019-06-14" FORMAT = "pcm" # 支持的音频格式：pcm, wav, mp3 def __init__(self, config: dict, delete_audio_file: bool = True): + super().__init__() self.secret_id = config.get("secret_id") self.secret_key = config.get("secret_key") self.output_dir = config.get("output_dir") - + self.delete_audio_file = delete_audio_file + # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" - - file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav" + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" + module_name = __name__.split(".")[-1] + file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - + with wave.open(file_path, "wb") as wf: wf.setnchannels(1) wf.setsampwidth(2) # 2 bytes = 16-bit wf.setframerate(16000) wf.writeframes(b"".join(pcm_data)) - + return file_path - @staticmethod - def decode_opus(opus_data: List[bytes]) -> bytes: - """将Opus音频数据解码为PCM数据""" - import opuslib_next - - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - - return b"".join(pcm_data) - - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """将语音数据转换为文本""" if not opus_data: - logger.bind(tag=TAG).warn("音频数据为空！") + logger.bind(tag=TAG).warning("音频数据为空！") return None, None + file_path = None try: # 检查配置是否已设置 if not self.secret_id or not self.secret_key: logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别") - return None, None + return None, file_path # 将Opus音频数据解码为PCM - pcm_data = self.decode_opus(opus_data) - + if self.audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + combined_pcm_data = b"".join(pcm_data) + + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + self.save_audio_to_file(pcm_data, session_id) + # 将音频数据转换为Base64编码 - base64_audio = base64.b64encode(pcm_data).decode('utf-8') + base64_audio = base64.b64encode(combined_pcm_data).decode("utf-8") # 构建请求体 request_body = self._build_request_body(base64_audio) @@ -98,15 +87,17 @@ class ASRProvider(ASRProviderBase): # 发送请求 start_time = time.time() result = self._send_request(request_body, timestamp, authorization) - + if result: - logger.bind(tag=TAG).debug(f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}") - - return result, None + logger.bind(tag=TAG).debug( + f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}" + ) + + return result, file_path except Exception as e: logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True) - return None, None + return None, file_path def _build_request_body(self, base64_audio: str) -> str: """构建请求体""" @@ -117,7 +108,7 @@ class ASRProvider(ASRProviderBase): "SourceType": 1, # 音频数据来源为语音文件 "VoiceFormat": self.FORMAT, # 音频格式 "Data": base64_audio, # Base64编码的音频数据 - "DataLen": len(base64_audio) # 数据长度 + "DataLen": len(base64_audio), # 数据长度 } return json.dumps(request_map) @@ -150,9 +141,11 @@ class ASRProvider(ASRProviderBase): action = "SentenceRecognition" # 接口名称 # 构建规范头部信息，注意顺序和格式 - canonical_headers = f"content-type:{content_type.lower()}\n" + \ - f"host:{host.lower()}\n" + \ - f"x-tc-action:{action.lower()}\n" + canonical_headers = ( + f"content-type:{content_type.lower()}\n" + + f"host:{host.lower()}\n" + + f"x-tc-action:{action.lower()}\n" + ) signed_headers = "content-type;host;x-tc-action" @@ -160,21 +153,25 @@ class ASRProvider(ASRProviderBase): payload_hash = self._sha256_hex(request_body) # 构建规范请求字符串 - canonical_request = f"{http_request_method}\n" + \ - f"{canonical_uri}\n" + \ - f"{canonical_query_string}\n" + \ - f"{canonical_headers}\n" + \ - f"{signed_headers}\n" + \ - f"{payload_hash}" + canonical_request = ( + f"{http_request_method}\n" + + f"{canonical_uri}\n" + + f"{canonical_query_string}\n" + + f"{canonical_headers}\n" + + f"{signed_headers}\n" + + f"{payload_hash}" + ) # 计算规范请求的哈希值 hashed_canonical_request = self._sha256_hex(canonical_request) # 构建待签名字符串 - string_to_sign = f"{algorithm}\n" + \ - f"{timestamp}\n" + \ - f"{credential_scope}\n" + \ - f"{hashed_canonical_request}" + string_to_sign = ( + f"{algorithm}\n" + + f"{timestamp}\n" + + f"{credential_scope}\n" + + f"{hashed_canonical_request}" + ) # 计算签名密钥 secret_date = self._hmac_sha256(f"TC3{self.secret_key}", date) @@ -182,13 +179,17 @@ class ASRProvider(ASRProviderBase): secret_signing = self._hmac_sha256(secret_service, "tc3_request") # 计算签名 - signature = self._bytes_to_hex(self._hmac_sha256(secret_signing, string_to_sign)) + signature = self._bytes_to_hex( + self._hmac_sha256(secret_signing, string_to_sign) + ) # 构建授权头 - authorization = f"{algorithm} " + \ - f"Credential={self.secret_id}/{credential_scope}, " + \ - f"SignedHeaders={signed_headers}, " + \ - f"Signature={signature}" + authorization = ( + f"{algorithm} " + + f"Credential={self.secret_id}/{credential_scope}, " + + f"SignedHeaders={signed_headers}, " + + f"Signature={signature}" + ) return timestamp, authorization @@ -196,7 +197,9 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error(f"生成认证头失败: {e}", exc_info=True) raise RuntimeError(f"生成认证头失败: {e}") - def _send_request(self, request_body: str, timestamp: str, authorization: str) -> Optional[str]: + def _send_request( + self, request_body: str, timestamp: str, authorization: str + ) -> Optional[str]: """发送请求到腾讯云API""" headers = { "Content-Type": "application/json; charset=utf-8", @@ -205,47 +208,47 @@ class ASRProvider(ASRProviderBase): "X-TC-Action": "SentenceRecognition", "X-TC-Version": self.API_VERSION, "X-TC-Timestamp": timestamp, - "X-TC-Region": "ap-shanghai" + "X-TC-Region": "ap-shanghai", } try: response = requests.post(self.API_URL, headers=headers, data=request_body) - + if not response.ok: raise IOError(f"请求失败: {response.status_code} {response.reason}") - + response_json = response.json() - + # 检查是否有错误 if "Response" in response_json and "Error" in response_json["Response"]: error = response_json["Response"]["Error"] error_code = error["Code"] error_message = error["Message"] raise IOError(f"API返回错误: {error_code}: {error_message}") - + # 提取识别结果 if "Response" in response_json and "Result" in response_json["Response"]: return response_json["Response"]["Result"] else: - logger.bind(tag=TAG).warn(f"响应中没有识别结果: {response_json}") + logger.bind(tag=TAG).warning(f"响应中没有识别结果: {response_json}") return "" - + except Exception as e: logger.bind(tag=TAG).error(f"发送请求失败: {e}", exc_info=True) return None def _sha256_hex(self, data: str) -> str: """计算字符串的SHA256哈希值""" - digest = hashlib.sha256(data.encode('utf-8')).digest() + digest = hashlib.sha256(data.encode("utf-8")).digest() return self._bytes_to_hex(digest) def _hmac_sha256(self, key, data: str) -> bytes: """计算HMAC-SHA256""" if isinstance(key, str): - key = key.encode('utf-8') - - return hmac.new(key, data.encode('utf-8'), hashlib.sha256).digest() + key = key.encode("utf-8") + + return hmac.new(key, data.encode("utf-8"), hashlib.sha256).digest() def _bytes_to_hex(self, bytes_data: bytes) -> str: """字节数组转十六进制字符串""" - return ''.join(f"{b:02x}" for b in bytes_data) \ No newline at end of file + return "".join(f"{b:02x}" for b in bytes_data) diff --git a/main/xiaozhi-server/core/providers/intent/base.py b/main/xiaozhi-server/core/providers/intent/base.py index 040dd025..f6df10dd 100644 --- a/main/xiaozhi-server/core/providers/intent/base.py +++ b/main/xiaozhi-server/core/providers/intent/base.py @@ -9,18 +9,6 @@ logger = setup_logging() class IntentProviderBase(ABC): def __init__(self, config): self.config = config - self.intent_options = [ - { - "name": "handle_exit_intent", - "desc": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候", - }, - { - "name": "play_music", - "desc": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图", - }, - {"name": "get_time", "desc": "获取今天日期或者当前时间信息"}, - {"name": "continue_chat", "desc": "继续聊天"}, - ] def set_llm(self, llm): self.llm = llm diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py index 05d01c10..2a3e023f 100644 --- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py +++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py @@ -15,57 +15,72 @@ class IntentProvider(IntentProviderBase): def __init__(self, config): super().__init__(config) self.llm = None - self.promot = self.get_intent_system_prompt() + self.promot = "" # 添加缓存管理 self.intent_cache = {} # 缓存意图识别结果 self.cache_expiry = 600 # 缓存有效期10分钟 self.cache_max_size = 100 # 最多缓存100个意图 + self.history_count = 4 # 默认使用最近4条对话记录 - def get_intent_system_prompt(self) -> str: + def get_intent_system_prompt(self, functions_list: str) -> str: """ - 根据配置的意图选项动态生成系统提示词 + 根据配置的意图选项和可用函数动态生成系统提示词 + Args: + functions: 可用的函数列表，JSON格式字符串 Returns: 格式化后的系统提示词 """ + # 构建函数说明部分 + functions_desc = "可用的函数列表：\n" + for func in functions_list: + func_info = func.get("function", {}) + name = func_info.get("name", "") + desc = func_info.get("description", "") + params = func_info.get("parameters", {}) + + functions_desc += f"\n函数名: {name}\n" + functions_desc += f"描述: {desc}\n" + + if params: + functions_desc += "参数:\n" + for param_name, param_info in params.get("properties", {}).items(): + param_desc = param_info.get("description", "") + param_type = param_info.get("type", "") + functions_desc += f"- {param_name} ({param_type}): {param_desc}\n" + + functions_desc += "---\n" + prompt = ( - "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图属于以下哪一类：\n" - "" - f"{str(self.intent_options)}" - "\n" - "处理步骤:" - "1. 思考意图类型，生成function_call格式" - "\n\n" - "返回格式示例：\n" - '1. 播放音乐意图: {"function_call": {"name": "play_music", "arguments": {"song_name": "音乐名称"}}}\n' - '2. 结束对话意图: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n' - '3. 获取当天日期时间: {"function_call": {"name": "get_time"}}\n' - '4. 继续聊天意图: {"function_call": {"name": "continue_chat"}}\n' - "\n" - "注意:\n" - '- 播放音乐：无歌名时，song_name设为"random"\n' - "- 如果没有明显的意图，应按照继续聊天意图处理\n" - "- 只返回纯JSON，不要任何其他内容\n" - "\n" - "示例分析:\n" + "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n" + f"{functions_desc}\n" + "处理步骤:\n" + "1. 分析用户输入，确定用户意图\n" + "2. 从可用函数列表中选择最匹配的函数\n" + "3. 如果找到匹配的函数，生成对应的function_call 格式\n" + '4. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n' + "返回格式要求：\n" + "1. 必须返回纯JSON格式\n" + "2. 必须包含function_call字段\n" + "3. function_call必须包含name字段\n" + "4. 如果函数需要参数，必须包含arguments字段\n\n" + "示例：\n" "```\n" - "用户: 你也太搞笑了\n" - '返回: {"function_call": {"name": "continue_chat"}}\n' - "```\n" - "```\n" - "用户: 现在是几号了?现在几点了？\n" + "用户: 现在几点了？\n" '返回: {"function_call": {"name": "get_time"}}\n' "```\n" "```\n" - "用户: 我们明天再聊吧\n" - '返回: {"function_call": {"name": "handle_exit_intent"}}\n' + "用户: 我想结束对话\n" + '返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n' "```\n" "```\n" - "用户: 播放中秋月\n" - '返回: {"function_call": {"name": "play_music", "arguments": {"song_name": "中秋月"}}}\n' - "```\n" - "```\n" - "可用的音乐名称:\n" + "用户: 你好啊\n" + '返回: {"function_call": {"name": "continue_chat"}}\n' + "```\n\n" + "注意：\n" + "1. 只返回JSON格式，不要包含任何其他文字\n" + '2. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n' + "3. 确保返回的JSON格式正确，包含所有必要的字段\n" ) return prompt @@ -90,6 +105,14 @@ class IntentProvider(IntentProviderBase): for key, _ in sorted_items[: len(sorted_items) - self.cache_max_size]: del self.intent_cache[key] + def replyResult(self, text: str, original_text: str): + llm_result = self.llm.response_no_stream( + system_prompt=text, + user_prompt="请根据以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。用户现在说：" + + original_text, + ) + return llm_result + async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str: if not self.llm: raise ValueError("LLM provider not set") @@ -118,22 +141,35 @@ class IntentProvider(IntentProviderBase): # 清理缓存 self.clean_cache() - # 构建用户最后一句话的提示 - msgStr = "" + if self.promot == "": + if hasattr(conn, "func_handler"): + functions = conn.func_handler.get_functions() + self.promot = self.get_intent_system_prompt(functions) - # 只使用最后两句即可 - if len(dialogue_history) >= 2: - # 保证最少有两句话的时候处理 - msgStr += f"{dialogue_history[-2].role}: {dialogue_history[-2].content}\n" - msgStr += f"{dialogue_history[-1].role}: {dialogue_history[-1].content}\n" - - msgStr += f"User: {text}\n" - user_prompt = f"当前的对话如下：\n{msgStr}" music_config = initialize_music_handler(conn) music_file_names = music_config["music_file_names"] - prompt_music = f"{self.promot}\n{music_file_names}\n" + prompt_music = f"{self.promot}\n{music_file_names}\n" + + devices = conn.config["plugins"]["home_assistant"].get("devices", []) + if len(devices) > 0: + hass_prompt = "\n下面是我家智能设备列表（位置，设备名，entity_id），可以通过homeassistant控制\n" + for device in devices: + hass_prompt += device + "\n" + prompt_music += hass_prompt + logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}") + # 构建用户对话历史的提示 + msgStr = "" + + # 获取最近的对话历史 + start_idx = max(0, len(dialogue_history) - self.history_count) + for i in range(start_idx, len(dialogue_history)): + msgStr += f"{dialogue_history[i].role}: {dialogue_history[i].content}\n" + + msgStr += f"User: {text}\n" + user_prompt = f"current dialogue:\n{msgStr}" + # 记录预处理完成时间 preprocess_time = time.time() - total_start_time logger.bind(tag=TAG).debug(f"意图识别预处理耗时: {preprocess_time:.4f}秒") @@ -179,9 +215,18 @@ class IntentProvider(IntentProviderBase): # 记录识别到的function call logger.bind(tag=TAG).info( - f"识别到function call: {function_name}, 参数: {function_args}" + f"llm 识别到意图: {function_name}, 参数: {function_args}" ) + # 如果是继续聊天，清理工具调用相关的历史消息 + if function_name == "continue_chat": + # 保留非工具相关的消息 + clean_history = [ + msg for msg in conn.dialogue.dialogue + if msg.role not in ["tool", "function"] + ] + conn.dialogue.dialogue = clean_history + # 添加到缓存 self.intent_cache[cache_key] = { "intent": intent, diff --git a/main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py b/main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py index 51efdc06..012035ee 100644 --- a/main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py +++ b/main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py @@ -2,10 +2,12 @@ from config.logger import setup_logging from http import HTTPStatus from dashscope import Application from core.providers.llm.base import LLMProviderBase +from core.utils.util import check_model_key TAG = __name__ logger = setup_logging() + class LLMProvider(LLMProviderBase): def __init__(self, config): self.api_key = config["api_key"] @@ -13,27 +15,32 @@ class LLMProvider(LLMProviderBase): self.base_url = config.get("base_url") self.is_No_prompt = config.get("is_no_prompt") self.memory_id = config.get("ali_memory_id") + check_model_key("AliBLLLM", self.api_key) def response(self, session_id, dialogue): try: # 处理dialogue if self.is_No_prompt: dialogue.pop(0) - logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的dialogue: {dialogue}") + logger.bind(tag=TAG).debug( + f"【阿里百练API服务】处理后的dialogue: {dialogue}" + ) # 构造调用参数 call_params = { "api_key": self.api_key, "app_id": self.app_id, "session_id": session_id, - "messages": dialogue + "messages": dialogue, } if self.memory_id != False: # 百练memory需要prompt参数 prompt = dialogue[-1].get("content") call_params["memory_id"] = self.memory_id call_params["prompt"] = prompt - logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的prompt: {prompt}") + logger.bind(tag=TAG).debug( + f"【阿里百练API服务】处理后的prompt: {prompt}" + ) responses = Application.call(**call_params) if responses.status_code != HTTPStatus.OK: @@ -44,9 +51,16 @@ class LLMProvider(LLMProviderBase): ) yield "【阿里百练API服务响应异常】" else: - logger.bind(tag=TAG).debug(f"【阿里百练API服务】构造参数: {call_params}") + logger.bind(tag=TAG).debug( + f"【阿里百练API服务】构造参数: {call_params}" + ) yield responses.output.text except Exception as e: logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}") yield "【LLM服务响应异常】" + + def response_with_functions(self, session_id, dialogue, functions=None): + logger.bind(tag=TAG).error( + f"阿里百练暂未实现完整的工具调用（function call），建议使用其他意图识别" + ) diff --git a/main/xiaozhi-server/core/providers/llm/base.py b/main/xiaozhi-server/core/providers/llm/base.py index d39a8589..97d0d8e7 100644 --- a/main/xiaozhi-server/core/providers/llm/base.py +++ b/main/xiaozhi-server/core/providers/llm/base.py @@ -35,4 +35,5 @@ class LLMProviderBase(ABC): """ # For providers that don't support functions, just return regular response for token in self.response(session_id, dialogue): - yield {"type": "content", "content": token} \ No newline at end of file + yield token, None + diff --git a/main/xiaozhi-server/core/providers/llm/coze/coze.py b/main/xiaozhi-server/core/providers/llm/coze/coze.py index b1b14d91..60a04667 100644 --- a/main/xiaozhi-server/core/providers/llm/coze/coze.py +++ b/main/xiaozhi-server/core/providers/llm/coze/coze.py @@ -1,12 +1,17 @@ from config.logger import setup_logging -import requests import json -import re from core.providers.llm.base import LLMProviderBase -import os + # official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py) from cozepy import COZE_CN_BASE_URL -from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType # noqa +from cozepy import ( + Coze, + TokenAuth, + Message, + ChatEventType, +) # noqa +from core.providers.llm.system_prompt import get_system_prompt_for_function +from core.utils.util import check_model_key TAG = __name__ logger = setup_logging() @@ -15,25 +20,23 @@ logger = setup_logging() class LLMProvider(LLMProviderBase): def __init__(self, config): self.personal_access_token = config.get("personal_access_token") - self.bot_id = config.get("bot_id") - self.user_id = config.get("user_id") + self.bot_id = str(config.get("bot_id")) + self.user_id = str(config.get("user_id")) self.session_conversation_map = {} # 存储session_id和conversation_id的映射 + check_model_key("CozeLLM", self.personal_access_token) def response(self, session_id, dialogue): coze_api_token = self.personal_access_token coze_api_base = COZE_CN_BASE_URL last_msg = next(m for m in reversed(dialogue) if m["role"] == "user") - + coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base) conversation_id = self.session_conversation_map.get(session_id) # 如果没有找到conversation_id，则创建新的对话 if not conversation_id: - conversation = coze.conversations.create( - messages=[ - ] - ) + conversation = coze.conversations.create(messages=[]) conversation_id = conversation.id self.session_conversation_map[session_id] = conversation_id # 更新映射 @@ -47,4 +50,24 @@ class LLMProvider(LLMProviderBase): ): if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA: print(event.message.content, end="", flush=True) - yield event.message.content \ No newline at end of file + yield event.message.content + + def response_with_functions(self, session_id, dialogue, functions=None): + if len(dialogue) == 2 and functions is not None and len(functions) > 0: + # 第一次调用llm， 取最后一条用户消息，附加tool提示词 + last_msg = dialogue[-1]["content"] + function_str = json.dumps(functions, ensure_ascii=False) + modify_msg = get_system_prompt_for_function(function_str) + last_msg + dialogue[-1]["content"] = modify_msg + + # 如果最后一个是 role="tool"，附加到user上 + if len(dialogue) > 1 and dialogue[-1]["role"] == "tool": + assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n" + while len(dialogue) > 1: + if dialogue[-1]["role"] == "user": + dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"] + break + dialogue.pop() + + for token in self.response(session_id, dialogue): + yield token, None diff --git a/main/xiaozhi-server/core/providers/llm/dify/dify.py b/main/xiaozhi-server/core/providers/llm/dify/dify.py index 26bb48b7..52ca9853 100644 --- a/main/xiaozhi-server/core/providers/llm/dify/dify.py +++ b/main/xiaozhi-server/core/providers/llm/dify/dify.py @@ -2,6 +2,8 @@ import json from config.logger import setup_logging import requests from core.providers.llm.base import LLMProviderBase +from core.providers.llm.system_prompt import get_system_prompt_for_function +from core.utils.util import check_model_key TAG = __name__ logger = setup_logging() @@ -13,6 +15,7 @@ class LLMProvider(LLMProviderBase): self.mode = config.get("mode", "chat-messages") self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip("/") self.session_conversation_map = {} # 存储session_id和conversation_id的映射 + check_model_key("DifyLLM", self.api_key) def response(self, session_id, dialogue): try: @@ -58,7 +61,10 @@ class LLMProvider(LLMProviderBase): self.session_conversation_map[session_id] = ( conversation_id # 更新映射 ) - if event.get("answer"): + # 过滤 message_replace 事件，此事件会全量推一次 + if event.get("event") != "message_replace" and event.get( + "answer" + ): yield event["answer"] elif self.mode == "workflows/run": for line in r.iter_lines(): @@ -73,9 +79,32 @@ class LLMProvider(LLMProviderBase): for line in r.iter_lines(): if line.startswith(b"data: "): event = json.loads(line[6:]) - if event.get("answer"): + # 过滤 message_replace 事件，此事件会全量推一次 + if event.get("event") != "message_replace" and event.get( + "answer" + ): yield event["answer"] except Exception as e: logger.bind(tag=TAG).error(f"Error in response generation: {e}") yield "【服务响应异常】" + + def response_with_functions(self, session_id, dialogue, functions=None): + if len(dialogue) == 2 and functions is not None and len(functions) > 0: + # 第一次调用llm， 取最后一条用户消息，附加tool提示词 + last_msg = dialogue[-1]["content"] + function_str = json.dumps(functions, ensure_ascii=False) + modify_msg = get_system_prompt_for_function(function_str) + last_msg + dialogue[-1]["content"] = modify_msg + + # 如果最后一个是 role="tool"，附加到user上 + if len(dialogue) > 1 and dialogue[-1]["role"] == "tool": + assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n" + while len(dialogue) > 1: + if dialogue[-1]["role"] == "user": + dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"] + break + dialogue.pop() + + for token in self.response(session_id, dialogue): + yield token, None diff --git a/main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py b/main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py index 38f9a2c9..b2e9d6b7 100644 --- a/main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py +++ b/main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py @@ -2,6 +2,7 @@ import json from config.logger import setup_logging import requests from core.providers.llm.base import LLMProviderBase +from core.utils.util import check_model_key TAG = __name__ logger = setup_logging() @@ -13,6 +14,7 @@ class LLMProvider(LLMProviderBase): self.base_url = config.get("base_url") self.detail = config.get("detail", False) self.variables = config.get("variables", {}) + check_model_key("FastGPTLLM", self.api_key) def response(self, session_id, dialogue): try: @@ -21,37 +23,36 @@ class LLMProvider(LLMProviderBase): # 发起流式请求 with requests.post( - f"{self.base_url}/chat/completions", - headers={"Authorization": f"Bearer {self.api_key}"}, - json={ - "stream": True, - "chatId": session_id, - "detail": self.detail, - "variables": self.variables, - "messages": [ - { - "role": "user", - "content": last_msg["content"] - } - ] - }, - stream=True + f"{self.base_url}/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "stream": True, + "chatId": session_id, + "detail": self.detail, + "variables": self.variables, + "messages": [{"role": "user", "content": last_msg["content"]}], + }, + stream=True, ) as r: for line in r.iter_lines(): if line: try: - if line.startswith(b'data: '): - if line[6:].decode('utf-8') == '[DONE]': + if line.startswith(b"data: "): + if line[6:].decode("utf-8") == "[DONE]": break data = json.loads(line[6:]) - if 'choices' in data and len(data['choices']) > 0: - delta = data['choices'][0].get('delta', {}) - if delta and 'content' in delta and delta['content'] is not None: - content = delta['content'] - if '' in content: + if "choices" in data and len(data["choices"]) > 0: + delta = data["choices"][0].get("delta", {}) + if ( + delta + and "content" in delta + and delta["content"] is not None + ): + content = delta["content"] + if "" in content: continue - if '' in content: + if "" in content: continue yield content @@ -62,4 +63,9 @@ class LLMProvider(LLMProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"Error in response generation: {e}") - yield "【服务响应异常】" \ No newline at end of file + yield "【服务响应异常】" + + def response_with_functions(self, session_id, dialogue, functions=None): + logger.bind(tag=TAG).error( + f"fastgpt暂未实现完整的工具调用（function call），建议使用其他意图识别" + ) diff --git a/main/xiaozhi-server/core/providers/llm/gemini/gemini.py b/main/xiaozhi-server/core/providers/llm/gemini/gemini.py index 93c7dcb5..d1cf238e 100644 --- a/main/xiaozhi-server/core/providers/llm/gemini/gemini.py +++ b/main/xiaozhi-server/core/providers/llm/gemini/gemini.py @@ -1,136 +1,205 @@ -import google.generativeai as genai -from core.utils.util import check_model_key -from core.providers.llm.base import LLMProviderBase -from config.logger import setup_logging +import os, json, uuid +from types import SimpleNamespace +from typing import Any, Dict, List + import requests -import json +from google import generativeai as genai +from google.generativeai import types, GenerationConfig + +from core.providers.llm.base import LLMProviderBase +from core.utils.util import check_model_key +from config.logger import setup_logging +from google.generativeai.types import GenerateContentResponse +from requests import RequestException + +log = setup_logging() TAG = __name__ -logger = setup_logging() + + +def test_proxy(proxy_url: str, test_url: str) -> bool: + try: + resp = requests.get(test_url, proxies={"http": proxy_url, "https": proxy_url}) + return 200 <= resp.status_code < 400 + except RequestException: + return False + + +def setup_proxy_env(http_proxy: str | None, https_proxy: str | None): + """ + 分别测试 HTTP 和 HTTPS 代理是否可用，并设置环境变量。 + 如果 HTTPS 代理不可用但 HTTP 可用，会将 HTTPS_PROXY 也指向 HTTP。 + """ + test_http_url = "http://www.google.com" + test_https_url = "https://www.google.com" + + ok_http = ok_https = False + + if http_proxy: + ok_http = test_proxy(http_proxy, test_http_url) + if ok_http: + os.environ["HTTP_PROXY"] = http_proxy + log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {http_proxy}") + else: + log.bind(tag=TAG).warning(f"配置提供的Gemini HTTP代理不可用: {http_proxy}") + + if https_proxy: + ok_https = test_proxy(https_proxy, test_https_url) + if ok_https: + os.environ["HTTPS_PROXY"] = https_proxy + log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {https_proxy}") + else: + log.bind(tag=TAG).warning( + f"配置提供的Gemini HTTPS代理不可用: {https_proxy}" + ) + + # 如果https_proxy不可用，但http_proxy可用且能走通https，则复用http_proxy作为https_proxy + if ok_http and not ok_https: + if test_proxy(http_proxy, test_https_url): + os.environ["HTTPS_PROXY"] = http_proxy + ok_https = True + log.bind(tag=TAG).info(f"复用HTTP代理作为HTTPS代理: {http_proxy}") + + if not ok_http and not ok_https: + log.bind(tag=TAG).error( + f"Gemini 代理设置失败: HTTP 和 HTTPS 代理都不可用，请检查配置" + ) + raise RuntimeError("HTTP 和 HTTPS 代理都不可用，请检查配置") + class LLMProvider(LLMProviderBase): - def __init__(self, config): - """初始化Gemini LLM Provider""" - self.model_name = config.get("model_name", "gemini-1.5-pro") - self.api_key = config.get("api_key") - self.http_proxy=config.get("http_proxy") - self.https_proxy = config.get("https_proxy") - have_key = check_model_key("LLM", self.api_key) + def __init__(self, cfg: Dict[str, Any]): + self.model_name = cfg.get("model_name", "gemini-2.0-flash") + self.api_key = cfg["api_key"] + http_proxy = cfg.get("http_proxy") + https_proxy = cfg.get("https_proxy") - if not have_key: - return + if not check_model_key("LLM", self.api_key): + raise ValueError("无效的Gemini API Key，请检查是否配置正确") - try: - # 初始化Gemini客户端 - # 配置代理（如果提供了代理配置） - self.proxies=None - if self.http_proxy is not "" or self.https_proxy is not "": + if http_proxy or https_proxy: + log.bind(tag=TAG).info( + f"检测到Gemini代理配置，开始测试代理连通性和设置代理环境..." + ) + setup_proxy_env(http_proxy, https_proxy) + log.bind(tag=TAG).info( + f"Gemini 代理设置成功 - HTTP: {http_proxy}, HTTPS: {https_proxy}" + ) + genai.configure(api_key=self.api_key) + self.model = genai.GenerativeModel(self.model_name) - self.proxies = { - "http": self.http_proxy, - "https": self.https_proxy, - } - logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}") - # 使用猴子补丁修改 google-generativeai 库的请求会话 + self.gen_cfg = GenerationConfig( + temperature=0.7, + top_p=0.9, + top_k=40, + max_output_tokens=2048, + ) - # 使用 session 对象配置 genai - - genai.configure(api_key=self.api_key) - self.model = genai.GenerativeModel(self.model_name) - - # 设置生成参数 - self.generation_config = { - "temperature": 0.7, - "top_p": 0.9, - "top_k": 40, - "max_output_tokens": 2048, - } - self.chat = None - except Exception as e: - logger.bind(tag=TAG).error(f"Gemini初始化失败: {e}") - self.model = None + @staticmethod + def _build_tools(funcs: List[Dict[str, Any]] | None): + if not funcs: + return None + return [ + types.Tool( + function_declarations=[ + types.FunctionDeclaration( + name=f["function"]["name"], + description=f["function"]["description"], + parameters=f["function"]["parameters"], + ) + for f in funcs + ] + ) + ] + # Gemini文档提到，无需维护session-id，直接用dialogue拼接而成 def response(self, session_id, dialogue): - """生成Gemini对话响应""" - if not self.model: - yield "【Gemini服务未正确初始化】" - return + yield from self._generate(dialogue, None) + + def response_with_functions(self, session_id, dialogue, functions=None): + yield from self._generate(dialogue, self._build_tools(functions)) + + def _generate(self, dialogue, tools): + role_map = {"assistant": "model", "user": "user"} + contents: list = [] + # 拼接对话 + for m in dialogue: + r = m["role"] + + if r == "assistant" and "tool_calls" in m: + tc = m["tool_calls"][0] + contents.append( + { + "role": "model", + "parts": [ + { + "function_call": { + "name": tc["function"]["name"], + "args": json.loads(tc["function"]["arguments"]), + } + } + ], + } + ) + continue + + if r == "tool": + contents.append( + { + "role": "model", + "parts": [{"text": str(m.get("content", ""))}], + } + ) + continue + + contents.append( + { + "role": role_map.get(r, "user"), + "parts": [{"text": str(m.get("content", ""))}], + } + ) + + stream: GenerateContentResponse = self.model.generate_content( + contents=contents, + generation_config=self.gen_cfg, + tools=tools, + stream=True, + ) try: - # 处理对话历史 - chat_history = [] - for msg in dialogue[:-1]: # 历史对话 - role = "model" if msg["role"] == "assistant" else "user" - content = msg["content"].strip() - if content: - chat_history.append({ - "role": role, - "parts": [{"text":content}] + for chunk in stream: + cand = chunk.candidates[0] + for part in cand.content.parts: + # a) 函数调用-通常是最后一段话才是函数调用 + if getattr(part, "function_call", None): + fc = part.function_call + yield None, [ + SimpleNamespace( + id=uuid.uuid4().hex, + type="function", + function=SimpleNamespace( + name=fc.name, + arguments=json.dumps( + dict(fc.args), ensure_ascii=False + ), + ), + ) + ] + return + # b) 普通文本 + if getattr(part, "text", None): + yield part.text if tools is None else (part.text, None) - }) + finally: + if tools is not None: + yield None, None # function‑mode 结束，返回哑包 - # 获取当前消息 - current_msg = dialogue[-1]["content"] - - # 构建请求体 - request_body = { - "contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}], - "generationConfig": self.generation_config - } - - # 构建请求URL - url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}" - - # 构建请求头 - headers = { - "Content-Type": "application/json", - } - - # 发送POST请求,经测试手动 request 无法使用 stream 模式 - if self.proxies: - response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies) - try: - data = response.json() # 直接解析JSON - if 'candidates' in data and data['candidates']: - yield data['candidates'][0]['content']['parts'][0]['text'] - else: - yield "未找到候选回复。" - except json.JSONDecodeError as e: - yield f"JSON解码错误：{e}" - except Exception as e: - yield f"发生错误：{e}" - else: - logger.bind(tag=TAG).info(f"Gemini stream mode ") - chat = self.model.start_chat(history=chat_history) - - # 发送消息并获取流式响应 - response = chat.send_message( - current_msg, - stream=True, - generation_config=self.generation_config - ) - # 处理流式响应 - for chunk in response: - if hasattr(chunk, 'text') and chunk.text: - yield chunk.text - - except Exception as e: - error_msg = str(e) - logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}") - - # 针对不同错误返回友好提示 - if "Rate limit" in error_msg: - yield "【Gemini服务请求太频繁,请稍后再试】" - elif "Invalid API key" in error_msg: - yield "【Gemini API key无效】" - else: - yield f"【Gemini服务响应异常: {error_msg}】" - - - - - except requests.exceptions.RequestException as e: - yield f"请求失败：{e}" - except json.JSONDecodeError as e: - yield f"JSON解码错误：{e}" - except Exception as e: - yield f"发生错误：{e}" + # 关闭stream，预留后续打断对话功能的功能方法，官方文档推荐打断对话要关闭上一个流，可以有效减少配额计费和资源占用 + @staticmethod + def _safe_finish_stream(stream: GenerateContentResponse): + if hasattr(stream, "resolve"): + stream.resolve() # Gemini SDK version ≥ 0.5.0 + elif hasattr(stream, "close"): + stream.close() # Gemini SDK version < 0.5.0 + else: + for _ in stream: # 兜底耗尽 + pass diff --git a/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py b/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py new file mode 100644 index 00000000..c436fc9e --- /dev/null +++ b/main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py @@ -0,0 +1,71 @@ +import requests +from requests.exceptions import RequestException +from config.logger import setup_logging +from core.providers.llm.base import LLMProviderBase + +TAG = __name__ +logger = setup_logging() + + +class LLMProvider(LLMProviderBase): + def __init__(self, config): + self.agent_id = config.get("agent_id") # 对应 agent_id + self.api_key = config.get("api_key") + self.base_url = config.get("base_url", config.get("url")) # 默认使用 base_url + self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL + + def response(self, session_id, dialogue): + try: + # home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可 + + # 提取最后一个 role 为 'user' 的 content + input_text = None + if isinstance(dialogue, list): # 确保 dialogue 是一个列表 + # 逆序遍历，找到最后一个 role 为 'user' 的消息 + for message in reversed(dialogue): + if message.get("role") == "user": # 找到 role 为 'user' 的消息 + input_text = message.get("content", "") + break # 找到后立即退出循环 + + # 构造请求数据 + payload = { + "text": input_text, + "agent_id": self.agent_id, + "conversation_id": session_id, # 使用 session_id 作为 conversation_id + } + # 设置请求头 + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + + # 发起 POST 请求 + response = requests.post(self.api_url, json=payload, headers=headers) + + # 检查请求是否成功 + response.raise_for_status() + + # 解析返回数据 + data = response.json() + speech = ( + data.get("response", {}) + .get("speech", {}) + .get("plain", {}) + .get("speech", "") + ) + + # 返回生成的内容 + if speech: + yield speech + else: + logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容") + + except RequestException as e: + logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}") + except Exception as e: + logger.bind(tag=TAG).error(f"生成响应时出错: {e}") + + def response_with_functions(self, session_id, dialogue, functions=None): + logger.bind(tag=TAG).error( + f"homeassistant不支持（function call），建议使用其他意图识别" + ) diff --git a/main/xiaozhi-server/core/providers/llm/ollama/ollama.py b/main/xiaozhi-server/core/providers/llm/ollama/ollama.py index 179a85b1..aad6354f 100644 --- a/main/xiaozhi-server/core/providers/llm/ollama/ollama.py +++ b/main/xiaozhi-server/core/providers/llm/ollama/ollama.py @@ -21,27 +21,67 @@ class LLMProvider(LLMProviderBase): api_key="ollama" # Ollama doesn't need an API key but OpenAI client requires one ) + # 检查是否是qwen3模型 + self.is_qwen3 = self.model_name and self.model_name.lower().startswith("qwen3") + def response(self, session_id, dialogue): try: + # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令 + if self.is_qwen3: + # 复制对话列表，避免修改原始对话 + dialogue_copy = dialogue.copy() + + # 找到最后一条用户消息 + for i in range(len(dialogue_copy) - 1, -1, -1): + if dialogue_copy[i]["role"] == "user": + # 在用户消息前添加/no_think指令 + dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"] + logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令") + break + + # 使用修改后的对话 + dialogue = dialogue_copy + responses = self.client.chat.completions.create( model=self.model_name, messages=dialogue, stream=True ) - is_active=True + is_active = True + # 用于处理跨chunk的标签 + buffer = "" + for chunk in responses: try: delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None content = delta.content if hasattr(delta, 'content') else '' + if content: - if '' in content: + # 将内容添加到缓冲区 + buffer += content + + # 处理缓冲区中的标签 + while '' in buffer and '' in buffer: + # 找到完整的标签并移除 + pre = buffer.split('', 1)[0] + post = buffer.split('', 1)[1] + buffer = pre + post + + # 处理只有开始标签的情况 + if '' in buffer: is_active = False - content = content.split('')[0] - if '' in content: + buffer = buffer.split('', 1)[0] + + # 处理只有结束标签的情况 + if '' in buffer: is_active = True - content = content.split('')[-1] - if is_active: - yield content + buffer = buffer.split('', 1)[1] + + # 如果当前处于活动状态且缓冲区有内容，则输出 + if is_active and buffer: + yield buffer + buffer = "" # 清空缓冲区 + except Exception as e: logger.bind(tag=TAG).error(f"Error processing chunk: {e}") @@ -51,6 +91,22 @@ class LLMProvider(LLMProviderBase): def response_with_functions(self, session_id, dialogue, functions=None): try: + # 如果是qwen3模型，在用户最后一条消息中添加/no_think指令 + if self.is_qwen3: + # 复制对话列表，避免修改原始对话 + dialogue_copy = dialogue.copy() + + # 找到最后一条用户消息 + for i in range(len(dialogue_copy) - 1, -1, -1): + if dialogue_copy[i]["role"] == "user": + # 在用户消息前添加/no_think指令 + dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"] + logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令") + break + + # 使用修改后的对话 + dialogue = dialogue_copy + stream = self.client.chat.completions.create( model=self.model_name, messages=dialogue, @@ -58,9 +114,50 @@ class LLMProvider(LLMProviderBase): tools=functions, ) + is_active = True + buffer = "" + for chunk in stream: - yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls + try: + delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None + content = delta.content if hasattr(delta, 'content') else None + tool_calls = delta.tool_calls if hasattr(delta, 'tool_calls') else None + + # 如果是工具调用，直接传递 + if tool_calls: + yield None, tool_calls + continue + + # 处理文本内容 + if content: + # 将内容添加到缓冲区 + buffer += content + + # 处理缓冲区中的标签 + while '' in buffer and '' in buffer: + # 找到完整的标签并移除 + pre = buffer.split('', 1)[0] + post = buffer.split('', 1)[1] + buffer = pre + post + + # 处理只有开始标签的情况 + if '' in buffer: + is_active = False + buffer = buffer.split('', 1)[0] + + # 处理只有结束标签的情况 + if '' in buffer: + is_active = True + buffer = buffer.split('', 1)[1] + + # 如果当前处于活动状态且缓冲区有内容，则输出 + if is_active and buffer: + yield buffer, None + buffer = "" # 清空缓冲区 + except Exception as e: + logger.bind(tag=TAG).error(f"Error processing function chunk: {e}") + continue except Exception as e: logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}") - yield {"type": "content", "content": f"【Ollama服务响应异常: {str(e)}】"} + yield f"【Ollama服务响应异常: {str(e)}】", None diff --git a/main/xiaozhi-server/core/providers/llm/openai/openai.py b/main/xiaozhi-server/core/providers/llm/openai/openai.py index c3732a22..a20067af 100644 --- a/main/xiaozhi-server/core/providers/llm/openai/openai.py +++ b/main/xiaozhi-server/core/providers/llm/openai/openai.py @@ -1,4 +1,5 @@ import openai +from openai.types import CompletionUsage from config.logger import setup_logging from core.utils.util import check_model_key from core.providers.llm.base import LLMProviderBase @@ -11,11 +12,19 @@ class LLMProvider(LLMProviderBase): def __init__(self, config): self.model_name = config.get("model_name") self.api_key = config.get("api_key") - if 'base_url' in config: + if "base_url" in config: self.base_url = config.get("base_url") else: self.base_url = config.get("url") - self.max_tokens = config.get("max_tokens", 500) + max_tokens = config.get("max_tokens") + if max_tokens is None or max_tokens == "": + max_tokens = 500 + + try: + max_tokens = int(max_tokens) + except (ValueError, TypeError): + max_tokens = 500 + self.max_tokens = max_tokens check_model_key("LLM", self.api_key) self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url) @@ -33,18 +42,22 @@ class LLMProvider(LLMProviderBase): for chunk in responses: try: # 检查是否存在有效的choice且content不为空 - delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None - content = delta.content if hasattr(delta, 'content') else '' + delta = ( + chunk.choices[0].delta + if getattr(chunk, "choices", None) + else None + ) + content = delta.content if hasattr(delta, "content") else "" except IndexError: - content = '' + content = "" if content: # 处理标签跨多个chunk的情况 - if '' in content: + if "" in content: is_active = False - content = content.split('')[0] - if '' in content: + content = content.split("")[0] + if "" in content: is_active = True - content = content.split('')[-1] + content = content.split("")[-1] if is_active: yield content @@ -54,15 +67,22 @@ class LLMProvider(LLMProviderBase): def response_with_functions(self, session_id, dialogue, functions=None): try: stream = self.client.chat.completions.create( - model=self.model_name, - messages=dialogue, - stream=True, - tools=functions + model=self.model_name, messages=dialogue, stream=True, tools=functions ) for chunk in stream: - yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls + # 检查是否存在有效的choice且content不为空 + if getattr(chunk, "choices", None): + yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls + # 存在 CompletionUsage 消息时，生成 Token 消耗 log + elif isinstance(getattr(chunk, 'usage', None), CompletionUsage): + usage_info = getattr(chunk, 'usage', None) + logger.bind(tag=TAG).info( + f"Token 消耗：输入 {getattr(usage_info, 'prompt_tokens', '未知')}，" + f"输出 {getattr(usage_info, 'completion_tokens', '未知')}，" + f"共计 {getattr(usage_info, 'total_tokens', '未知')}" + ) except Exception as e: logger.bind(tag=TAG).error(f"Error in function call streaming: {e}") - yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"} + yield f"【OpenAI服务响应异常: {e}】", None diff --git a/main/xiaozhi-server/core/providers/llm/system_prompt.py b/main/xiaozhi-server/core/providers/llm/system_prompt.py new file mode 100644 index 00000000..606f0e4b --- /dev/null +++ b/main/xiaozhi-server/core/providers/llm/system_prompt.py @@ -0,0 +1,103 @@ +def get_system_prompt_for_function(functions: str) -> str: + """ + 生成系统提示信息 + :param functions: 可用的函数列表 + :return: 系统提示信息 + """ + + SYSTEM_PROMPT = f""" +==== + +TOOL USE + +You have access to a set of tools that are executed upon the user's approval. You can use one tool per message, and will receive the result of that tool use in the user's response. +You use tools step-by-step to accomplish a given task, with each tool use informed by the result of the previous tool use. + +# Tool Use Formatting + +Tool use is formatted using JSON-style tags. The tool name is enclosed in opening and closing tags, and each parameter is similarly enclosed within its own set of tags. +Here's the structure: + + +{{ + "name": "function name", + "arguments": {{ + "param1": "value1", + "param2": "value2", + // Add more parameters as needed, if parameters are required, you must provide them + }} +}} + + +For example: +if you got tool as follow + +{{ + "type": "function", + "function": {{ + "name": "handle_exit_intent", + "description": "当用户想结束对话或需要退出系统时调用", + "parameters": {{ + "type": "object", + "properties": {{ + "say_goodbye": {{ + "type": "string", + "description": "和用户友好结束对话的告别语", + }} + }}, + "required": ["say_goodbye"], + }}, + }}, +}} + +you should respond with the following format: + + +{{ + "name": "handle_exit_intent", + "arguments": {{ + "say_goodbye": "再见，祝您生活愉快！" + }} +}} + + + +Always adhere to this format for the tool use to ensure proper parsing and execution. + +# Tools + +{functions} + +# Tool Use Guidelines + +1. Tools must be called in a separate message, Do not add thoughts when calling tools. The message must start with and end with , with the tool invocation JSON data in between. No additional response content is needed. +2. Choose the most appropriate tool based on the task and the tool descriptions provided. Assess if you need additional information to proceed, and which of the available tools would be most effective for gathering this information. + For example using the list_files tool is more effective than running a command like \`ls\` in the terminal. It's critical that you think about each available tool and use the one that best fits the current step in the task. +3. If multiple actions are needed, use one tool at a time per message to accomplish the task iteratively, with each tool use being informed by the result of the previous tool use. Do not assume the outcome of any tool use. + Each step must be informed by the previous step's result. +4. Formulate your tool use using the JSON format specified for each tool. +5. After each tool use, the user will respond with the result of that tool use. This result will provide you with the necessary information to continue your task or make further decisions. This response may include: +- Information about whether the tool succeeded or failed, along with any reasons for failure. +- Linter errors that may have arisen due to the changes you made, which you'll need to address. +- New terminal output in reaction to the changes, which you may need to consider or act upon. +- Any other relevant feedback or information related to the tool use. +6. ALWAYS wait for user confirmation after each tool use before proceeding. Never assume the success of a tool use without explicit confirmation of the result from the user. +7. Tool calls should contain no extra information. Only after receiving the tool's response should you integrate it into a complete reply. + +It is crucial to proceed step-by-step, waiting for the user's message after each tool use before moving forward with the task. This approach allows you to: +1. Confirm the success of each step before proceeding. +2. Address any issues or errors that arise immediately. +3. Adapt your approach based on new information or unexpected results. +4. Ensure that each action builds correctly on the previous ones. + +By waiting for and carefully considering the user's response after each tool use, you can react accordingly and make informed decisions about how to proceed with the task. This iterative process helps ensure the overall success and accuracy of your work. + +==== + +USER CHAT CONTENT + +The following additional message is the user's chat message, and should be followed to the best of your ability without interfering with the TOOL USE guidelines. + +""" + + return SYSTEM_PROMPT \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/memory/base.py b/main/xiaozhi-server/core/providers/memory/base.py index 2f88a6a0..19dcabb8 100644 --- a/main/xiaozhi-server/core/providers/memory/base.py +++ b/main/xiaozhi-server/core/providers/memory/base.py @@ -4,6 +4,7 @@ from config.logger import setup_logging TAG = __name__ logger = setup_logging() + class MemoryProviderBase(ABC): def __init__(self, config): self.config = config @@ -20,6 +21,6 @@ class MemoryProviderBase(ABC): """Query memories for specific role based on similarity""" return "please implement query method" - def init_memory(self, role_id, llm): - self.role_id = role_id + def init_memory(self, role_id, llm, **kwargs): + self.role_id = role_id self.llm = llm diff --git a/main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py b/main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py index 9a82efe5..6ef3bc8c 100644 --- a/main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py +++ b/main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py @@ -6,13 +6,14 @@ from core.utils.util import check_model_key TAG = __name__ + class MemoryProvider(MemoryProviderBase): - def __init__(self, config): + def __init__(self, config, summary_memory=None): super().__init__(config) self.api_key = config.get("api_key", "") self.api_version = config.get("api_version", "v1.1") have_key = check_model_key("Mem0ai", self.api_key) - if not have_key : + if not have_key: self.use_mem0 = False return else: @@ -30,54 +31,55 @@ class MemoryProvider(MemoryProviderBase): return None if len(msgs) < 2: return None - + try: # Format the content as a message list for mem0 messages = [ {"role": message.role, "content": message.content} - for message in msgs if message.role != "system" + for message in msgs + if message.role != "system" ] - result = self.client.add(messages, user_id=self.role_id, output_format=self.api_version) + result = self.client.add( + messages, user_id=self.role_id, output_format=self.api_version + ) logger.bind(tag=TAG).debug(f"Save memory result: {result}") except Exception as e: logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}") return None - async def query_memory(self, query: str)-> str: + async def query_memory(self, query: str) -> str: if not self.use_mem0: return "" try: results = self.client.search( - query, - user_id=self.role_id, - output_format=self.api_version + query, user_id=self.role_id, output_format=self.api_version ) - if not results or 'results' not in results: + if not results or "results" not in results: return "" - + # Format each memory entry with its update time up to minutes memories = [] - for entry in results['results']: - timestamp = entry.get('updated_at', '') + for entry in results["results"]: + timestamp = entry.get("updated_at", "") if timestamp: try: # Parse and reformat the timestamp - dt = timestamp.split('.')[0] # Remove milliseconds - formatted_time = dt.replace('T', ' ') + dt = timestamp.split(".")[0] # Remove milliseconds + formatted_time = dt.replace("T", " ") except: formatted_time = timestamp - memory = entry.get('memory', '') + memory = entry.get("memory", "") if timestamp and memory: # Store tuple of (timestamp, formatted_string) for sorting memories.append((timestamp, f"[{formatted_time}] {memory}")) - + # Sort by timestamp in descending order (newest first) memories.sort(key=lambda x: x[0], reverse=True) - + # Extract only the formatted strings memories_str = "\n".join(f"- {memory[1]}" for memory in memories) logger.bind(tag=TAG).debug(f"Query results: {memories_str}") return memories_str except Exception as e: logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}") - return "" \ No newline at end of file + return "" diff --git a/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py b/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py index c6643861..a916e6c9 100644 --- a/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py +++ b/main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py @@ -3,7 +3,9 @@ import time import json import os import yaml -from core.utils.util import get_project_dir +from config.config_loader import get_project_dir +from config.manage_api_client import save_mem_local_short + short_term_memory_prompt = """ # 时空记忆编织者 @@ -71,11 +73,23 @@ short_term_memory_prompt = """ ``` """ +short_term_memory_prompt_only_content = """ +你是一个经验丰富的记忆总结者，擅长将对话内容进行总结摘要，遵循以下规则： +1、总结user的重要信息，以便在未来的对话中提供更个性化的服务 +2、不要重复总结，不要遗忘之前记忆，除非原来的记忆超过了1800字内，否则不要遗忘、不要压缩用户的历史记忆 +3、用户操控的设备音量、播放音乐、天气、退出、不想对话等和用户本身无关的内容，这些信息不需要加入到总结中 +4、不要把设备操控的成果结果和失败结果加入到总结中，也不要把用户的一些废话加入到总结中 +5、不要为了总结而总结，如果用户的聊天没有意义，请返回原来的历史记录也是可以的 +6、只需要返回总结摘要，严格控制在1800字内 +7、不要包含代码、xml，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容 +""" + + def extract_json_data(json_code): start = json_code.find("```json") # 从start开始找到下一个```结束 - end = json_code.find("```", start+1) - #print("start:", start, "end:", end) + end = json_code.find("```", start + 1) + # print("start:", start, "end:", end) if start == -1 or end == -1: try: jsonData = json.loads(json_code) @@ -83,74 +97,89 @@ def extract_json_data(json_code): except Exception as e: print("Error:", e) return "" - jsonData = json_code[start+7:end] + jsonData = json_code[start + 7 : end] return jsonData + TAG = __name__ + class MemoryProvider(MemoryProviderBase): - def __init__(self, config): + def __init__(self, config, summary_memory): super().__init__(config) self.short_momery = "" - self.memory_path = get_project_dir() + 'data/.memory.yaml' - self.load_memory() + self.save_to_file = True + self.memory_path = get_project_dir() + "data/.memory.yaml" + self.load_memory(summary_memory) + + def init_memory( + self, role_id, llm, summary_memory=None, save_to_file=True, **kwargs + ): + super().init_memory(role_id, llm, **kwargs) + self.save_to_file = save_to_file + self.load_memory(summary_memory) + + def load_memory(self, summary_memory): + # api获取到总结记忆后直接返回 + if summary_memory or not self.save_to_file: + self.short_momery = summary_memory + return - def init_memory(self, role_id, llm): - super().init_memory(role_id, llm) - self.load_memory() - - def load_memory(self): all_memory = {} if os.path.exists(self.memory_path): - with open(self.memory_path, 'r', encoding='utf-8') as f: + with open(self.memory_path, "r", encoding="utf-8") as f: all_memory = yaml.safe_load(f) or {} if self.role_id in all_memory: self.short_momery = all_memory[self.role_id] - + def save_memory_to_file(self): all_memory = {} if os.path.exists(self.memory_path): - with open(self.memory_path, 'r', encoding='utf-8') as f: - all_memory = yaml.safe_load(f) or {} + with open(self.memory_path, "r", encoding="utf-8") as f: + all_memory = yaml.safe_load(f) or {} all_memory[self.role_id] = self.short_momery - with open(self.memory_path, 'w', encoding='utf-8') as f: + with open(self.memory_path, "w", encoding="utf-8") as f: yaml.dump(all_memory, f, allow_unicode=True) - + async def save_memory(self, msgs): if self.llm is None: logger.bind(tag=TAG).error("LLM is not set for memory provider") return None - + if len(msgs) < 2: return None - + msgStr = "" for msg in msgs: if msg.role == "user": msgStr += f"User: {msg.content}\n" - elif msg.role== "assistant": + elif msg.role == "assistant": msgStr += f"Assistant: {msg.content}\n" - if len(self.short_momery) > 0: - msgStr+="历史记忆：\n" - msgStr+=self.short_momery - - #当前时间 + if self.short_momery and len(self.short_momery) > 0: + msgStr += "历史记忆：\n" + msgStr += self.short_momery + + # 当前时间 time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) msgStr += f"当前时间：{time_str}" - result = self.llm.response_no_stream(short_term_memory_prompt, msgStr) - - json_str = extract_json_data(result) - try: - json_data = json.loads(json_str) # 检查json格式是否正确 - self.short_momery = json_str - except Exception as e: - print("Error:", e) - - self.save_memory_to_file() + if self.save_to_file: + result = self.llm.response_no_stream(short_term_memory_prompt, msgStr) + json_str = extract_json_data(result) + try: + json.loads(json_str) # 检查json格式是否正确 + self.short_momery = json_str + self.save_memory_to_file() + except Exception as e: + print("Error:", e) + else: + result = self.llm.response_no_stream( + short_term_memory_prompt_only_content, msgStr + ) + save_mem_local_short(self.role_id, result) logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}") return self.short_momery - - async def query_memory(self, query: str)-> str: - return self.short_momery \ No newline at end of file + + async def query_memory(self, query: str) -> str: + return self.short_momery diff --git a/main/xiaozhi-server/core/providers/memory/nomem/nomem.py b/main/xiaozhi-server/core/providers/memory/nomem/nomem.py index c9349626..51523be4 100644 --- a/main/xiaozhi-server/core/providers/memory/nomem/nomem.py +++ b/main/xiaozhi-server/core/providers/memory/nomem/nomem.py @@ -1,18 +1,20 @@ -''' +""" 不使用记忆，可以选择此模块 -''' +""" + from ..base import MemoryProviderBase, logger TAG = __name__ + class MemoryProvider(MemoryProviderBase): - def __init__(self, config): + def __init__(self, config, summary_memory=None): super().__init__(config) - + async def save_memory(self, msgs): logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.") return None - async def query_memory(self, query: str)-> str: + async def query_memory(self, query: str) -> str: logger.bind(tag=TAG).debug("nomem mode: No memory query is performed.") - return "" \ No newline at end of file + return "" diff --git a/main/xiaozhi-server/core/providers/vad/base.py b/main/xiaozhi-server/core/providers/vad/base.py new file mode 100644 index 00000000..1d8d4c8d --- /dev/null +++ b/main/xiaozhi-server/core/providers/vad/base.py @@ -0,0 +1,9 @@ +from abc import ABC, abstractmethod +from typing import Optional + + +class VADProviderBase(ABC): + @abstractmethod + def is_vad(self, conn, data) -> bool: + """检测音频数据中的语音活动""" + pass diff --git a/main/xiaozhi-server/core/providers/vad/silero.py b/main/xiaozhi-server/core/providers/vad/silero.py new file mode 100644 index 00000000..8ec1f6eb --- /dev/null +++ b/main/xiaozhi-server/core/providers/vad/silero.py @@ -0,0 +1,71 @@ +import time +import numpy as np +import torch +import opuslib_next +from config.logger import setup_logging +from core.providers.vad.base import VADProviderBase + +TAG = __name__ +logger = setup_logging() + + +class VADProvider(VADProviderBase): + def __init__(self, config): + logger.bind(tag=TAG).info("SileroVAD", config) + self.model, self.utils = torch.hub.load( + repo_or_dir=config["model_dir"], + source="local", + model="silero_vad", + force_reload=False, + ) + (get_speech_timestamps, _, _, _, _) = self.utils + + self.decoder = opuslib_next.Decoder(16000, 1) + + # 处理空字符串的情况 + threshold = config.get("threshold", "0.5") + min_silence_duration_ms = config.get("min_silence_duration_ms", "1000") + + self.vad_threshold = float(threshold) if threshold else 0.5 + self.silence_threshold_ms = ( + int(min_silence_duration_ms) if min_silence_duration_ms else 1000 + ) + + def is_vad(self, conn, opus_packet): + try: + pcm_frame = self.decoder.decode(opus_packet, 960) + conn.client_audio_buffer.extend(pcm_frame) # 将新数据加入缓冲区 + + # 处理缓冲区中的完整帧（每次处理512采样点） + client_have_voice = False + while len(conn.client_audio_buffer) >= 512 * 2: + # 提取前512个采样点（1024字节） + chunk = conn.client_audio_buffer[: 512 * 2] + conn.client_audio_buffer = conn.client_audio_buffer[512 * 2 :] + + # 转换为模型需要的张量格式 + audio_int16 = np.frombuffer(chunk, dtype=np.int16) + audio_float32 = audio_int16.astype(np.float32) / 32768.0 + audio_tensor = torch.from_numpy(audio_float32) + + # 检测语音活动 + with torch.no_grad(): + speech_prob = self.model(audio_tensor, 16000).item() + client_have_voice = speech_prob >= self.vad_threshold + + # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话 + if conn.client_have_voice and not client_have_voice: + stop_duration = ( + time.time() * 1000 - conn.client_have_voice_last_time + ) + if stop_duration >= self.silence_threshold_ms: + conn.client_voice_stop = True + if client_have_voice: + conn.client_have_voice = True + conn.client_have_voice_last_time = time.time() * 1000 + + return client_have_voice + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).info(f"解码错误: {e}") + except Exception as e: + logger.bind(tag=TAG).error(f"Error processing audio packet: {e}") diff --git a/main/xiaozhi-server/core/utils/auth_code_gen.py b/main/xiaozhi-server/core/utils/auth_code_gen.py deleted file mode 100644 index c61024dd..00000000 --- a/main/xiaozhi-server/core/utils/auth_code_gen.py +++ /dev/null @@ -1,97 +0,0 @@ -import random -import threading -import time -from typing import Set - -class AuthCodeGenerator: - _instance = None - _instance_lock = threading.Lock() - - def __new__(cls): - if not cls._instance: - with cls._instance_lock: - if not cls._instance: - cls._instance = super(AuthCodeGenerator, cls).__new__(cls) - # 初始化随机种子 - random.seed(time.time()) - return cls._instance - - def __init__(self): - # 确保 __init__ 只被调用一次 - if not hasattr(self, '_initialized'): - self._used_codes: Set[str] = set() - self._code_timestamps = {} - self._lock = threading.Lock() - self._code_timeout = 3 * 24 * 60 * 60 - self._initialized = True - - @classmethod - def get_instance(cls): - """获取AuthCodeGenerator的单例实例""" - return cls() - - def generate_code(self) -> str: - """ - 生成6位数字认证码，确保不重复 - 返回: 6位数字字符串 - """ - with self._lock: - self._clean_expired_codes() # 清理过期code - while True: - # 使用时间戳和已用码数量作为种子，确保每次生成不同的随机数 - seed = int(time.time() * 1000) + len(self._used_codes) - random.seed(seed) - - # 生成6位随机数字 - code = ''.join(str(random.randint(0, 9)) for _ in range(6)) - - # 检查是否已存在 - if code not in self._used_codes: - self._used_codes.add(code) - self._code_timestamps[code] = time.time() - return code - - def remove_code(self, code: str) -> bool: - """ - 删除已使用的认证码 - 参数: - code: 要删除的认证码 - 返回: - bool: 删除成功返回True，码不存在返回False - """ - print('remove_code', code) - with self._lock: - if code in self._used_codes: - self._used_codes.remove(code) - if code in self._code_timestamps: - del self._code_timestamps[code] - return True - return False - - def is_code_used(self, code: str) -> bool: - """ - 检查认证码是否已被使用 - 参数: - code: 要检查的认证码 - 返回: - bool: 如果码存在返回True，否则返回False - """ - with self._lock: - return code in self._used_codes - - def clear_codes(self): - """清空所有已使用的认证码""" - with self._lock: - self._used_codes.clear() - self._code_timestamps.clear() - - def _clean_expired_codes(self): - """清理过期的认证码""" - current_time = time.time() - expired_codes = [ - code for code, timestamp in self._code_timestamps.items() - if (current_time - timestamp) > self._code_timeout - ] - for code in expired_codes: - self._used_codes.remove(code) - del self._code_timestamps[code] diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py index d4e66bb6..2ee30d4a 100644 --- a/main/xiaozhi-server/core/utils/dialogue.py +++ b/main/xiaozhi-server/core/utils/dialogue.py @@ -4,7 +4,14 @@ from datetime import datetime class Message: - def __init__(self, role: str, content: str = None, uniq_id: str = None, tool_calls = None, tool_call_id=None): + def __init__( + self, + role: str, + content: str = None, + uniq_id: str = None, + tool_calls=None, + tool_call_id=None, + ): self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4()) self.role = role self.content = content @@ -16,7 +23,7 @@ class Dialogue: def __init__(self): self.dialogue: List[Message] = [] # 获取当前时间 - self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + self.current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") def put(self, message: Message): self.dialogue.append(message) @@ -25,7 +32,15 @@ class Dialogue: if m.tool_calls is not None: dialogue.append({"role": m.role, "tool_calls": m.tool_calls}) elif m.role == "tool": - dialogue.append({"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content}) + dialogue.append( + { + "role": m.role, + "tool_call_id": ( + str(uuid.uuid4()) if m.tool_call_id is None else m.tool_call_id + ), + "content": m.content, + } + ) else: dialogue.append({"role": m.role, "content": m.content}) @@ -44,23 +59,24 @@ class Dialogue: else: self.put(Message(role="system", content=new_content)) - def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]: + def get_llm_dialogue_with_memory( + self, memory_str: str = None + ) -> List[Dict[str, str]]: if memory_str is None or len(memory_str) == 0: return self.get_llm_dialogue() - + # 构建带记忆的对话 dialogue = [] - + # 添加系统提示和记忆 system_message = next( (msg for msg in self.dialogue if msg.role == "system"), None ) - if system_message: enhanced_system_prompt = ( f"{system_message.content}\n\n" - f"相关记忆：\n{memory_str}" + f"以下是用户的历史记忆：\n```\n{memory_str}\n```" ) dialogue.append({"role": "system", "content": enhanced_system_prompt}) diff --git a/main/xiaozhi-server/core/utils/lock_manager.py b/main/xiaozhi-server/core/utils/lock_manager.py deleted file mode 100644 index f1db6fc7..00000000 --- a/main/xiaozhi-server/core/utils/lock_manager.py +++ /dev/null @@ -1,39 +0,0 @@ -import asyncio -from typing import Dict -from config.logger import setup_logging - -TAG = __name__ -logger = setup_logging() - -class FileLockManager: - _instance = None - _locks: Dict[str, asyncio.Lock] = {} - - def __new__(cls): - if cls._instance is None: - cls._instance = super(FileLockManager, cls).__new__(cls) - return cls._instance - - @classmethod - def get_lock(cls, file_path: str) -> asyncio.Lock: - """获取指定文件的锁""" - if file_path not in cls._locks: - cls._locks[file_path] = asyncio.Lock() - return cls._locks[file_path] - - @classmethod - async def acquire_lock(cls, file_path: str): - """获取锁""" - lock = cls.get_lock(file_path) - await lock.acquire() - logger.bind(tag=TAG).debug(f"Acquired lock for {file_path}") - - @classmethod - def release_lock(cls, file_path: str): - """释放锁""" - if file_path in cls._locks: - try: - cls._locks[file_path].release() - logger.bind(tag=TAG).debug(f"Released lock for {file_path}") - except RuntimeError as e: - logger.bind(tag=TAG).warning(f"Failed to release lock for {file_path}: {e}") diff --git a/main/xiaozhi-server/core/utils/memory.py b/main/xiaozhi-server/core/utils/memory.py index c750e9aa..ff43b220 100644 --- a/main/xiaozhi-server/core/utils/memory.py +++ b/main/xiaozhi-server/core/utils/memory.py @@ -2,16 +2,17 @@ import os import sys import importlib from config.logger import setup_logging -from core.utils.util import read_config, get_project_dir logger = setup_logging() + def create_instance(class_name, *args, **kwargs): - if os.path.exists(os.path.join('core', 'providers', 'memory', class_name, f'{class_name}.py')): - lib_name = f'core.providers.memory.{class_name}.{class_name}' + if os.path.exists( + os.path.join("core", "providers", "memory", class_name, f"{class_name}.py") + ): + lib_name = f"core.providers.memory.{class_name}.{class_name}" if lib_name not in sys.modules: - sys.modules[lib_name] = importlib.import_module(f'{lib_name}') + sys.modules[lib_name] = importlib.import_module(f"{lib_name}") return sys.modules[lib_name].MemoryProvider(*args, **kwargs) raise ValueError(f"不支持的记忆服务类型: {class_name}") - diff --git a/main/xiaozhi-server/core/utils/output_counter.py b/main/xiaozhi-server/core/utils/output_counter.py new file mode 100644 index 00000000..1e7ae8df --- /dev/null +++ b/main/xiaozhi-server/core/utils/output_counter.py @@ -0,0 +1,50 @@ +import datetime +from typing import Dict, Tuple + +# 全局字典，用于存储每个设备的每日输出字数 +_device_daily_output: Dict[Tuple[str, datetime.date], int] = {} +# 记录最后一次检查的日期 +_last_check_date: datetime.date = None + + +def reset_device_output(): + """ + 重置所有设备的每日输出字数 + 每天0点调用此函数 + """ + _device_daily_output.clear() + + +def get_device_output(device_id: str) -> int: + """ + 获取设备当日的输出字数 + """ + current_date = datetime.datetime.now().date() + return _device_daily_output.get((device_id, current_date), 0) + + +def add_device_output(device_id: str, char_count: int): + """ + 增加设备的输出字数 + """ + current_date = datetime.datetime.now().date() + global _last_check_date + + # 如果是第一次调用或者日期发生变化，清空计数器 + if _last_check_date is None or _last_check_date != current_date: + _device_daily_output.clear() + _last_check_date = current_date + + current_count = _device_daily_output.get((device_id, current_date), 0) + _device_daily_output[(device_id, current_date)] = current_count + char_count + + +def check_device_output_limit(device_id: str, max_output_size: int) -> bool: + """ + 检查设备是否超过输出限制 + :return: True 如果超过限制，False 如果未超过 + """ + if not device_id: + return False + current_output = get_device_output(device_id) + return current_output >= max_output_size diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 14d71fb2..0b7e46a1 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -1,19 +1,40 @@ -import os import json -import yaml import socket import subprocess -import logging import re +import os +import numpy as np import requests +import opuslib_next +from pydub import AudioSegment +from typing import Dict, Any +from core.utils import tts, llm, intent, memory, vad, asr +import copy - -def get_project_dir(): - """获取项目根目录""" - return ( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - + "/" - ) +TAG = __name__ +emoji_map = { + "neutral": "😶", + "happy": "🙂", + "laughing": "😆", + "funny": "😂", + "sad": "😔", + "angry": "😠", + "crying": "😭", + "loving": "😍", + "embarrassed": "😳", + "surprised": "😲", + "shocked": "😱", + "thinking": "🤔", + "winking": "😉", + "cool": "😎", + "relaxed": "😌", + "delicious": "🤤", + "kissy": "😘", + "confident": "😏", + "sleepy": "😴", + "silly": "😜", + "confused": "🙄", +} def get_local_ip(): @@ -72,7 +93,7 @@ def is_private_ip(ip_addr): return False # IP address format error or insufficient segments -def get_ip_info(ip_addr): +def get_ip_info(ip_addr, logger): try: if is_private_ip(ip_addr): ip_addr = "" @@ -81,16 +102,10 @@ def get_ip_info(ip_addr): ip_info = {"city": resp.get("city")} return ip_info except Exception as e: - logging.error(f"Error getting client ip info: {e}") + logger.bind(tag=TAG).error(f"Error getting client ip info: {e}") return {} -def read_config(config_path): - with open(config_path, "r", encoding="utf-8") as file: - config = yaml.safe_load(file) - return config - - def write_json_file(file_path, data): """将数据写入 JSON 文件""" with open(file_path, "w", encoding="utf-8") as file: @@ -103,13 +118,14 @@ def is_punctuation_or_emoji(char): punctuation_set = { "，", ",", # 中文逗号 + 英文逗号 - "。", - ".", # 中文句号 + 英文句号 - "！", - "!", # 中文感叹号 + 英文感叹号 "-", "－", # 英文连字符 + 中文全角横线 "、", # 中文顿号 + "“", + "”", + '"', # 中文双引号 + 英文引号 + "：", + ":", # 中文冒号 + 英文冒号 } if char.isspace() or char in punctuation_set: return True @@ -169,15 +185,30 @@ def remove_punctuation_and_length(text): def check_model_key(modelType, modelKey): if "你" in modelKey: - logging.error( - "你还没配置" - + modelType - + "的密钥，请在配置文件中配置密钥，否则无法正常工作" + raise ValueError( + "你还没配置" + modelType + "的密钥，请检查一下所使用的LLM是否配置了密钥" ) - return False return True +def parse_string_to_list(value, separator=";"): + """ + 将输入值转换为列表 + Args: + value: 输入值，可以是 None、字符串或列表 + separator: 分隔符，默认为分号 + Returns: + list: 处理后的列表 + """ + if value is None or value == "": + return [] + elif isinstance(value, str): + return [item.strip() for item in value.split(separator) if item.strip()] + elif isinstance(value, list): + return value + return [] + + def check_ffmpeg_installed(): ffmpeg_installed = False try: @@ -208,7 +239,755 @@ def check_ffmpeg_installed(): def extract_json_from_string(input_string): """提取字符串中的 JSON 部分""" pattern = r"(\{.*\})" - match = re.search(pattern, input_string) + match = re.search(pattern, input_string, re.DOTALL) # 添加 re.DOTALL if match: return match.group(1) # 返回提取的 JSON 字符串 return None + + +def initialize_modules( + logger, + config: Dict[str, Any], + init_vad=False, + init_asr=False, + init_llm=False, + init_tts=False, + init_memory=False, + init_intent=False, +) -> Dict[str, Any]: + """ + 初始化所有模块组件 + + Args: + config: 配置字典 + + Returns: + Dict[str, Any]: 包含所有初始化后的模块的字典 + """ + modules = {} + + # 初始化TTS模块 + if init_tts: + select_tts_module = config["selected_module"]["TTS"] + tts_type = ( + select_tts_module + if "type" not in config["TTS"][select_tts_module] + else config["TTS"][select_tts_module]["type"] + ) + modules["tts"] = tts.create_instance( + tts_type, + config["TTS"][select_tts_module], + str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"), + ) + logger.bind(tag=TAG).info(f"初始化组件: tts成功 {select_tts_module}") + + # 初始化LLM模块 + if init_llm: + select_llm_module = config["selected_module"]["LLM"] + llm_type = ( + select_llm_module + if "type" not in config["LLM"][select_llm_module] + else config["LLM"][select_llm_module]["type"] + ) + modules["llm"] = llm.create_instance( + llm_type, + config["LLM"][select_llm_module], + ) + logger.bind(tag=TAG).info(f"初始化组件: llm成功 {select_llm_module}") + + # 初始化Intent模块 + if init_intent: + select_intent_module = config["selected_module"]["Intent"] + intent_type = ( + select_intent_module + if "type" not in config["Intent"][select_intent_module] + else config["Intent"][select_intent_module]["type"] + ) + modules["intent"] = intent.create_instance( + intent_type, + config["Intent"][select_intent_module], + ) + logger.bind(tag=TAG).info(f"初始化组件: intent成功 {select_intent_module}") + + # 初始化Memory模块 + if init_memory: + select_memory_module = config["selected_module"]["Memory"] + memory_type = ( + select_memory_module + if "type" not in config["Memory"][select_memory_module] + else config["Memory"][select_memory_module]["type"] + ) + modules["memory"] = memory.create_instance( + memory_type, + config["Memory"][select_memory_module], + config.get("summaryMemory", None), + ) + logger.bind(tag=TAG).info(f"初始化组件: memory成功 {select_memory_module}") + + # 初始化VAD模块 + if init_vad: + select_vad_module = config["selected_module"]["VAD"] + vad_type = ( + select_vad_module + if "type" not in config["VAD"][select_vad_module] + else config["VAD"][select_vad_module]["type"] + ) + modules["vad"] = vad.create_instance( + vad_type, + config["VAD"][select_vad_module], + ) + logger.bind(tag=TAG).info(f"初始化组件: vad成功 {select_vad_module}") + + # 初始化ASR模块 + if init_asr: + select_asr_module = config["selected_module"]["ASR"] + asr_type = ( + select_asr_module + if "type" not in config["ASR"][select_asr_module] + else config["ASR"][select_asr_module]["type"] + ) + modules["asr"] = asr.create_instance( + asr_type, + config["ASR"][select_asr_module], + str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"), + ) + logger.bind(tag=TAG).info(f"初始化组件: asr成功 {select_asr_module}") + return modules + + +def analyze_emotion(text): + """ + 分析文本情感并返回对应的emoji名称（支持中英文） + """ + if not text or not isinstance(text, str): + return "neutral" + + original_text = text + text = text.lower().strip() + + # 检查是否包含现有emoji + for emotion, emoji in emoji_map.items(): + if emoji in original_text: + return emotion + + # 标点符号分析 + has_exclamation = "!" in original_text or "！" in original_text + has_question = "?" in original_text or "？" in original_text + has_ellipsis = "..." in original_text or "…" in original_text + + # 定义情感关键词映射（中英文扩展版） + emotion_keywords = { + "happy": [ + "开心", + "高兴", + "快乐", + "愉快", + "幸福", + "满意", + "棒", + "好", + "不错", + "完美", + "棒极了", + "太好了", + "好呀", + "好的", + "happy", + "joy", + "great", + "good", + "nice", + "awesome", + "fantastic", + "wonderful", + ], + "laughing": [ + "哈哈", + "哈哈哈", + "呵呵", + "嘿嘿", + "嘻嘻", + "笑死", + "太好笑了", + "笑死我了", + "lol", + "lmao", + "haha", + "hahaha", + "hehe", + "rofl", + "funny", + "laugh", + ], + "funny": [ + "搞笑", + "滑稽", + "逗", + "幽默", + "笑点", + "段子", + "笑话", + "太逗了", + "hilarious", + "joke", + "comedy", + ], + "sad": [ + "伤心", + "难过", + "悲哀", + "悲伤", + "忧郁", + "郁闷", + "沮丧", + "失望", + "想哭", + "难受", + "不开心", + "唉", + "呜呜", + "sad", + "upset", + "unhappy", + "depressed", + "sorrow", + "gloomy", + ], + "angry": [ + "生气", + "愤怒", + "气死", + "讨厌", + "烦人", + "可恶", + "烦死了", + "恼火", + "暴躁", + "火大", + "愤怒", + "气炸了", + "angry", + "mad", + "annoyed", + "furious", + "pissed", + "hate", + ], + "crying": [ + "哭泣", + "泪流", + "大哭", + "伤心欲绝", + "泪目", + "流泪", + "哭死", + "哭晕", + "想哭", + "泪崩", + "cry", + "crying", + "tears", + "sob", + "weep", + ], + "loving": [ + "爱你", + "喜欢", + "爱", + "亲爱的", + "宝贝", + "么么哒", + "抱抱", + "想你", + "思念", + "最爱", + "亲亲", + "喜欢你", + "love", + "like", + "adore", + "darling", + "sweetie", + "honey", + "miss you", + "heart", + ], + "embarrassed": [ + "尴尬", + "不好意思", + "害羞", + "脸红", + "难为情", + "社死", + "丢脸", + "出丑", + "embarrassed", + "awkward", + "shy", + "blush", + ], + "surprised": [ + "惊讶", + "吃惊", + "天啊", + "哇塞", + "哇", + "居然", + "竟然", + "没想到", + "出乎意料", + "surprise", + "wow", + "omg", + "oh my god", + "amazing", + "unbelievable", + ], + "shocked": [ + "震惊", + "吓到", + "惊呆了", + "不敢相信", + "震撼", + "吓死", + "恐怖", + "害怕", + "吓人", + "shocked", + "shocking", + "scared", + "frightened", + "terrified", + "horror", + ], + "thinking": [ + "思考", + "考虑", + "想一下", + "琢磨", + "沉思", + "冥想", + "想", + "思考中", + "在想", + "think", + "thinking", + "consider", + "ponder", + "meditate", + ], + "winking": [ + "调皮", + "眨眼", + "你懂的", + "坏笑", + "邪恶", + "奸笑", + "使眼色", + "wink", + "teasing", + "naughty", + "mischievous", + ], + "cool": [ + "酷", + "帅", + "厉害", + "棒极了", + "真棒", + "牛逼", + "强", + "优秀", + "杰出", + "出色", + "完美", + "cool", + "awesome", + "amazing", + "great", + "impressive", + "perfect", + ], + "relaxed": [ + "放松", + "舒服", + "惬意", + "悠闲", + "轻松", + "舒适", + "安逸", + "自在", + "relax", + "relaxed", + "comfortable", + "cozy", + "chill", + "peaceful", + ], + "delicious": [ + "好吃", + "美味", + "香", + "馋", + "可口", + "香甜", + "大餐", + "大快朵颐", + "流口水", + "垂涎", + "delicious", + "yummy", + "tasty", + "yum", + "appetizing", + "mouthwatering", + ], + "kissy": [ + "亲亲", + "么么", + "吻", + "mua", + "muah", + "亲一下", + "飞吻", + "kiss", + "xoxo", + "hug", + "muah", + "smooch", + ], + "confident": [ + "自信", + "肯定", + "确定", + "毫无疑问", + "当然", + "必须的", + "毫无疑问", + "确信", + "坚信", + "confident", + "sure", + "certain", + "definitely", + "positive", + ], + "sleepy": [ + "困", + "睡觉", + "晚安", + "想睡", + "好累", + "疲惫", + "疲倦", + "困了", + "想休息", + "睡意", + "sleep", + "sleepy", + "tired", + "exhausted", + "bedtime", + "good night", + ], + "silly": [ + "傻", + "笨", + "呆", + "憨", + "蠢", + "二", + "憨憨", + "傻乎乎", + "呆萌", + "silly", + "stupid", + "dumb", + "foolish", + "goofy", + "ridiculous", + ], + "confused": [ + "疑惑", + "不明白", + "不懂", + "困惑", + "疑问", + "为什么", + "怎么回事", + "啥意思", + "不清楚", + "confused", + "puzzled", + "doubt", + "question", + "what", + "why", + "how", + ], + } + + # 特殊句型判断（中英文） + # 赞美他人 + if any( + phrase in text + for phrase in [ + "你真", + "你好", + "您真", + "你真棒", + "你好厉害", + "你太强了", + "你真好", + "你真聪明", + "you are", + "you're", + "you look", + "you seem", + "so smart", + "so kind", + ] + ): + return "loving" + # 自我赞美 + if any( + phrase in text + for phrase in [ + "我真", + "我最", + "我太棒了", + "我厉害", + "我聪明", + "我优秀", + "i am", + "i'm", + "i feel", + "so good", + "so happy", + ] + ): + return "cool" + # 晚安/睡觉相关 + if any( + phrase in text + for phrase in [ + "睡觉", + "晚安", + "睡了", + "好梦", + "休息了", + "去睡了", + "sleep", + "good night", + "bedtime", + "go to bed", + ] + ): + return "sleepy" + # 疑问句 + if has_question and not has_exclamation: + return "thinking" + # 强烈情感（感叹号） + if has_exclamation and not has_question: + # 检查是否是积极内容 + positive_words = ( + emotion_keywords["happy"] + + emotion_keywords["laughing"] + + emotion_keywords["cool"] + ) + if any(word in text for word in positive_words): + return "laughing" + # 检查是否是消极内容 + negative_words = ( + emotion_keywords["angry"] + + emotion_keywords["sad"] + + emotion_keywords["crying"] + ) + if any(word in text for word in negative_words): + return "angry" + return "surprised" + # 省略号（表示犹豫或思考） + if has_ellipsis: + return "thinking" + + # 关键词匹配（带权重） + emotion_scores = {emotion: 0 for emotion in emoji_map.keys()} + + # 给匹配到的关键词加分 + for emotion, keywords in emotion_keywords.items(): + for keyword in keywords: + if keyword in text: + emotion_scores[emotion] += 1 + + # 给长文本中的重复关键词额外加分 + if len(text) > 20: # 长文本 + for emotion, keywords in emotion_keywords.items(): + for keyword in keywords: + emotion_scores[emotion] += text.count(keyword) * 0.5 + + # 根据分数选择最可能的情感 + max_score = max(emotion_scores.values()) + if max_score == 0: + return "happy" # 默认 + + # 可能有多个情感同分，根据上下文选择最合适的 + top_emotions = [e for e, s in emotion_scores.items() if s == max_score] + + # 如果多个情感同分，使用以下优先级 + priority_order = [ + "laughing", + "crying", + "angry", + "surprised", + "shocked", # 强烈情感优先 + "loving", + "happy", + "funny", + "cool", # 积极情感 + "sad", + "embarrassed", + "confused", # 消极情感 + "thinking", + "winking", + "relaxed", # 中性情感 + "delicious", + "kissy", + "confident", + "sleepy", + "silly", # 特殊场景 + ] + + for emotion in priority_order: + if emotion in top_emotions: + return emotion + + return top_emotions[0] # 如果都不在优先级列表里，返回第一个 + + +def audio_to_data(audio_file_path, is_opus=True): + # 获取文件后缀名 + file_type = os.path.splitext(audio_file_path)[1] + if file_type: + file_type = file_type.lstrip(".") + # 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞 + audio = AudioSegment.from_file( + audio_file_path, format=file_type, parameters=["-nostdin"] + ) + + # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配） + audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) + + # 音频时长(秒) + duration = len(audio) / 1000.0 + + # 获取原始PCM数据（16位小端） + raw_data = audio.raw_data + + # 初始化Opus编码器 + encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO) + + # 编码参数 + frame_duration = 60 # 60ms per frame + frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame + + datas = [] + # 按帧处理所有音频数据（包括最后一帧可能补零） + for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample + # 获取当前帧的二进制数据 + chunk = raw_data[i : i + frame_size * 2] + + # 如果最后一帧不足，补零 + if len(chunk) < frame_size * 2: + chunk += b"\x00" * (frame_size * 2 - len(chunk)) + + if is_opus: + # 转换为numpy数组处理 + np_frame = np.frombuffer(chunk, dtype=np.int16) + # 编码Opus数据 + frame_data = encoder.encode(np_frame.tobytes(), frame_size) + else: + frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk) + + datas.append(frame_data) + + return datas, duration + + +def check_vad_update(before_config, new_config): + if ( + new_config.get("selected_module") is None + or new_config["selected_module"].get("VAD") is None + ): + return False + update_vad = False + current_vad_module = before_config["selected_module"]["VAD"] + new_vad_module = new_config["selected_module"]["VAD"] + current_vad_type = ( + current_vad_module + if "type" not in before_config["VAD"][current_vad_module] + else before_config["VAD"][current_vad_module]["type"] + ) + new_vad_type = ( + new_vad_module + if "type" not in new_config["VAD"][new_vad_module] + else new_config["VAD"][new_vad_module]["type"] + ) + update_vad = current_vad_type != new_vad_type + return update_vad + + +def check_asr_update(before_config, new_config): + if ( + new_config.get("selected_module") is None + or new_config["selected_module"].get("ASR") is None + ): + return False + update_asr = False + current_asr_module = before_config["selected_module"]["ASR"] + new_asr_module = new_config["selected_module"]["ASR"] + current_asr_type = ( + current_asr_module + if "type" not in before_config["ASR"][current_asr_module] + else before_config["ASR"][current_asr_module]["type"] + ) + new_asr_type = ( + new_asr_module + if "type" not in new_config["ASR"][new_asr_module] + else new_config["ASR"][new_asr_module]["type"] + ) + update_asr = current_asr_type != new_asr_type + return update_asr + + +def filter_sensitive_info(config: dict) -> dict: + """ + 过滤配置中的敏感信息 + Args: + config: 原始配置字典 + Returns: + 过滤后的配置字典 + """ + sensitive_keys = [ + "api_key", + "personal_access_token", + "access_token", + "token", + "secret", + "access_key_secret", + "secret_key", + ] + + def _filter_dict(d: dict) -> dict: + filtered = {} + for k, v in d.items(): + if any(sensitive in k.lower() for sensitive in sensitive_keys): + filtered[k] = "***" + elif isinstance(v, dict): + filtered[k] = _filter_dict(v) + elif isinstance(v, list): + filtered[k] = [_filter_dict(i) if isinstance(i, dict) else i for i in v] + else: + filtered[k] = v + return filtered + + return _filter_dict(copy.deepcopy(config)) diff --git a/main/xiaozhi-server/core/utils/vad.py b/main/xiaozhi-server/core/utils/vad.py index 2079b354..b1dd1faf 100644 --- a/main/xiaozhi-server/core/utils/vad.py +++ b/main/xiaozhi-server/core/utils/vad.py @@ -1,77 +1,19 @@ -from abc import ABC, abstractmethod +import importlib +import os +import sys +from core.providers.vad.base import VADProviderBase from config.logger import setup_logging -import opuslib_next -import time -import numpy as np -import torch TAG = __name__ logger = setup_logging() -class VAD(ABC): - @abstractmethod - def is_vad(self, conn, data): - """检测音频数据中的语音活动""" - pass +def create_instance(class_name: str, *args, **kwargs) -> VADProviderBase: + """工厂方法创建VAD实例""" + if os.path.exists(os.path.join("core", "providers", "vad", f"{class_name}.py")): + lib_name = f"core.providers.vad.{class_name}" + if lib_name not in sys.modules: + sys.modules[lib_name] = importlib.import_module(f"{lib_name}") + return sys.modules[lib_name].VADProvider(*args, **kwargs) -class SileroVAD(VAD): - def __init__(self, config): - logger.bind(tag=TAG).info("SileroVAD", config) - self.model, self.utils = torch.hub.load(repo_or_dir=config["model_dir"], - source='local', - model='silero_vad', - force_reload=False) - (get_speech_timestamps, _, _, _, _) = self.utils - - self.decoder = opuslib_next.Decoder(16000, 1) - self.vad_threshold = config.get("threshold") - self.silence_threshold_ms = config.get("min_silence_duration_ms") - - def is_vad(self, conn, opus_packet): - try: - pcm_frame = self.decoder.decode(opus_packet, 960) - conn.client_audio_buffer.extend(pcm_frame) # 将新数据加入缓冲区 - - # 处理缓冲区中的完整帧（每次处理512采样点） - client_have_voice = False - while len(conn.client_audio_buffer) >= 512 * 2: - # 提取前512个采样点（1024字节） - chunk = conn.client_audio_buffer[:512 * 2] - conn.client_audio_buffer = conn.client_audio_buffer[512 * 2:] - - # 转换为模型需要的张量格式 - audio_int16 = np.frombuffer(chunk, dtype=np.int16) - audio_float32 = audio_int16.astype(np.float32) / 32768.0 - audio_tensor = torch.from_numpy(audio_float32) - - # 检测语音活动 - speech_prob = self.model(audio_tensor, 16000).item() - client_have_voice = speech_prob >= self.vad_threshold - - # 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话 - if conn.client_have_voice and not client_have_voice: - stop_duration = time.time() * 1000 - conn.client_have_voice_last_time - if stop_duration >= self.silence_threshold_ms: - conn.client_voice_stop = True - if client_have_voice: - conn.client_have_voice = True - conn.client_have_voice_last_time = time.time() * 1000 - - return client_have_voice - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).info(f"解码错误: {e}") - except Exception as e: - logger.bind(tag=TAG).error(f"Error processing audio packet: {e}") - - -def create_instance(class_name, *args, **kwargs) -> VAD: - # 获取类对象 - cls_map = { - "SileroVAD": SileroVAD, - # 可扩展其他SileroVAD实现 - } - - if cls := cls_map.get(class_name): - return cls(*args, **kwargs) - raise ValueError(f"不支持的SileroVAD类型: {class_name}") + raise ValueError(f"不支持的VAD类型: {class_name}，请检查该配置的type是否设置正确") diff --git a/main/xiaozhi-server/core/websocket_server.py b/main/xiaozhi-server/core/websocket_server.py index 21c022be..3f9e2fd7 100644 --- a/main/xiaozhi-server/core/websocket_server.py +++ b/main/xiaozhi-server/core/websocket_server.py @@ -2,8 +2,8 @@ import asyncio import websockets from config.logger import setup_logging from core.connection import ConnectionHandler -from core.utils.util import get_local_ip -from core.utils import asr, vad, llm, tts, memory, intent +from core.utils.util import initialize_modules, check_vad_update, check_asr_update +from config.config_loader import get_config_from_api TAG = __name__ @@ -12,109 +12,112 @@ class WebSocketServer: def __init__(self, config: dict): self.config = config self.logger = setup_logging() - self._vad, self._asr, self._llm, self._memory, self.intent = ( - self._create_processing_instances() - ) - self.active_connections = set() # 添加全局连接记录 - - def _create_processing_instances(self): - memory_cls_name = self.config["selected_module"].get( - "Memory", "nomem" - ) # 默认使用nomem - has_memory_cfg = ( - self.config.get("Memory") and memory_cls_name in self.config["Memory"] - ) - memory_cfg = self.config["Memory"][memory_cls_name] if has_memory_cfg else {} - - """创建处理模块实例""" - return ( - vad.create_instance( - self.config["selected_module"]["VAD"], - self.config["VAD"][self.config["selected_module"]["VAD"]], - ), - asr.create_instance( - ( - self.config["selected_module"]["ASR"] - if not "type" - in self.config["ASR"][self.config["selected_module"]["ASR"]] - else self.config["ASR"][self.config["selected_module"]["ASR"]][ - "type" - ] - ), - self.config["ASR"][self.config["selected_module"]["ASR"]], - self.config["delete_audio"], - ), - llm.create_instance( - ( - self.config["selected_module"]["LLM"] - if not "type" - in self.config["LLM"][self.config["selected_module"]["LLM"]] - else self.config["LLM"][self.config["selected_module"]["LLM"]][ - "type" - ] - ), - self.config["LLM"][self.config["selected_module"]["LLM"]], - ), - memory.create_instance(memory_cls_name, memory_cfg), - intent.create_instance( - ( - self.config["selected_module"]["Intent"] - if not "type" - in self.config["Intent"][self.config["selected_module"]["Intent"]] - else self.config["Intent"][ - self.config["selected_module"]["Intent"] - ]["type"] - ), - self.config["Intent"][self.config["selected_module"]["Intent"]], - ), + self.config_lock = asyncio.Lock() + modules = initialize_modules( + self.logger, + self.config, + "VAD" in self.config["selected_module"], + "ASR" in self.config["selected_module"], + "LLM" in self.config["selected_module"], + "TTS" in self.config["selected_module"], + "Memory" in self.config["selected_module"], + "Intent" in self.config["selected_module"], ) + self._vad = modules["vad"] if "vad" in modules else None + self._asr = modules["asr"] if "asr" in modules else None + self._tts = modules["tts"] if "tts" in modules else None + self._llm = modules["llm"] if "llm" in modules else None + self._intent = modules["intent"] if "intent" in modules else None + self._memory = modules["memory"] if "memory" in modules else None + self.active_connections = set() async def start(self): server_config = self.config["server"] - host = server_config["ip"] - port = server_config["port"] + host = server_config.get("ip", "0.0.0.0") + port = int(server_config.get("port", 8000)) - self.logger.bind(tag=TAG).info( - "Server is running at ws://{}:{}/xiaozhi/v1/", get_local_ip(), port - ) - self.logger.bind(tag=TAG).info( - "=======上面的地址是websocket协议地址，请勿用浏览器访问=======" - ) - self.logger.bind(tag=TAG).info( - "如想测试websocket请用谷歌浏览器打开test目录下的test_page.html" - ) - self.logger.bind(tag=TAG).info( - "=============================================================\n" - ) - async with websockets.serve(self._handle_connection, host, port): + async with websockets.serve( + self._handle_connection, host, port, process_request=self._http_response + ): await asyncio.Future() async def _handle_connection(self, websocket): """处理新连接，每次创建独立的ConnectionHandler""" # 创建ConnectionHandler时传入当前server实例 - _tts = tts.create_instance( - ( - self.config["selected_module"]["TTS"] - if not "type" - in self.config["TTS"][self.config["selected_module"]["TTS"]] - else self.config["TTS"][self.config["selected_module"]["TTS"]][ - "type" - ] - ), - self.config["TTS"][self.config["selected_module"]["TTS"]], - self.config["delete_audio"], - ) handler = ConnectionHandler( self.config, self._vad, self._asr, self._llm, - _tts, + self._tts, self._memory, - self.intent, + self._intent, + self, # 传入server实例 ) self.active_connections.add(handler) try: await handler.handle_connection(websocket) finally: self.active_connections.discard(handler) + + async def _http_response(self, websocket, request_headers): + # 检查是否为 WebSocket 升级请求 + if request_headers.headers.get("connection", "").lower() == "upgrade": + # 如果是 WebSocket 请求，返回 None 允许握手继续 + return None + else: + # 如果是普通 HTTP 请求，返回 "server is running" + return websocket.respond(200, "Server is running\n") + + async def update_config(self) -> bool: + """更新服务器配置并重新初始化组件 + + Returns: + bool: 更新是否成功 + """ + try: + async with self.config_lock: + # 重新获取配置 + new_config = get_config_from_api(self.config) + if new_config is None: + self.logger.bind(tag=TAG).error("获取新配置失败") + return False + self.logger.bind(tag=TAG).info(f"获取新配置成功") + # 检查 VAD 和 ASR 类型是否需要更新 + update_vad = check_vad_update(self.config, new_config) + update_asr = check_asr_update(self.config, new_config) + self.logger.bind(tag=TAG).info( + f"检查VAD和ASR类型是否需要更新: {update_vad} {update_asr}" + ) + # 更新配置 + self.config = new_config + # 重新初始化组件 + modules = initialize_modules( + self.logger, + new_config, + update_vad, + update_asr, + "LLM" in new_config["selected_module"], + "TTS" in new_config["selected_module"], + "Memory" in new_config["selected_module"], + "Intent" in new_config["selected_module"], + ) + + # 更新组件实例 + if "vad" in modules: + self._vad = modules["vad"] + if "asr" in modules: + self._asr = modules["asr"] + if "tts" in modules: + self._tts = modules["tts"] + if "llm" in modules: + self._llm = modules["llm"] + if "intent" in modules: + self._intent = modules["intent"] + if "memory" in modules: + self._memory = modules["memory"] + self.logger.bind(tag=TAG).info(f"更新配置任务执行完毕") + return True + except Exception as e: + self.logger.bind(tag=TAG).error(f"更新服务器配置失败: {str(e)}") + return False diff --git a/main/xiaozhi-server/docker-compose.yml b/main/xiaozhi-server/docker-compose.yml index 609fcbc1..f66f9eea 100644 --- a/main/xiaozhi-server/docker-compose.yml +++ b/main/xiaozhi-server/docker-compose.yml @@ -1,16 +1,4 @@ -# 1、安装mysql -# |- 如果本机已经安装了MySQL，可以直接在数据库中创建名为`xiaozhi_esp32_server`的数据库。 -# |- 如果还没有MySQL，你可以通过docker安装mysql,执行以下一句话 -# |- docker run --name xiaozhi-esp32-server-db -e MYSQL_ROOT_PASSWORD=123456 -p 3306:3306 -e MYSQL_DATABASE=xiaozhi_esp32_server -e MYSQL_INITDB_ARGS="--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci" -d mysql:latest -# |- 如果你的mysql账号和密码有修改过，记得修改下方数据库的账号和密码 -# |- 记得修改下方SPRING_DATASOURCE_DRUID_URL的IP，ip不能写127.0.0.1或localhost，否则容器无法访问，要写你电脑局域网ip - -# 2、安装redis -# |- 如果本机已经安装了Redis，看一下你安装的redis端口、密码，然后修改下方redis的地址和端口 -# |- 如果还没有Redis，你可以通过docker安装redis,执行以下一句话 -# |- docker run --name xiaozhi-esp32-server-redis -d -p 6379:6379 redis -# |- 记得修改SPRING_DATA_REDIS_HOST的IP，ip不能写127.0.0.1或localhost，否则容器无法访问，要写你电脑局域网ip - +# Docker安装Server version: '3' services: @@ -25,24 +13,10 @@ services: ports: # ws服务端 - "8000:8000" + # ota服务端 + - "8002:8002" volumes: # 配置文件目录 - ./data:/opt/xiaozhi-esp32-server/data # 模型文件挂接，很重要 - - ./models/SenseVoiceSmall/model.pt:/opt/xiaozhi-esp32-server/models/SenseVoiceSmall/model.pt - -# #智控台还没开发好，还不能完全使用，会报很多错误，如果是非技术人员，请不要启用智控台服务 -# xiaozhi-esp32-server-web: -# image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:web_latest -# container_name: xiaozhi-esp32-server-web -# restart: always -# ports: -# - "8002:8002" -# environment: -# - TZ=Asia/Shanghai -# ##记得改mysql和redis IP 密码 -# - SPRING_DATASOURCE_DRUID_URL=jdbc:mysql://192.168.1.20:3306/xiaozhi_esp32_server?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai -# - SPRING_DATASOURCE_DRUID_USERNAME=root -# - SPRING_DATASOURCE_DRUID_PASSWORD=123456 -# - SPRING_DATA_REDIS_HOST=192.168.1.20 -# - SPRING_DATA_REDIS_PORT=6379 + - ./models/SenseVoiceSmall/model.pt:/opt/xiaozhi-esp32-server/models/SenseVoiceSmall/model.pt \ No newline at end of file diff --git a/main/xiaozhi-server/docker-compose_all.yml b/main/xiaozhi-server/docker-compose_all.yml new file mode 100644 index 00000000..9339329e --- /dev/null +++ b/main/xiaozhi-server/docker-compose_all.yml @@ -0,0 +1,88 @@ +# Docker安装全模块 + +version: '3' +services: + # Server模块 + xiaozhi-esp32-server: + image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:server_latest + container_name: xiaozhi-esp32-server + depends_on: + - xiaozhi-esp32-server-db + - xiaozhi-esp32-server-redis + restart: always + networks: + - default + ports: + # ws服务端 + - "8000:8000" + security_opt: + - seccomp:unconfined + environment: + - TZ=Asia/Shanghai + volumes: + # 配置文件目录 + - ./data:/opt/xiaozhi-esp32-server/data + # 模型文件挂接，很重要 + - ./models/SenseVoiceSmall/model.pt:/opt/xiaozhi-esp32-server/models/SenseVoiceSmall/model.pt + + # manager-api和manager-web模块 + xiaozhi-esp32-server-web: + image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:web_latest + container_name: xiaozhi-esp32-server-web + restart: always + networks: + - default + depends_on: + xiaozhi-esp32-server-db: + condition: service_healthy + xiaozhi-esp32-server-redis: + condition: service_healthy + ports: + # 智控台 + - "8002:8002" + environment: + - TZ=Asia/Shanghai + - SPRING_DATASOURCE_DRUID_URL=jdbc:mysql://xiaozhi-esp32-server-db:3306/xiaozhi_esp32_server?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true&connectTimeout=30000&socketTimeout=30000&autoReconnect=true&failOverReadOnly=false&maxReconnects=10 + - SPRING_DATASOURCE_DRUID_USERNAME=root + - SPRING_DATASOURCE_DRUID_PASSWORD=123456 + - SPRING_DATA_REDIS_HOST=xiaozhi-esp32-server-redis + - SPRING_DATA_REDIS_PORT=6379 + volumes: + # 配置文件目录 + - ./uploadfile:/uploadfile + + xiaozhi-esp32-server-db: + image: mysql:latest + container_name: xiaozhi-esp32-server-db + healthcheck: + test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ] + timeout: 45s + interval: 10s + retries: 10 + restart: always + networks: + - default + expose: + - 3306 + volumes: + - ./mysql/data:/var/lib/mysql + environment: + - TZ=Asia/Shanghai + - MYSQL_ROOT_PASSWORD=123456 + - MYSQL_DATABASE=xiaozhi_esp32_server + - MYSQL_INITDB_ARGS="--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci" + xiaozhi-esp32-server-redis: + image: redis + expose: + - 6379 + container_name: xiaozhi-esp32-server-redis + restart: always + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 3 + networks: + - default +networks: + default: diff --git a/main/xiaozhi-server/mcp_server_settings.json b/main/xiaozhi-server/mcp_server_settings.json index a38bbdfb..a1bddf06 100644 --- a/main/xiaozhi-server/mcp_server_settings.json +++ b/main/xiaozhi-server/mcp_server_settings.json @@ -3,9 +3,19 @@ "在data目录下创建.mcp_server_settings.json文件，可以选择下面的MCP服务，也可以自行添加新的MCP服务。", "后面不断测试补充好用的mcp服务，欢迎大家一起补充。", "记得删除注释行,des属性仅为说明,不会被解析。", - "des和link属性，仅为说明安装方式，方便大家查看原始链接，不是必须项。" + "des和link属性，仅为说明安装方式，方便大家查看原始链接，不是必须项。", + "当前支持stdio/sse两种模式。" ], "mcpServers": { + "Home Assistant": { + "command": "mcp-proxy", + "args": [ + "http://YOUR_HA_HOST/mcp_server/sse" + ], + "env": { + "API_ACCESS_TOKEN": "YOUR_API_ACCESS_TOKEN" + } + }, "filesystem": { "command": "npx", "args": [ diff --git a/main/xiaozhi-server/performance_tester.py b/main/xiaozhi-server/performance_tester.py index 8bae2df0..80503166 100644 --- a/main/xiaozhi-server/performance_tester.py +++ b/main/xiaozhi-server/performance_tester.py @@ -1,16 +1,17 @@ -import time -import aiohttp import asyncio +import logging +import os +import statistics +import time +from typing import Dict + +import aiohttp from tabulate import tabulate -from typing import Dict, List + +from config.settings import load_config +from core.utils.asr import create_instance as create_stt_instance from core.utils.llm import create_instance as create_llm_instance from core.utils.tts import create_instance as create_tts_instance -from core.utils.util import read_config -import statistics -from config.settings import get_config_file -import inspect -import os -import logging # 设置全局日志级别为WARNING，抑制INFO级别日志 logging.basicConfig(level=logging.WARNING) @@ -18,17 +19,26 @@ logging.basicConfig(level=logging.WARNING) class AsyncPerformanceTester: def __init__(self): - self.config = read_config(get_config_file()) + self.config = load_config() self.test_sentences = self.config.get("module_test", {}).get( "test_sentences", - ["你好，请介绍一下你自己", "What's the weather like today?", - "请用100字概括量子计算的基本原理和应用前景"] + [ + "你好，请介绍一下你自己", + "What's the weather like today?", + "请用100字概括量子计算的基本原理和应用前景", + ], ) - self.results = { - "llm": {}, - "tts": {}, - "combinations": [] - } + + self.test_wav_list = [] + self.wav_root = r"config/assets" + for file_name in os.listdir(self.wav_root): + file_path = os.path.join(self.wav_root, file_name) + # 检查文件大小是否大于300KB + if os.path.getsize(file_path) > 300 * 1024: # 300KB = 300 * 1024 bytes + with open(file_path, "rb") as f: + self.test_wav_list.append(f.read()) + + self.results = {"llm": {}, "tts": {}, "stt": {}, "combinations": []} async def _check_ollama_service(self, base_url: str, model_name: str) -> bool: """异步检查Ollama服务状态""" @@ -46,7 +56,9 @@ class AsyncPerformanceTester: data = await response.json() models = data.get("models", []) if not any(model["name"] == model_name for model in models): - print(f"🚫 Ollama模型 {model_name} 未找到，请先使用 ollama pull {model_name} 下载") + print( + f"🚫 Ollama模型 {model_name} 未找到，请先使用 ollama pull {model_name} 下载" + ) return False else: print(f"🚫 无法获取Ollama模型列表") @@ -62,17 +74,16 @@ class AsyncPerformanceTester: logging.getLogger("core.providers.tts.base").setLevel(logging.WARNING) token_fields = ["access_token", "api_key", "token"] - if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in - token_fields): + if any( + field in config + and any(x in config[field] for x in ["你的", "placeholder"]) + for field in token_fields + ): print(f"⏭️ TTS {tts_name} 未配置access_token/api_key，已跳过") return {"name": tts_name, "type": "tts", "errors": 1} - module_type = config.get('type', tts_name) - tts = create_tts_instance( - module_type, - config, - delete_audio_file=True - ) + module_type = config.get("type", tts_name) + tts = create_tts_instance(module_type, config, delete_audio_file=True) print(f"🎵 测试 TTS: {tts_name}") @@ -103,20 +114,71 @@ class AsyncPerformanceTester: "name": tts_name, "type": "tts", "avg_time": total_time / test_count, - "errors": 0 + "errors": 0, } except Exception as e: print(f"⚠️ {tts_name} 测试失败: {str(e)}") return {"name": tts_name, "type": "tts", "errors": 1} + async def _test_stt(self, stt_name: str, config: Dict) -> Dict: + """异步测试单个STT性能""" + try: + logging.getLogger("core.providers.asr.base").setLevel(logging.WARNING) + token_fields = ["access_token", "api_key", "token"] + if any( + field in config + and any(x in config[field] for x in ["你的", "placeholder"]) + for field in token_fields + ): + print(f"⏭️ STT {stt_name} 未配置access_token/api_key，已跳过") + return {"name": stt_name, "type": "stt", "errors": 1} + + module_type = config.get("type", stt_name) + stt = create_stt_instance(module_type, config, delete_audio_file=True) + stt.audio_format = "pcm" + + print(f"🎵 测试 STT: {stt_name}") + + text, _ = await stt.speech_to_text([self.test_wav_list[0]], "1") + + if text is None: + print(f"❌ {stt_name} 连接失败") + return {"name": stt_name, "type": "stt", "errors": 1} + + total_time = 0 + test_count = len(self.test_wav_list) + + for i, sentence in enumerate(self.test_wav_list, 1): + start = time.time() + text, _ = await stt.speech_to_text([sentence], "1") + duration = time.time() - start + total_time += duration + + if text: + print(f"✓ {stt_name} [{i}/{test_count}]") + else: + print(f"✗ {stt_name} [{i}/{test_count}]") + return {"name": stt_name, "type": "stt", "errors": 1} + + return { + "name": stt_name, + "type": "stt", + "avg_time": total_time / test_count, + "errors": 0, + } + + except Exception as e: + print(f"⚠️ {stt_name} 测试失败: {str(e)}") + return {"name": stt_name, "type": "stt", "errors": 1} + async def _test_llm(self, llm_name: str, config: Dict) -> Dict: """异步测试单个LLM性能""" try: # 对于Ollama，跳过api_key检查并进行特殊处理 if llm_name == "Ollama": - base_url = config.get('base_url', 'http://localhost:11434') - model_name = config.get('model_name') + base_url = config.get("base_url", "http://localhost:11434") + model_name = config.get("model_name") if not model_name: print(f"🚫 Ollama未配置model_name") return {"name": llm_name, "type": "llm", "errors": 1} @@ -124,21 +186,27 @@ class AsyncPerformanceTester: if not await self._check_ollama_service(base_url, model_name): return {"name": llm_name, "type": "llm", "errors": 1} else: - if "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]): + if "api_key" in config and any( + x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"] + ): print(f"🚫 跳过未配置的LLM: {llm_name}") return {"name": llm_name, "type": "llm", "errors": 1} # 获取实际类型（兼容旧配置） - module_type = config.get('type', llm_name) + module_type = config.get("type", llm_name) llm = create_llm_instance(module_type, config) # 统一使用UTF-8编码 - test_sentences = [s.encode('utf-8').decode('utf-8') for s in self.test_sentences] + test_sentences = [ + s.encode("utf-8").decode("utf-8") for s in self.test_sentences + ] # 创建所有句子的测试任务 sentence_tasks = [] for sentence in test_sentences: - sentence_tasks.append(self._test_single_sentence(llm_name, llm, sentence)) + sentence_tasks.append( + self._test_single_sentence(llm_name, llm, sentence) + ) # 并发执行所有句子测试 sentence_results = await asyncio.gather(*sentence_tasks) @@ -166,9 +234,15 @@ class AsyncPerformanceTester: "type": "llm", "avg_response": sum(response_times) / len(response_times), "avg_first_token": sum(first_token_times) / len(first_token_times), - "std_first_token": statistics.stdev(first_token_times) if len(first_token_times) > 1 else 0, - "std_response": statistics.stdev(response_times) if len(response_times) > 1 else 0, - "errors": 0 + "std_first_token": ( + statistics.stdev(first_token_times) + if len(first_token_times) > 1 + else 0 + ), + "std_response": ( + statistics.stdev(response_times) if len(response_times) > 1 else 0 + ), + "errors": 0, } except Exception as e: print(f"LLM {llm_name} 测试失败: {str(e)}") @@ -184,8 +258,10 @@ class AsyncPerformanceTester: async def process_response(): nonlocal first_token_received, first_token_time - for chunk in llm.response("perf_test", [{"role": "user", "content": sentence}]): - if not first_token_received and chunk.strip() != '': + for chunk in llm.response( + "perf_test", [{"role": "user", "content": sentence}] + ): + if not first_token_received and chunk.strip() != "": first_token_time = time.time() - sentence_start first_token_received = True print(f"✓ {llm_name} 首个Token: {first_token_time:.3f}s") @@ -199,13 +275,15 @@ class AsyncPerformanceTester: print(f"✓ {llm_name} 完成响应: {response_time:.3f}s") if first_token_time is None: - first_token_time = response_time # 如果没有检测到first token，使用总响应时间 + first_token_time = ( + response_time # 如果没有检测到first token，使用总响应时间 + ) return { "name": llm_name, "type": "llm", "first_token_time": first_token_time, - "response_time": response_time + "response_time": response_time, } except Exception as e: print(f"⚠️ {llm_name} 句子测试失败: {str(e)}") @@ -214,42 +292,71 @@ class AsyncPerformanceTester: def _generate_combinations(self): """生成最佳组合建议""" valid_llms = [ - k for k, v in self.results["llm"].items() + k + for k, v in self.results["llm"].items() if v["errors"] == 0 and v["avg_first_token"] >= 0.05 ] valid_tts = [k for k, v in self.results["tts"].items() if v["errors"] == 0] + valid_stt = [k for k, v in self.results["stt"].items() if v["errors"] == 0] # 找出基准值 - min_first_token = min([self.results["llm"][llm]["avg_first_token"] for llm in valid_llms]) if valid_llms else 1 - min_tts_time = min([self.results["tts"][tts]["avg_time"] for tts in valid_tts]) if valid_tts else 1 + min_first_token = ( + min([self.results["llm"][llm]["avg_first_token"] for llm in valid_llms]) + if valid_llms + else 1 + ) + min_tts_time = ( + min([self.results["tts"][tts]["avg_time"] for tts in valid_tts]) + if valid_tts + else 1 + ) + min_stt_time = ( + min([self.results["stt"][stt]["avg_time"] for stt in valid_stt]) + if valid_stt + else 1 + ) for llm in valid_llms: for tts in valid_tts: - # 计算相对性能分数（越小越好） - llm_score = self.results["llm"][llm]["avg_first_token"] / min_first_token - tts_score = self.results["tts"][tts]["avg_time"] / min_tts_time + for stt in valid_stt: + # 计算相对性能分数（越小越好） + llm_score = ( + self.results["llm"][llm]["avg_first_token"] / min_first_token + ) + tts_score = self.results["tts"][tts]["avg_time"] / min_tts_time + stt_score = self.results["stt"][stt]["avg_time"] / min_stt_time - # 计算稳定性分数（标准差/平均值，越小越稳定） - llm_stability = self.results["llm"][llm]["std_first_token"] / self.results["llm"][llm][ - "avg_first_token"] + # 计算稳定性分数（标准差/平均值，越小越稳定） + llm_stability = ( + self.results["llm"][llm]["std_first_token"] + / self.results["llm"][llm]["avg_first_token"] + ) - # 综合得分（考虑性能和稳定性） - # 性能权重0.7，稳定性权重0.3 - llm_final_score = llm_score * 0.7 + llm_stability * 0.3 + # 综合得分（考虑性能和稳定性） + # LLM得分： 性能权重(70%) + 稳定性权重(30%) + llm_final_score = llm_score * 0.7 + llm_stability * 0.3 - # 总分 = LLM得分(70%) + TTS得分(30%) - total_score = llm_final_score * 0.7 + tts_score * 0.3 + # 总分 = LLM得分(70%) + TTS得分(30%) + STT得分(30%) + total_score = ( + llm_final_score * 0.7 + tts_score * 0.3 + stt_score * 0.3 + ) - self.results["combinations"].append({ - "llm": llm, - "tts": tts, - "score": total_score, - "details": { - "llm_first_token": self.results["llm"][llm]["avg_first_token"], - "llm_stability": llm_stability, - "tts_time": self.results["tts"][tts]["avg_time"] - } - }) + self.results["combinations"].append( + { + "llm": llm, + "tts": tts, + "stt": stt, + "score": total_score, + "details": { + "llm_first_token": self.results["llm"][llm][ + "avg_first_token" + ], + "llm_stability": llm_stability, + "tts_time": self.results["tts"][tts]["avg_time"], + "stt_time": self.results["stt"][stt]["avg_time"], + }, + } + ) # 分数越小越好 self.results["combinations"].sort(key=lambda x: x["score"]) @@ -260,64 +367,98 @@ class AsyncPerformanceTester: for name, data in self.results["llm"].items(): if data["errors"] == 0: stability = data["std_first_token"] / data["avg_first_token"] - llm_table.append([ - name, # 不需要固定宽度，让tabulate自己处理对齐 - f"{data['avg_first_token']:.3f}秒", - f"{data['avg_response']:.3f}秒", - f"{stability:.3f}" - ]) + llm_table.append( + [ + name, # 不需要固定宽度，让tabulate自己处理对齐 + f"{data['avg_first_token']:.3f}秒", + f"{data['avg_response']:.3f}秒", + f"{stability:.3f}", + ] + ) if llm_table: - print("\nLLM 性能排行:") - print(tabulate( - llm_table, - headers=["模型名称", "首字耗时", "总耗时", "稳定性"], - tablefmt="github", - colalign=("left", "right", "right", "right"), - disable_numparse=True - )) + print("\nLLM 性能排行:\n") + print( + tabulate( + llm_table, + headers=["模型名称", "首字耗时", "总耗时", "稳定性"], + tablefmt="github", + colalign=("left", "right", "right", "right"), + disable_numparse=True, + ) + ) else: print("\n⚠️ 没有可用的LLM模块进行测试。") tts_table = [] for name, data in self.results["tts"].items(): if data["errors"] == 0: - tts_table.append([ - name, # 不需要固定宽度 - f"{data['avg_time']:.3f}秒" - ]) + tts_table.append([name, f"{data['avg_time']:.3f}秒"]) # 不需要固定宽度 if tts_table: - print("\nTTS 性能排行:") - print(tabulate( - tts_table, - headers=["模型名称", "合成耗时"], - tablefmt="github", - colalign=("left", "right"), - disable_numparse=True - )) + print("\nTTS 性能排行:\n") + print( + tabulate( + tts_table, + headers=["模型名称", "合成耗时"], + tablefmt="github", + colalign=("left", "right"), + disable_numparse=True, + ) + ) else: print("\n⚠️ 没有可用的TTS模块进行测试。") - if self.results["combinations"]: - print("\n推荐配置组合 (得分越小越好):") - combo_table = [] - for combo in self.results["combinations"][:5]: - combo_table.append([ - f"{combo['llm']} + {combo['tts']}", # 不需要固定宽度 - f"{combo['score']:.3f}", - f"{combo['details']['llm_first_token']:.3f}秒", - f"{combo['details']['llm_stability']:.3f}", - f"{combo['details']['tts_time']:.3f}秒" - ]) + stt_table = [] + for name, data in self.results["stt"].items(): + if data["errors"] == 0: + stt_table.append([name, f"{data['avg_time']:.3f}秒"]) # 不需要固定宽度 - print(tabulate( - combo_table, - headers=["组合方案", "综合得分", "LLM首字耗时", "稳定性", "TTS合成耗时"], - tablefmt="github", - colalign=("left", "right", "right", "right", "right"), - disable_numparse=True - )) + if stt_table: + print("\nSTT 性能排行:\n") + print( + tabulate( + stt_table, + headers=["模型名称", "合成耗时"], + tablefmt="github", + colalign=("left", "right"), + disable_numparse=True, + ) + ) + else: + print("\n⚠️ 没有可用的STT模块进行测试。") + + if self.results["combinations"]: + print("\n推荐配置组合 (得分越小越好):\n") + combo_table = [] + for combo in self.results["combinations"][:]: + combo_table.append( + [ + f"{combo['llm']} + {combo['tts']} + {combo['stt']}", # 不需要固定宽度 + f"{combo['score']:.3f}", + f"{combo['details']['llm_first_token']:.3f}秒", + f"{combo['details']['llm_stability']:.3f}", + f"{combo['details']['tts_time']:.3f}秒", + f"{combo['details']['stt_time']:.3f}秒", + ] + ) + + print( + tabulate( + combo_table, + headers=[ + "组合方案", + "综合得分", + "LLM首字耗时", + "稳定性", + "TTS合成耗时", + "STT合成耗时", + ], + tablefmt="github", + colalign=("left", "right", "right", "right", "right", "right"), + disable_numparse=True, + ) + ) else: print("\n⚠️ 没有可用的模块组合建议。") @@ -327,8 +468,12 @@ class AsyncPerformanceTester: if result["errors"] == 0: if result["type"] == "llm": self.results["llm"][result["name"]] = result - else: + elif result["type"] == "tts": self.results["tts"][result["name"]] = result + elif result["type"] == "stt": + self.results["stt"][result["name"]] = result + else: + pass async def run(self): """执行全量异步测试""" @@ -338,50 +483,83 @@ class AsyncPerformanceTester: all_tasks = [] # LLM测试任务 - for llm_name, config in self.config.get("LLM", {}).items(): - # 检查配置有效性 - if llm_name == "CozeLLM": - if any(x in config.get("bot_id", "") for x in ["你的"]) \ - or any(x in config.get("user_id", "") for x in ["你的"]): - print(f"⏭️ LLM {llm_name} 未配置bot_id/user_id，已跳过") - continue - elif "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]): - print(f"⏭️ LLM {llm_name} 未配置api_key，已跳过") - continue - - # 对于Ollama，先检查服务状态 - if llm_name == "Ollama": - base_url = config.get('base_url', 'http://localhost:11434') - model_name = config.get('model_name') - if not model_name: - print(f"🚫 Ollama未配置model_name") + if self.config.get("LLM") is not None: + for llm_name, config in self.config.get("LLM", {}).items(): + # 检查配置有效性 + if llm_name == "CozeLLM": + if any(x in config.get("bot_id", "") for x in ["你的"]) or any( + x in config.get("user_id", "") for x in ["你的"] + ): + print(f"⏭️ LLM {llm_name} 未配置bot_id/user_id，已跳过") + continue + elif "api_key" in config and any( + x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"] + ): + print(f"⏭️ LLM {llm_name} 未配置api_key，已跳过") continue - if not await self._check_ollama_service(base_url, model_name): - continue + # 对于Ollama，先检查服务状态 + if llm_name == "Ollama": + base_url = config.get("base_url", "http://localhost:11434") + model_name = config.get("model_name") + if not model_name: + print(f"🚫 Ollama未配置model_name") + continue - print(f"📋 添加LLM测试任务: {llm_name}") - module_type = config.get('type', llm_name) - llm = create_llm_instance(module_type, config) + if not await self._check_ollama_service(base_url, model_name): + continue - # 为每个句子创建独立任务 - for sentence in self.test_sentences: - sentence = sentence.encode('utf-8').decode('utf-8') - all_tasks.append(self._test_single_sentence(llm_name, llm, sentence)) + print(f"📋 添加LLM测试任务: {llm_name}") + module_type = config.get("type", llm_name) + llm = create_llm_instance(module_type, config) + + # 为每个句子创建独立任务 + for sentence in self.test_sentences: + sentence = sentence.encode("utf-8").decode("utf-8") + all_tasks.append( + self._test_single_sentence(llm_name, llm, sentence) + ) # TTS测试任务 - for tts_name, config in self.config.get("TTS", {}).items(): - token_fields = ["access_token", "api_key", "token"] - if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in - token_fields): - print(f"⏭️ TTS {tts_name} 未配置access_token/api_key，已跳过") - continue - print(f"🎵 添加TTS测试任务: {tts_name}") - all_tasks.append(self._test_tts(tts_name, config)) + if self.config.get("TTS") is not None: + for tts_name, config in self.config.get("TTS", {}).items(): + token_fields = ["access_token", "api_key", "token"] + if any( + field in config + and any(x in config[field] for x in ["你的", "placeholder"]) + for field in token_fields + ): + print(f"⏭️ TTS {tts_name} 未配置access_token/api_key，已跳过") + continue + print(f"🎵 添加TTS测试任务: {tts_name}") + all_tasks.append(self._test_tts(tts_name, config)) + + # STT测试任务 + if len(self.test_wav_list) >= 1: + if self.config.get("ASR") is not None: + for stt_name, config in self.config.get("ASR", {}).items(): + token_fields = ["access_token", "api_key", "token"] + if any( + field in config + and any(x in config[field] for x in ["你的", "placeholder"]) + for field in token_fields + ): + print(f"⏭️ ASR {stt_name} 未配置access_token/api_key，已跳过") + continue + print(f"🎵 添加ASR测试任务: {stt_name}") + all_tasks.append(self._test_stt(stt_name, config)) + else: + print(f"\n⚠️ {self.wav_root} 路径下没有音频文件，已跳过STT测试任务") print( - f"\n✅ 找到 {len([t for t in all_tasks if 'test_single_sentence' in str(t)]) / len(self.test_sentences):.0f} 个可用LLM模块") - print(f"✅ 找到 {len([t for t in all_tasks if '_test_tts' in str(t)])} 个可用TTS模块") + f"\n✅ 找到 {len([t for t in all_tasks if 'test_single_sentence' in str(t)]) / len(self.test_sentences):.0f} 个可用LLM模块" + ) + print( + f"✅ 找到 {len([t for t in all_tasks if '_test_tts' in str(t)])} 个可用TTS模块" + ) + print( + f"✅ 找到 {len([t for t in all_tasks if '_test_stt' in str(t)])} 个可用STT模块" + ) print("\n⏳ 开始并发测试所有模块...\n") # 并发执行所有测试任务 @@ -389,7 +567,11 @@ class AsyncPerformanceTester: # 处理LLM结果 llm_results = {} - for result in [r for r in all_results if r and isinstance(r, dict) and r.get("type") == "llm"]: + for result in [ + r + for r in all_results + if r and isinstance(r, dict) and r.get("type") == "llm" + ]: llm_name = result["name"] if llm_name not in llm_results: llm_results[llm_name] = { @@ -397,9 +579,11 @@ class AsyncPerformanceTester: "type": "llm", "first_token_times": [], "response_times": [], - "errors": 0 + "errors": 0, } - llm_results[llm_name]["first_token_times"].append(result["first_token_time"]) + llm_results[llm_name]["first_token_times"].append( + result["first_token_time"] + ) llm_results[llm_name]["response_times"].append(result["response_time"]) # 计算LLM平均值和标准差 @@ -408,19 +592,41 @@ class AsyncPerformanceTester: self.results["llm"][llm_name] = { "name": llm_name, "type": "llm", - "avg_response": sum(data["response_times"]) / len(data["response_times"]), - "avg_first_token": sum(data["first_token_times"]) / len(data["first_token_times"]), - "std_first_token": statistics.stdev(data["first_token_times"]) if len( - data["first_token_times"]) > 1 else 0, - "std_response": statistics.stdev(data["response_times"]) if len(data["response_times"]) > 1 else 0, - "errors": 0 + "avg_response": sum(data["response_times"]) + / len(data["response_times"]), + "avg_first_token": sum(data["first_token_times"]) + / len(data["first_token_times"]), + "std_first_token": ( + statistics.stdev(data["first_token_times"]) + if len(data["first_token_times"]) > 1 + else 0 + ), + "std_response": ( + statistics.stdev(data["response_times"]) + if len(data["response_times"]) > 1 + else 0 + ), + "errors": 0, } # 处理TTS结果 - for result in [r for r in all_results if r and isinstance(r, dict) and r.get("type") == "tts"]: + for result in [ + r + for r in all_results + if r and isinstance(r, dict) and r.get("type") == "tts" + ]: if result["errors"] == 0: self.results["tts"][result["name"]] = result + # 处理STT结果 + for result in [ + r + for r in all_results + if r and isinstance(r, dict) and r.get("type") == "stt" + ]: + if result["errors"] == 0: + self.results["stt"][result["name"]] = result + # 生成组合建议并打印结果 print("\n📊 生成测试报告...") self._generate_combinations() @@ -433,4 +639,4 @@ async def main(): if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/main/xiaozhi-server/plugins_func/functions/get_news.py b/main/xiaozhi-server/plugins_func/functions/get_news_from_chinanews.py similarity index 62% rename from main/xiaozhi-server/plugins_func/functions/get_news.py rename to main/xiaozhi-server/plugins_func/functions/get_news_from_chinanews.py index 85b203dc..64d499b1 100644 --- a/main/xiaozhi-server/plugins_func/functions/get_news.py +++ b/main/xiaozhi-server/plugins_func/functions/get_news_from_chinanews.py @@ -8,10 +8,10 @@ from plugins_func.register import register_function, ToolType, ActionResponse, A TAG = __name__ logger = setup_logging() -GET_NEWS_FUNCTION_DESC = { +GET_NEWS_FROM_CHINANEWS_FUNCTION_DESC = { "type": "function", "function": { - "name": "get_news", + "name": "get_news_from_chinanews", "description": ( "获取最新新闻，随机选择一条新闻进行播报。" "用户可以指定新闻类型，如社会新闻、科技新闻、国际新闻等。" @@ -23,20 +23,20 @@ GET_NEWS_FUNCTION_DESC = { "properties": { "category": { "type": "string", - "description": "新闻类别，例如社会、科技、国际。可选参数，如果不提供则使用默认类别" + "description": "新闻类别，例如社会、科技、国际。可选参数，如果不提供则使用默认类别", }, "detail": { "type": "boolean", - "description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容" + "description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容", }, "lang": { "type": "string", - "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN" - } + "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN", + }, }, - "required": ["lang"] - } - } + "required": ["lang"], + }, + }, } @@ -51,18 +51,30 @@ def fetch_news_from_rss(rss_url): # 查找所有item元素（新闻条目） news_items = [] - for item in root.findall('.//item'): - title = item.find('title').text if item.find('title') is not None else "无标题" - link = item.find('link').text if item.find('link') is not None else "#" - description = item.find('description').text if item.find('description') is not None else "无描述" - pubDate = item.find('pubDate').text if item.find('pubDate') is not None else "未知时间" + for item in root.findall(".//item"): + title = ( + item.find("title").text if item.find("title") is not None else "无标题" + ) + link = item.find("link").text if item.find("link") is not None else "#" + description = ( + item.find("description").text + if item.find("description") is not None + else "无描述" + ) + pubDate = ( + item.find("pubDate").text + if item.find("pubDate") is not None + else "未知时间" + ) - news_items.append({ - 'title': title, - 'link': link, - 'description': description, - 'pubDate': pubDate - }) + news_items.append( + { + "title": title, + "link": link, + "description": description, + "pubDate": pubDate, + } + ) return news_items except Exception as e: @@ -76,18 +88,24 @@ def fetch_news_detail(url): response = requests.get(url) response.raise_for_status() - soup = BeautifulSoup(response.content, 'html.parser') + soup = BeautifulSoup(response.content, "html.parser") # 尝试提取正文内容 (这里的选择器需要根据实际网站结构调整) - content_div = soup.select_one('.content_desc, .content, article, .article-content') + content_div = soup.select_one( + ".content_desc, .content, article, .article-content" + ) if content_div: - paragraphs = content_div.find_all('p') - content = '\n'.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()]) + paragraphs = content_div.find_all("p") + content = "\n".join( + [p.get_text().strip() for p in paragraphs if p.get_text().strip()] + ) return content else: # 如果找不到特定的内容区域，尝试获取所有段落 - paragraphs = soup.find_all('p') - content = '\n'.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()]) + paragraphs = soup.find_all("p") + content = "\n".join( + [p.get_text().strip() for p in paragraphs if p.get_text().strip()] + ) return content[:2000] # 限制长度 except Exception as e: logger.bind(tag=TAG).error(f"获取新闻详情失败: {e}") @@ -111,7 +129,7 @@ def map_category(category_text): "财经": "finance", "财经新闻": "finance", "金融": "finance", - "经济": "finance" + "经济": "finance", } # 转换为小写并去除空格 @@ -121,20 +139,36 @@ def map_category(category_text): return category_map.get(normalized_category, category_text) -@register_function('get_news', GET_NEWS_FUNCTION_DESC, ToolType.SYSTEM_CTL) -def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_CN"): +@register_function( + "get_news_from_chinanews", + GET_NEWS_FROM_CHINANEWS_FUNCTION_DESC, + ToolType.SYSTEM_CTL, +) +def get_news_from_chinanews( + conn, category: str = None, detail: bool = False, lang: str = "zh_CN" +): """获取新闻并随机选择一条进行播报，或获取上一条新闻的详细内容""" try: # 如果detail为True，获取上一条新闻的详细内容 if detail: - if not hasattr(conn, 'last_news_link') or not conn.last_news_link or 'link' not in conn.last_news_link: - return ActionResponse(Action.REQLLM, "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", None) + if ( + not hasattr(conn, "last_news_link") + or not conn.last_news_link + or "link" not in conn.last_news_link + ): + return ActionResponse( + Action.REQLLM, + "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", + None, + ) - link = conn.last_news_link.get('link') - title = conn.last_news_link.get('title', '未知标题') + link = conn.last_news_link.get("link") + title = conn.last_news_link.get("title", "未知标题") - if link == '#': - return ActionResponse(Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None) + if link == "#": + return ActionResponse( + Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None + ) logger.bind(tag=TAG).debug(f"获取新闻详情: {title}, URL={link}") @@ -142,8 +176,11 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C detail_content = fetch_news_detail(link) if not detail_content or detail_content == "无法获取详细内容": - return ActionResponse(Action.REQLLM, - f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", None) + return ActionResponse( + Action.REQLLM, + f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", + None, + ) # 构建详情报告 detail_report = ( @@ -158,8 +195,10 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C # 否则，获取新闻列表并随机选择一条 # 从配置中获取RSS URL - rss_config = conn.config["plugins"]["get_news"] - default_rss_url = rss_config.get("default_rss_url", "https://www.chinanews.com.cn/rss/society.xml") + rss_config = conn.config["plugins"]["get_news_from_chinanews"] + default_rss_url = rss_config.get( + "default_rss_url", "https://www.chinanews.com.cn/rss/society.xml" + ) # 将用户输入的类别映射到配置中的类别键 mapped_category = map_category(category) @@ -169,23 +208,27 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C if mapped_category and mapped_category in rss_config.get("category_urls", {}): rss_url = rss_config["category_urls"][mapped_category] - logger.bind(tag=TAG).info(f"获取新闻: 原始类别={category}, 映射类别={mapped_category}, URL={rss_url}") + logger.bind(tag=TAG).info( + f"获取新闻: 原始类别={category}, 映射类别={mapped_category}, URL={rss_url}" + ) # 获取新闻列表 news_items = fetch_news_from_rss(rss_url) if not news_items: - return ActionResponse(Action.REQLLM, "抱歉，未能获取到新闻信息，请稍后再试。", None) + return ActionResponse( + Action.REQLLM, "抱歉，未能获取到新闻信息，请稍后再试。", None + ) # 随机选择一条新闻 selected_news = random.choice(news_items) # 保存当前新闻链接到连接对象，以便后续查询详情 - if not hasattr(conn, 'last_news_link'): + if not hasattr(conn, "last_news_link"): conn.last_news_link = {} conn.last_news_link = { - 'link': selected_news.get('link', '#'), - 'title': selected_news.get('title', '未知标题') + "link": selected_news.get("link", "#"), + "title": selected_news.get("title", "未知标题"), } # 构建新闻报告 @@ -203,4 +246,6 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C except Exception as e: logger.bind(tag=TAG).error(f"获取新闻出错: {e}") - return ActionResponse(Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None) \ No newline at end of file + return ActionResponse( + Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None + ) diff --git a/main/xiaozhi-server/plugins_func/functions/get_news_from_newsnow.py b/main/xiaozhi-server/plugins_func/functions/get_news_from_newsnow.py new file mode 100644 index 00000000..1b98539d --- /dev/null +++ b/main/xiaozhi-server/plugins_func/functions/get_news_from_newsnow.py @@ -0,0 +1,214 @@ +import random +import requests +import json +from config.logger import setup_logging +from plugins_func.register import register_function, ToolType, ActionResponse, Action +from markitdown import MarkItDown + +TAG = __name__ +logger = setup_logging() + +# 新闻来源字典，包含名称和对应的API ID +NEWS_SOURCES = { + "thepaper": "澎湃新闻", + "baidu": "百度热搜", + "cls-depth": "财联社", +} + + +# 动态生成新闻源描述 +def generate_news_sources_description(): + sources_desc = [] + for source_id, source_name in NEWS_SOURCES.items(): + sources_desc.append(f"{source_name}({source_id})") + return "、".join(sources_desc) + + +GET_NEWS_FROM_NEWSNOW_FUNCTION_DESC = { + "type": "function", + "function": { + "name": "get_news_from_newsnow", + "description": ( + "获取最新新闻，随机选择一条新闻进行播报。" + f"用户可以选择不同的新闻源，如{generate_news_sources_description()}等。" + "如果没有指定，默认从澎湃新闻获取。" + "用户可以要求获取详细内容，此时会获取新闻的详细内容。" + ), + "parameters": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": f"新闻源，例如{generate_news_sources_description()}等。可选参数，如果不提供则使用默认新闻源", + }, + "detail": { + "type": "boolean", + "description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容", + }, + "lang": { + "type": "string", + "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN", + }, + }, + "required": ["lang"], + }, + }, +} + + +def fetch_news_from_api(conn, source="thepaper"): + """从API获取新闻列表""" + try: + api_url = f"https://newsnow.busiyi.world/api/s?id={source}" + if conn.config["plugins"].get("get_news_from_newsnow") and conn.config[ + "plugins" + ]["get_news_from_newsnow"].get("url"): + api_url = conn.config["plugins"]["get_news_from_newsnow"]["url"] + source + + response = requests.get(api_url, timeout=10) + response.raise_for_status() + + data = response.json() + + if "items" in data: + return data["items"] + else: + logger.bind(tag=TAG).error(f"获取新闻API响应格式错误: {data}") + return [] + + except Exception as e: + logger.bind(tag=TAG).error(f"获取新闻API失败: {e}") + return [] + + +def fetch_news_detail(url): + """获取新闻详情页内容并使用MarkItDown清理HTML""" + try: + response = requests.get(url, timeout=10) + response.raise_for_status() + + # 使用MarkItDown清理HTML内容 + md = MarkItDown(enable_plugins=False) + result = md.convert(response) + + # 获取清理后的文本内容 + clean_text = result.text_content + + # 如果清理后的内容为空，返回提示信息 + if not clean_text or len(clean_text.strip()) == 0: + logger.bind(tag=TAG).warning(f"清理后的新闻内容为空: {url}") + return "无法解析新闻详情内容，可能是网站结构特殊或内容受限。" + + return clean_text + except Exception as e: + logger.bind(tag=TAG).error(f"获取新闻详情失败: {e}") + return "无法获取详细内容" + + +@register_function( + "get_news_from_newsnow", + GET_NEWS_FROM_NEWSNOW_FUNCTION_DESC, + ToolType.SYSTEM_CTL, +) +def get_news_from_newsnow( + conn, source: str = "thepaper", detail: bool = False, lang: str = "zh_CN" +): + """获取新闻并随机选择一条进行播报，或获取上一条新闻的详细内容""" + try: + # 如果detail为True，获取上一条新闻的详细内容 + detail = str(detail).lower() == "true" + if detail: + if ( + not hasattr(conn, "last_newsnow_link") + or not conn.last_newsnow_link + or "url" not in conn.last_newsnow_link + ): + return ActionResponse( + Action.REQLLM, + "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", + None, + ) + + url = conn.last_newsnow_link.get("url") + title = conn.last_newsnow_link.get("title", "未知标题") + source_id = conn.last_newsnow_link.get("source_id", "thepaper") + source_name = NEWS_SOURCES.get(source_id, "未知来源") + + if not url or url == "#": + return ActionResponse( + Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None + ) + + logger.bind(tag=TAG).debug( + f"获取新闻详情: {title}, 来源: {source_name}, URL={url}" + ) + + # 获取新闻详情 + detail_content = fetch_news_detail(url) + + if not detail_content or detail_content == "无法获取详细内容": + return ActionResponse( + Action.REQLLM, + f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", + None, + ) + + # 构建详情报告 + detail_report = ( + f"根据下列数据，用{lang}回应用户的新闻详情查询请求：\n\n" + f"新闻标题: {title}\n" + # f"新闻来源: {source_name}\n" + f"详细内容: {detail_content}\n\n" + f"(请对上述新闻内容进行总结，提取关键信息，以自然、流畅的方式向用户播报，" + f"不要提及这是总结，就像是在讲述一个完整的新闻故事)" + ) + + return ActionResponse(Action.REQLLM, detail_report, None) + + # 否则，获取新闻列表并随机选择一条 + # 验证新闻源是否有效，如果无效则使用默认源 + if source not in NEWS_SOURCES: + logger.bind(tag=TAG).warning(f"无效的新闻源: {source}，使用默认源thepaper") + source = "thepaper" + + source_name = NEWS_SOURCES.get(source, "澎湃新闻") + logger.bind(tag=TAG).info(f"获取新闻: 新闻源={source}({source_name})") + + # 获取新闻列表 + news_items = fetch_news_from_api(conn, source) + + if not news_items: + return ActionResponse( + Action.REQLLM, + f"抱歉，未能从{source_name}获取到新闻信息，请稍后再试或尝试其他新闻源。", + None, + ) + + # 随机选择一条新闻 + selected_news = random.choice(news_items) + + # 保存当前新闻链接到连接对象，以便后续查询详情 + if not hasattr(conn, "last_newsnow_link"): + conn.last_newsnow_link = {} + conn.last_newsnow_link = { + "url": selected_news.get("url", "#"), + "title": selected_news.get("title", "未知标题"), + "source_id": source, + } + + # 构建新闻报告 + news_report = ( + f"根据下列数据，用{lang}回应用户的新闻查询请求：\n\n" + f"新闻标题: {selected_news['title']}\n" + # f"新闻来源: {source_name}\n" + f"(请以自然、流畅的方式向用户播报这条新闻标题，" + f"提示用户可以要求获取详细内容，此时会获取新闻的详细内容。)" + ) + + return ActionResponse(Action.REQLLM, news_report, None) + + except Exception as e: + logger.bind(tag=TAG).error(f"获取新闻出错: {e}") + return ActionResponse( + Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None + ) diff --git a/main/xiaozhi-server/plugins_func/functions/get_weather.py b/main/xiaozhi-server/plugins_func/functions/get_weather.py index 060274d9..75c15c7b 100644 --- a/main/xiaozhi-server/plugins_func/functions/get_weather.py +++ b/main/xiaozhi-server/plugins_func/functions/get_weather.py @@ -2,6 +2,7 @@ import requests from bs4 import BeautifulSoup from config.logger import setup_logging from plugins_func.register import register_function, ToolType, ActionResponse, Action +from core.utils.util import get_ip_info TAG = __name__ logger = setup_logging() @@ -12,55 +13,104 @@ GET_WEATHER_FUNCTION_DESC = { "name": "get_weather", "description": ( "获取某个地点的天气，用户应提供一个位置，比如用户说杭州天气，参数为：杭州。" - "如果用户说的是省份，默认用省会城市。如果用户说的不是省份或城市而是一个地名，" - "默认用该地所在省份的省会城市。" + "如果用户说的是省份，默认用省会城市。如果用户说的不是省份或城市而是一个地名，默认用该地所在省份的省会城市。" + "如果用户没有指明地点，说“天气怎么样”，”今天天气如何“，location参数为空" ), "parameters": { "type": "object", "properties": { "location": { "type": "string", - "description": "地点名，例如杭州。可选参数，如果不提供则不传" + "description": "地点名，例如杭州。可选参数，如果不提供则不传", }, "lang": { "type": "string", - "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN" - } + "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN", + }, }, - "required": ["lang"] - } - } + "required": ["lang"], + }, + }, } HEADERS = { - 'User-Agent': ( - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' - '(KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36' + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36" ) } # 天气代码 https://dev.qweather.com/docs/resource/icons/#weather-icons WEATHER_CODE_MAP = { - "100": "晴", "101": "多云", "102": "少云", "103": "晴间多云", "104": "阴", - "150": "晴", "151": "多云", "152": "少云", "153": "晴间多云", - "300": "阵雨", "301": "强阵雨", "302": "雷阵雨", "303": "强雷阵雨", "304": "雷阵雨伴有冰雹", - "305": "小雨", "306": "中雨", "307": "大雨", "308": "极端降雨", "309": "毛毛雨/细雨", - "310": "暴雨", "311": "大暴雨", "312": "特大暴雨", "313": "冻雨", "314": "小到中雨", - "315": "中到大雨", "316": "大到暴雨", "317": "暴雨到大暴雨", "318": "大暴雨到特大暴雨", - "350": "阵雨", "351": "强阵雨", "399": "雨", - "400": "小雪", "401": "中雪", "402": "大雪", "403": "暴雪", "404": "雨夹雪", - "405": "雨雪天气", "406": "阵雨夹雪", "407": "阵雪", "408": "小到中雪", "409": "中到大雪", "410": "大到暴雪", - "456": "阵雨夹雪", "457": "阵雪", "499": "雪", - "500": "薄雾", "501": "雾", "502": "霾", "503": "扬沙", "504": "浮尘", - "507": "沙尘暴", "508": "强沙尘暴", - "509": "浓雾", "510": "强浓雾", "511": "中度霾", "512": "重度霾", "513": "严重霾", "514": "大雾", "515": "特强浓雾", - "900": "热", "901": "冷", "999": "未知" + "100": "晴", + "101": "多云", + "102": "少云", + "103": "晴间多云", + "104": "阴", + "150": "晴", + "151": "多云", + "152": "少云", + "153": "晴间多云", + "300": "阵雨", + "301": "强阵雨", + "302": "雷阵雨", + "303": "强雷阵雨", + "304": "雷阵雨伴有冰雹", + "305": "小雨", + "306": "中雨", + "307": "大雨", + "308": "极端降雨", + "309": "毛毛雨/细雨", + "310": "暴雨", + "311": "大暴雨", + "312": "特大暴雨", + "313": "冻雨", + "314": "小到中雨", + "315": "中到大雨", + "316": "大到暴雨", + "317": "暴雨到大暴雨", + "318": "大暴雨到特大暴雨", + "350": "阵雨", + "351": "强阵雨", + "399": "雨", + "400": "小雪", + "401": "中雪", + "402": "大雪", + "403": "暴雪", + "404": "雨夹雪", + "405": "雨雪天气", + "406": "阵雨夹雪", + "407": "阵雪", + "408": "小到中雪", + "409": "中到大雪", + "410": "大到暴雪", + "456": "阵雨夹雪", + "457": "阵雪", + "499": "雪", + "500": "薄雾", + "501": "雾", + "502": "霾", + "503": "扬沙", + "504": "浮尘", + "507": "沙尘暴", + "508": "强沙尘暴", + "509": "浓雾", + "510": "强浓雾", + "511": "中度霾", + "512": "重度霾", + "513": "严重霾", + "514": "大雾", + "515": "特强浓雾", + "900": "热", + "901": "冷", + "999": "未知", } -def fetch_city_info(location, api_key): - url = f"https://geoapi.qweather.com/v2/city/lookup?key={api_key}&location={location}&lang=zh" + +def fetch_city_info(location, api_key, api_host): + url = f"https://{api_host}/geo/v2/city/lookup?key={api_key}&location={location}&lang=zh" response = requests.get(url, headers=HEADERS).json() - return response.get('location', [])[0] if response.get('location') else None + return response.get("location", [])[0] if response.get("location") else None def fetch_weather_page(url): @@ -72,10 +122,14 @@ def parse_weather_info(soup): city_name = soup.select_one("h1.c-submenu__location").get_text(strip=True) current_abstract = soup.select_one(".c-city-weather-current .current-abstract") - current_abstract = current_abstract.get_text(strip=True) if current_abstract else "未知" + current_abstract = ( + current_abstract.get_text(strip=True) if current_abstract else "未知" + ) current_basic = {} - for item in soup.select(".c-city-weather-current .current-basic .current-basic___item"): + for item in soup.select( + ".c-city-weather-current .current-basic .current-basic___item" + ): parts = item.get_text(strip=True, separator=" ").split(" ") if len(parts) == 2: key, value = parts[1], parts[0] @@ -84,7 +138,9 @@ def parse_weather_info(soup): temps_list = [] for row in soup.select(".city-forecast-tabs__row")[:7]: # 取前7天的数据 date = row.select_one(".date-bg .date").get_text(strip=True) - weather_code = row.select_one(".date-bg .icon")["src"].split("/")[-1].split(".")[0] + weather_code = ( + row.select_one(".date-bg .icon")["src"].split("/")[-1].split(".")[0] + ) weather = WEATHER_CODE_MAP.get(weather_code, "未知") temps = [span.get_text(strip=True) for span in row.select(".tmp-cont .temp")] high_temp, low_temp = (temps[0], temps[-1]) if len(temps) >= 2 else (None, None) @@ -93,31 +149,47 @@ def parse_weather_info(soup): return city_name, current_abstract, current_basic, temps_list -@register_function('get_weather', GET_WEATHER_FUNCTION_DESC, ToolType.SYSTEM_CTL) +@register_function("get_weather", GET_WEATHER_FUNCTION_DESC, ToolType.SYSTEM_CTL) def get_weather(conn, location: str = None, lang: str = "zh_CN"): - api_key = conn.config["plugins"]["get_weather"]["api_key"] + api_host = conn.config["plugins"]["get_weather"].get("api_host", "mj7p3y7naa.re.qweatherapi.com") + api_key = conn.config["plugins"]["get_weather"].get("api_key", "a861d0d5e7bf4ee1a83d9a9e4f96d4da") default_location = conn.config["plugins"]["get_weather"]["default_location"] - location = location or conn.client_ip_info.get("city") or default_location - logger.bind(tag=TAG).debug(f"获取天气: {location}") - - city_info = fetch_city_info(location, api_key) + client_ip = conn.client_ip + # 优先使用用户提供的location参数 + if not location: + # 通过客户端IP解析城市 + if client_ip: + # 动态解析IP对应的城市信息 + ip_info = get_ip_info(client_ip, logger) + location = ip_info.get("city") if ip_info and "city" in ip_info else None + else: + # 若IP解析失败或无IP，使用默认位置 + location = default_location + city_info = fetch_city_info(location, api_key, api_host) if not city_info: - return ActionResponse(Action.REQLLM, f"未找到相关的城市: {location}，请确认地点是否正确", None) - - soup = fetch_weather_page(city_info['fxLink']) + return ActionResponse( + Action.REQLLM, f"未找到相关的城市: {location}，请确认地点是否正确", None + ) + soup = fetch_weather_page(city_info["fxLink"]) if not soup: return ActionResponse(Action.REQLLM, None, "请求失败") - city_name, current_abstract, current_basic, temps_list = parse_weather_info(soup) - weather_report = f"根据下列数据，用{lang}回应用户的查询天气请求：\n{city_name}未来7天天气:\n" - for i, (date, weather, high, low) in enumerate(temps_list): - if high and low: - weather_report += f"{date}: {low}到{high}, {weather}\n" - weather_report += ( - f"当前天气: {current_abstract}\n" - f"当前天气参数: {current_basic}\n" - f"(确保只报告指定单日的天气情况，除非未来会出现异常天气；或者用户明确要求想要了解多日天气，如果未指定，默认报告今天的天气。" - "参数为0的值不需要报告给用户，每次都报告体感温度，根据语境选择合适的参数内容告知用户，并对参数给出相应评价)" - ) - return ActionResponse(Action.REQLLM, weather_report, None) \ No newline at end of file + weather_report = f"您查询的位置是：{city_name}\n\n当前天气: {current_abstract}\n" + + # 添加有效的当前天气参数 + if current_basic: + weather_report += "详细参数：\n" + for key, value in current_basic.items(): + if value != "0": # 过滤无效值 + weather_report += f" · {key}: {value}\n" + + # 添加7天预报 + weather_report += "\n未来7天预报：\n" + for date, weather, high, low in temps_list: + weather_report += f"{date}: {weather}，气温 {low}~{high}\n" + + # 提示语 + weather_report += "\n（如需某一天的具体天气，请告诉我日期）" + + return ActionResponse(Action.REQLLM, weather_report, None) diff --git a/main/xiaozhi-server/plugins_func/functions/handle_exit_intent.py b/main/xiaozhi-server/plugins_func/functions/handle_exit_intent.py index e4769d00..affa8af2 100644 --- a/main/xiaozhi-server/plugins_func/functions/handle_exit_intent.py +++ b/main/xiaozhi-server/plugins_func/functions/handle_exit_intent.py @@ -1,34 +1,43 @@ -from plugins_func.register import register_function,ToolType, ActionResponse, Action +from plugins_func.register import register_function, ToolType, ActionResponse, Action from config.logger import setup_logging TAG = __name__ logger = setup_logging() handle_exit_intent_function_desc = { - "type": "function", - "function": { - "name": "handle_exit_intent", - "description": "当用户想结束对话或需要退出系统时调用", - "parameters": { - "type": "object", - "properties": { - "say_goodbye": { - "type": "string", - "description": "和用户友好结束对话的告别语" - } - }, - "required": ["say_goodbye"] - } + "type": "function", + "function": { + "name": "handle_exit_intent", + "description": "当用户想结束对话或需要退出系统时调用", + "parameters": { + "type": "object", + "properties": { + "say_goodbye": { + "type": "string", + "description": "和用户友好结束对话的告别语", } - } + }, + "required": ["say_goodbye"], + }, + }, +} -@register_function('handle_exit_intent', handle_exit_intent_function_desc, ToolType.SYSTEM_CTL) -def handle_exit_intent(conn, say_goodbye: str): + +@register_function( + "handle_exit_intent", handle_exit_intent_function_desc, ToolType.SYSTEM_CTL +) +def handle_exit_intent(conn, say_goodbye: str | None = None): # 处理退出意图 try: + if say_goodbye is None: + say_goodbye = "再见，祝您生活愉快！" conn.close_after_chat = True logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}") - return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye) + return ActionResponse( + action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye + ) except Exception as e: logger.bind(tag=TAG).error(f"处理退出意图错误: {e}") - return ActionResponse(action=Action.NONE, result="退出意图处理失败", response="") \ No newline at end of file + return ActionResponse( + action=Action.NONE, result="退出意图处理失败", response="" + ) diff --git a/main/xiaozhi-server/plugins_func/functions/hass_init.py b/main/xiaozhi-server/plugins_func/functions/hass_init.py index 17846cfa..3a5458d3 100644 --- a/main/xiaozhi-server/plugins_func/functions/hass_init.py +++ b/main/xiaozhi-server/plugins_func/functions/hass_init.py @@ -8,10 +8,12 @@ HASS_CACHE = {} def append_devices_to_prompt(conn): - if conn.use_function_call_mode: - funcs = conn.config["Intent"]["function_call"].get("functions", []) + if conn.intent_type == "function_call": + funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get( + "functions", [] + ) if "hass_get_state" in funcs or "hass_set_state" in funcs: - prompt = "下面是我家智能设备，可以通过homeassistant控制\n" + prompt = "\n下面是我家智能设备列表（位置，设备名，entity_id），可以通过homeassistant控制\n" devices = conn.config["plugins"]["home_assistant"].get("devices", []) if len(devices) == 0: return @@ -25,8 +27,10 @@ def append_devices_to_prompt(conn): def initialize_hass_handler(conn): global HASS_CACHE if HASS_CACHE == {}: - if conn.use_function_call_mode: - funcs = conn.config["Intent"]["function_call"].get("functions", []) + if conn.load_function_plugin: + funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get( + "functions", [] + ) if "hass_get_state" in funcs or "hass_set_state" in funcs: HASS_CACHE["base_url"] = conn.config["plugins"]["home_assistant"].get( "base_url" diff --git a/main/xiaozhi-server/plugins_func/functions/plugin_loader.py b/main/xiaozhi-server/plugins_func/functions/plugin_loader.py index 4747d997..7041d7a0 100644 --- a/main/xiaozhi-server/plugins_func/functions/plugin_loader.py +++ b/main/xiaozhi-server/plugins_func/functions/plugin_loader.py @@ -1,51 +1,56 @@ -from plugins_func.register import register_function,ToolType, ActionResponse, Action -from config.logger import setup_logging - -TAG = __name__ -logger = setup_logging() +from plugins_func.register import register_function, ToolType, ActionResponse, Action plugin_loader_function_desc = { - "type": "function", - "function": { - "name": "plugin_loader", - "description": "当用户想加载或卸载插件/function时，调用此函数：支持的插件列表为[plugins]", - "parameters": { - "type": "object", - "properties": { - "oper": { - "type": "string", - "description": "load or unload" - }, - "name":{ - "type": "string", - "description": "要加载或卸载的插件名字" - } - }, - "required": ["oper","name"] - } - } - } + "type": "function", + "function": { + "name": "plugin_loader", + "description": "当用户想加载或卸载插件/function时，调用此函数：支持的插件列表为[plugins]", + "parameters": { + "type": "object", + "properties": { + "oper": {"type": "string", "description": "load or unload"}, + "name": {"type": "string", "description": "要加载或卸载的插件名字"}, + }, + "required": ["oper", "name"], + }, + }, +} -@register_function('plugin_loader', plugin_loader_function_desc, ToolType.SYSTEM_CTL) + +@register_function("plugin_loader", plugin_loader_function_desc, ToolType.SYSTEM_CTL) def plugin_loader(conn, oper: str, name: str): """插件加载""" if oper not in ["load", "unload"]: - return ActionResponse(action=Action.RESPONSE, result="插件操作失败", response="不支持的操作") - + return ActionResponse( + action=Action.RESPONSE, result="插件操作失败", response="不支持的操作" + ) + cur_support = conn.func_handler.current_support_functions() if oper == "load": if name in cur_support: - return ActionResponse(action=Action.RESPONSE, result="插件加载失败", response=f"{name}插件已加载,无需重复加载") + return ActionResponse( + action=Action.RESPONSE, + result="插件加载失败", + response=f"{name}插件已加载,无需重复加载", + ) func = conn.func_handler.function_registry.register_function(name) if not func: - return ActionResponse(action=Action.RESPONSE, result="插件加载失败", response="插件未找到") + return ActionResponse( + action=Action.RESPONSE, result="插件加载失败", response="插件未找到" + ) res = f"{name}插件加载成功" else: if name not in cur_support: - return ActionResponse(action=Action.RESPONSE, result="插件卸载失败", response=f"{name}插件未加载") + return ActionResponse( + action=Action.RESPONSE, + result="插件卸载失败", + response=f"{name}插件未加载", + ) bOK = conn.func_handler.function_registry.unregister_function(name) if not bOK: - return ActionResponse(action=Action.RESPONSE, result="插件卸载失败", response="插件未找到") + return ActionResponse( + action=Action.RESPONSE, result="插件卸载失败", response="插件未找到" + ) res = f"{name}插件卸载成功" conn.func_handler.upload_functions_desc() return ActionResponse(action=Action.RESPONSE, result="插件操作成功", response=res) diff --git a/main/xiaozhi-server/plugins_func/register.py b/main/xiaozhi-server/plugins_func/register.py index fd990760..b96f2f29 100644 --- a/main/xiaozhi-server/plugins_func/register.py +++ b/main/xiaozhi-server/plugins_func/register.py @@ -10,7 +10,10 @@ class ToolType(Enum): NONE = (1, "调用完工具后，不做其他操作") WAIT = (2, "调用工具，等待函数返回") CHANGE_SYS_PROMPT = (3, "修改系统提示词，切换角色性格或职责") - SYSTEM_CTL = (4, "系统控制，影响正常的对话流程，如退出、播放音乐等，需要传递conn参数") + SYSTEM_CTL = ( + 4, + "系统控制，影响正常的对话流程，如退出、播放音乐等，需要传递conn参数", + ) IOT_CTL = (5, "IOT设备控制，需要传递conn参数") MCP_CLIENT = (6, "MCP客户端") @@ -30,12 +33,14 @@ class Action(Enum): self.code = code self.message = message + class ActionResponse: def __init__(self, action: Action, result, response): self.action = action # 动作类型 self.result = result # 动作产生的结果 self.response = response # 直接回复的内容 + class FunctionItem: def __init__(self, name, description, func, type): self.name = name @@ -43,40 +48,49 @@ class FunctionItem: self.func = func self.type = type + class DeviceTypeRegistry: """设备类型注册表，用于管理IOT设备类型及其函数""" + def __init__(self): self.type_functions = {} # type_signature -> {func_name: FunctionItem} - + def generate_device_type_id(self, descriptor): """通过设备能力描述生成类型ID""" properties = sorted(descriptor["properties"].keys()) methods = sorted(descriptor["methods"].keys()) # 使用属性和方法的组合作为设备类型的唯一标识 - type_signature = f"{descriptor['name']}:{','.join(properties)}:{','.join(methods)}" + type_signature = ( + f"{descriptor['name']}:{','.join(properties)}:{','.join(methods)}" + ) return type_signature - + def get_device_functions(self, type_id): """获取设备类型对应的所有函数""" return self.type_functions.get(type_id, {}) - + def register_device_type(self, type_id, functions): """注册设备类型及其函数""" if type_id not in self.type_functions: self.type_functions[type_id] = functions + # 初始化函数注册字典 all_function_registry = {} device_type_registry = DeviceTypeRegistry() + def register_function(name, desc, type=None): """注册函数到函数注册字典的装饰器""" + def decorator(func): all_function_registry[name] = FunctionItem(name, desc, func, type) logger.bind(tag=TAG).debug(f"函数 '{name}' 已加载，可以注册使用") return func + return decorator + class FunctionRegistry: def __init__(self): self.function_registry = {} @@ -89,9 +103,9 @@ class FunctionRegistry: self.logger.bind(tag=TAG).error(f"函数 '{name}' 未找到") return None self.function_registry[name] = func - self.logger.bind(tag=TAG).info(f"函数 '{name}' 注册成功") + self.logger.bind(tag=TAG).debug(f"函数 '{name}' 注册成功") return func - + def unregister_function(self, name): # 注销函数，检测是否存在 if name not in self.function_registry: @@ -103,9 +117,9 @@ class FunctionRegistry: def get_function(self, name): return self.function_registry.get(name) - + def get_all_functions(self): return self.function_registry - + def get_all_function_desc(self): - return [func.description for _, func in self.function_registry.items()] \ No newline at end of file + return [func.description for _, func in self.function_registry.items()] diff --git a/main/xiaozhi-server/requirements.txt b/main/xiaozhi-server/requirements.txt index 66128cff..9df7dde1 100755 --- a/main/xiaozhi-server/requirements.txt +++ b/main/xiaozhi-server/requirements.txt @@ -22,6 +22,12 @@ mem0ai==0.1.62 bs4==0.0.2 modelscope==1.23.2 sherpa_onnx==1.11.0 -mcp==1.4.1 +mcp==1.8.1 cnlunar==0.2.0 PySocks==1.7.1 +dashscope==1.23.1 +baidu-aip==4.16.13 +chardet==5.2.0 +aioconsole==0.8.1 +markitdown==0.1.1 +mcp-proxy==0.6.0 diff --git a/main/xiaozhi-server/test/abbreviated_version/app.js b/main/xiaozhi-server/test/abbreviated_version/app.js index 59b75625..b2286524 100644 --- a/main/xiaozhi-server/test/abbreviated_version/app.js +++ b/main/xiaozhi-server/test/abbreviated_version/app.js @@ -992,7 +992,9 @@ function connectToServer() { if(connectButton.id === "connectButton") { connectButton.textContent = '断开'; - connectButton.onclick = disconnectFromServer; + // connectButton.onclick = disconnectFromServer; + connectButton.removeEventListener("click", connectToServer); + connectButton.addEventListener("click", disconnectFromServer); } if(messageInput.id === "messageInput") { @@ -1011,7 +1013,9 @@ function connectToServer() { if(connectButton.id === "connectButton") { connectButton.textContent = '连接'; - connectButton.onclick = connectToServer; + // connectButton.onclick = connectToServer; + connectButton.removeEventListener("click", disconnectFromServer); + connectButton.addEventListener("click", connectToServer); } if(messageInput.id === "messageInput") { @@ -1175,7 +1179,7 @@ function updateStatus(message, type = 'info') { // 处理文本消息 function handleTextMessage(message) { if (message.type === 'hello') { - console.log(`服务器回应：${message.message}`); + console.log(`服务器回应：${JSON.stringify(message, null, 2)}`); } else if (message.type === 'tts') { // TTS状态消息 if (message.state === 'start') { diff --git a/main/xiaozhi-server/test/test_page.html b/main/xiaozhi-server/test/test_page.html index 2735fb5a..d9b2eff6 100644 --- a/main/xiaozhi-server/test/test_page.html +++ b/main/xiaozhi-server/test/test_page.html @@ -42,6 +42,50 @@ display: flex; align-items: center; gap: 10px; + padding: 10px 0; + } + + .section h2 .toggle-button { + margin-left: auto; + padding: 4px 12px; + font-size: 12px; + border-radius: 4px; + cursor: pointer; + height: 28px; + line-height: 20px; + } + + .device-info { + display: flex; + align-items: center; + gap: 20px; + margin-left: 20px; + padding: 0 15px; + background-color: #f9f9f9; + border-radius: 4px; + height: 28px; + line-height: 28px; + } + + .device-info span { + color: #666; + font-size: 13px; + } + + .device-info strong { + color: #333; + font-weight: 500; + } + + .config-panel { + display: none; + transition: all 0.3s ease; + margin-top: 5px; + padding: 5px 0; + } + + .config-panel.expanded { + display: block; } .control-panel { @@ -70,7 +114,8 @@ cursor: not-allowed; } - #serverUrl { + #serverUrl, + #otaUrl { flex-grow: 1; padding: 8px; border: 1px solid #ddd; @@ -316,6 +361,76 @@ gap: 20px; margin-top: 10px; } + + .config-item { + display: flex; + align-items: center; + margin-bottom: 8px; + width: 100%; + } + + .config-item label { + width: 100px; + text-align: right; + margin-right: 10px; + color: #666; + } + + .config-item input { + flex-grow: 1; + padding: 6px; + border: 1px solid #ddd; + border-radius: 5px; + } + + .control-panel { + display: flex; + flex-direction: column; + gap: 10px; + margin-top: 10px; + } + + .connection-controls { + display: flex; + gap: 10px; + align-items: center; + width: 100%; + } + + .connection-controls input { + flex: 1; + padding: 8px; + border: 1px solid #ddd; + border-radius: 5px; + min-width: 200px; + } + + .connection-controls button { + white-space: nowrap; + padding: 8px 15px; + } + + .connection-status { + display: flex; + align-items: center; + gap: 20px; + margin-left: 20px; + padding: 0 15px; + background-color: #f9f9f9; + border-radius: 4px; + height: 28px; + line-height: 28px; + } + + .connection-status span { + color: #666; + font-size: 13px; + } + + .connection-status .status { + color: #333; + font-weight: 500; + } @@ -326,17 +441,56 @@
正在加载Opus库...
+
-

WebSocket连接 未连接

-
- +

+ 设备配置 + + MAC: + 客户端: web_test_client + + +

+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+ +
+

+ 连接信息 + + OTA: ota未连接 + WS: ws未连接 + +

+
+ +
-

消息发送

@@ -475,22 +629,36 @@ // 日志函数 function log(message, type = 'info') { - const logEntry = document.createElement('div'); - logEntry.className = `log-entry log-${type}`; - logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`; - if (type === 'error') { - logEntry.style.color = 'red'; - } else if (type === 'debug') { - logEntry.style.color = 'gray'; - return; - } else if (type === 'warning') { - logEntry.style.color = 'orange'; - } else if (type === 'success') { - logEntry.style.color = 'green'; - } else { - logEntry.style.color = 'black'; - } - logContainer.appendChild(logEntry); + // 将消息按换行符分割成多行 + const lines = message.split('\n'); + const now = new Date(); + // const timestamp = `[${now.toLocaleTimeString()}] `; + const timestamp = `[${now.toLocaleTimeString()}.${now.getMilliseconds().toString().padStart(3, '0')}] `; + // 为每一行创建日志条目 + lines.forEach((line, index) => { + const logEntry = document.createElement('div'); + logEntry.className = `log-entry log-${type}`; + // 如果是第一条日志，显示时间戳 + const prefix = index === 0 ? timestamp : ' '.repeat(timestamp.length); + logEntry.textContent = `${prefix}${line}`; + // logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`; + // logEntry.style 保留起始的空格 + logEntry.style.whiteSpace = 'pre'; + if (type === 'error') { + logEntry.style.color = 'red'; + } else if (type === 'debug') { + logEntry.style.color = 'gray'; + return; + } else if (type === 'warning') { + logEntry.style.color = 'orange'; + } else if (type === 'success') { + logEntry.style.color = 'green'; + } else { + logEntry.style.color = 'black'; + } + logContainer.appendChild(logEntry); + }); + logContainer.scrollTop = logContainer.scrollHeight; } @@ -540,16 +708,16 @@ // 开始音频缓冲过程 function startAudioBuffering() { if (isAudioBuffering || isAudioPlaying) return; - + isAudioBuffering = true; log("开始音频缓冲...", 'info'); - + // 先尝试初始化解码器，以便在播放时已准备好 initOpusDecoder().catch(error => { log(`预初始化Opus解码器失败: ${error.message}`, 'warning'); // 继续缓冲，我们会在播放时再次尝试初始化 }); - + // 设置超时，如果在一定时间内没有收集到足够的音频包，就开始播放 setTimeout(() => { if (isAudioBuffering && audioBufferQueue.length > 0) { @@ -557,14 +725,14 @@ playBufferedAudio(); } }, 300); // 300ms超时 - + // 监控缓冲进度 const bufferCheckInterval = setInterval(() => { if (!isAudioBuffering) { clearInterval(bufferCheckInterval); return; } - + // 当累积了足够的音频包，开始播放 if (audioBufferQueue.length >= BUFFER_THRESHOLD) { clearInterval(bufferCheckInterval); @@ -577,10 +745,10 @@ // 播放已缓冲的音频 function playBufferedAudio() { if (isAudioPlaying || audioBufferQueue.length === 0) return; - + isAudioPlaying = true; isAudioBuffering = false; - + // 确保Opus解码器已初始化 const initDecoderAndPlay = async () => { try { @@ -591,7 +759,7 @@ }); log('创建音频上下文，采样率: ' + SAMPLE_RATE + 'Hz', 'debug'); } - + // 确保解码器已初始化 if (!opusDecoder) { log('初始化Opus解码器...', 'info'); @@ -607,7 +775,7 @@ return; } } - + // 创建流式播放上下文 if (!streamingContext) { streamingContext = { @@ -617,16 +785,16 @@ source: null, // 当前音频源 totalSamples: 0, // 累积的总样本数 lastPlayTime: 0, // 上次播放的时间戳 - + // 将Opus数据解码为PCM - decodeOpusFrames: async function(opusFrames) { + decodeOpusFrames: async function (opusFrames) { if (!opusDecoder) { log('Opus解码器未初始化，无法解码', 'error'); return; } - + let decodedSamples = []; - + for (const frame of opusFrames) { try { // 使用Opus解码器解码 @@ -640,12 +808,12 @@ log("Opus解码失败: " + error.message, 'error'); } } - + if (decodedSamples.length > 0) { // 添加到解码队列 this.queue.push(...decodedSamples); this.totalSamples += decodedSamples.length; - + // 如果累积了至少0.2秒的音频，开始播放 const minSamples = SAMPLE_RATE * MIN_AUDIO_DURATION; if (!this.playing && this.queue.length >= minSamples) { @@ -655,50 +823,50 @@ log('没有成功解码的样本', 'warning'); } }, - + // 开始播放音频 - startPlaying: function() { + startPlaying: function () { if (this.playing || this.queue.length === 0) return; - + this.playing = true; - + // 创建新的音频缓冲区 const minPlaySamples = Math.min(this.queue.length, SAMPLE_RATE); // 最多播放1秒 const currentSamples = this.queue.splice(0, minPlaySamples); - + const audioBuffer = audioContext.createBuffer(CHANNELS, currentSamples.length, SAMPLE_RATE); audioBuffer.copyToChannel(new Float32Array(currentSamples), 0); - + // 创建音频源 this.source = audioContext.createBufferSource(); this.source.buffer = audioBuffer; - + // 创建增益节点用于平滑过渡 const gainNode = audioContext.createGain(); - + // 应用淡入淡出效果避免爆音 const fadeDuration = 0.02; // 20毫秒 gainNode.gain.setValueAtTime(0, audioContext.currentTime); gainNode.gain.linearRampToValueAtTime(1, audioContext.currentTime + fadeDuration); - + const duration = audioBuffer.duration; if (duration > fadeDuration * 2) { gainNode.gain.setValueAtTime(1, audioContext.currentTime + duration - fadeDuration); gainNode.gain.linearRampToValueAtTime(0, audioContext.currentTime + duration); } - + // 连接节点并开始播放 this.source.connect(gainNode); gainNode.connect(audioContext.destination); - + this.lastPlayTime = audioContext.currentTime; log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / SAMPLE_RATE).toFixed(2)} 秒`, 'info'); - + // 播放结束后的处理 this.source.onended = () => { this.source = null; this.playing = false; - + // 如果队列中还有数据或者缓冲区有新数据，继续播放 if (this.queue.length > 0) { setTimeout(() => this.startPlaying(), 10); @@ -729,26 +897,26 @@ }, 500); // 500ms超时 } }; - + this.source.start(); } }; } - + // 开始处理缓冲的数据 const frames = [...audioBufferQueue]; audioBufferQueue = []; // 清空缓冲队列 - + // 解码并播放 await streamingContext.decodeOpusFrames(frames); - + } catch (error) { log(`播放已缓冲的音频出错: ${error.message}`, 'error'); isAudioPlaying = false; streamingContext = null; } }; - + // 执行初始化和播放 initDecoderAndPlay(); } @@ -766,7 +934,7 @@ // 初始化Opus解码器 - 确保完全初始化完成后才返回 async function initOpusDecoder() { if (opusDecoder) return opusDecoder; // 已经初始化 - + try { // 检查ModuleInstance是否存在 if (typeof window.ModuleInstance === 'undefined') { @@ -778,9 +946,9 @@ throw new Error('Opus库未加载，ModuleInstance和Module对象都不存在'); } } - + const mod = window.ModuleInstance; - + // 创建解码器对象 opusDecoder = { channels: CHANNELS, @@ -788,55 +956,55 @@ frameSize: FRAME_SIZE, module: mod, decoderPtr: null, // 初始为null - + // 初始化解码器 - init: function() { + init: function () { if (this.decoderPtr) return true; // 已经初始化 - + // 获取解码器大小 const decoderSize = mod._opus_decoder_get_size(this.channels); log(`Opus解码器大小: ${decoderSize}字节`, 'debug'); - + // 分配内存 this.decoderPtr = mod._malloc(decoderSize); if (!this.decoderPtr) { throw new Error("无法分配解码器内存"); } - + // 初始化解码器 const err = mod._opus_decoder_init( this.decoderPtr, this.rate, this.channels ); - + if (err < 0) { this.destroy(); // 清理资源 throw new Error(`Opus解码器初始化失败: ${err}`); } - + log("Opus解码器初始化成功", 'success'); return true; }, - + // 解码方法 - decode: function(opusData) { + decode: function (opusData) { if (!this.decoderPtr) { if (!this.init()) { throw new Error("解码器未初始化且无法初始化"); } } - + try { const mod = this.module; - + // 为Opus数据分配内存 const opusPtr = mod._malloc(opusData.length); mod.HEAPU8.set(opusData, opusPtr); - + // 为PCM输出分配内存 const pcmPtr = mod._malloc(this.frameSize * 2); // Int16 = 2字节 - + // 解码 const decodedSamples = mod._opus_decode( this.decoderPtr, @@ -846,46 +1014,46 @@ this.frameSize, 0 // 不使用FEC ); - + if (decodedSamples < 0) { mod._free(opusPtr); mod._free(pcmPtr); throw new Error(`Opus解码失败: ${decodedSamples}`); } - + // 复制解码后的数据 const decodedData = new Int16Array(decodedSamples); for (let i = 0; i < decodedSamples; i++) { decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i]; } - + // 释放内存 mod._free(opusPtr); mod._free(pcmPtr); - + return decodedData; } catch (error) { log(`Opus解码错误: ${error.message}`, 'error'); return new Int16Array(0); } }, - + // 销毁方法 - destroy: function() { + destroy: function () { if (this.decoderPtr) { this.module._free(this.decoderPtr); this.decoderPtr = null; } } }; - + // 初始化解码器 if (!opusDecoder.init()) { throw new Error("Opus解码器初始化失败"); } - + return opusDecoder; - + } catch (error) { log(`Opus解码器初始化失败: ${error.message}`, 'error'); opusDecoder = null; // 重置为null，以便下次重试 @@ -1118,25 +1286,100 @@ } // 连接WebSocket服务器 - function connectToServer() { + async function connectToServer() { const url = serverUrlInput.value.trim(); if (url === '') return; try { + // 获取并验证配置 + const config = getConfig(); + if (!validateConfig(config)) { + return; + } + // 检查URL格式 if (!url.startsWith('ws://') && !url.startsWith('wss://')) { log('URL格式错误，必须以ws://或wss://开头', 'error'); return; } + // 先检查OTA状态 + log('正在检查OTA状态...', 'info'); + const otaUrl = document.getElementById('otaUrl').value.trim(); + + try { + const otaResponse = await fetch(otaUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Device-Id': config.deviceId, + 'Client-Id': config.clientId + }, + body: JSON.stringify({ + "version": 0, + "uuid": "", + "application": { + "name": "xiaozhi-web-test", + "version": "1.0.0", + "compile_time": "2025-04-16 10:00:00", + "idf_version": "4.4.3", + "elf_sha256": "1234567890abcdef1234567890abcdef1234567890abcdef" + }, + "ota": { + "label": "xiaozhi-web-test", + }, + "board": { + "type": "xiaozhi-web-test", + "ssid": "xiaozhi-web-test", + "rssi": 0, + "channel": 0, + "ip": "192.168.1.1", + "mac": config.deviceMac + }, + "flash_size": 0, + "minimum_free_heap_size": 0, + "mac_address": config.deviceMac, + "chip_model_name": "", + "chip_info": { + "model": 0, + "cores": 0, + "revision": 0, + "features": 0 + }, + "partition_table": [ + { + "label": "", + "type": 0, + "subtype": 0, + "address": 0, + "size": 0 + } + ] + }) + }); + + if (!otaResponse.ok) { + throw new Error(`OTA检查失败: ${otaResponse.status} ${otaResponse.statusText}`); + } + + const otaResult = await otaResponse.json(); + log(`OTA检查结果: ${JSON.stringify(otaResult)}`, 'info'); + + log('OTA检查通过，开始连接WebSocket...', 'success'); + document.getElementById('otaStatus').textContent = 'ota已连接'; + document.getElementById('otaStatus').style.color = 'green'; + } catch (error) { + log(`OTA检查错误: ${error.message}`, 'error'); + document.getElementById('otaStatus').textContent = 'ota未连接'; + document.getElementById('otaStatus').style.color = 'red'; + } + // 使用自定义WebSocket实现以添加认证头信息 - // 注意：浏览器原生WebSocket不支持自定义头，需要通过服务器添加一个代理层 - // 但我们可以通过URL参数模拟认证信息 let connUrl = new URL(url); // 添加认证参数 - connUrl.searchParams.append('device_id', 'web_test_device'); - connUrl.searchParams.append('device_mac', '00:11:22:33:44:55'); + connUrl.searchParams.append('device-id', config.deviceId); + connUrl.searchParams.append('client-id', config.clientId); log(`正在连接: ${connUrl.toString()}`, 'info'); websocket = new WebSocket(connUrl.toString()); @@ -1146,14 +1389,16 @@ websocket.onopen = async () => { log(`已连接到服务器: ${url}`, 'success'); - connectionStatus.textContent = '已连接'; + connectionStatus.textContent = 'ws已连接'; connectionStatus.style.color = 'green'; // 连接成功后发送hello消息 await sendHelloMessage(); connectButton.textContent = '断开'; - connectButton.onclick = disconnectFromServer; + connectButton.removeEventListener('click', connectToServer); + connectButton.addEventListener('click', disconnectFromServer); + // connectButton.onclick = disconnectFromServer; messageInput.disabled = false; sendTextButton.disabled = false; @@ -1165,11 +1410,13 @@ websocket.onclose = () => { log('已断开连接', 'info'); - connectionStatus.textContent = '已断开'; + connectionStatus.textContent = 'ws已断开'; connectionStatus.style.color = 'red'; connectButton.textContent = '连接'; - connectButton.onclick = connectToServer; + connectButton.removeEventListener('click', disconnectFromServer); + connectButton.addEventListener('click', connectToServer); + // connectButton.onclick = connectToServer; messageInput.disabled = true; sendTextButton.disabled = true; recordButton.disabled = true; @@ -1178,7 +1425,7 @@ websocket.onerror = (error) => { log(`WebSocket错误: ${error.message || '未知错误'}`, 'error'); - connectionStatus.textContent = '连接错误'; + connectionStatus.textContent = 'ws未连接'; connectionStatus.style.color = 'red'; }; @@ -1189,7 +1436,7 @@ const message = JSON.parse(event.data); if (message.type === 'hello') { - log(`服务器回应：${message.message}`, 'info'); + log(`服务器回应：${JSON.stringify(message, null, 2)}`, 'success'); } else if (message.type === 'tts') { // TTS状态消息 if (message.state === 'start') { @@ -1244,11 +1491,11 @@ } }; - connectionStatus.textContent = '正在连接...'; + connectionStatus.textContent = 'ws未连接'; connectionStatus.style.color = 'orange'; } catch (error) { log(`连接错误: ${error.message}`, 'error'); - connectionStatus.textContent = '连接失败'; + connectionStatus.textContent = 'ws未连接'; } } @@ -1257,13 +1504,15 @@ if (!websocket || websocket.readyState !== WebSocket.OPEN) return; try { + const config = getConfig(); + // 设置设备信息 const helloMessage = { type: 'hello', - device_id: 'web_test_device', - device_name: 'Web测试设备', - device_mac: '00:11:22:33:44:55', - token: 'your-token1' // 使用config.yaml中配置的token + device_id: config.deviceId, + device_name: config.deviceName, + device_mac: config.deviceMac, + token: config.token }; log('发送hello握手消息', 'info'); @@ -1333,11 +1582,64 @@ } } + // 生成随机MAC地址 + function generateRandomMac() { + const hexDigits = '0123456789ABCDEF'; + let mac = ''; + for (let i = 0; i < 6; i++) { + if (i > 0) mac += ':'; + for (let j = 0; j < 2; j++) { + mac += hexDigits.charAt(Math.floor(Math.random() * 16)); + } + } + return mac; + } + // 初始化事件监听器 function initEventListeners() { connectButton.addEventListener('click', connectToServer); document.getElementById('authTestButton').addEventListener('click', testAuthentication); + // 设备配置面板折叠/展开 + const toggleButton = document.getElementById('toggleConfig'); + const configPanel = document.getElementById('configPanel'); + const deviceMacInput = document.getElementById('deviceMac'); + const clientIdInput = document.getElementById('clientId'); + const displayMac = document.getElementById('displayMac'); + const displayClient = document.getElementById('displayClient'); + + // 从localStorage加载MAC地址，如果没有则生成新的 + let savedMac = localStorage.getItem('deviceMac'); + if (!savedMac) { + savedMac = generateRandomMac(); + localStorage.setItem('deviceMac', savedMac); + } + deviceMacInput.value = savedMac; + displayMac.textContent = savedMac; + + // 更新显示的值 + function updateDisplayValues() { + const newMac = deviceMacInput.value; + displayMac.textContent = newMac; + displayClient.textContent = clientIdInput.value; + // 保存MAC地址到localStorage + localStorage.setItem('deviceMac', newMac); + } + + // 监听输入变化 + deviceMacInput.addEventListener('input', updateDisplayValues); + clientIdInput.addEventListener('input', updateDisplayValues); + + // 初始更新显示值 + updateDisplayValues(); + + // 切换面板显示 + toggleButton.addEventListener('click', () => { + const isExpanded = configPanel.classList.contains('expanded'); + configPanel.classList.toggle('expanded'); + toggleButton.textContent = isExpanded ? '编辑' : '收起'; + }); + // 标签页切换 const tabs = document.querySelectorAll('.tab'); tabs.forEach(tab => { @@ -1372,12 +1674,14 @@ async function testAuthentication() { log('开始测试认证...', 'info'); + const config = getConfig(); + // 显示服务器配置 log('-------- 服务器认证配置检查 --------', 'info'); log('请确认config.yaml中的auth配置：', 'info'); log('1. server.auth.enabled 为 false 或服务器已正确配置认证', 'info'); log('2. 如果启用了认证，请确认使用了正确的token', 'info'); - log('3. 或者在allowed_devices中添加了测试设备MAC：00:11:22:33:44:55', 'info'); + log(`3. 或者在allowed_devices中添加了测试设备MAC：${config.deviceMac}`, 'info'); const serverUrl = serverUrlInput.value.trim(); if (!serverUrl) { @@ -1418,9 +1722,9 @@ log('测试2: 尝试带token参数连接...', 'info'); let url = new URL(serverUrl); - url.searchParams.append('token', 'your-token1'); - url.searchParams.append('device_id', 'web_test_device'); - url.searchParams.append('device_mac', '00:11:22:33:44:55'); + url.searchParams.append('token', config.token); + url.searchParams.append('device_id', config.deviceId); + url.searchParams.append('device_mac', config.deviceMac); const ws2 = new WebSocket(url.toString()); @@ -1430,9 +1734,9 @@ // 尝试发送hello消息 const helloMsg = { type: 'hello', - device_id: 'web_test_device', - device_mac: '00:11:22:33:44:55', - token: 'your-token1' + device_id: config.deviceId, + device_mac: config.deviceMac, + token: config.token }; ws2.send(JSON.stringify(helloMsg)); @@ -2074,19 +2378,19 @@ if (opusData.length > 0) { // 将数据添加到缓冲队列 audioBufferQueue.push(opusData); - + // 如果收到的是第一个音频包，开始缓冲过程 if (audioBufferQueue.length === 1 && !isAudioBuffering && !isAudioPlaying) { startAudioBuffering(); } } else { log('收到空音频数据帧，可能是结束标志', 'warning'); - + // 如果缓冲队列中有数据且没有在播放，立即开始播放 if (audioBufferQueue.length > 0 && !isAudioPlaying) { playBufferedAudio(); } - + // 如果正在播放，发送结束信号 if (isAudioPlaying && streamingContext) { streamingContext.endOfStream = true; @@ -2097,6 +2401,31 @@ } } + // 获取配置值 + function getConfig() { + const deviceMac = document.getElementById('deviceMac').value.trim(); + return { + deviceId: deviceMac, // 使用MAC地址作为deviceId + deviceName: document.getElementById('deviceName').value.trim(), + deviceMac: deviceMac, + clientId: document.getElementById('clientId').value.trim(), + token: document.getElementById('token').value.trim() + }; + } + + // 验证配置 + function validateConfig(config) { + if (!config.deviceMac) { + log('设备MAC地址不能为空', 'error'); + return false; + } + if (!config.clientId) { + log('客户端ID不能为空', 'error'); + return false; + } + return true; + } + initApp();