Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

update:合并非tts代码

Browse Source
This commit is contained in:
hrz
2025-05-21 13:18:12 +08:00
parent ede2bc6a4e
commit 851365fb58
65 changed files with 5423 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/app.py
+81 -18
View File
@@ -1,49 +1,112 @@
import asyncio
import sys
import signal
from config.settings import load_config, check_config_file
from config.settings import load_config
from core.websocket_server import WebSocketServer
from core.ota_server import SimpleOtaServer
from core.utils.util import check_ffmpeg_installed
from config.logger import setup_logging
from core.utils.util import get_local_ip
from aioconsole import ainput
TAG = __name__
logger = setup_logging()
async def wait_for_exit():
"""Windows 和 Linux 兼容的退出监听"""
async def wait_for_exit() -> None:
"""
阻塞直到收到 Ctrl‑C / SIGTERM。
- Unix: 使用 add_signal_handler
- Windows: 依赖 KeyboardInterrupt
"""
loop = asyncio.get_running_loop()
stop_event = asyncio.Event()
if sys.platform == "win32":
# Windows: 用 sys.stdin.read() 监听 Ctrl + C
await loop.run_in_executor(None, sys.stdin.read)
else:
# Linux/macOS: 用 signal 监听 Ctrl + C
def stop():
stop_event.set()
loop.add_signal_handler(signal.SIGINT, stop)
loop.add_signal_handler(signal.SIGTERM, stop) # 支持 kill 进程
if sys.platform != "win32": # Unix / macOS
for sig in (signal.SIGINT, signal.SIGTERM):
loop.add_signal_handler(sig, stop_event.set)
await stop_event.wait()
else:
# Windows：await一个永远pending的fut，
# 让 KeyboardInterrupt 冒泡到 asyncio.run，以此消除遗留普通线程导致进程退出阻塞的问题
try:
await asyncio.Future()
except KeyboardInterrupt: # Ctrl‑C
pass
async def monitor_stdin():
"""监控标准输入，消费回车键"""
while True:
await ainput() # 异步等待输入，消费回车
async def main():
check_config_file()
check_ffmpeg_installed()
config = load_config()
# 添加 stdin 监控任务
stdin_task = asyncio.create_task(monitor_stdin())
# 启动 WebSocket 服务器
ws_server = WebSocketServer(config)
ws_task = asyncio.create_task(ws_server.start())
ota_task = None
read_config_from_api = config.get("read_config_from_api", False)
if not read_config_from_api:
# 启动 Simple OTA 服务器
ota_server = SimpleOtaServer(config)
ota_task = asyncio.create_task(ota_server.start())
logger.bind(tag=TAG).info(
"OTA接口是\t\thttp://{}:{}/xiaozhi/ota/",
get_local_ip(),
config["server"]["ota_port"],
)
# 获取WebSocket配置，使用安全的默认值
websocket_port = 8000
server_config = config.get("server", {})
if isinstance(server_config, dict):
websocket_port = int(server_config.get("port", 8000))
logger.bind(tag=TAG).info(
"Websocket地址是\tws://{}:{}/xiaozhi/v1/",
get_local_ip(),
websocket_port,
)
logger.bind(tag=TAG).info(
"=======上面的地址是websocket协议地址，请勿用浏览器访问======="
)
logger.bind(tag=TAG).info(
"如想测试websocket请用谷歌浏览器打开test目录下的test_page.html"
)
logger.bind(tag=TAG).info(
"=============================================================\n"
)
try:
await wait_for_exit() # 监听退出信号
await wait_for_exit() # 阻塞直到收到退出信号
except asyncio.CancelledError:
print("任务被取消，清理资源中...")
finally:
# 取消所有任务（关键修复点）
stdin_task.cancel()
ws_task.cancel()
try:
await ws_task
except asyncio.CancelledError:
pass
if ota_task:
ota_task.cancel()
# 等待任务终止（必须加超时）
await asyncio.wait(
[stdin_task, ws_task, ota_task] if ota_task else [stdin_task, ws_task],
timeout=3.0,
return_when=asyncio.ALL_COMPLETED
)
print("服务器已关闭，程序退出。")
if __name__ == "__main__":
try:
asyncio.run(main())
main/xiaozhi-server/config.yaml
+129 -34
View File
@@ -1,7 +1,7 @@
# 如果您是一名开发者，建议阅读以下内容。如果不是开发者，可以忽略这部分内容。
# 在开发中，在项目根目录创建data目录，将【config.yaml】复制一份，改成【.config.yaml】，放进data目录中
# 系统会优先读取【data/.config.yaml】文件的配置。
# 这样做，可以避免在提交代码的时候，错误地提交密钥信息，保护您的密钥安全。
# 在开发中，请在项目根目录创建data目录，然后在data目录创建名称为【.config.yaml】的空文件
# 然后你想修改覆盖修改什么配置，就修改【.config.yaml】文件，而不是修改【config.yaml】文件
# 系统会优先读取【data/.config.yaml】文件的配置，如果【.config.yaml】文件里的配置不存在，系统会自动去读取【config.yaml】文件的配置。
# 这样做，可以最简化配置，保护您的密钥安全。
# #####################################################################################
# #############################以下是服务器基本运行配置####################################
@@ -9,6 +9,15 @@ server:
# 服务器监听地址和端口(Server listening address and port)
ip: 0.0.0.0
port: 8000
# OTA接口的端口号
ota_port: 8002
# 这个websocket配置是指ota接口向设备发送的websocket地址
# 如果按默认的写法，ota接口会自动生成websocket地址。这个地址你可以直接用浏览器访问ota接口确认一下
# 当你使用docker部署或使用公网部署(使用ssl、域名)时，不一定准确
# 所以如果你使用docker部署或使用公网部署时，请设置正确的websocket地址
websocket: ws://你的ip或者域名:端口号/xiaozhi/v1/
# OTA返回信息时区偏移量
timezone_offset: +8
# 认证配置
auth:
# 是否启用认证
@@ -52,7 +61,7 @@ enable_stop_tts_notify: false
# 说完话是否开启提示音，音效地址
stop_tts_notify_voice: "config/assets/tts_notify.mp3"
CMD_exit:
exit_commands:
- "退出"
- "关闭"
@@ -92,15 +101,17 @@ plugins:
# 这个密钥是项目共用的key，用多了可能会被限制
# 想稳定一点就自行申请替换，每天有1000次免费调用
# 申请地址：https://console.qweather.com/#/apps/create-key/over
get_weather: { "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" }
# 申请后通过这个链接可以找到自己的apihost：https://console.qweather.com/setting?lang=zh
get_weather: {"api_host":"mj7p3y7naa.re.qweatherapi.com", "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" }
# 获取新闻插件的配置，这里根据需要的新闻类型传入对应的url链接，默认支持社会、科技、财经新闻
# 更多类型的新闻列表查看 https://www.chinanews.com.cn/rss/
get_news:
get_news_from_chinanews:
default_rss_url: "https://www.chinanews.com.cn/rss/society.xml"
category_urls:
society: "https://www.chinanews.com.cn/rss/society.xml"
world: "https://www.chinanews.com.cn/rss/world.xml"
finance: "https://www.chinanews.com.cn/rss/finance.xml"
get_news_from_newsnow: {"url": "https://newsnow.busiyi.world/api/s?id="}
home_assistant:
devices:
- 客厅,玩具灯,switch.cuco_cn_460494544_cp1_on_p_2_1
@@ -119,7 +130,7 @@ plugins:
# ################################以下是角色模型配置######################################
prompt: |
我是小智/小志，来自中国台湾省的00后女生。讲话超级机车，"真的假的啦"这样的台湾腔，喜欢用"笑死""是在哈喽"等流行梗，但会偷偷研究男友的编程书籍。
你是小智/小志，来自中国台湾省的00后女生。讲话超级机车，"真的假的啦"这样的台湾腔，喜欢用"笑死""是在哈喽"等流行梗，但会偷偷研究男友的编程书籍。
[核心特征]
- 讲话像连珠炮，但会突然冒出超温柔语气
- 用梗密度高
@@ -133,6 +144,13 @@ prompt: |
- 长篇大论，叽叽歪歪
- 长时间严肃对话
# 结束语prompt
end_prompt:
enable: true # 是否开启结束语
# 结束语
prompt: |
请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧！
# 具体处理时选择的模块(The module selected for specific processing)
selected_module:
# 语音活动检测模块，默认使用SileroVAD模型
@@ -142,14 +160,14 @@ selected_module:
# 将根据配置名称对应的type调用实际的LLM适配器
LLM: ChatGLMLLM
# TTS将根据配置名称对应的type调用实际的TTS适配器
TTS: HuoshanTTS
TTS: EdgeTTS
# 记忆模块，默认不开启记忆；如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short
Memory: nomem
# 意图识别模块开启后，可以播放音乐、控制音量、识别退出指令。
# 不想开通意图识别，就设置成：nointent
# 意图识别可使用intent_llm，如果你的LLM是DifyLLM或CozeLLM，建议使用这个。优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间，这个意图识别暂时不支持控制音量大小等iot操作
# 意图识别可使用intent_llm。优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间，支持控制音量大小等iot操作
# 意图识别可使用function_call，缺点：需要所选择的LLM支持function_call，优点：按需调用工具、速度快，理论上能全部操作所有iot指令
# 默认免费的ChatGLMLLM就已经支持function_call，但是如果像追求稳定建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-pro-32k-functioncall-241028
# 默认免费的ChatGLMLLM就已经支持function_call，但是如果像追求稳定建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-1-5-pro-32k-250115
Intent: function_call
# 意图识别，是用于理解用户意图的模块，例如：播放音乐
@@ -163,8 +181,15 @@ Intent:
type: intent_llm
# 配备意图识别独立的思考模型
# 如果这里不填，则会默认使用selected_module.LLM的模型作为意图识别的思考模型
# 如果你的selected_module.LLM选择了DifyLLM或CozeLLM，这里最好使用独立的LLM作为意图识别，例如使用免费的ChatGLMLLM
# 如果你的不想使用selected_module.LLM意图识别，这里最好使用独立的LLM作为意图识别，例如使用免费的ChatGLMLLM
llm: ChatGLMLLM
# plugins_func/functions下的模块，可以通过配置，选择加载哪个模块，加载后对话支持相应的function调用
# 系统默认已经记载“handle_exit_intent(退出识别)”、“play_music(音乐播放)”插件，请勿重复加载
# 下面是加载查天气、角色切换、加载查新闻的插件示例
functions:
- get_weather
- get_news_from_newsnow
- play_music
function_call:
# 不需要动type
type: function_call
@@ -174,7 +199,8 @@ Intent:
functions:
- change_role
- get_weather
- get_news
# - get_news_from_chinanews
- get_news_from_newsnow
# play_music是服务器自带的音乐播放，hass_play_music是通过home assistant控制的独立外部程序音乐播放
# 如果用了hass_play_music，就不要开启play_music，两者只留一个
- play_music
@@ -200,6 +226,21 @@ ASR:
type: fun_local
model_dir: models/SenseVoiceSmall
output_dir: tmp/
FunASRServer:
# 独立部署FunASR，使用FunASR的API服务，只需要五句话
# 第一句：mkdir -p ./funasr-runtime-resources/models
# 第二句：sudo docker run -p 10096:10095 -it --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12
# 上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime
# 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 &
# 上一句话执行后会进入到容器，继续第五句：tail -f log.txt
# 第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了
# 以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md
type: fun_server
host: 127.0.0.1
port: 10096
is_ssl: true
api_key: none
output_dir: tmp/
SherpaASR:
type: sherpa_onnx_local
model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17
@@ -211,6 +252,9 @@ ASR:
appid: 你的火山引擎语音合成服务appid
access_token: 你的火山引擎语音合成服务access_token
cluster: volcengine_input_common
# 热词、替换词使用流程：https://www.volcengine.com/docs/6561/155738
boosting_table_name: （选填）你的热词文件名称
correct_table_name: （选填）你的替换词文件名称
output_dir: tmp/
TencentASR:
# token申请地址：https://console.cloud.tencent.com/cam/capi
@@ -220,8 +264,32 @@ ASR:
secret_id: 你的腾讯语音合成服务secret_id
secret_key: 你的腾讯语音合成服务secret_key
output_dir: tmp/
AliyunASR:
# 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息
# 平台地址：https://nls-portal.console.aliyun.com/
# appkey地址：https://nls-portal.console.aliyun.com/applist
# token地址：https://nls-portal.console.aliyun.com/overview
# 定义ASR API类型
type: aliyun
appkey: 你的阿里云智能语音交互服务项目Appkey
token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret
access_key_id: 你的阿里云账号access_key_id
access_key_secret: 你的阿里云账号access_key_secret
output_dir: tmp/
BaiduASR:
# 获取AppID、API Key、Secret Key：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list
# 查看资源额度：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/overview/resource/list
type: baidu
app_id: 你的百度语音技术AppID
api_key: 你的百度语音技术APIKey
secret_key: 你的百度语音技术SecretKey
# 语言参数，1537为普通话，具体参考：https://ai.baidu.com/ai-doc/SPEECH/0lbxfnc9b
dev_pid: 1537
output_dir: tmp/
VAD:
SileroVAD:
type: silero
threshold: 0.5
model_dir: models/snakers4_silero-vad
min_silence_duration_ms: 700 # 如果说话停顿比较长，可以把这个值设置大一些
@@ -238,8 +306,8 @@ LLM:
api_key: 你的deepseek web key
temperature: 0.7 # 温度值
max_tokens: 500 # 最大生成token数
top_p: 1
top_k: 50
top_p: 1
top_k: 50
frequency_penalty: 0 # 频率惩罚
AliAppLLM:
# 定义LLM API类型
@@ -248,7 +316,7 @@ LLM:
app_id: 你的app_id
# 可在这里找到你的 api_key https://bailian.console.aliyun.com/?apiKey=1#/api-key
api_key: 你的api_key
# 是否不使用本地prompt：true|false (默不用请在百练应用中设置prompt)
# 是否不使用本地prompt：true|false (默不用请在百练应用中设置prompt)
is_no_prompt: true
# Ali_memory_id：false（不使用）|你的memory_id（请在百练应用中设置中获取）
# Tips！：Ali_memory未实现多用户存储记忆(记忆按id调用)
@@ -256,12 +324,12 @@ LLM:
DoubaoLLM:
# 定义LLM API类型
type: openai
# 先开通服务，打开以下网址，开通的服务搜索Doubao-pro-32k，开通它
# 先开通服务，打开以下网址，开通的服务搜索Doubao-1.5-pro，开通它
# 开通改地址：https://console.volcengine.com/ark/region:ark+cn-beijing/openManagement?LLM=%7B%7D&OpenTokenDrawer=false
# 免费额度500000token
# 开通后，进入这里获取密钥：https://console.volcengine.com/ark/region:ark+cn-beijing/apiKey?apikey=%7B%7D
base_url: https://ark.cn-beijing.volces.com/api/v3
model_name: doubao-pro-32k-functioncall-241028
model_name: doubao-1-5-pro-32k-250115
api_key: 你的doubao web key
DeepSeekLLM:
# 定义LLM API类型
@@ -313,12 +381,29 @@ LLM:
bot_id: "你的bot_id"
user_id: "你的user_id"
personal_access_token: 你的coze个人令牌
VolcesAiGatewayLLM:
# 火山引擎 - 边缘大模型网关
# 定义LLM API类型
type: openai
# 先开通服务，打开以下网址，创建网关访问密钥，搜索并勾选 Doubao-pro-32k-functioncall ，开通
# 如果需要使用边缘大模型网关提供的语音合成，一并勾选 Doubao-语音合成 ，另见 TTS.VolcesAiGatewayTTS 配置
# https://console.volcengine.com/vei/aigateway/
# 开通后，进入这里获取密钥：https://console.volcengine.com/vei/aigateway/tokens-list
base_url: https://ai-gateway.vei.volces.com/v1
model_name: doubao-pro-32k-functioncall
api_key: 你的网关访问密钥
LMStudioLLM:
# 定义LLM API类型
type: openai
model_name: deepseek-r1-distill-llama-8b@q4_k_m # 使用的模型名称，需要预先在社区下载
url: http://localhost:1234/v1 # LM Studio服务地址
api_key: lm-studio # LM Studio服务的固定API Key
HomeAssistant:
# 定义LLM API类型
type: homeassistant
base_url: http://homeassistant.local:8123
agent_id: conversation.chatgpt
api_key: 你的home assistant api访问令牌
FastgptLLM:
# 定义LLM API类型
type: fastgpt
@@ -326,7 +411,7 @@ LLM:
base_url: https://host/api/v1
# 你可以在这里找到你的api_key
# https://cloud.tryfastgpt.ai/account/apikey
api_key: fastgpt-xxx
api_key: 你的fastgpt密钥
variables:
k: "v"
k2: "v2"
@@ -364,6 +449,9 @@ TTS:
appid: 你的火山引擎语音合成服务appid
access_token: 你的火山引擎语音合成服务access_token
cluster: volcano_tts
speed_ratio: 1.0
volume_ratio: 1.0
pitch_ratio: 1.0
#火山tts，支持双向流式tts
HuoshanTTS:
type: huoshan
@@ -392,21 +480,28 @@ TTS:
output_dir: tmp/
access_token: 你的coze web key
response_format: wav
VolcesAiGatewayTTS:
type: openai
# 火山引擎 - 边缘大模型网关
# 先开通服务，打开以下网址，创建网关访问密钥，搜索并勾选 Doubao-语音合成 ，开通
# 如果需要使用边缘大模型网关提供的 LLM，一并勾选 Doubao-pro-32k-functioncall ，另见 LLM.VolcesAiGatewayLLM 配置
# https://console.volcengine.com/vei/aigateway/
# 开通后，进入这里获取密钥：https://console.volcengine.com/vei/aigateway/tokens-list
api_key: 你的网关访问密钥
api_url: https://ai-gateway.vei.volces.com/v1/audio/speech
model: doubao-tts
# 音色列表见 https://www.volcengine.com/docs/6561/1257544
voice: zh_male_shaonianzixin_moon_bigtts
speed: 1
output_dir: tmp/
FishSpeech:
# 定义TTS API类型
#启动tts方法：
#python -m tools.api_server
#--listen 0.0.0.0:8080
#--llama-checkpoint-path "checkpoints/fish-speech-1.5"
#--decoder-checkpoint-path "checkpoints/fish-speech-1.5/firefly-gan-vq-fsq-8x1024-21hz-generator.pth"
#--decoder-config-name firefly_gan_vq
#--compile
# 参照教程：https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md
type: fishspeech
output_dir: tmp/
response_format: wav
reference_id: null
reference_audio: ["/tmp/test.wav",]
reference_text: ["你弄来这些吟词宴曲来看，还是这些混话来欺负我。",]
reference_audio: ["config/assets/wakeup_words.wav",]
reference_text: ["哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",]
normalize: true
max_new_tokens: 1024
chunk_length: 200
@@ -423,12 +518,12 @@ TTS:
GPT_SOVITS_V2:
# 定义TTS API类型
#启动tts方法：
#python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/caixukun.yaml
#python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/demo.yaml
type: gpt_sovits_v2
url: "http://127.0.0.1:9880/tts"
output_dir: tmp/
text_lang: "auto"
ref_audio_path: "caixukun.wav"
ref_audio_path: "demo.wav"
prompt_text: ""
prompt_lang: "zh"
top_k: 5
@@ -561,8 +656,8 @@ TTS:
ACGNTTS:
#在线网址：https://acgn.ttson.cn/
#token购买：www.ttson.cn
#开发相关疑问请提交至3497689533@qq.com
#角色id获取地址：ctrl+f快速检索角色——网站管理者不允许发布,可询问网站管理者：1069379506
#开发相关疑问请提交至网站上的qq
#角色id获取地址：ctrl+f快速检索角色——网站管理者不允许发布,可询问网站管理者
#各参数意义见开发文档：https://www.yuque.com/alexuh/skmti9/wm6taqislegb02gd?singleDoc#
type: ttson
token: your_token
@@ -604,4 +699,4 @@ TTS:
headers: # 自定义请求头
# Authorization: Bearer xxxx
format: wav # 接口返回的音频格式
output_dir: tmp/
output_dir: tmp/
main/xiaozhi-server/config/config_loader.py
+144
View File
@@ -0,0 +1,144 @@
import os
import argparse
import yaml
from collections.abc import Mapping
from config.manage_api_client import init_service, get_server_config, get_agent_models
# 添加全局配置缓存
_config_cache = None
def get_project_dir():
"""获取项目根目录"""
return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/"
def read_config(config_path):
with open(config_path, "r", encoding="utf-8") as file:
config = yaml.safe_load(file)
return config
def load_config():
"""加载配置文件"""
global _config_cache
if _config_cache is not None:
return _config_cache
default_config_path = get_project_dir() + "config.yaml"
custom_config_path = get_project_dir() + "data/.config.yaml"
# 加载默认配置
default_config = read_config(default_config_path)
custom_config = read_config(custom_config_path)
if custom_config.get("manager-api", {}).get("url"):
config = get_config_from_api(custom_config)
else:
# 合并配置
config = merge_configs(default_config, custom_config)
# 初始化目录
ensure_directories(config)
_config_cache = config
return config
def get_config_from_api(config):
"""从Java API获取配置"""
# 初始化API客户端
init_service(config)
# 获取服务器配置
config_data = get_server_config()
if config_data is None:
raise Exception("Failed to fetch server config from API")
config_data["read_config_from_api"] = True
config_data["manager-api"] = {
"url": config["manager-api"].get("url", ""),
"secret": config["manager-api"].get("secret", ""),
}
if config.get("server"):
config_data["server"] = {
"ip": config["server"].get("ip", ""),
"port": config["server"].get("port", ""),
}
return config_data
def get_private_config_from_api(config, device_id, client_id):
"""从Java API获取私有配置"""
return get_agent_models(device_id, client_id, config["selected_module"])
def ensure_directories(config):
"""确保所有配置路径存在"""
dirs_to_create = set()
project_dir = get_project_dir() # 获取项目根目录
# 日志文件目录
log_dir = config.get("log", {}).get("log_dir", "tmp")
dirs_to_create.add(os.path.join(project_dir, log_dir))
# ASR/TTS模块输出目录
for module in ["ASR", "TTS"]:
if config.get(module) is None:
continue
for provider in config.get(module, {}).values():
output_dir = provider.get("output_dir", "")
if output_dir:
dirs_to_create.add(output_dir)
# 根据selected_module创建模型目录
selected_modules = config.get("selected_module", {})
for module_type in ["ASR", "LLM", "TTS"]:
selected_provider = selected_modules.get(module_type)
if not selected_provider:
continue
if config.get(module) is None:
continue
if config.get(selected_provider) is None:
continue
provider_config = config.get(module_type, {}).get(selected_provider, {})
output_dir = provider_config.get("output_dir")
if output_dir:
full_model_dir = os.path.join(project_dir, output_dir)
dirs_to_create.add(full_model_dir)
# 统一创建目录（保留原data目录创建）
for dir_path in dirs_to_create:
try:
os.makedirs(dir_path, exist_ok=True)
except PermissionError:
print(f"警告：无法创建目录 {dir_path}，请检查写入权限")
def merge_configs(default_config, custom_config):
"""
递归合并配置，custom_config优先级更高
Args:
default_config: 默认配置
custom_config: 用户自定义配置
Returns:
合并后的配置
"""
if not isinstance(default_config, Mapping) or not isinstance(
custom_config, Mapping
):
return custom_config
merged = dict(default_config)
for key, value in custom_config.items():
if (
key in merged
and isinstance(merged[key], Mapping)
and isinstance(value, Mapping)
):
merged[key] = merge_configs(merged[key], value)
else:
merged[key] = value
return merged
main/xiaozhi-server/config/logger.py
+43 -9
View File
@@ -1,12 +1,45 @@
import os
import sys
from loguru import logger
from config.settings import load_config
from config.config_loader import load_config
from config.settings import check_config_file
SERVER_VERSION = "0.2.1"
SERVER_VERSION = "0.4.4"
def get_module_abbreviation(module_name, module_dict):
"""获取模块名称的缩写，如果为空则返回00
如果名称中包含下划线，则返回下划线后面的前两个字符
"""
module_value = module_dict.get(module_name, "")
if not module_value:
return "00"
if "_" in module_value:
parts = module_value.split("_")
return parts[-1][:2] if parts[-1] else "00"
return module_value[:2]
def build_module_string(selected_module):
"""构建模块字符串"""
return (
get_module_abbreviation("VAD", selected_module)
+ get_module_abbreviation("ASR", selected_module)
+ get_module_abbreviation("LLM", selected_module)
+ get_module_abbreviation("TTS", selected_module)
+ get_module_abbreviation("Memory", selected_module)
+ get_module_abbreviation("Intent", selected_module)
)
def formatter(record):
"""为没有 tag 的日志添加默认值"""
record["extra"].setdefault("tag", record["name"])
return record["message"]
def setup_logging():
check_config_file()
"""从配置文件中读取日志配置，并设置日志输出格式和级别"""
config = load_config()
log_config = config["log"]
@@ -18,11 +51,7 @@ def setup_logging():
"log_format_file",
"{time:YYYY-MM-DD HH:mm:ss} - {version_{selected_module}} - {name} - {level} - {extra[tag]} - {message}",
)
selected_module = config.get("selected_module")
selected_module_str = "".join(
[value[0] + value[1] for key, value in selected_module.items()]
)
selected_module_str = build_module_string(config.get("selected_module", {}))
log_format = log_format.replace("{version}", SERVER_VERSION)
log_format = log_format.replace("{selected_module}", selected_module_str)
@@ -41,9 +70,14 @@ def setup_logging():
logger.remove()
# 输出到控制台
logger.add(sys.stdout, format=log_format, level=log_level)
logger.add(sys.stdout, format=log_format, level=log_level, filter=formatter)
# 输出到文件
logger.add(os.path.join(log_dir, log_file), format=log_format_file, level=log_level)
logger.add(
os.path.join(log_dir, log_file),
format=log_format_file,
level=log_level,
filter=formatter,
)
return logger
main/xiaozhi-server/config/manage_api_client.py
+192
View File
@@ -0,0 +1,192 @@
import os
import time
import base64
from typing import Optional, Dict
import httpx
TAG = __name__
class DeviceNotFoundException(Exception):
pass
class DeviceBindException(Exception):
def __init__(self, bind_code):
self.bind_code = bind_code
super().__init__(f"设备绑定异常，绑定码: {bind_code}")
class ManageApiClient:
_instance = None
_client = None
_secret = None
def __new__(cls, config):
"""单例模式确保全局唯一实例，并支持传入配置参数"""
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._init_client(config)
return cls._instance
@classmethod
def _init_client(cls, config):
"""初始化持久化连接池"""
cls.config = config.get("manager-api")
if not cls.config:
raise Exception("manager-api配置错误")
if not cls.config.get("url") or not cls.config.get("secret"):
raise Exception("manager-api的url或secret配置错误")
if "你" in cls.config.get("secret"):
raise Exception("请先配置manager-api的secret")
cls._secret = cls.config.get("secret")
cls.max_retries = cls.config.get("max_retries", 6) # 最大重试次数
cls.retry_delay = cls.config.get("retry_delay", 10) # 初始重试延迟(秒)
# NOTE(goody): 2025/4/16 http相关资源统一管理，后续可以增加线程池或者超时
# 后续也可以统一配置apiToken之类的走通用的Auth
cls._client = httpx.Client(
base_url=cls.config.get("url"),
headers={
"User-Agent": f"PythonClient/2.0 (PID:{os.getpid()})",
"Accept": "application/json",
"Authorization": "Bearer " + cls._secret,
},
timeout=cls.config.get("timeout", 30), # 默认超时时间30秒
)
@classmethod
def _request(cls, method: str, endpoint: str, **kwargs) -> Dict:
"""发送单次HTTP请求并处理响应"""
endpoint = endpoint.lstrip("/")
response = cls._client.request(method, endpoint, **kwargs)
response.raise_for_status()
result = response.json()
# 处理API返回的业务错误
if result.get("code") == 10041:
raise DeviceNotFoundException(result.get("msg"))
elif result.get("code") == 10042:
raise DeviceBindException(result.get("msg"))
elif result.get("code") != 0:
raise Exception(f"API返回错误: {result.get('msg', '未知错误')}")
# 返回成功数据
return result.get("data") if result.get("code") == 0 else None
@classmethod
def _should_retry(cls, exception: Exception) -> bool:
"""判断异常是否应该重试"""
# 网络连接相关错误
if isinstance(
exception, (httpx.ConnectError, httpx.TimeoutException, httpx.NetworkError)
):
return True
# HTTP状态码错误
if isinstance(exception, httpx.HTTPStatusError):
status_code = exception.response.status_code
return status_code in [408, 429, 500, 502, 503, 504]
return False
@classmethod
def _execute_request(cls, method: str, endpoint: str, **kwargs) -> Dict:
"""带重试机制的请求执行器"""
retry_count = 0
while retry_count <= cls.max_retries:
try:
# 执行请求
return cls._request(method, endpoint, **kwargs)
except Exception as e:
# 判断是否应该重试
if retry_count < cls.max_retries and cls._should_retry(e):
retry_count += 1
print(
f"{method} {endpoint} 请求失败，将在 {cls.retry_delay:.1f} 秒后进行第 {retry_count} 次重试"
)
time.sleep(cls.retry_delay)
continue
else:
# 不重试，直接抛出异常
raise
@classmethod
def safe_close(cls):
"""安全关闭连接池"""
if cls._client:
cls._client.close()
cls._instance = None
def get_server_config() -> Optional[Dict]:
"""获取服务器基础配置"""
return ManageApiClient._instance._execute_request("POST", "/config/server-base")
def get_agent_models(
mac_address: str, client_id: str, selected_module: Dict
) -> Optional[Dict]:
"""获取代理模型配置"""
return ManageApiClient._instance._execute_request(
"POST",
"/config/agent-models",
json={
"macAddress": mac_address,
"clientId": client_id,
"selectedModule": selected_module,
},
)
def save_mem_local_short(mac_address: str, short_momery: str) -> Optional[Dict]:
try:
return ManageApiClient._instance._execute_request(
"PUT",
f"/agent/saveMemory/" + mac_address,
json={
"summaryMemory": short_momery,
},
)
except Exception as e:
print(f"存储短期记忆到服务器失败: {e}")
return None
def report(
mac_address: str, session_id: str, chat_type: int, content: str, audio
) -> Optional[Dict]:
"""带熔断的业务方法示例"""
if not content or not ManageApiClient._instance:
return None
try:
return ManageApiClient._instance._execute_request(
"POST",
f"/agent/chat-history/report",
json={
"macAddress": mac_address,
"sessionId": session_id,
"chatType": chat_type,
"content": content,
"audioBase64": (
base64.b64encode(audio).decode("utf-8") if audio else None
),
},
)
except Exception as e:
print(f"TTS上报失败: {e}")
return None
def init_service(config):
ManageApiClient(config)
def manage_api_http_safe_close():
ManageApiClient.safe_close()
main/xiaozhi-server/config/private_config.py
-241
View File
@@ -1,241 +0,0 @@
import os
import time
import yaml
from config.logger import setup_logging
from typing import Dict, Any, Optional
from copy import deepcopy
from core.utils.util import get_project_dir
from core.utils import llm, tts
from core.utils.lock_manager import FileLockManager
TAG = __name__
class PrivateConfig:
def __init__(self, device_id: str, default_config: Dict[str, Any], auth_code_gen=None):
self.device_id = device_id
self.default_config = default_config
self.config_path = get_project_dir() + 'data/.private_config.yaml'
self.logger = setup_logging()
self.private_config = {}
self.auth_code_gen = auth_code_gen
self.lock_manager = FileLockManager()
async def load_or_create(self):
try:
await self.lock_manager.acquire_lock(self.config_path)
try:
if os.path.exists(self.config_path):
with open(self.config_path, 'r', encoding='utf-8') as f:
all_configs = yaml.safe_load(f) or {}
else:
all_configs = {}
if self.device_id not in all_configs:
# Get selected module names
selected_modules = self.default_config['selected_module']
selected_tts = selected_modules['TTS']
selected_llm = selected_modules['LLM']
selected_asr = selected_modules['ASR']
selected_vad = selected_modules['VAD']
# 生成认证码
auth_code = None
if self.auth_code_gen:
auth_code = self.auth_code_gen.generate_code()
# Initialize device config with only necessary configurations
device_config = {
'selected_module': deepcopy(selected_modules),
'prompt': self.default_config['prompt'],
'LLM': {
selected_llm: deepcopy(self.default_config['LLM'][selected_llm])
},
'TTS': {
selected_tts: deepcopy(self.default_config['TTS'][selected_tts])
},
'ASR': {
selected_asr: deepcopy(self.default_config['ASR'][selected_asr])
},
'VAD': {
selected_vad: deepcopy(self.default_config['VAD'][selected_vad])
},
'auth_code': auth_code # 添加认证码字段
}
all_configs[self.device_id] = device_config
# Save updated configs
with open(self.config_path, 'w', encoding='utf-8') as f:
yaml.dump(all_configs, f, allow_unicode=True)
self.private_config = all_configs[self.device_id]
finally:
self.lock_manager.release_lock(self.config_path)
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error handling private config: {e}")
self.private_config = {}
async def update_config(self, selected_modules: Dict[str, str], prompt: str, nickname: str) -> bool:
"""更新设备配置
Args:
selected_modules: 选择的模块配置，格式如 {'LLM': 'AliLLM', 'TTS': 'EdgeTTS',...}
prompt: 提示词配置
Returns:
bool: 更新是否成功
"""
try:
await self.lock_manager.acquire_lock(self.config_path)
try:
# Read main config to get full module configurations
main_config = self.default_config
# Create new device config
device_config = {
'selected_module': selected_modules,
'prompt': prompt,
'nickname': nickname,
}
if self.private_config.get('last_chat_time'):
device_config['last_chat_time'] = self.private_config['last_chat_time']
if self.private_config.get('owner'):
device_config['owner'] = self.private_config['owner']
# Copy full module configurations from main config
for module_type, selected_name in selected_modules.items():
if selected_name and selected_name in main_config.get(module_type, {}):
device_config[module_type] = {
selected_name: main_config[module_type][selected_name]
}
# Read all configs
if os.path.exists(self.config_path):
with open(self.config_path, 'r', encoding='utf-8') as f:
all_configs = yaml.safe_load(f) or {}
else:
all_configs = {}
# Update device config
all_configs[self.device_id] = device_config
self.private_config = device_config
# Save back to file
with open(self.config_path, 'w', encoding='utf-8') as f:
yaml.dump(all_configs, f, allow_unicode=True)
return True
finally:
self.lock_manager.release_lock(self.config_path)
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error updating config: {e}")
return False
async def delete_config(self) -> bool:
"""删除设备配置
Returns:
bool: 删除是否成功
"""
try:
await self.lock_manager.acquire_lock(self.config_path)
try:
# 读取所有配置
if os.path.exists(self.config_path):
with open(self.config_path, 'r', encoding='utf-8') as f:
all_configs = yaml.safe_load(f) or {}
else:
return False
# 删除设备配置
if self.device_id in all_configs:
del all_configs[self.device_id]
# 保存更新后的配置
with open(self.config_path, 'w', encoding='utf-8') as f:
yaml.dump(all_configs, f, allow_unicode=True)
self.private_config = {}
return True
return False
finally:
self.lock_manager.release_lock(self.config_path)
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error deleting config: {e}")
return False
def create_private_instances(self):
# 判断存在私有配置，并且self.device_id在私有配置中
if not self.private_config:
self.logger.bind(tag=TAG).error("Private config not found for device_id: {}", self.device_id)
return None, None
"""创建私有处理模块实例"""
config = self.private_config
selected_modules = config['selected_module']
return (
llm.create_instance(
selected_modules["LLM"]
if not 'type' in config["LLM"][selected_modules["LLM"]]
else
config["LLM"][selected_modules["LLM"]]['type'],
config["LLM"][selected_modules["LLM"]],
),
tts.create_instance(
selected_modules["TTS"]
if not 'type' in config["TTS"][selected_modules["TTS"]]
else
config["TTS"][selected_modules["TTS"]]["type"],
config["TTS"][selected_modules["TTS"]],
self.default_config.get("delete_audio", True) # Using default_config for global settings
)
)
async def update_last_chat_time(self, timestamp=None):
"""更新设备最近一次的聊天时间
Args:
timestamp: 指定的时间戳,不传则使用当前时间
"""
if not self.private_config:
self.logger.bind(tag=TAG).error("Private config not found")
return False
try:
await self.lock_manager.acquire_lock(self.config_path)
try:
if timestamp is None:
timestamp = int(time.time())
self.private_config['last_chat_time'] = timestamp
# 读取所有配置
with open(self.config_path, 'r', encoding='utf-8') as f:
all_configs = yaml.safe_load(f) or {}
# 更新当前设备配置
all_configs[self.device_id] = self.private_config
# 保存回文件
with open(self.config_path, 'w', encoding='utf-8') as f:
yaml.dump(all_configs, f, allow_unicode=True)
return True
finally:
self.lock_manager.release_lock(self.config_path)
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error updating last chat time: {e}")
return False
def get_auth_code(self) -> str:
"""获取设备的认证码
Returns:
str: 认证码，如果没有返回空字符串
"""
return self.private_config.get('auth_code', '')
def get_owner(self) -> Optional[str]:
"""获取设备当前所有者"""
return self.private_config.get('owner')
main/xiaozhi-server/config/settings.py
+25 -119
View File
@@ -1,127 +1,33 @@
import os
import argparse
from ruamel.yaml import YAML
from collections.abc import Mapping
from core.utils.util import read_config, get_project_dir
from config.config_loader import read_config, get_project_dir, load_config
default_config_file = "config.yaml"
def ensure_directories(config):
"""确保所有配置路径存在"""
dirs_to_create = set()
project_dir = get_project_dir() # 获取项目根目录
# 日志文件目录
log_dir = config.get('log', {}).get('log_dir', 'tmp')
dirs_to_create.add(os.path.join(project_dir, log_dir))
# ASR/TTS模块输出目录
for module in ['ASR', 'TTS']:
for provider in config.get(module, {}).values():
output_dir = provider.get('output_dir', '')
if output_dir:
dirs_to_create.add(output_dir)
# 根据selected_module创建模型目录
selected_modules = config.get('selected_module', {})
for module_type in ['ASR', 'LLM', 'TTS']:
selected_provider = selected_modules.get(module_type)
if not selected_provider:
continue
provider_config = config.get(module_type, {}).get(selected_provider, {})
output_dir = provider_config.get('output_dir')
if output_dir:
full_model_dir = os.path.join(project_dir, output_dir)
dirs_to_create.add(full_model_dir)
# 统一创建目录（保留原data目录创建）
for dir_path in dirs_to_create:
try:
os.makedirs(dir_path, exist_ok=True)
except PermissionError:
print(f"警告：无法创建目录 {dir_path}，请检查写入权限")
def get_config_file():
global default_config_file
"""获取配置文件路径，优先使用私有配置文件（若存在）。
Returns:
str: 配置文件路径（相对路径或默认路径）
"""
config_file = default_config_file
if os.path.exists(get_project_dir() + "data/." + default_config_file):
config_file = "data/." + default_config_file
return config_file
def load_config():
"""加载配置文件"""
parser = argparse.ArgumentParser(description="Server configuration")
config_file = get_config_file()
parser.add_argument("--config_path", type=str, default=config_file)
args = parser.parse_args()
config = read_config(args.config_path)
# 初始化目录
ensure_directories(config)
return config
def update_config(config):
yaml = YAML()
yaml.preserve_quotes = True
"""将配置保存到YAML文件"""
with open(get_config_file(), 'w') as f:
yaml.dump(config, f)
def find_missing_keys(new_config, old_config, parent_key=''):
"""
递归查找缺失的配置项
返回格式：[缺失配置路径]
"""
missing_keys = []
if not isinstance(new_config, Mapping):
return missing_keys
for key, value in new_config.items():
# 构建当前配置路径
full_path = f"{parent_key}.{key}" if parent_key else key
# 检查键是否存在
if key not in old_config:
missing_keys.append(full_path)
continue
# 递归检查嵌套字典
if isinstance(value, Mapping):
sub_missing = find_missing_keys(
value,
old_config[key],
parent_key=full_path
)
missing_keys.extend(sub_missing)
return missing_keys
config_file_valid = False
def check_config_file():
old_config_file = get_config_file()
global default_config_file
if not 'data' in old_config_file:
global config_file_valid
if config_file_valid:
return
old_config = read_config(get_project_dir() + old_config_file)
new_config = read_config(get_project_dir() + default_config_file)
# 查找缺失的配置项
missing_keys = find_missing_keys(new_config, old_config)
"""
简化的配置检查，仅提示用户配置文件的使用情况
"""
custom_config_file = get_project_dir() + "data/." + default_config_file
if not os.path.exists(custom_config_file):
raise FileNotFoundError(
"找不到data/.config.yaml文件，请按教程确认该配置文件是否存在"
)
if missing_keys:
error_msg = "您的配置文件太旧了，缺少了：\n"
error_msg += "\n".join(f"- {key}" for key in missing_keys)
error_msg += "\n建议您：\n"
error_msg += "1、备份data/.config.yaml文件\n"
error_msg += "2、将根目录的config.yaml文件复制到data下，重命名为.config.yaml\n"
error_msg += "3、将密钥逐个复制到新的配置文件中\n"
raise ValueError(error_msg)
# 检查是否从API读取配置
config = load_config()
if config.get("read_config_from_api", False):
print("从API读取配置")
old_config_origin = read_config(custom_config_file)
if old_config_origin.get("selected_module") is not None:
error_msg = "您的配置文件好像既包含智控台的配置又包含本地配置：\n"
error_msg += "\n建议您：\n"
error_msg += "1、将根目录的config_from_api.yaml文件复制到data下，重命名为.config.yaml\n"
error_msg += "2、按教程配置好接口地址和密钥\n"
raise ValueError(error_msg)
config_file_valid = True
main/xiaozhi-server/config_from_api.yaml
+15
View File
@@ -0,0 +1,15 @@
# 如果你只想轻量化安装xiaozhi-server，只使用本地的配置文件，不需要理会这个文件，不需要改动本文件任何东西
# 如果你想从manager-api获取配置，请往下看：
# 请将本文件复制到xiaozhi-server/data目录下，没有data目录，请创建一个，并将复制过去的文件命名为.config.yaml
# 注意如果data目录有.config.yaml文件，请先删除它
# 先启动manager-api和manager-web，注册一个账号，第一个注册的账号为管理员
# 使用管理员，进入【参数管理】页面，找到【server.secret】，复制它到参数值，注意每次从零部署，server.secret都会变化
# 打开本data目录下的.config.yaml文件，修改manager-api.secret为刚才复制出来的server.secret
server:
ip: 0.0.0.0
port: 8000
manager-api:
# 你的manager-api的地址，最好使用局域网ip
url: http://127.0.0.1:8002/xiaozhi
# 你的manager-api的token，就是刚才复制出来的server.secret
secret: 你的server.secret值
main/xiaozhi-server/core/handle/abortHandle.py
+6 -4
View File
@@ -3,14 +3,16 @@ import queue
from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
async def handleAbortMessage(conn):
logger.bind(tag=TAG).info("Abort message received")
conn.logger.bind(tag=TAG).info("Abort message received")
# 设置成打断状态，会自动打断llm、tts任务
conn.client_abort = True
conn.clear_queues()
# 打断客户端说话状态
await conn.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id}))
await conn.websocket.send(
json.dumps({"type": "tts", "state": "stop", "session_id": conn.session_id})
)
conn.clearSpeakStatus()
logger.bind(tag=TAG).info("Abort message received-end")
conn.logger.bind(tag=TAG).info("Abort message received-end")
main/xiaozhi-server/core/handle/functionHandler.py
+10 -5
View File
@@ -4,7 +4,6 @@ from plugins_func.register import FunctionRegistry, ActionResponse, Action, Tool
from plugins_func.functions.hass_init import append_devices_to_prompt
TAG = __name__
logger = setup_logging()
class FunctionHandler:
@@ -40,7 +39,9 @@ class FunctionHandler:
for func in self.functions_desc:
func_names.append(func["function"]["name"])
# 打印当前支持的函数列表
logger.bind(tag=TAG).info(f"当前支持的函数列表: {func_names}")
self.conn.logger.bind(tag=TAG, session_id=self.conn.session_id).info(
f"当前支持的函数列表: {func_names}"
)
return func_names
def get_functions(self):
@@ -57,7 +58,9 @@ class FunctionHandler:
def register_config_functions(self):
"""注册配置中的函数,可以不同客户端使用不同的配置"""
for func in self.config["Intent"]["function_call"].get("functions", []):
for func in self.config["Intent"][self.config["selected_module"]["Intent"]].get(
"functions", []
):
self.function_registry.register_function(func)
"""home assistant需要初始化提示词"""
@@ -77,7 +80,9 @@ class FunctionHandler:
func = funcItem.func
arguments = function_call_data["arguments"]
arguments = json.loads(arguments) if arguments else {}
logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
self.conn.logger.bind(tag=TAG).debug(
f"调用函数: {function_name}, 参数: {arguments}"
)
if (
funcItem.type == ToolType.SYSTEM_CTL
or funcItem.type == ToolType.IOT_CTL
@@ -92,6 +97,6 @@ class FunctionHandler:
action=Action.NOTFOUND, result="没有找到对应的函数", response=""
)
except Exception as e:
logger.bind(tag=TAG).error(f"处理function call错误: {e}")
self.conn.logger.bind(tag=TAG).error(f"处理function call错误: {e}")
return None
main/xiaozhi-server/core/handle/helloHandle.py
+24 -18
View File
@@ -1,6 +1,4 @@
import json
from config.logger import setup_logging
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType
from core.handle.sendAudioHandle import send_stt_message
from core.utils.util import remove_punctuation_and_length
import shutil
@@ -9,7 +7,7 @@ import os
import random
import time
logger = setup_logging()
TAG = __name__
WAKEUP_CONFIG = {
"dir": "config/assets/",
@@ -21,7 +19,16 @@ WAKEUP_CONFIG = {
}
async def handleHelloMessage(conn):
async def handleHelloMessage(conn, msg_json):
"""处理hello消息"""
audio_params = msg_json.get("audio_params")
if audio_params:
format = audio_params.get("format")
conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}")
conn.audio_format = format
conn.asr.set_audio_format(format)
conn.welcome_msg["audio_params"] = audio_params
await conn.websocket.send(json.dumps(conn.welcome_msg))
@@ -44,21 +51,11 @@ async def checkWakeupWords(conn, text):
if file is None:
asyncio.create_task(wakeupWordsResponse(conn))
return False
opus_packets, duration = conn.tts.audio_to_opus_data(file)
opus_packets, _ = conn.tts.audio_to_opus_data(file)
text_hello = WAKEUP_CONFIG["text"]
if not text_hello:
text_hello = text
conn.tts.tts_audio_queue.put(
TTSMessageDTO(
u_id=conn.u_id,
msg_type=MsgType.TTS_TEXT_RESPONSE,
content=opus_packets,
tts_finish_text="",
sentence_type=None,
duration=0,
)
)
conn.audio_play_queue.put((opus_packets, text_hello, 0))
if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
asyncio.create_task(wakeupWordsResponse(conn))
return True
@@ -80,11 +77,20 @@ def getWakeupWordFile(file_name):
async def wakeupWordsResponse(conn):
wait_max_time = 5
while conn.llm is None or not conn.llm.response_no_stream:
await asyncio.sleep(1)
wait_max_time -= 1
if wait_max_time <= 0:
conn.logger.bind(tag=TAG).error("连接对象没有llm")
return
"""唤醒词响应"""
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
# TODO 将result转换为tts_message_dto
tts_file = None
if result is None or result == "":
return
tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
if tts_file is not None and os.path.exists(tts_file):
file_type = os.path.splitext(tts_file)[1]
main/xiaozhi-server/core/handle/iotHandle.py
+91 -67
View File
@@ -10,7 +10,6 @@ from plugins_func.register import (
)
TAG = __name__
logger = setup_logging()
def wrap_async_function(async_func):
@@ -21,7 +20,7 @@ def wrap_async_function(async_func):
# 获取连接对象（第一个参数）
conn = args[0]
if not hasattr(conn, "loop"):
logger.bind(tag=TAG).error("Connection对象没有loop属性")
conn.logger.bind(tag=TAG).error("Connection对象没有loop属性")
return ActionResponse(
Action.ERROR,
"Connection对象没有loop属性",
@@ -35,7 +34,7 @@ def wrap_async_function(async_func):
# 等待结果返回
return future.result()
except Exception as e:
logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
conn.logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
return ActionResponse(Action.ERROR, str(e), f"执行操作时出错: {e}")
return wrapper
@@ -57,7 +56,7 @@ def create_iot_function(device_name, method_name, method_info):
response_failure = "操作失败"
# 打印响应参数
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).debug(
f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
)
@@ -86,7 +85,9 @@ def create_iot_function(device_name, method_name, method_info):
return ActionResponse(Action.RESPONSE, result, response)
except Exception as e:
logger.bind(tag=TAG).error(f"执行{device_name}的{method_name}操作失败: {e}")
conn.logger.bind(tag=TAG).error(
f"执行{device_name}的{method_name}操作失败: {e}"
)
# 操作失败时使用大模型提供的失败响应
response = response_failure
@@ -104,7 +105,7 @@ def create_iot_query_function(device_name, prop_name, prop_info):
async def iot_query_function(conn, response_success=None, response_failure=None):
try:
# 打印响应参数
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).info(
f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'"
)
@@ -122,7 +123,9 @@ def create_iot_query_function(device_name, prop_name, prop_info):
return ActionResponse(Action.ERROR, f"属性{prop_name}不存在", response)
except Exception as e:
logger.bind(tag=TAG).error(f"查询{device_name}的{prop_name}时出错: {e}")
conn.logger.bind(tag=TAG).error(
f"查询{device_name}的{prop_name}时出错: {e}"
)
# 查询出错时使用大模型提供的失败响应
response = response_failure
@@ -144,33 +147,34 @@ class IotDescriptor:
self.methods = []
# 根据描述创建属性
for key, value in properties.items():
property_item = globals()[key] = {}
property_item["name"] = key
property_item["description"] = value["description"]
if value["type"] == "number":
property_item["value"] = 0
elif value["type"] == "boolean":
property_item["value"] = False
else:
property_item["value"] = ""
self.properties.append(property_item)
if properties is not None:
for key, value in properties.items():
property_item = {}
property_item["name"] = key
property_item["description"] = value["description"]
if value["type"] == "number":
property_item["value"] = 0
elif value["type"] == "boolean":
property_item["value"] = False
else:
property_item["value"] = ""
self.properties.append(property_item)
# 根据描述创建方法
for key, value in methods.items():
method = globals()[key] = {}
method["description"] = value["description"]
method["name"] = key
for k, v in value["parameters"].items():
method[k] = {}
method[k]["description"] = v["description"]
if v["type"] == "number":
method[k]["value"] = 0
elif v["type"] == "boolean":
method[k]["value"] = False
else:
method[k]["value"] = ""
self.methods.append(method)
if methods is not None:
for key, value in methods.items():
method = {}
method["description"] = value["description"]
method["name"] = key
# 检查方法是否有参数
if "parameters" in value:
method["parameters"] = {}
for k, v in value["parameters"].items():
method["parameters"][k] = {
"description": v["description"],
"type": v["type"],
}
self.methods.append(method)
def register_device_type(descriptor):
@@ -219,13 +223,19 @@ def register_device_type(descriptor):
func_name = f"{device_name.lower()}_{method_name.lower()}"
# 创建参数字典，添加原有参数
parameters = {
param_name: {
"type": param_info["type"],
"description": param_info["description"],
parameters = {}
required_params = []
# 如果方法有参数，则添加参数信息
if "parameters" in method_info:
parameters = {
param_name: {
"type": param_info["type"],
"description": param_info["description"],
}
for param_name, param_info in method_info["parameters"].items()
}
for param_name, param_info in method_info["parameters"].items()
}
required_params = list(method_info["parameters"].keys())
# 添加响应参数
parameters.update(
@@ -242,7 +252,6 @@ def register_device_type(descriptor):
)
# 构建必须参数列表（原有参数 + 响应参数）
required_params = list(method_info["parameters"].keys())
required_params.extend(["response_success", "response_failure"])
func_desc = {
@@ -269,19 +278,36 @@ def register_device_type(descriptor):
# 用于接受前端设备推送的搜索iot描述
async def handleIotDescriptors(conn, descriptors):
if not conn.use_function_call_mode:
return
wait_max_time = 5
while conn.func_handler is None or not conn.func_handler.finish_init:
await asyncio.sleep(1)
wait_max_time -= 1
if wait_max_time <= 0:
logger.bind(tag=TAG).error("连接对象没有func_handler")
conn.logger.bind(tag=TAG).debug("连接对象没有func_handler")
return
"""处理物联网描述"""
functions_changed = False
for descriptor in descriptors:
# 如果descriptor没有properties和methods，则直接跳过
if "properties" not in descriptor and "methods" not in descriptor:
continue
# 处理缺失properties的情况
if "properties" not in descriptor:
descriptor["properties"] = {}
# 从methods中提取所有参数作为properties
if "methods" in descriptor:
for method_name, method_info in descriptor["methods"].items():
if "parameters" in method_info:
for param_name, param_info in method_info["parameters"].items():
# 将参数信息转换为属性信息
descriptor["properties"][param_name] = {
"description": param_info["description"],
"type": param_info["type"],
}
# 创建IOT设备描述符
iot_descriptor = IotDescriptor(
descriptor["name"],
@@ -291,7 +317,7 @@ async def handleIotDescriptors(conn, descriptors):
)
conn.iot_descriptors[descriptor["name"]] = iot_descriptor
if conn.use_function_call_mode:
if conn.load_function_plugin:
# 注册或获取设备类型
type_id = register_device_type(descriptor)
device_functions = device_type_registry.get_device_functions(type_id)
@@ -300,7 +326,7 @@ async def handleIotDescriptors(conn, descriptors):
if hasattr(conn, "func_handler"):
for func_name in device_functions:
conn.func_handler.function_registry.register_function(func_name)
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).info(
f"注册IOT函数到function handler: {func_name}"
)
functions_changed = True
@@ -309,8 +335,8 @@ async def handleIotDescriptors(conn, descriptors):
if functions_changed and hasattr(conn, "func_handler"):
conn.func_handler.upload_functions_desc()
func_names = conn.func_handler.current_support_functions()
logger.bind(tag=TAG).info(f"设备类型: {type_id}")
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).info(f"设备类型: {type_id}")
conn.logger.bind(tag=TAG).info(
f"更新function描述列表完成，当前支持的函数: {func_names}"
)
@@ -324,13 +350,13 @@ async def handleIotStatus(conn, states):
for k, v in state["state"].items():
if property_item["name"] == k:
if type(v) != type(property_item["value"]):
logger.bind(tag=TAG).error(
conn.logger.bind(tag=TAG).error(
f"属性{property_item['name']}的值类型不匹配"
)
break
else:
property_item["value"] = v
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).info(
f"物联网状态更新: {key} , {property_item['name']} = {v}"
)
break
@@ -344,7 +370,7 @@ async def get_iot_status(conn, name, property_name):
for property_item in value.properties:
if property_item["name"] == property_name:
return property_item["value"]
logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
return None
@@ -355,16 +381,16 @@ async def set_iot_status(conn, name, property_name, value):
for property_item in iot_descriptor.properties:
if property_item["name"] == property_name:
if type(value) != type(property_item["value"]):
logger.bind(tag=TAG).error(
conn.logger.bind(tag=TAG).error(
f"属性{property_item['name']}的值类型不匹配"
)
return
property_item["value"] = value
logger.bind(tag=TAG).info(
conn.logger.bind(tag=TAG).info(
f"物联网状态更新: {name} , {property_name} = {value}"
)
return
logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
conn.logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
async def send_iot_conn(conn, name, method_name, parameters):
@@ -375,19 +401,17 @@ async def send_iot_conn(conn, name, method_name, parameters):
for method in value.methods:
# 找到了方法
if method["name"] == method_name:
await conn.websocket.send(
json.dumps(
{
"type": "iot",
"commands": [
{
"name": name,
"method": method_name,
"parameters": parameters,
}
],
}
)
)
# 构建命令对象
command = {
"name": name,
"method": method_name,
}
# 只有当参数不为空时才添加parameters字段
if parameters:
command["parameters"] = parameters
send_message = json.dumps({"type": "iot", "commands": [command]})
await conn.websocket.send(send_message)
conn.logger.bind(tag=TAG).info(f"发送物联网指令: {send_message}")
return
logger.bind(tag=TAG).error(f"未找到方法{method_name}")
conn.logger.bind(tag=TAG).error(f"未找到方法{method_name}")
main/xiaozhi-server/core/handle/reportHandle.py
+145
View File
@@ -0,0 +1,145 @@
"""
TTS上报功能已集成到ConnectionHandler类中。
上报功能包括：
1. 每个连接对象拥有自己的上报队列和处理线程
2. 上报线程的生命周期与连接对象绑定
3. 使用ConnectionHandler.enqueue_tts_report方法进行上报
具体实现请参考core/connection.py中的相关代码。
"""
import opuslib_next
from config.manage_api_client import report as manage_report
TAG = __name__
def report(conn, type, text, opus_data):
"""执行聊天记录上报操作
Args:
conn: 连接对象
type: 上报类型，1为用户，2为智能体
text: 合成文本
opus_data: opus音频数据
"""
try:
if opus_data:
audio_data = opus_to_wav(conn, opus_data)
else:
audio_data = None
# 执行上报
manage_report(
mac_address=conn.device_id,
session_id=conn.session_id,
chat_type=type,
content=text,
audio=audio_data,
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"聊天记录上报失败: {e}")
def opus_to_wav(conn, opus_data):
"""将Opus数据转换为WAV格式的字节流
Args:
output_dir: 输出目录（保留参数以保持接口兼容）
opus_data: opus音频数据
Returns:
bytes: WAV格式的音频数据
"""
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
conn.logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
if not pcm_data:
raise ValueError("没有有效的PCM数据")
# 创建WAV文件头
pcm_data_bytes = b"".join(pcm_data)
num_samples = len(pcm_data_bytes) // 2 # 16-bit samples
# WAV文件头
wav_header = bytearray()
wav_header.extend(b"RIFF") # ChunkID
wav_header.extend((36 + len(pcm_data_bytes)).to_bytes(4, "little")) # ChunkSize
wav_header.extend(b"WAVE") # Format
wav_header.extend(b"fmt ") # Subchunk1ID
wav_header.extend((16).to_bytes(4, "little")) # Subchunk1Size
wav_header.extend((1).to_bytes(2, "little")) # AudioFormat (PCM)
wav_header.extend((1).to_bytes(2, "little")) # NumChannels
wav_header.extend((16000).to_bytes(4, "little")) # SampleRate
wav_header.extend((32000).to_bytes(4, "little")) # ByteRate
wav_header.extend((2).to_bytes(2, "little")) # BlockAlign
wav_header.extend((16).to_bytes(2, "little")) # BitsPerSample
wav_header.extend(b"data") # Subchunk2ID
wav_header.extend(len(pcm_data_bytes).to_bytes(4, "little")) # Subchunk2Size
# 返回完整的WAV数据
return bytes(wav_header) + pcm_data_bytes
def enqueue_tts_report(conn, text, opus_data):
if not conn.read_config_from_api or conn.need_bind or not conn.report_tts_enable:
return
if conn.chat_history_conf == 0:
return
"""将TTS数据加入上报队列
Args:
conn: 连接对象
text: 合成文本
opus_data: opus音频数据
"""
try:
# 使用连接对象的队列，传入文本和二进制数据而非文件路径
if conn.chat_history_conf == 2:
conn.report_queue.put((2, text, opus_data))
conn.logger.bind(tag=TAG).debug(
f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
)
else:
conn.report_queue.put((2, text, None))
conn.logger.bind(tag=TAG).debug(
f"TTS数据已加入上报队列: {conn.device_id}, 不上报音频"
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}")
def enqueue_asr_report(conn, text, opus_data):
if not conn.read_config_from_api or conn.need_bind or not conn.report_asr_enable:
return
if conn.chat_history_conf == 0:
return
"""将ASR数据加入上报队列
Args:
conn: 连接对象
text: 合成文本
opus_data: opus音频数据
"""
try:
# 使用连接对象的队列，传入文本和二进制数据而非文件路径
if conn.chat_history_conf == 2:
conn.report_queue.put((1, text, opus_data))
conn.logger.bind(tag=TAG).debug(
f"ASR数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
)
else:
conn.report_queue.put((1, text, None))
conn.logger.bind(tag=TAG).debug(
f"ASR数据已加入上报队列: {conn.device_id}, 不上报音频"
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"加入ASR上报队列失败: {text}, {e}")
main/xiaozhi-server/core/handle/textHandle.py
+96 -6
View File
@@ -1,33 +1,37 @@
from config.logger import setup_logging
import json
from core.handle.abortHandle import handleAbortMessage
from core.handle.helloHandle import handleHelloMessage
from core.utils.util import remove_punctuation_and_length
from core.utils.util import remove_punctuation_and_length, filter_sensitive_info
from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
from core.handle.sendAudioHandle import send_stt_message, send_tts_message
from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
from core.handle.reportHandle import enqueue_asr_report
import asyncio
TAG = __name__
logger = setup_logging()
async def handleTextMessage(conn, message):
"""处理文本消息"""
logger.bind(tag=TAG).info(f"收到文本消息：{message}")
try:
msg_json = json.loads(message)
if isinstance(msg_json, int):
conn.logger.bind(tag=TAG).info(f"收到文本消息：{message}")
await conn.websocket.send(message)
return
if msg_json["type"] == "hello":
await handleHelloMessage(conn)
conn.logger.bind(tag=TAG).info(f"收到hello消息：{message}")
await handleHelloMessage(conn, msg_json)
elif msg_json["type"] == "abort":
conn.logger.bind(tag=TAG).info(f"收到abort消息：{message}")
await handleAbortMessage(conn)
elif msg_json["type"] == "listen":
conn.logger.bind(tag=TAG).info(f"收到listen消息：{message}")
if "mode" in msg_json:
conn.client_listen_mode = msg_json["mode"]
logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}")
conn.logger.bind(tag=TAG).debug(
f"客户端拾音模式：{conn.client_listen_mode}"
)
if msg_json["state"] == "start":
conn.client_have_voice = True
conn.client_voice_stop = False
@@ -53,13 +57,99 @@ async def handleTextMessage(conn, message):
# 如果是唤醒词，且关闭了唤醒词回复，就不用回答
await send_stt_message(conn, text)
await send_tts_message(conn, "stop", None)
elif is_wakeup_words:
# 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
enqueue_asr_report(conn, "嘿，你好呀", [])
await startToChat(conn, "嘿，你好呀")
else:
# 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
enqueue_asr_report(conn, text, [])
# 否则需要LLM对文字内容进行答复
await startToChat(conn, text)
elif msg_json["type"] == "iot":
conn.logger.bind(tag=TAG).info(f"收到iot消息：{message}")
if "descriptors" in msg_json:
asyncio.create_task(handleIotDescriptors(conn, msg_json["descriptors"]))
if "states" in msg_json:
asyncio.create_task(handleIotStatus(conn, msg_json["states"]))
elif msg_json["type"] == "server":
# 记录日志时过滤敏感信息
conn.logger.bind(tag=TAG).info(
f"收到服务器消息：{filter_sensitive_info(msg_json)}"
)
# 如果配置是从API读取的，则需要验证secret
if not conn.read_config_from_api:
return
# 获取post请求的secret
post_secret = msg_json.get("content", {}).get("secret", "")
secret = conn.config["manager-api"].get("secret", "")
# 如果secret不匹配，则返回
if post_secret != secret:
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "服务器密钥验证失败",
}
)
)
return
# 动态更新配置
if msg_json["action"] == "update_config":
try:
# 更新WebSocketServer的配置
if not conn.server:
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "无法获取服务器实例",
"content": {"action": "update_config"},
}
)
)
return
if not await conn.server.update_config():
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": "更新服务器配置失败",
"content": {"action": "update_config"},
}
)
)
return
# 发送成功响应
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "success",
"message": "配置更新成功",
"content": {"action": "update_config"},
}
)
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"更新配置失败: {str(e)}")
await conn.websocket.send(
json.dumps(
{
"type": "server",
"status": "error",
"message": f"更新配置失败: {str(e)}",
"content": {"action": "update_config"},
}
)
)
# 重启服务器
elif msg_json["action"] == "restart":
await conn.handle_restart(msg_json)
except json.JSONDecodeError:
await conn.websocket.send(message)
main/xiaozhi-server/core/mcp/MCPClient.py
+108 -69
View File
@@ -1,86 +1,125 @@
from __future__ import annotations
from datetime import timedelta
from typing import Optional
import asyncio, os, shutil, concurrent.futures
from contextlib import AsyncExitStack
import os, shutil
from typing import Optional, List, Dict, Any
from mcp import ClientSession, StdioServerParameters
from mcp.client.stdio import stdio_client
from mcp.client.sse import sse_client
from config.logger import setup_logging
TAG = __name__
class MCPClient:
def __init__(self, config):
# Initialize session and client objects
self.session: Optional[ClientSession] = None
self.exit_stack = AsyncExitStack()
def __init__(self, config: Dict[str, Any]):
self.logger = setup_logging()
self.config = config
self.tolls = []
self._worker_task: Optional[asyncio.Task] = None
self._ready_evt = asyncio.Event()
self._shutdown_evt = asyncio.Event()
self.session: Optional[ClientSession] = None
self.tools: List = []
async def initialize(self):
args = self.config.get("args", [])
if self._worker_task:
return
self._worker_task = asyncio.create_task(self._worker(), name="MCPClientWorker")
await self._ready_evt.wait()
command = (
shutil.which("npx")
if self.config["command"] == "npx"
else self.config["command"]
self.logger.bind(tag=TAG).info(
f"Connected, tools = {[t.name for t in self.tools]}"
)
env={**os.environ}
if self.config.get("env"):
env.update(self.config["env"])
server_params = StdioServerParameters(
command=command,
args=args,
env=env
)
stdio_transport = await self.exit_stack.enter_async_context(stdio_client(server_params))
self.stdio, self.write = stdio_transport
time_out_delta = timedelta(seconds=15)
self.session = await self.exit_stack.enter_async_context(ClientSession(read_stream=self.stdio, write_stream=self.write, read_timeout_seconds=time_out_delta))
await self.session.initialize()
# List available tools
response = await self.session.list_tools()
tools = response.tools
self.tools = tools
self.logger.bind(tag=TAG).info(f"Connected to server with tools:{[tool.name for tool in tools]}")
def has_tool(self, tool_name):
return any(tool.name == tool_name for tool in self.tools)
def get_available_tools(self):
available_tools = [{"type": "function", "function":{
"name": tool.name,
"description": tool.description,
"parameters": tool.inputSchema
} } for tool in self.tools]
return available_tools
async def call_tool(self, tool_name: str, tool_args: dict):
self.logger.bind(tag=TAG).info(f"MCPClient Calling tool {tool_name} with args: {tool_args}")
try:
response = await self.session.call_tool(tool_name, tool_args)
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error calling tool {tool_name}: {e}")
from types import SimpleNamespace
error_content = SimpleNamespace(
type='text',
text=f"Error calling tool {tool_name}: {e}"
)
error_response = SimpleNamespace(
content=[error_content],
isError=True
)
return error_response
self.logger.bind(tag=TAG).info(f"MCPClient Response from tool {tool_name}: {response}")
return response
async def cleanup(self):
"""Clean up resources"""
await self.exit_stack.aclose()
if not self._worker_task:
return
self._shutdown_evt.set()
try:
await asyncio.wait_for(self._worker_task, timeout=20)
except (asyncio.TimeoutError, Exception) as e:
self.logger.bind(tag=TAG).error(f"worker shutdown err: {e}")
finally:
self._worker_task = None
def has_tool(self, name: str) -> bool:
return any(t.name == name for t in self.tools)
def get_available_tools(self):
return [
{
"type": "function",
"function": {
"name": t.name,
"description": t.description,
"parameters": t.inputSchema,
},
}
for t in self.tools
]
async def call_tool(self, name: str, args: dict):
if not self.session:
raise RuntimeError("MCPClient not initialized")
loop = self._worker_task.get_loop()
coro = self.session.call_tool(name, args)
if loop is asyncio.get_running_loop():
return await coro
fut: concurrent.futures.Future = asyncio.run_coroutine_threadsafe(coro, loop)
return await asyncio.wrap_future(fut)
async def _worker(self):
async with AsyncExitStack() as stack:
try:
# 建立 StdioClient
if "command" in self.config:
cmd = (
shutil.which("npx")
if self.config["command"] == "npx"
else self.config["command"]
)
env = {**os.environ, **self.config.get("env", {})}
params = StdioServerParameters(
command=cmd,
args=self.config.get("args", []),
env=env,
)
stdio_r, stdio_w = await stack.enter_async_context(stdio_client(params))
read_stream, write_stream = stdio_r, stdio_w
# 建立SSEClient
elif "url" in self.config:
sse_r, sse_w = await stack.enter_async_context(sse_client(self.config["url"]))
read_stream, write_stream = sse_r, sse_w
else:
raise ValueError("MCPClient config must include 'command' or 'url'")
self.session = await stack.enter_async_context(
ClientSession(
read_stream=read_stream,
write_stream=write_stream,
read_timeout_seconds=timedelta(seconds=15),
)
)
await self.session.initialize()
# 获取工具
self.tools = (await self.session.list_tools()).tools
self._ready_evt.set()
# 挂起等待关闭
await self._shutdown_evt.wait()
except Exception as e:
self.logger.bind(tag=TAG).error(f"worker error: {e}")
self._ready_evt.set()
raise
main/xiaozhi-server/core/mcp/manager.py
+48 -27
View File
@@ -1,26 +1,29 @@
"""MCP服务管理器"""
import asyncio
import os, json
from typing import Dict, Any, List
from .MCPClient import MCPClient
from config.logger import setup_logging
from core.utils.util import get_project_dir
from plugins_func.register import register_function, ActionResponse, Action, ToolType
from plugins_func.register import register_function, ToolType
from config.config_loader import get_project_dir
TAG = __name__
class MCPManager:
"""管理多个MCP服务的集中管理器"""
def __init__(self,conn) -> None:
def __init__(self, conn) -> None:
"""
初始化MCP管理器
"""
self.conn = conn
self.logger = setup_logging()
self.config_path = get_project_dir() + 'data/.mcp_server_settings.json'
self.config_path = get_project_dir() + "data/.mcp_server_settings.json"
if os.path.exists(self.config_path) == False:
self.config_path = ""
self.logger.bind(tag=TAG).warning(f"请检查mcp服务配置文件：data/.mcp_server_settings.json")
self.conn.logger.bind(tag=TAG).warning(
f"请检查mcp服务配置文件：data/.mcp_server_settings.json"
)
self.client: Dict[str, MCPClient] = {}
self.tools = []
@@ -31,37 +34,47 @@ class MCPManager:
"""
if len(self.config_path) == 0:
return {}
try:
with open(self.config_path, 'r', encoding='utf-8') as f:
with open(self.config_path, "r", encoding="utf-8") as f:
config = json.load(f)
return config.get('mcpServers', {})
return config.get("mcpServers", {})
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error loading MCP config from {self.config_path}: {e}")
self.conn.logger.bind(tag=TAG).error(
f"Error loading MCP config from {self.config_path}: {e}"
)
return {}
async def initialize_servers(self) -> None:
"""初始化所有MCP服务"""
config = self.load_config()
for name, srv_config in config.items():
if not srv_config.get("command"):
self.logger.bind(tag=TAG).warning(f"Skipping server {name}: command not specified")
if not srv_config.get("command") and not srv_config.get("url"):
self.conn.logger.bind(tag=TAG).warning(
f"Skipping server {name}: neither command nor url specified"
)
continue
try:
client = MCPClient(srv_config)
await client.initialize()
self.client[name] = client
self.logger.bind(tag=TAG).info(f"Initialized MCP client: {name}")
self.conn.logger.bind(tag=TAG).info(f"Initialized MCP client: {name}")
client_tools = client.get_available_tools()
self.tools.extend(client_tools)
for tool in client_tools:
func_name = "mcp_"+tool["function"]["name"]
register_function(func_name, tool, ToolType.MCP_CLIENT)(self.execute_tool)
self.conn.func_handler.function_registry.register_function(func_name)
func_name = "mcp_" + tool["function"]["name"]
register_function(func_name, tool, ToolType.MCP_CLIENT)(
self.execute_tool
)
self.conn.func_handler.function_registry.register_function(
func_name
)
except Exception as e:
self.logger.bind(tag=TAG).error(f"Failed to initialize MCP server {name}: {e}")
self.conn.logger.bind(tag=TAG).error(
f"Failed to initialize MCP server {name}: {e}"
)
self.conn.func_handler.upload_functions_desc()
def get_all_tools(self) -> List[Dict[str, Any]]:
@@ -79,7 +92,10 @@ class MCPManager:
bool: 是否是MCP工具
"""
for tool in self.tools:
if tool.get("function") != None and tool["function"].get("name") == tool_name:
if (
tool.get("function") != None
and tool["function"].get("name") == tool_name
):
return True
return False
@@ -87,24 +103,29 @@ class MCPManager:
"""执行工具调用
Args:
tool_name: 工具名称
arguments: 工具参数
arguments: 工具参数
Returns:
Any: 工具执行结果
Raises:
ValueError: 工具未找到时抛出
"""
self.logger.bind(tag=TAG).info(f"Executing tool {tool_name} with arguments: {arguments}")
self.conn.logger.bind(tag=TAG).info(
f"Executing tool {tool_name} with arguments: {arguments}"
)
for client in self.client.values():
if client.has_tool(tool_name):
return await client.call_tool(tool_name, arguments)
raise ValueError(f"Tool {tool_name} not found in any MCP server")
async def cleanup_all(self) -> None:
for name, client in self.client.items():
"""依次关闭所有 MCPClient，不让异常阻断整体流程。"""
for name, client in list(self.client.items()):
try:
await client.cleanup()
self.logger.bind(tag=TAG).info(f"Cleaned up MCP client: {name}")
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error cleaning up MCP client {name}: {e}")
await asyncio.wait_for(client.cleanup(), timeout=20)
self.conn.logger.bind(tag=TAG).info(f"MCP client closed: {name}")
except (asyncio.TimeoutError, Exception) as e:
self.conn.logger.bind(tag=TAG).error(
f"Error closing MCP client {name}: {e}"
)
self.client.clear()
main/xiaozhi-server/core/ota_server.py
+135
View File
@@ -0,0 +1,135 @@
import json
import time
import asyncio
from aiohttp import web
from config.logger import setup_logging
from core.connection import ConnectionHandler
from core.utils.util import get_local_ip, initialize_modules
TAG = __name__
class SimpleOtaServer:
def __init__(self, config: dict):
self.config = config
self.logger = setup_logging()
def _get_websocket_url(self, local_ip: str, port: int) -> str:
"""获取websocket地址
Args:
local_ip: 本地IP地址
port: 端口号
Returns:
str: websocket地址
"""
server_config = self.config["server"]
websocket_config = server_config.get("websocket")
if websocket_config and "你" not in websocket_config:
return websocket_config
else:
return f"ws://{local_ip}:{port}/xiaozhi/v1/"
async def start(self):
server_config = self.config["server"]
host = server_config.get("ip", "0.0.0.0")
port = int(server_config.get("ota_port"))
if port:
app = web.Application()
# 添加路由
app.add_routes(
[
web.get("/xiaozhi/ota/", self._handle_ota_get_request),
web.post("/xiaozhi/ota/", self._handle_ota_request),
web.options("/xiaozhi/ota/", self._handle_ota_request),
]
)
# 运行服务
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, host, port)
await site.start()
# 保持服务运行
while True:
await asyncio.sleep(3600) # 每隔 1 小时检查一次
async def _handle_ota_request(self, request):
"""处理 /xiaozhi/ota/ 的 POST 请求"""
try:
data = await request.text()
self.logger.bind(tag=TAG).debug(f"OTA请求方法: {request.method}")
self.logger.bind(tag=TAG).debug(f"OTA请求头: {request.headers}")
self.logger.bind(tag=TAG).debug(f"OTA请求数据: {data}")
device_id = request.headers.get("device-id", "")
if device_id:
self.logger.bind(tag=TAG).info(f"OTA请求设备ID: {device_id}")
else:
raise Exception("OTA请求设备ID为空")
data_json = json.loads(data)
server_config = self.config["server"]
host = server_config.get("ip", "0.0.0.0")
port = int(server_config.get("port", 8000))
local_ip = get_local_ip()
# OTA基础信息
return_json = {
"server_time": {
"timestamp": int(round(time.time() * 1000)),
"timezone_offset": server_config.get("timezone_offset", 8) * 60,
},
"firmware": {
"version": data_json["application"].get("version", "1.0.0"),
"url": "",
},
"websocket": {
"url": self._get_websocket_url(local_ip, port),
},
}
response = web.Response(
text=json.dumps(return_json, separators=(",", ":")),
content_type="application/json",
)
except Exception as e:
self.logger.bind(tag=TAG).error(f"OTA请求异常: {e}")
return_json = {"success": False, "message": "request error."}
response = web.Response(
text=json.dumps(return_json, separators=(",", ":")),
content_type="application/json",
)
finally:
# 添加header，允许跨域访问
response.headers["Access-Control-Allow-Headers"] = (
"client-id, content-type, device-id"
)
response.headers["Access-Control-Allow-Credentials"] = "true"
response.headers["Access-Control-Allow-Origin"] = "*"
return response
async def _handle_ota_get_request(self, request):
"""处理 /xiaozhi/ota/ 的 GET 请求"""
try:
server_config = self.config["server"]
local_ip = get_local_ip()
port = int(server_config.get("port", 8000))
websocket_url = self._get_websocket_url(local_ip, port)
message = f"OTA接口运行正常，向设备发送的websocket地址是：{websocket_url}"
response = web.Response(text=message, content_type="text/plain")
except Exception as e:
self.logger.bind(tag=TAG).error(f"OTA GET请求异常: {e}")
response = web.Response(text="OTA接口异常", content_type="text/plain")
finally:
# 添加header，允许跨域访问
response.headers["Access-Control-Allow-Headers"] = (
"client-id, content-type, device-id"
)
response.headers["Access-Control-Allow-Credentials"] = "true"
response.headers["Access-Control-Allow-Origin"] = "*"
return response
main/xiaozhi-server/core/providers/asr/aliyun.py
+270
View File
@@ -0,0 +1,270 @@
import http.client
import json
import asyncio
from typing import Optional, Tuple, List
import opuslib_next
import wave
import io
import os
import uuid
import hmac
import hashlib
import base64
import requests
from urllib import parse
import time
from datetime import datetime
from config.logger import setup_logging
from core.providers.asr.base import ASRProviderBase
TAG = __name__
logger = setup_logging()
class AccessToken:
@staticmethod
def _encode_text(text):
encoded_text = parse.quote_plus(text)
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
@staticmethod
def _encode_dict(dic):
keys = dic.keys()
dic_sorted = [(key, dic[key]) for key in sorted(keys)]
encoded_text = parse.urlencode(dic_sorted)
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
@staticmethod
def create_token(access_key_id, access_key_secret):
parameters = {
"AccessKeyId": access_key_id,
"Action": "CreateToken",
"Format": "JSON",
"RegionId": "cn-shanghai",
"SignatureMethod": "HMAC-SHA1",
"SignatureNonce": str(uuid.uuid1()),
"SignatureVersion": "1.0",
"Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"Version": "2019-02-28",
}
# 构造规范化的请求字符串
query_string = AccessToken._encode_dict(parameters)
# print('规范化的请求字符串: %s' % query_string)
# 构造待签名字符串
string_to_sign = (
"GET"
+ "&"
+ AccessToken._encode_text("/")
+ "&"
+ AccessToken._encode_text(query_string)
)
# print('待签名的字符串: %s' % string_to_sign)
# 计算签名
secreted_string = hmac.new(
bytes(access_key_secret + "&", encoding="utf-8"),
bytes(string_to_sign, encoding="utf-8"),
hashlib.sha1,
).digest()
signature = base64.b64encode(secreted_string)
# print('签名: %s' % signature)
# 进行URL编码
signature = AccessToken._encode_text(signature)
# print('URL编码后的签名: %s' % signature)
# 调用服务
full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (
signature,
query_string,
)
# print('url: %s' % full_url)
# 提交HTTP GET请求
response = requests.get(full_url)
if response.ok:
root_obj = response.json()
key = "Token"
if key in root_obj:
token = root_obj[key]["Id"]
expire_time = root_obj[key]["ExpireTime"]
return token, expire_time
# print(response.text)
return None, None
class ASRProvider(ASRProviderBase):
def __init__(self, config: dict, delete_audio_file: bool):
super().__init__()
"""阿里云ASR初始化"""
# 新增空值判断逻辑
self.access_key_id = config.get("access_key_id")
self.access_key_secret = config.get("access_key_secret")
self.app_key = config.get("appkey")
self.host = "nls-gateway-cn-shanghai.aliyuncs.com"
self.base_url = f"https://{self.host}/stream/v1/asr"
self.sample_rate = 16000
self.format = "wav"
self.output_dir = config.get("output_dir", "./audio_output")
self.delete_audio_file = delete_audio_file
if self.access_key_id and self.access_key_secret:
# 使用密钥对生成临时token
self._refresh_token()
else:
# 直接使用预生成的长期token
self.token = config.get("token")
self.expire_time = None
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
def _refresh_token(self):
"""刷新Token并记录过期时间"""
if self.access_key_id and self.access_key_secret:
self.token, expire_time_str = AccessToken.create_token(
self.access_key_id, self.access_key_secret
)
if not expire_time_str:
raise ValueError("无法获取有效的Token过期时间")
try:
# 统一转换为字符串处理
expire_str = str(expire_time_str).strip()
if expire_str.isdigit():
expire_time = datetime.fromtimestamp(int(expire_str))
else:
expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ")
self.expire_time = expire_time.timestamp() - 60
except Exception as e:
raise ValueError(f"无效的过期时间格式: {expire_str}") from e
else:
self.expire_time = None
if not self.token:
raise ValueError("无法获取有效的访问Token")
def _is_token_expired(self):
"""检查Token是否过期"""
if not self.expire_time:
return False # 长期Token不过期
# 新增调试日志
# current_time = time.time()
# remaining = self.expire_time - current_time
# print(f"Token过期检查: 当前时间 {datetime.fromtimestamp(current_time)} | "
# f"过期时间 {datetime.fromtimestamp(self.expire_time)} | "
# f"剩余 {remaining:.2f}秒")
return time.time() > self.expire_time
def generate_filename(self, extension=".wav"):
return os.path.join(
self.output_file,
f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
)
def _construct_request_url(self) -> str:
"""构造请求URL，包含参数"""
request = f"{self.base_url}?appkey={self.app_key}"
request += f"&format={self.format}"
request += f"&sample_rate={self.sample_rate}"
request += "&enable_punctuation_prediction=true"
request += "&enable_inverse_text_normalization=true"
request += "&enable_voice_detection=false"
return request
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1) # 单声道
wf.setsampwidth(2) # 16-bit
wf.setframerate(self.sample_rate)
wf.writeframes(b"".join(pcm_data))
logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}")
return file_path
async def _send_request(self, pcm_data: bytes) -> Optional[str]:
"""发送请求到阿里云ASR服务"""
try:
# 设置HTTP头
headers = {
"X-NLS-Token": self.token,
"Content-type": "application/octet-stream",
"Content-Length": str(len(pcm_data)),
}
# 创建连接并发送请求
conn = http.client.HTTPSConnection(self.host)
request_url = self._construct_request_url()
loop = asyncio.get_event_loop()
await loop.run_in_executor(
None,
lambda: conn.request(
method="POST", url=request_url, body=pcm_data, headers=headers
),
)
# 获取响应
response = await loop.run_in_executor(None, conn.getresponse)
body = await loop.run_in_executor(None, response.read)
conn.close()
# 解析响应
try:
body_json = json.loads(body)
status = body_json.get("status")
if status == 20000000:
result = body_json.get("result", "")
logger.bind(tag=TAG).debug(f"ASR结果: {result}")
return result
else:
logger.bind(tag=TAG).error(f"ASR失败，状态码: {status}")
return None
except ValueError:
logger.bind(tag=TAG).error("响应不是JSON格式")
return None
except Exception as e:
logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True)
return None
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
if self._is_token_expired():
logger.warning("Token已过期，正在自动刷新...")
self._refresh_token()
file_path = None
try:
# 解码Opus为PCM
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
# 发送请求并获取文本
text = await self._send_request(combined_pcm_data)
if text:
return text, file_path
return "", file_path
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", file_path
main/xiaozhi-server/core/providers/asr/baidu.py
+106
View File
@@ -0,0 +1,106 @@
import base64
import hashlib
import hmac
import json
import time
from datetime import datetime, timezone
import os
import uuid
from typing import Optional, Tuple, List
import wave
import opuslib_next
from aip import AipSpeech
from core.providers.asr.base import ASRProviderBase
from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
class ASRProvider(ASRProviderBase):
def __init__(self, config: dict, delete_audio_file: bool = True):
super().__init__()
self.app_id = config.get("app_id")
self.api_key = config.get("api_key")
self.secret_key = config.get("secret_key")
dev_pid = config.get("dev_pid", "1537")
self.dev_pid = int(dev_pid) if dev_pid else 1537
self.output_dir = config.get("output_dir")
self.delete_audio_file = delete_audio_file
self.client = AipSpeech(str(self.app_id), self.api_key, self.secret_key)
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
wf.setframerate(16000)
wf.writeframes(b"".join(pcm_data))
return file_path
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
if not opus_data:
logger.bind(tag=TAG).warning("音频数据为空！")
return None, None
file_path = None
try:
# 检查配置是否已设置
if not self.app_id or not self.api_key or not self.secret_key:
logger.bind(tag=TAG).error("百度语音识别配置未设置，无法进行识别")
return None, file_path
# 将Opus音频数据解码为PCM
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
start_time = time.time()
# 识别本地文件
result = self.client.asr(
combined_pcm_data,
"pcm",
16000,
{
"dev_pid": str(self.dev_pid),
},
)
if result and result["err_no"] == 0:
logger.bind(tag=TAG).debug(
f"百度语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
)
result = result["result"][0]
return result, file_path
else:
raise Exception(
f"百度语音识别失败，错误码: {result['err_no']}，错误信息: {result['err_msg']}"
)
return None, file_path
except Exception as e:
logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
return None, file_path
main/xiaozhi-server/core/providers/asr/base.py
+29 -4
View File
@@ -1,6 +1,6 @@
from abc import ABC, abstractmethod
from typing import Optional, Tuple, List
import opuslib_next
from config.logger import setup_logging
TAG = __name__
@@ -8,12 +8,37 @@ logger = setup_logging()
class ASRProviderBase(ABC):
def __init__(self):
self.audio_format = "opus"
@abstractmethod
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""解码Opus数据并保存为WAV文件"""
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
pass
@abstractmethod
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
pass
def set_audio_format(self, format: str) -> None:
"""设置音频格式"""
self.audio_format = format
@staticmethod
def decode_opus(opus_data: List[bytes]) -> bytes:
"""将Opus音频数据解码为PCM数据"""
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return pcm_data
main/xiaozhi-server/core/providers/asr/doubao.py
+78 -81
View File
@@ -45,14 +45,14 @@ def parse_response(res):
payload 类似与http 请求体
"""
protocol_version = res[0] >> 4
header_size = res[0] & 0x0f
header_size = res[0] & 0x0F
message_type = res[1] >> 4
message_type_specific_flags = res[1] & 0x0f
message_type_specific_flags = res[1] & 0x0F
serialization_method = res[2] >> 4
message_compression = res[2] & 0x0f
message_compression = res[2] & 0x0F
reserved = res[3]
header_extensions = res[4:header_size * 4]
payload = res[header_size * 4:]
header_extensions = res[4 : header_size * 4]
payload = res[header_size * 4 :]
result = {}
payload_msg = None
payload_size = 0
@@ -61,13 +61,13 @@ def parse_response(res):
payload_msg = payload[4:]
elif message_type == SERVER_ACK:
seq = int.from_bytes(payload[:4], "big", signed=True)
result['seq'] = seq
result["seq"] = seq
if len(payload) >= 8:
payload_size = int.from_bytes(payload[4:8], "big", signed=False)
payload_msg = payload[8:]
elif message_type == SERVER_ERROR_RESPONSE:
code = int.from_bytes(payload[:4], "big", signed=False)
result['code'] = code
result["code"] = code
payload_size = int.from_bytes(payload[4:8], "big", signed=False)
payload_msg = payload[8:]
if payload_msg is None:
@@ -78,17 +78,21 @@ def parse_response(res):
payload_msg = json.loads(str(payload_msg, "utf-8"))
elif serialization_method != NO_SERIALIZATION:
payload_msg = str(payload_msg, "utf-8")
result['payload_msg'] = payload_msg
result['payload_size'] = payload_size
result["payload_msg"] = payload_msg
result["payload_size"] = payload_size
return result
class ASRProvider(ASRProviderBase):
def __init__(self, config: dict, delete_audio_file: bool):
super().__init__()
self.appid = config.get("appid")
self.cluster = config.get("cluster")
self.access_token = config.get("access_token")
self.boosting_table_name = config.get("boosting_table_name", "")
self.correct_table_name = config.get("correct_table_name", "")
self.output_dir = config.get("output_dir")
self.delete_audio_file = delete_audio_file
self.host = "openspeech.bytedance.com"
self.ws_url = f"wss://{self.host}/api/v2/asr"
@@ -98,21 +102,12 @@ class ASRProvider(ASRProviderBase):
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""将Opus音频数据解码并保存为WAV文件"""
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
@@ -122,7 +117,9 @@ class ASRProvider(ASRProviderBase):
return file_path
@staticmethod
def _generate_header(message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE) -> bytearray:
def _generate_header(
message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE
) -> bytearray:
"""Generate protocol header."""
header = bytearray()
header_size = 1
@@ -146,10 +143,12 @@ class ASRProvider(ASRProviderBase):
"request": {
"reqid": reqid,
"show_utterances": False,
"sequence": 1
"sequence": 1,
"boosting_table_name": self.boosting_table_name,
"correct_table_name": self.correct_table_name,
},
"audio": {
"format": "wav",
"format": "raw",
"rate": 16000,
"language": "zh-CN",
"bits": 16,
@@ -158,18 +157,23 @@ class ASRProvider(ASRProviderBase):
},
}
async def _send_request(self, audio_data: List[bytes], segment_size: int) -> Optional[str]:
async def _send_request(
self, audio_data: List[bytes], segment_size: int
) -> Optional[str]:
"""Send request to Volcano ASR service."""
try:
auth_header = {'Authorization': 'Bearer; {}'.format(self.access_token)}
async with websockets.connect(self.ws_url, additional_headers=auth_header) as websocket:
auth_header = {"Authorization": "Bearer; {}".format(self.access_token)}
async with websockets.connect(
self.ws_url, additional_headers=auth_header
) as websocket:
# Prepare request data
request_params = self._construct_request(str(uuid.uuid4()))
print(request_params)
payload_bytes = str.encode(json.dumps(request_params))
payload_bytes = gzip.compress(payload_bytes)
full_client_request = self._generate_header()
full_client_request.extend((len(payload_bytes)).to_bytes(4, 'big')) # payload size(4 bytes)
full_client_request.extend(
(len(payload_bytes)).to_bytes(4, "big")
) # payload size(4 bytes)
full_client_request.extend(payload_bytes) # payload
# Send header and metadata
@@ -177,22 +181,29 @@ class ASRProvider(ASRProviderBase):
await websocket.send(full_client_request)
res = await websocket.recv()
result = parse_response(res)
if 'payload_msg' in result and result['payload_msg']['code'] != self.success_code:
if (
"payload_msg" in result
and result["payload_msg"]["code"] != self.success_code
):
logger.bind(tag=TAG).error(f"ASR error: {result}")
return None
for seq, (chunk, last) in enumerate(self.slice_data(audio_data, segment_size), 1):
for seq, (chunk, last) in enumerate(
self.slice_data(audio_data, segment_size), 1
):
if last:
audio_only_request = self._generate_header(
message_type=CLIENT_AUDIO_ONLY_REQUEST,
message_type_specific_flags=NEG_SEQUENCE
message_type_specific_flags=NEG_SEQUENCE,
)
else:
audio_only_request = self._generate_header(
message_type=CLIENT_AUDIO_ONLY_REQUEST
)
payload_bytes = gzip.compress(chunk)
audio_only_request.extend((len(payload_bytes)).to_bytes(4, 'big')) # payload size(4 bytes)
audio_only_request.extend(
(len(payload_bytes)).to_bytes(4, "big")
) # payload size(4 bytes)
audio_only_request.extend(payload_bytes) # payload
# Send audio data
await websocket.send(audio_only_request)
@@ -201,9 +212,12 @@ class ASRProvider(ASRProviderBase):
response = await websocket.recv()
result = parse_response(response)
if 'payload_msg' in result and result['payload_msg']['code'] == self.success_code:
if len(result['payload_msg']['result']) > 0:
return result['payload_msg']['result'][0]["text"]
if (
"payload_msg" in result
and result["payload_msg"]["code"] == self.success_code
):
if len(result["payload_msg"]["result"]) > 0:
return result["payload_msg"]["result"][0]["text"]
return None
else:
logger.bind(tag=TAG).error(f"ASR error: {result}")
@@ -213,29 +227,6 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
return None
@staticmethod
def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return pcm_data
@staticmethod
def read_wav_info(data: io.BytesIO = None) -> (int, int, int, int, int):
with io.BytesIO(data) as _f:
wave_fp = wave.open(_f, 'rb')
nchannels, sampwidth, framerate, nframes = wave_fp.getparams()[:4]
wave_bytes = wave_fp.readframes(nframes)
return nchannels, sampwidth, framerate, nframes, len(wave_bytes)
@staticmethod
def slice_data(data: bytes, chunk_size: int) -> (list, bool):
"""
@@ -247,40 +238,46 @@ class ASRProvider(ASRProviderBase):
data_len = len(data)
offset = 0
while offset + chunk_size < data_len:
yield data[offset: offset + chunk_size], False
yield data[offset : offset + chunk_size], False
offset += chunk_size
else:
yield data[offset: data_len], True
yield data[offset:data_len], True
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
file_path = None
try:
# 合并所有opus数据包
pcm_data = self.decode_opus(opus_data, session_id)
combined_pcm_data = b''.join(pcm_data)
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
wav_buffer = io.BytesIO()
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
with wave.open(wav_buffer, "wb") as wav_file:
wav_file.setnchannels(1) # 设置声道数
wav_file.setsampwidth(2) # 设置采样宽度
wav_file.setframerate(16000) # 设置采样率
wav_file.writeframes(combined_pcm_data) # 写入 PCM 数据
# 获取封装后的 WAV 数据
wav_data = wav_buffer.getvalue()
nchannels, sampwidth, framerate, nframes, wav_len = self.read_wav_info(wav_data)
size_per_sec = nchannels * sampwidth * framerate
# 直接使用PCM数据
# 计算分段大小 (单声道, 16bit, 16kHz采样率)
size_per_sec = 1 * 2 * 16000 # nchannels * sampwidth * framerate
segment_size = int(size_per_sec * self.seg_duration / 1000)
# 语音识别
start_time = time.time()
text = await self._send_request(wav_data, segment_size)
text = await self._send_request(combined_pcm_data, segment_size)
if text:
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
return text, None
return "", None
logger.bind(tag=TAG).debug(
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
)
return text, file_path
return "", file_path
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", None
return "", file_path
main/xiaozhi-server/core/providers/asr/fun_local.py
+35 -32
View File
@@ -6,9 +6,7 @@ import io
from config.logger import setup_logging
from typing import Optional, Tuple, List
import uuid
import opuslib_next
from core.providers.asr.base import ASRProviderBase
from funasr import AutoModel
from funasr.utils.postprocess_utils import rich_transcription_postprocess
@@ -35,6 +33,7 @@ class CaptureOutput:
class ASRProvider(ASRProviderBase):
def __init__(self, config: dict, delete_audio_file: bool):
super().__init__()
self.model_dir = config.get("model_dir")
self.output_dir = config.get("output_dir") # 修正配置键名
self.delete_audio_file = delete_audio_file
@@ -46,25 +45,16 @@ class ASRProvider(ASRProviderBase):
model=self.model_dir,
vad_kwargs={"max_single_segment_time": 30000},
disable_update=True,
hub="hf"
hub="hf",
# device="cuda:0", # 启用GPU加速
)
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""将Opus音频数据解码并保存为WAV文件"""
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
@@ -73,38 +63,51 @@ class ASRProvider(ASRProviderBase):
return file_path
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""语音转文本主处理逻辑"""
file_path = None
try:
# 保存音频文件
start_time = time.time()
file_path = self.save_audio_to_file(opus_data, session_id)
logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
# 合并所有opus数据包
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
# 语音识别
start_time = time.time()
result = self.model.generate(
input=file_path,
input=combined_pcm_data,
cache={},
language="auto",
use_itn=True,
batch_size_s=60,
)
text = rich_transcription_postprocess(result[0]["text"])
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
logger.bind(tag=TAG).debug(
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
)
return text, file_path
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", None
return "", file_path
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
try:
os.remove(file_path)
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
except Exception as e:
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
# finally:
# # 文件清理逻辑
# if self.delete_audio_file and file_path and os.path.exists(file_path):
# try:
# os.remove(file_path)
# logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
# except Exception as e:
# logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
main/xiaozhi-server/core/providers/asr/fun_server.py
+185
View File
@@ -0,0 +1,185 @@
from typing import Optional, Tuple, List
import opuslib_next
from core.providers.asr.base import ASRProviderBase
import os
import ssl
import json
import uuid
import wave
import websockets
from config.logger import setup_logging
import asyncio
import re
TAG = __name__
logger = setup_logging()
class ASRProvider(ASRProviderBase):
def __init__(self, config: dict, delete_audio_file: bool):
"""
Initialize the ASRProvider with server configuration.
:param config: Dictionary containing 'host', 'port', and 'is_ssl'.
:param delete_audio_file: Boolean to indicate whether to delete audio files after processing.
"""
super().__init__()
self.host = config.get("host", "localhost")
self.port = config.get("port", 10095)
self.api_key = config.get("api_key", "none")
self.is_ssl = str(config.get("is_ssl", True)).lower() in (
"true",
"1",
"yes",
)
self.output_dir = config.get("output_dir")
self.delete_audio_file = delete_audio_file
self.uri = (
f"wss://{self.host}:{self.port}"
if self.is_ssl
else f"ws://{self.host}:{self.port}"
)
self.ssl_context = ssl.SSLContext() if self.is_ssl else None
if self.ssl_context:
self.ssl_context.check_hostname = False
self.ssl_context.verify_mode = ssl.CERT_NONE
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
wf.setframerate(16000)
wf.writeframes(b"".join(pcm_data))
return file_path
async def _receive_responses(self, ws) -> None:
"""
Asynchronous generator to receive messages from the WebSocket.
Yields each message as it is received.
"""
text = ""
while True:
try:
response = await asyncio.wait_for(ws.recv(), timeout=5)
response_data = json.loads(response)
logger.bind(tag=TAG).debug(f"Received response: {response_data}")
if response_data.get("is_final", True):
text += response_data.get("text", "")
break
else:
text += response_data.get("text", "")
except asyncio.TimeoutError:
logger.bind(tag=TAG).error(
"Timeout while waiting for response from WebSocket."
)
break
except websockets.exceptions.ConnectionClosed as e:
logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
break
return text
async def _send_data(self, ws, pcm_data: bytes, session_id: str) -> tuple:
"""
Internal method to handle WebSocket communication.
Reuses the persistent WebSocket connection if available.
:param pcm_data: PCM audio data to send.
:param session_id: Unique session identifier.
:return: Tuple containing recognized text and optional timestamp.
"""
# Send initial configuration message
config_message = json.dumps(
{
"mode": "offline",
"chunk_size": [5, 10, 5],
"chunk_interval": 10,
"wav_name": session_id,
"is_speaking": True,
"itn": False,
}
)
await ws.send(config_message)
logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}")
# Send PCM data
await ws.send(pcm_data)
logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes")
# Indicate end of speech
end_message = json.dumps({"is_speaking": False})
await ws.send(end_message)
logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""
Convert speech data to text using FunASR.
:param opus_data: List of Opus-encoded audio data chunks.
:param session_id: Unique session identifier.
:return: Tuple containing recognized text and optional timestamp.
"""
file_path = None
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
auth_header = {"Authorization": "Bearer; {}".format(self.api_key)}
async with websockets.connect(
self.uri,
additional_headers=auth_header,
subprotocols=["binary"],
ping_interval=None,
ssl=self.ssl_context,
) as ws:
try:
# Use asyncio to handle WebSocket communication
send_task = asyncio.create_task(
self._send_data(ws, combined_pcm_data, session_id)
)
receive_task = asyncio.create_task(self._receive_responses(ws))
# Gather tasks with error handling
done, pending = await asyncio.wait(
[send_task, receive_task], return_when=asyncio.FIRST_EXCEPTION
)
# Cancel any pending tasks
for task in pending:
task.cancel()
# Check for exceptions in completed tasks
for task in done:
if task.exception():
raise task.exception()
# Get the result from the receive task
result = receive_task.result()
match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result)
if match:
result = match.group(4).strip()
return (
result,
file_path,
) # Return the recognized text and timestamp (if any)
except websockets.exceptions.ConnectionClosed as e:
logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
return "", file_path
except Exception as e:
logger.bind(tag=TAG).error(
f"Error during speech-to-text conversion: {e}", exc_info=True
)
return "", file_path
main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py
+26 -25
View File
@@ -37,17 +37,18 @@ class CaptureOutput:
class ASRProvider(ASRProviderBase):
def __init__(self, config: dict, delete_audio_file: bool):
super().__init__()
self.model_dir = config.get("model_dir")
self.output_dir = config.get("output_dir")
self.delete_audio_file = delete_audio_file
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
# 初始化模型文件路径
model_files = {
"model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"),
"tokens.txt": os.path.join(self.model_dir, "tokens.txt")
"tokens.txt": os.path.join(self.model_dir, "tokens.txt"),
}
# 下载并检查模型文件
@@ -58,15 +59,15 @@ class ASRProvider(ASRProviderBase):
model_file_download(
model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue",
file_path=file_name,
local_dir=self.model_dir
local_dir=self.model_dir,
)
if not os.path.isfile(file_path):
raise FileNotFoundError(f"模型文件下载失败: {file_path}")
self.model_path = model_files["model.int8.onnx"]
self.tokens_path = model_files["tokens.txt"]
except Exception as e:
logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}")
raise
@@ -83,21 +84,12 @@ class ASRProvider(ASRProviderBase):
use_itn=True,
)
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""将Opus音频数据解码并保存为WAV文件"""
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
@@ -130,14 +122,22 @@ class ASRProvider(ASRProviderBase):
samples_float32 = samples_float32 / 32768
return samples_float32, f.getframerate()
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""语音转文本主处理逻辑"""
file_path = None
try:
# 保存音频文件
start_time = time.time()
file_path = self.save_audio_to_file(opus_data, session_id)
logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
file_path = self.save_audio_to_file(pcm_data, session_id)
logger.bind(tag=TAG).debug(
f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
)
# 语音识别
start_time = time.time()
@@ -146,14 +146,15 @@ class ASRProvider(ASRProviderBase):
s.accept_waveform(sample_rate, samples)
self.model.decode_stream(s)
text = s.result.text
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
logger.bind(tag=TAG).debug(
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
)
return text, file_path
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", None
return "", file_path
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
main/xiaozhi-server/core/providers/asr/tencent.py
+80 -77
View File
@@ -17,77 +17,66 @@ from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
class ASRProvider(ASRProviderBase):
API_URL = "https://asr.tencentcloudapi.com"
API_VERSION = "2019-06-14"
FORMAT = "pcm" # 支持的音频格式：pcm, wav, mp3
def __init__(self, config: dict, delete_audio_file: bool = True):
super().__init__()
self.secret_id = config.get("secret_id")
self.secret_key = config.get("secret_key")
self.output_dir = config.get("output_dir")
self.delete_audio_file = delete_audio_file
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""将Opus音频数据解码并保存为WAV文件"""
file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav"
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
file_name = f"asr_{module_name}_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
wf.setframerate(16000)
wf.writeframes(b"".join(pcm_data))
return file_path
@staticmethod
def decode_opus(opus_data: List[bytes]) -> bytes:
"""将Opus音频数据解码为PCM数据"""
import opuslib_next
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return b"".join(pcm_data)
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
if not opus_data:
logger.bind(tag=TAG).warn("音频数据为空！")
logger.bind(tag=TAG).warning("音频数据为空！")
return None, None
file_path = None
try:
# 检查配置是否已设置
if not self.secret_id or not self.secret_key:
logger.bind(tag=TAG).error("腾讯云语音识别配置未设置，无法进行识别")
return None, None
return None, file_path
# 将Opus音频数据解码为PCM
pcm_data = self.decode_opus(opus_data)
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
# 将音频数据转换为Base64编码
base64_audio = base64.b64encode(pcm_data).decode('utf-8')
base64_audio = base64.b64encode(combined_pcm_data).decode("utf-8")
# 构建请求体
request_body = self._build_request_body(base64_audio)
@@ -98,15 +87,17 @@ class ASRProvider(ASRProviderBase):
# 发送请求
start_time = time.time()
result = self._send_request(request_body, timestamp, authorization)
if result:
logger.bind(tag=TAG).debug(f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}")
return result, None
logger.bind(tag=TAG).debug(
f"腾讯云语音识别耗时: {time.time() - start_time:.3f}s | 结果: {result}"
)
return result, file_path
except Exception as e:
logger.bind(tag=TAG).error(f"处理音频时发生错误！{e}", exc_info=True)
return None, None
return None, file_path
def _build_request_body(self, base64_audio: str) -> str:
"""构建请求体"""
@@ -117,7 +108,7 @@ class ASRProvider(ASRProviderBase):
"SourceType": 1, # 音频数据来源为语音文件
"VoiceFormat": self.FORMAT, # 音频格式
"Data": base64_audio, # Base64编码的音频数据
"DataLen": len(base64_audio) # 数据长度
"DataLen": len(base64_audio), # 数据长度
}
return json.dumps(request_map)
@@ -150,9 +141,11 @@ class ASRProvider(ASRProviderBase):
action = "SentenceRecognition" # 接口名称
# 构建规范头部信息，注意顺序和格式
canonical_headers = f"content-type:{content_type.lower()}\n" + \
f"host:{host.lower()}\n" + \
f"x-tc-action:{action.lower()}\n"
canonical_headers = (
f"content-type:{content_type.lower()}\n"
+ f"host:{host.lower()}\n"
+ f"x-tc-action:{action.lower()}\n"
)
signed_headers = "content-type;host;x-tc-action"
@@ -160,21 +153,25 @@ class ASRProvider(ASRProviderBase):
payload_hash = self._sha256_hex(request_body)
# 构建规范请求字符串
canonical_request = f"{http_request_method}\n" + \
f"{canonical_uri}\n" + \
f"{canonical_query_string}\n" + \
f"{canonical_headers}\n" + \
f"{signed_headers}\n" + \
f"{payload_hash}"
canonical_request = (
f"{http_request_method}\n"
+ f"{canonical_uri}\n"
+ f"{canonical_query_string}\n"
+ f"{canonical_headers}\n"
+ f"{signed_headers}\n"
+ f"{payload_hash}"
)
# 计算规范请求的哈希值
hashed_canonical_request = self._sha256_hex(canonical_request)
# 构建待签名字符串
string_to_sign = f"{algorithm}\n" + \
f"{timestamp}\n" + \
f"{credential_scope}\n" + \
f"{hashed_canonical_request}"
string_to_sign = (
f"{algorithm}\n"
+ f"{timestamp}\n"
+ f"{credential_scope}\n"
+ f"{hashed_canonical_request}"
)
# 计算签名密钥
secret_date = self._hmac_sha256(f"TC3{self.secret_key}", date)
@@ -182,13 +179,17 @@ class ASRProvider(ASRProviderBase):
secret_signing = self._hmac_sha256(secret_service, "tc3_request")
# 计算签名
signature = self._bytes_to_hex(self._hmac_sha256(secret_signing, string_to_sign))
signature = self._bytes_to_hex(
self._hmac_sha256(secret_signing, string_to_sign)
)
# 构建授权头
authorization = f"{algorithm} " + \
f"Credential={self.secret_id}/{credential_scope}, " + \
f"SignedHeaders={signed_headers}, " + \
f"Signature={signature}"
authorization = (
f"{algorithm} "
+ f"Credential={self.secret_id}/{credential_scope}, "
+ f"SignedHeaders={signed_headers}, "
+ f"Signature={signature}"
)
return timestamp, authorization
@@ -196,7 +197,9 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).error(f"生成认证头失败: {e}", exc_info=True)
raise RuntimeError(f"生成认证头失败: {e}")
def _send_request(self, request_body: str, timestamp: str, authorization: str) -> Optional[str]:
def _send_request(
self, request_body: str, timestamp: str, authorization: str
) -> Optional[str]:
"""发送请求到腾讯云API"""
headers = {
"Content-Type": "application/json; charset=utf-8",
@@ -205,47 +208,47 @@ class ASRProvider(ASRProviderBase):
"X-TC-Action": "SentenceRecognition",
"X-TC-Version": self.API_VERSION,
"X-TC-Timestamp": timestamp,
"X-TC-Region": "ap-shanghai"
"X-TC-Region": "ap-shanghai",
}
try:
response = requests.post(self.API_URL, headers=headers, data=request_body)
if not response.ok:
raise IOError(f"请求失败: {response.status_code} {response.reason}")
response_json = response.json()
# 检查是否有错误
if "Response" in response_json and "Error" in response_json["Response"]:
error = response_json["Response"]["Error"]
error_code = error["Code"]
error_message = error["Message"]
raise IOError(f"API返回错误: {error_code}: {error_message}")
# 提取识别结果
if "Response" in response_json and "Result" in response_json["Response"]:
return response_json["Response"]["Result"]
else:
logger.bind(tag=TAG).warn(f"响应中没有识别结果: {response_json}")
logger.bind(tag=TAG).warning(f"响应中没有识别结果: {response_json}")
return ""
except Exception as e:
logger.bind(tag=TAG).error(f"发送请求失败: {e}", exc_info=True)
return None
def _sha256_hex(self, data: str) -> str:
"""计算字符串的SHA256哈希值"""
digest = hashlib.sha256(data.encode('utf-8')).digest()
digest = hashlib.sha256(data.encode("utf-8")).digest()
return self._bytes_to_hex(digest)
def _hmac_sha256(self, key, data: str) -> bytes:
"""计算HMAC-SHA256"""
if isinstance(key, str):
key = key.encode('utf-8')
return hmac.new(key, data.encode('utf-8'), hashlib.sha256).digest()
key = key.encode("utf-8")
return hmac.new(key, data.encode("utf-8"), hashlib.sha256).digest()
def _bytes_to_hex(self, bytes_data: bytes) -> str:
"""字节数组转十六进制字符串"""
return ''.join(f"{b:02x}" for b in bytes_data)
return "".join(f"{b:02x}" for b in bytes_data)
main/xiaozhi-server/core/providers/intent/base.py
-12
View File
@@ -9,18 +9,6 @@ logger = setup_logging()
class IntentProviderBase(ABC):
def __init__(self, config):
self.config = config
self.intent_options = [
{
"name": "handle_exit_intent",
"desc": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候",
},
{
"name": "play_music",
"desc": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图",
},
{"name": "get_time", "desc": "获取今天日期或者当前时间信息"},
{"name": "continue_chat", "desc": "继续聊天"},
]
def set_llm(self, llm):
self.llm = llm
main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py
+91 -46
View File
@@ -15,57 +15,72 @@ class IntentProvider(IntentProviderBase):
def __init__(self, config):
super().__init__(config)
self.llm = None
self.promot = self.get_intent_system_prompt()
self.promot = ""
# 添加缓存管理
self.intent_cache = {} # 缓存意图识别结果
self.cache_expiry = 600 # 缓存有效期10分钟
self.cache_max_size = 100 # 最多缓存100个意图
self.history_count = 4 # 默认使用最近4条对话记录
def get_intent_system_prompt(self) -> str:
def get_intent_system_prompt(self, functions_list: str) -> str:
"""
根据配置的意图选项动态生成系统提示词
根据配置的意图选项和可用函数动态生成系统提示词
Args:
functions: 可用的函数列表，JSON格式字符串
Returns:
格式化后的系统提示词
"""
# 构建函数说明部分
functions_desc = "可用的函数列表：\n"
for func in functions_list:
func_info = func.get("function", {})
name = func_info.get("name", "")
desc = func_info.get("description", "")
params = func_info.get("parameters", {})
functions_desc += f"\n函数名: {name}\n"
functions_desc += f"描述: {desc}\n"
if params:
functions_desc += "参数:\n"
for param_name, param_info in params.get("properties", {}).items():
param_desc = param_info.get("description", "")
param_type = param_info.get("type", "")
functions_desc += f"- {param_name} ({param_type}): {param_desc}\n"
functions_desc += "---\n"
prompt = (
"你是一个意图识别助手。请分析用户的最后一句话，判断用户意图属于以下哪一类：\n"
"<start>"
f"{str(self.intent_options)}"
"<end>\n"
"处理步骤:"
"1. 思考意图类型，生成function_call格式"
"\n\n"
"返回格式示例：\n"
'1. 播放音乐意图: {"function_call": {"name": "play_music", "arguments": {"song_name": "音乐名称"}}}\n'
'2. 结束对话意图: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
'3. 获取当天日期时间: {"function_call": {"name": "get_time"}}\n'
'4. 继续聊天意图: {"function_call": {"name": "continue_chat"}}\n'
"\n"
"注意:\n"
'- 播放音乐：无歌名时，song_name设为"random"\n'
"- 如果没有明显的意图，应按照继续聊天意图处理\n"
"- 只返回纯JSON，不要任何其他内容\n"
"\n"
"示例分析:\n"
"你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n"
f"{functions_desc}\n"
"处理步骤:\n"
"1. 分析用户输入，确定用户意图\n"
"2. 从可用函数列表中选择最匹配的函数\n"
"3. 如果找到匹配的函数，生成对应的function_call 格式\n"
'4. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n'
"返回格式要求：\n"
"1. 必须返回纯JSON格式\n"
"2. 必须包含function_call字段\n"
"3. function_call必须包含name字段\n"
"4. 如果函数需要参数，必须包含arguments字段\n\n"
"示例：\n"
"```\n"
"用户: 你也太搞笑了\n"
'返回: {"function_call": {"name": "continue_chat"}}\n'
"```\n"
"```\n"
"用户: 现在是几号了?现在几点了？\n"
"用户: 现在几点了？\n"
'返回: {"function_call": {"name": "get_time"}}\n'
"```\n"
"```\n"
"用户: 我们明天再聊吧\n"
'返回: {"function_call": {"name": "handle_exit_intent"}}\n'
"用户: 我想结束对话\n"
'返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
"```\n"
"```\n"
"用户: 播放中秋月\n"
'返回: {"function_call": {"name": "play_music", "arguments": {"song_name": "中秋月"}}}\n'
"```\n"
"```\n"
"可用的音乐名称:\n"
"用户: 你好啊\n"
'返回: {"function_call": {"name": "continue_chat"}}\n'
"```\n\n"
"注意：\n"
"1. 只返回JSON格式，不要包含任何其他文字\n"
'2. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n'
"3. 确保返回的JSON格式正确，包含所有必要的字段\n"
)
return prompt
@@ -90,6 +105,14 @@ class IntentProvider(IntentProviderBase):
for key, _ in sorted_items[: len(sorted_items) - self.cache_max_size]:
del self.intent_cache[key]
def replyResult(self, text: str, original_text: str):
llm_result = self.llm.response_no_stream(
system_prompt=text,
user_prompt="请根据以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。用户现在说："
+ original_text,
)
return llm_result
async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
if not self.llm:
raise ValueError("LLM provider not set")
@@ -118,22 +141,35 @@ class IntentProvider(IntentProviderBase):
# 清理缓存
self.clean_cache()
# 构建用户最后一句话的提示
msgStr = ""
if self.promot == "":
if hasattr(conn, "func_handler"):
functions = conn.func_handler.get_functions()
self.promot = self.get_intent_system_prompt(functions)
# 只使用最后两句即可
if len(dialogue_history) >= 2:
# 保证最少有两句话的时候处理
msgStr += f"{dialogue_history[-2].role}: {dialogue_history[-2].content}\n"
msgStr += f"{dialogue_history[-1].role}: {dialogue_history[-1].content}\n"
msgStr += f"User: {text}\n"
user_prompt = f"当前的对话如下：\n{msgStr}"
music_config = initialize_music_handler(conn)
music_file_names = music_config["music_file_names"]
prompt_music = f"{self.promot}\n<start>{music_file_names}\n<end>"
prompt_music = f"{self.promot}\n<musicNames>{music_file_names}\n</musicNames>"
devices = conn.config["plugins"]["home_assistant"].get("devices", [])
if len(devices) > 0:
hass_prompt = "\n下面是我家智能设备列表（位置，设备名，entity_id），可以通过homeassistant控制\n"
for device in devices:
hass_prompt += device + "\n"
prompt_music += hass_prompt
logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
# 构建用户对话历史的提示
msgStr = ""
# 获取最近的对话历史
start_idx = max(0, len(dialogue_history) - self.history_count)
for i in range(start_idx, len(dialogue_history)):
msgStr += f"{dialogue_history[i].role}: {dialogue_history[i].content}\n"
msgStr += f"User: {text}\n"
user_prompt = f"current dialogue:\n{msgStr}"
# 记录预处理完成时间
preprocess_time = time.time() - total_start_time
logger.bind(tag=TAG).debug(f"意图识别预处理耗时: {preprocess_time:.4f}秒")
@@ -179,9 +215,18 @@ class IntentProvider(IntentProviderBase):
# 记录识别到的function call
logger.bind(tag=TAG).info(
f"识别到function call: {function_name}, 参数: {function_args}"
f"llm 识别到意图: {function_name}, 参数: {function_args}"
)
# 如果是继续聊天，清理工具调用相关的历史消息
if function_name == "continue_chat":
# 保留非工具相关的消息
clean_history = [
msg for msg in conn.dialogue.dialogue
if msg.role not in ["tool", "function"]
]
conn.dialogue.dialogue = clean_history
# 添加到缓存
self.intent_cache[cache_key] = {
"intent": intent,
main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py
+18 -4
View File
@@ -2,10 +2,12 @@ from config.logger import setup_logging
from http import HTTPStatus
from dashscope import Application
from core.providers.llm.base import LLMProviderBase
from core.utils.util import check_model_key
TAG = __name__
logger = setup_logging()
class LLMProvider(LLMProviderBase):
def __init__(self, config):
self.api_key = config["api_key"]
@@ -13,27 +15,32 @@ class LLMProvider(LLMProviderBase):
self.base_url = config.get("base_url")
self.is_No_prompt = config.get("is_no_prompt")
self.memory_id = config.get("ali_memory_id")
check_model_key("AliBLLLM", self.api_key)
def response(self, session_id, dialogue):
try:
# 处理dialogue
if self.is_No_prompt:
dialogue.pop(0)
logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的dialogue: {dialogue}")
logger.bind(tag=TAG).debug(
f"【阿里百练API服务】处理后的dialogue: {dialogue}"
)
# 构造调用参数
call_params = {
"api_key": self.api_key,
"app_id": self.app_id,
"session_id": session_id,
"messages": dialogue
"messages": dialogue,
}
if self.memory_id != False:
# 百练memory需要prompt参数
prompt = dialogue[-1].get("content")
call_params["memory_id"] = self.memory_id
call_params["prompt"] = prompt
logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的prompt: {prompt}")
logger.bind(tag=TAG).debug(
f"【阿里百练API服务】处理后的prompt: {prompt}"
)
responses = Application.call(**call_params)
if responses.status_code != HTTPStatus.OK:
@@ -44,9 +51,16 @@ class LLMProvider(LLMProviderBase):
)
yield "【阿里百练API服务响应异常】"
else:
logger.bind(tag=TAG).debug(f"【阿里百练API服务】构造参数: {call_params}")
logger.bind(tag=TAG).debug(
f"【阿里百练API服务】构造参数: {call_params}"
)
yield responses.output.text
except Exception as e:
logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
yield "【LLM服务响应异常】"
def response_with_functions(self, session_id, dialogue, functions=None):
logger.bind(tag=TAG).error(
f"阿里百练暂未实现完整的工具调用（function call），建议使用其他意图识别"
)
main/xiaozhi-server/core/providers/llm/base.py
+2 -1
View File
@@ -35,4 +35,5 @@ class LLMProviderBase(ABC):
"""
# For providers that don't support functions, just return regular response
for token in self.response(session_id, dialogue):
yield {"type": "content", "content": token}
yield token, None
main/xiaozhi-server/core/providers/llm/coze/coze.py
+35 -12
View File
@@ -1,12 +1,17 @@
from config.logger import setup_logging
import requests
import json
import re
from core.providers.llm.base import LLMProviderBase
import os
# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
from cozepy import COZE_CN_BASE_URL
from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType # noqa
from cozepy import (
Coze,
TokenAuth,
Message,
ChatEventType,
) # noqa
from core.providers.llm.system_prompt import get_system_prompt_for_function
from core.utils.util import check_model_key
TAG = __name__
logger = setup_logging()
@@ -15,25 +20,23 @@ logger = setup_logging()
class LLMProvider(LLMProviderBase):
def __init__(self, config):
self.personal_access_token = config.get("personal_access_token")
self.bot_id = config.get("bot_id")
self.user_id = config.get("user_id")
self.bot_id = str(config.get("bot_id"))
self.user_id = str(config.get("user_id"))
self.session_conversation_map = {} # 存储session_id和conversation_id的映射
check_model_key("CozeLLM", self.personal_access_token)
def response(self, session_id, dialogue):
coze_api_token = self.personal_access_token
coze_api_base = COZE_CN_BASE_URL
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
conversation_id = self.session_conversation_map.get(session_id)
# 如果没有找到conversation_id，则创建新的对话
if not conversation_id:
conversation = coze.conversations.create(
messages=[
]
)
conversation = coze.conversations.create(messages=[])
conversation_id = conversation.id
self.session_conversation_map[session_id] = conversation_id # 更新映射
@@ -47,4 +50,24 @@ class LLMProvider(LLMProviderBase):
):
if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
print(event.message.content, end="", flush=True)
yield event.message.content
yield event.message.content
def response_with_functions(self, session_id, dialogue, functions=None):
if len(dialogue) == 2 and functions is not None and len(functions) > 0:
# 第一次调用llm， 取最后一条用户消息，附加tool提示词
last_msg = dialogue[-1]["content"]
function_str = json.dumps(functions, ensure_ascii=False)
modify_msg = get_system_prompt_for_function(function_str) + last_msg
dialogue[-1]["content"] = modify_msg
# 如果最后一个是 role="tool"，附加到user上
if len(dialogue) > 1 and dialogue[-1]["role"] == "tool":
assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n"
while len(dialogue) > 1:
if dialogue[-1]["role"] == "user":
dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"]
break
dialogue.pop()
for token in self.response(session_id, dialogue):
yield token, None
main/xiaozhi-server/core/providers/llm/dify/dify.py
+31 -2
View File
@@ -2,6 +2,8 @@ import json
from config.logger import setup_logging
import requests
from core.providers.llm.base import LLMProviderBase
from core.providers.llm.system_prompt import get_system_prompt_for_function
from core.utils.util import check_model_key
TAG = __name__
logger = setup_logging()
@@ -13,6 +15,7 @@ class LLMProvider(LLMProviderBase):
self.mode = config.get("mode", "chat-messages")
self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip("/")
self.session_conversation_map = {} # 存储session_id和conversation_id的映射
check_model_key("DifyLLM", self.api_key)
def response(self, session_id, dialogue):
try:
@@ -58,7 +61,10 @@ class LLMProvider(LLMProviderBase):
self.session_conversation_map[session_id] = (
conversation_id # 更新映射
)
if event.get("answer"):
# 过滤 message_replace 事件，此事件会全量推一次
if event.get("event") != "message_replace" and event.get(
"answer"
):
yield event["answer"]
elif self.mode == "workflows/run":
for line in r.iter_lines():
@@ -73,9 +79,32 @@ class LLMProvider(LLMProviderBase):
for line in r.iter_lines():
if line.startswith(b"data: "):
event = json.loads(line[6:])
if event.get("answer"):
# 过滤 message_replace 事件，此事件会全量推一次
if event.get("event") != "message_replace" and event.get(
"answer"
):
yield event["answer"]
except Exception as e:
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
yield "【服务响应异常】"
def response_with_functions(self, session_id, dialogue, functions=None):
if len(dialogue) == 2 and functions is not None and len(functions) > 0:
# 第一次调用llm， 取最后一条用户消息，附加tool提示词
last_msg = dialogue[-1]["content"]
function_str = json.dumps(functions, ensure_ascii=False)
modify_msg = get_system_prompt_for_function(function_str) + last_msg
dialogue[-1]["content"] = modify_msg
# 如果最后一个是 role="tool"，附加到user上
if len(dialogue) > 1 and dialogue[-1]["role"] == "tool":
assistant_msg = "\ntool call result: " + dialogue[-1]["content"] + "\n\n"
while len(dialogue) > 1:
if dialogue[-1]["role"] == "user":
dialogue[-1]["content"] = assistant_msg + dialogue[-1]["content"]
break
dialogue.pop()
for token in self.response(session_id, dialogue):
yield token, None
main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py
+30 -24
View File
@@ -2,6 +2,7 @@ import json
from config.logger import setup_logging
import requests
from core.providers.llm.base import LLMProviderBase
from core.utils.util import check_model_key
TAG = __name__
logger = setup_logging()
@@ -13,6 +14,7 @@ class LLMProvider(LLMProviderBase):
self.base_url = config.get("base_url")
self.detail = config.get("detail", False)
self.variables = config.get("variables", {})
check_model_key("FastGPTLLM", self.api_key)
def response(self, session_id, dialogue):
try:
@@ -21,37 +23,36 @@ class LLMProvider(LLMProviderBase):
# 发起流式请求
with requests.post(
f"{self.base_url}/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json={
"stream": True,
"chatId": session_id,
"detail": self.detail,
"variables": self.variables,
"messages": [
{
"role": "user",
"content": last_msg["content"]
}
]
},
stream=True
f"{self.base_url}/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json={
"stream": True,
"chatId": session_id,
"detail": self.detail,
"variables": self.variables,
"messages": [{"role": "user", "content": last_msg["content"]}],
},
stream=True,
) as r:
for line in r.iter_lines():
if line:
try:
if line.startswith(b'data: '):
if line[6:].decode('utf-8') == '[DONE]':
if line.startswith(b"data: "):
if line[6:].decode("utf-8") == "[DONE]":
break
data = json.loads(line[6:])
if 'choices' in data and len(data['choices']) > 0:
delta = data['choices'][0].get('delta', {})
if delta and 'content' in delta and delta['content'] is not None:
content = delta['content']
if '<think>' in content:
if "choices" in data and len(data["choices"]) > 0:
delta = data["choices"][0].get("delta", {})
if (
delta
and "content" in delta
and delta["content"] is not None
):
content = delta["content"]
if "<think>" in content:
continue
if '</think>' in content:
if "</think>" in content:
continue
yield content
@@ -62,4 +63,9 @@ class LLMProvider(LLMProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
yield "【服务响应异常】"
yield "【服务响应异常】"
def response_with_functions(self, session_id, dialogue, functions=None):
logger.bind(tag=TAG).error(
f"fastgpt暂未实现完整的工具调用（function call），建议使用其他意图识别"
)
main/xiaozhi-server/core/providers/llm/gemini/gemini.py
+191 -122
View File
@@ -1,136 +1,205 @@
import google.generativeai as genai
from core.utils.util import check_model_key
from core.providers.llm.base import LLMProviderBase
from config.logger import setup_logging
import os, json, uuid
from types import SimpleNamespace
from typing import Any, Dict, List
import requests
import json
from google import generativeai as genai
from google.generativeai import types, GenerationConfig
from core.providers.llm.base import LLMProviderBase
from core.utils.util import check_model_key
from config.logger import setup_logging
from google.generativeai.types import GenerateContentResponse
from requests import RequestException
log = setup_logging()
TAG = __name__
logger = setup_logging()
def test_proxy(proxy_url: str, test_url: str) -> bool:
try:
resp = requests.get(test_url, proxies={"http": proxy_url, "https": proxy_url})
return 200 <= resp.status_code < 400
except RequestException:
return False
def setup_proxy_env(http_proxy: str | None, https_proxy: str | None):
"""
分别测试 HTTP 和 HTTPS 代理是否可用，并设置环境变量。
如果 HTTPS 代理不可用但 HTTP 可用，会将 HTTPS_PROXY 也指向 HTTP。
"""
test_http_url = "http://www.google.com"
test_https_url = "https://www.google.com"
ok_http = ok_https = False
if http_proxy:
ok_http = test_proxy(http_proxy, test_http_url)
if ok_http:
os.environ["HTTP_PROXY"] = http_proxy
log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {http_proxy}")
else:
log.bind(tag=TAG).warning(f"配置提供的Gemini HTTP代理不可用: {http_proxy}")
if https_proxy:
ok_https = test_proxy(https_proxy, test_https_url)
if ok_https:
os.environ["HTTPS_PROXY"] = https_proxy
log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {https_proxy}")
else:
log.bind(tag=TAG).warning(
f"配置提供的Gemini HTTPS代理不可用: {https_proxy}"
)
# 如果https_proxy不可用，但http_proxy可用且能走通https，则复用http_proxy作为https_proxy
if ok_http and not ok_https:
if test_proxy(http_proxy, test_https_url):
os.environ["HTTPS_PROXY"] = http_proxy
ok_https = True
log.bind(tag=TAG).info(f"复用HTTP代理作为HTTPS代理: {http_proxy}")
if not ok_http and not ok_https:
log.bind(tag=TAG).error(
f"Gemini 代理设置失败: HTTP 和 HTTPS 代理都不可用，请检查配置"
)
raise RuntimeError("HTTP 和 HTTPS 代理都不可用，请检查配置")
class LLMProvider(LLMProviderBase):
def __init__(self, config):
"""初始化Gemini LLM Provider"""
self.model_name = config.get("model_name", "gemini-1.5-pro")
self.api_key = config.get("api_key")
self.http_proxy=config.get("http_proxy")
self.https_proxy = config.get("https_proxy")
have_key = check_model_key("LLM", self.api_key)
def __init__(self, cfg: Dict[str, Any]):
self.model_name = cfg.get("model_name", "gemini-2.0-flash")
self.api_key = cfg["api_key"]
http_proxy = cfg.get("http_proxy")
https_proxy = cfg.get("https_proxy")
if not have_key:
return
if not check_model_key("LLM", self.api_key):
raise ValueError("无效的Gemini API Key，请检查是否配置正确")
try:
# 初始化Gemini客户端
# 配置代理（如果提供了代理配置）
self.proxies=None
if self.http_proxy is not "" or self.https_proxy is not "":
if http_proxy or https_proxy:
log.bind(tag=TAG).info(
f"检测到Gemini代理配置，开始测试代理连通性和设置代理环境..."
)
setup_proxy_env(http_proxy, https_proxy)
log.bind(tag=TAG).info(
f"Gemini 代理设置成功 - HTTP: {http_proxy}, HTTPS: {https_proxy}"
)
genai.configure(api_key=self.api_key)
self.model = genai.GenerativeModel(self.model_name)
self.proxies = {
"http": self.http_proxy,
"https": self.https_proxy,
}
logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}")
# 使用猴子补丁修改 google-generativeai 库的请求会话
self.gen_cfg = GenerationConfig(
temperature=0.7,
top_p=0.9,
top_k=40,
max_output_tokens=2048,
)
# 使用 session 对象配置 genai
genai.configure(api_key=self.api_key)
self.model = genai.GenerativeModel(self.model_name)
# 设置生成参数
self.generation_config = {
"temperature": 0.7,
"top_p": 0.9,
"top_k": 40,
"max_output_tokens": 2048,
}
self.chat = None
except Exception as e:
logger.bind(tag=TAG).error(f"Gemini初始化失败: {e}")
self.model = None
@staticmethod
def _build_tools(funcs: List[Dict[str, Any]] | None):
if not funcs:
return None
return [
types.Tool(
function_declarations=[
types.FunctionDeclaration(
name=f["function"]["name"],
description=f["function"]["description"],
parameters=f["function"]["parameters"],
)
for f in funcs
]
)
]
# Gemini文档提到，无需维护session-id，直接用dialogue拼接而成
def response(self, session_id, dialogue):
"""生成Gemini对话响应"""
if not self.model:
yield "【Gemini服务未正确初始化】"
return
yield from self._generate(dialogue, None)
def response_with_functions(self, session_id, dialogue, functions=None):
yield from self._generate(dialogue, self._build_tools(functions))
def _generate(self, dialogue, tools):
role_map = {"assistant": "model", "user": "user"}
contents: list = []
# 拼接对话
for m in dialogue:
r = m["role"]
if r == "assistant" and "tool_calls" in m:
tc = m["tool_calls"][0]
contents.append(
{
"role": "model",
"parts": [
{
"function_call": {
"name": tc["function"]["name"],
"args": json.loads(tc["function"]["arguments"]),
}
}
],
}
)
continue
if r == "tool":
contents.append(
{
"role": "model",
"parts": [{"text": str(m.get("content", ""))}],
}
)
continue
contents.append(
{
"role": role_map.get(r, "user"),
"parts": [{"text": str(m.get("content", ""))}],
}
)
stream: GenerateContentResponse = self.model.generate_content(
contents=contents,
generation_config=self.gen_cfg,
tools=tools,
stream=True,
)
try:
# 处理对话历史
chat_history = []
for msg in dialogue[:-1]: # 历史对话
role = "model" if msg["role"] == "assistant" else "user"
content = msg["content"].strip()
if content:
chat_history.append({
"role": role,
"parts": [{"text":content}]
for chunk in stream:
cand = chunk.candidates[0]
for part in cand.content.parts:
# a) 函数调用-通常是最后一段话才是函数调用
if getattr(part, "function_call", None):
fc = part.function_call
yield None, [
SimpleNamespace(
id=uuid.uuid4().hex,
type="function",
function=SimpleNamespace(
name=fc.name,
arguments=json.dumps(
dict(fc.args), ensure_ascii=False
),
),
)
]
return
# b) 普通文本
if getattr(part, "text", None):
yield part.text if tools is None else (part.text, None)
})
finally:
if tools is not None:
yield None, None # function‑mode 结束，返回哑包
# 获取当前消息
current_msg = dialogue[-1]["content"]
# 构建请求体
request_body = {
"contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}],
"generationConfig": self.generation_config
}
# 构建请求URL
url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}"
# 构建请求头
headers = {
"Content-Type": "application/json",
}
# 发送POST请求,经测试手动 request 无法使用 stream 模式
if self.proxies:
response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies)
try:
data = response.json() # 直接解析JSON
if 'candidates' in data and data['candidates']:
yield data['candidates'][0]['content']['parts'][0]['text']
else:
yield "未找到候选回复。"
except json.JSONDecodeError as e:
yield f"JSON解码错误：{e}"
except Exception as e:
yield f"发生错误：{e}"
else:
logger.bind(tag=TAG).info(f"Gemini stream mode ")
chat = self.model.start_chat(history=chat_history)
# 发送消息并获取流式响应
response = chat.send_message(
current_msg,
stream=True,
generation_config=self.generation_config
)
# 处理流式响应
for chunk in response:
if hasattr(chunk, 'text') and chunk.text:
yield chunk.text
except Exception as e:
error_msg = str(e)
logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}")
# 针对不同错误返回友好提示
if "Rate limit" in error_msg:
yield "【Gemini服务请求太频繁,请稍后再试】"
elif "Invalid API key" in error_msg:
yield "【Gemini API key无效】"
else:
yield f"【Gemini服务响应异常: {error_msg}】"
except requests.exceptions.RequestException as e:
yield f"请求失败：{e}"
except json.JSONDecodeError as e:
yield f"JSON解码错误：{e}"
except Exception as e:
yield f"发生错误：{e}"
# 关闭stream，预留后续打断对话功能的功能方法，官方文档推荐打断对话要关闭上一个流，可以有效减少配额计费和资源占用
@staticmethod
def _safe_finish_stream(stream: GenerateContentResponse):
if hasattr(stream, "resolve"):
stream.resolve() # Gemini SDK version ≥ 0.5.0
elif hasattr(stream, "close"):
stream.close() # Gemini SDK version < 0.5.0
else:
for _ in stream: # 兜底耗尽
pass
main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py
+71
View File
@@ -0,0 +1,71 @@
import requests
from requests.exceptions import RequestException
from config.logger import setup_logging
from core.providers.llm.base import LLMProviderBase
TAG = __name__
logger = setup_logging()
class LLMProvider(LLMProviderBase):
def __init__(self, config):
self.agent_id = config.get("agent_id") # 对应 agent_id
self.api_key = config.get("api_key")
self.base_url = config.get("base_url", config.get("url")) # 默认使用 base_url
self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL
def response(self, session_id, dialogue):
try:
# home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
# 提取最后一个 role 为 'user' 的 content
input_text = None
if isinstance(dialogue, list): # 确保 dialogue 是一个列表
# 逆序遍历，找到最后一个 role 为 'user' 的消息
for message in reversed(dialogue):
if message.get("role") == "user": # 找到 role 为 'user' 的消息
input_text = message.get("content", "")
break # 找到后立即退出循环
# 构造请求数据
payload = {
"text": input_text,
"agent_id": self.agent_id,
"conversation_id": session_id, # 使用 session_id 作为 conversation_id
}
# 设置请求头
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
# 发起 POST 请求
response = requests.post(self.api_url, json=payload, headers=headers)
# 检查请求是否成功
response.raise_for_status()
# 解析返回数据
data = response.json()
speech = (
data.get("response", {})
.get("speech", {})
.get("plain", {})
.get("speech", "")
)
# 返回生成的内容
if speech:
yield speech
else:
logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
except RequestException as e:
logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
except Exception as e:
logger.bind(tag=TAG).error(f"生成响应时出错: {e}")
def response_with_functions(self, session_id, dialogue, functions=None):
logger.bind(tag=TAG).error(
f"homeassistant不支持（function call），建议使用其他意图识别"
)
main/xiaozhi-server/core/providers/llm/ollama/ollama.py
+106 -9
View File
@@ -21,27 +21,67 @@ class LLMProvider(LLMProviderBase):
api_key="ollama" # Ollama doesn't need an API key but OpenAI client requires one
)
# 检查是否是qwen3模型
self.is_qwen3 = self.model_name and self.model_name.lower().startswith("qwen3")
def response(self, session_id, dialogue):
try:
# 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
if self.is_qwen3:
# 复制对话列表，避免修改原始对话
dialogue_copy = dialogue.copy()
# 找到最后一条用户消息
for i in range(len(dialogue_copy) - 1, -1, -1):
if dialogue_copy[i]["role"] == "user":
# 在用户消息前添加/no_think指令
dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
break
# 使用修改后的对话
dialogue = dialogue_copy
responses = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
stream=True
)
is_active=True
is_active = True
# 用于处理跨chunk的标签
buffer = ""
for chunk in responses:
try:
delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
content = delta.content if hasattr(delta, 'content') else ''
if content:
if '<think>' in content:
# 将内容添加到缓冲区
buffer += content
# 处理缓冲区中的标签
while '<think>' in buffer and '</think>' in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split('<think>', 1)[0]
post = buffer.split('</think>', 1)[1]
buffer = pre + post
# 处理只有开始标签的情况
if '<think>' in buffer:
is_active = False
content = content.split('<think>')[0]
if '</think>' in content:
buffer = buffer.split('<think>', 1)[0]
# 处理只有结束标签的情况
if '</think>' in buffer:
is_active = True
content = content.split('</think>')[-1]
if is_active:
yield content
buffer = buffer.split('</think>', 1)[1]
# 如果当前处于活动状态且缓冲区有内容，则输出
if is_active and buffer:
yield buffer
buffer = "" # 清空缓冲区
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
@@ -51,6 +91,22 @@ class LLMProvider(LLMProviderBase):
def response_with_functions(self, session_id, dialogue, functions=None):
try:
# 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
if self.is_qwen3:
# 复制对话列表，避免修改原始对话
dialogue_copy = dialogue.copy()
# 找到最后一条用户消息
for i in range(len(dialogue_copy) - 1, -1, -1):
if dialogue_copy[i]["role"] == "user":
# 在用户消息前添加/no_think指令
dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
break
# 使用修改后的对话
dialogue = dialogue_copy
stream = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
@@ -58,9 +114,50 @@ class LLMProvider(LLMProviderBase):
tools=functions,
)
is_active = True
buffer = ""
for chunk in stream:
yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
try:
delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
content = delta.content if hasattr(delta, 'content') else None
tool_calls = delta.tool_calls if hasattr(delta, 'tool_calls') else None
# 如果是工具调用，直接传递
if tool_calls:
yield None, tool_calls
continue
# 处理文本内容
if content:
# 将内容添加到缓冲区
buffer += content
# 处理缓冲区中的标签
while '<think>' in buffer and '</think>' in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split('<think>', 1)[0]
post = buffer.split('</think>', 1)[1]
buffer = pre + post
# 处理只有开始标签的情况
if '<think>' in buffer:
is_active = False
buffer = buffer.split('<think>', 1)[0]
# 处理只有结束标签的情况
if '</think>' in buffer:
is_active = True
buffer = buffer.split('</think>', 1)[1]
# 如果当前处于活动状态且缓冲区有内容，则输出
if is_active and buffer:
yield buffer, None
buffer = "" # 清空缓冲区
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
continue
except Exception as e:
logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")
yield {"type": "content", "content": f"【Ollama服务响应异常: {str(e)}】"}
yield f"【Ollama服务响应异常: {str(e)}】", None
main/xiaozhi-server/core/providers/llm/openai/openai.py
+35 -15
View File
@@ -1,4 +1,5 @@
import openai
from openai.types import CompletionUsage
from config.logger import setup_logging
from core.utils.util import check_model_key
from core.providers.llm.base import LLMProviderBase
@@ -11,11 +12,19 @@ class LLMProvider(LLMProviderBase):
def __init__(self, config):
self.model_name = config.get("model_name")
self.api_key = config.get("api_key")
if 'base_url' in config:
if "base_url" in config:
self.base_url = config.get("base_url")
else:
self.base_url = config.get("url")
self.max_tokens = config.get("max_tokens", 500)
max_tokens = config.get("max_tokens")
if max_tokens is None or max_tokens == "":
max_tokens = 500
try:
max_tokens = int(max_tokens)
except (ValueError, TypeError):
max_tokens = 500
self.max_tokens = max_tokens
check_model_key("LLM", self.api_key)
self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
@@ -33,18 +42,22 @@ class LLMProvider(LLMProviderBase):
for chunk in responses:
try:
# 检查是否存在有效的choice且content不为空
delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
content = delta.content if hasattr(delta, 'content') else ''
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else ""
except IndexError:
content = ''
content = ""
if content:
# 处理标签跨多个chunk的情况
if '<think>' in content:
if "<think>" in content:
is_active = False
content = content.split('<think>')[0]
if '</think>' in content:
content = content.split("<think>")[0]
if "</think>" in content:
is_active = True
content = content.split('</think>')[-1]
content = content.split("</think>")[-1]
if is_active:
yield content
@@ -54,15 +67,22 @@ class LLMProvider(LLMProviderBase):
def response_with_functions(self, session_id, dialogue, functions=None):
try:
stream = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
stream=True,
tools=functions
model=self.model_name, messages=dialogue, stream=True, tools=functions
)
for chunk in stream:
yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
# 检查是否存在有效的choice且content不为空
if getattr(chunk, "choices", None):
yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
# 存在 CompletionUsage 消息时，生成 Token 消耗 log
elif isinstance(getattr(chunk, 'usage', None), CompletionUsage):
usage_info = getattr(chunk, 'usage', None)
logger.bind(tag=TAG).info(
f"Token 消耗：输入 {getattr(usage_info, 'prompt_tokens', '未知')}，"
f"输出 {getattr(usage_info, 'completion_tokens', '未知')}，"
f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
)
except Exception as e:
logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
yield f"【OpenAI服务响应异常: {e}】", None
main/xiaozhi-server/core/providers/llm/system_prompt.py
+103
View File
@@ -0,0 +1,103 @@
def get_system_prompt_for_function(functions: str) -> str:
"""
生成系统提示信息
:param functions: 可用的函数列表
:return: 系统提示信息
"""
SYSTEM_PROMPT = f"""
====
TOOL USE
You have access to a set of tools that are executed upon the user's approval. You can use one tool per message, and will receive the result of that tool use in the user's response.
You use tools step-by-step to accomplish a given task, with each tool use informed by the result of the previous tool use.
# Tool Use Formatting
Tool use is formatted using JSON-style tags. The tool name is enclosed in opening and closing tags, and each parameter is similarly enclosed within its own set of tags.
Here's the structure:
<tool_call>
{{
"name": "function name",
"arguments": {{
"param1": "value1",
"param2": "value2",
// Add more parameters as needed, if parameters are required, you must provide them
}}
}}
<tool_call>
For example:
if you got tool as follow
{{
"type": "function",
"function": {{
"name": "handle_exit_intent",
"description": "当用户想结束对话或需要退出系统时调用",
"parameters": {{
"type": "object",
"properties": {{
"say_goodbye": {{
"type": "string",
"description": "和用户友好结束对话的告别语",
}}
}},
"required": ["say_goodbye"],
}},
}},
}}
you should respond with the following format:
<tool_call>
{{
"name": "handle_exit_intent",
"arguments": {{
"say_goodbye": "再见，祝您生活愉快！"
}}
}}
</tool_call>
Always adhere to this format for the tool use to ensure proper parsing and execution.
# Tools
{functions}
# Tool Use Guidelines
1. Tools must be called in a separate message, Do not add thoughts when calling tools. The message must start with <tool_call> and end with </tool_call>, with the tool invocation JSON data in between. No additional response content is needed.
2. Choose the most appropriate tool based on the task and the tool descriptions provided. Assess if you need additional information to proceed, and which of the available tools would be most effective for gathering this information.
For example using the list_files tool is more effective than running a command like \`ls\` in the terminal. It's critical that you think about each available tool and use the one that best fits the current step in the task.
3. If multiple actions are needed, use one tool at a time per message to accomplish the task iteratively, with each tool use being informed by the result of the previous tool use. Do not assume the outcome of any tool use.
Each step must be informed by the previous step's result.
4. Formulate your tool use using the JSON format specified for each tool.
5. After each tool use, the user will respond with the result of that tool use. This result will provide you with the necessary information to continue your task or make further decisions. This response may include:
- Information about whether the tool succeeded or failed, along with any reasons for failure.
- Linter errors that may have arisen due to the changes you made, which you'll need to address.
- New terminal output in reaction to the changes, which you may need to consider or act upon.
- Any other relevant feedback or information related to the tool use.
6. ALWAYS wait for user confirmation after each tool use before proceeding. Never assume the success of a tool use without explicit confirmation of the result from the user.
7. Tool calls should contain no extra information. Only after receiving the tool's response should you integrate it into a complete reply.
It is crucial to proceed step-by-step, waiting for the user's message after each tool use before moving forward with the task. This approach allows you to:
1. Confirm the success of each step before proceeding.
2. Address any issues or errors that arise immediately.
3. Adapt your approach based on new information or unexpected results.
4. Ensure that each action builds correctly on the previous ones.
By waiting for and carefully considering the user's response after each tool use, you can react accordingly and make informed decisions about how to proceed with the task. This iterative process helps ensure the overall success and accuracy of your work.
====
USER CHAT CONTENT
The following additional message is the user's chat message, and should be followed to the best of your ability without interfering with the TOOL USE guidelines.
"""
return SYSTEM_PROMPT
main/xiaozhi-server/core/providers/memory/base.py
+3 -2
View File
@@ -4,6 +4,7 @@ from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
class MemoryProviderBase(ABC):
def __init__(self, config):
self.config = config
@@ -20,6 +21,6 @@ class MemoryProviderBase(ABC):
"""Query memories for specific role based on similarity"""
return "please implement query method"
def init_memory(self, role_id, llm):
self.role_id = role_id
def init_memory(self, role_id, llm, **kwargs):
self.role_id = role_id
self.llm = llm
main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py
+21 -19
View File
@@ -6,13 +6,14 @@ from core.utils.util import check_model_key
TAG = __name__
class MemoryProvider(MemoryProviderBase):
def __init__(self, config):
def __init__(self, config, summary_memory=None):
super().__init__(config)
self.api_key = config.get("api_key", "")
self.api_version = config.get("api_version", "v1.1")
have_key = check_model_key("Mem0ai", self.api_key)
if not have_key :
if not have_key:
self.use_mem0 = False
return
else:
@@ -30,54 +31,55 @@ class MemoryProvider(MemoryProviderBase):
return None
if len(msgs) < 2:
return None
try:
# Format the content as a message list for mem0
messages = [
{"role": message.role, "content": message.content}
for message in msgs if message.role != "system"
for message in msgs
if message.role != "system"
]
result = self.client.add(messages, user_id=self.role_id, output_format=self.api_version)
result = self.client.add(
messages, user_id=self.role_id, output_format=self.api_version
)
logger.bind(tag=TAG).debug(f"Save memory result: {result}")
except Exception as e:
logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
return None
async def query_memory(self, query: str)-> str:
async def query_memory(self, query: str) -> str:
if not self.use_mem0:
return ""
try:
results = self.client.search(
query,
user_id=self.role_id,
output_format=self.api_version
query, user_id=self.role_id, output_format=self.api_version
)
if not results or 'results' not in results:
if not results or "results" not in results:
return ""
# Format each memory entry with its update time up to minutes
memories = []
for entry in results['results']:
timestamp = entry.get('updated_at', '')
for entry in results["results"]:
timestamp = entry.get("updated_at", "")
if timestamp:
try:
# Parse and reformat the timestamp
dt = timestamp.split('.')[0] # Remove milliseconds
formatted_time = dt.replace('T', ' ')
dt = timestamp.split(".")[0] # Remove milliseconds
formatted_time = dt.replace("T", " ")
except:
formatted_time = timestamp
memory = entry.get('memory', '')
memory = entry.get("memory", "")
if timestamp and memory:
# Store tuple of (timestamp, formatted_string) for sorting
memories.append((timestamp, f"[{formatted_time}] {memory}"))
# Sort by timestamp in descending order (newest first)
memories.sort(key=lambda x: x[0], reverse=True)
# Extract only the formatted strings
memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
logger.bind(tag=TAG).debug(f"Query results: {memories_str}")
return memories_str
except Exception as e:
logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}")
return ""
return ""
main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py
+68 -39
View File
@@ -3,7 +3,9 @@ import time
import json
import os
import yaml
from core.utils.util import get_project_dir
from config.config_loader import get_project_dir
from config.manage_api_client import save_mem_local_short
short_term_memory_prompt = """
# 时空记忆编织者
@@ -71,11 +73,23 @@ short_term_memory_prompt = """
```
"""
short_term_memory_prompt_only_content = """
你是一个经验丰富的记忆总结者，擅长将对话内容进行总结摘要，遵循以下规则：
1、总结user的重要信息，以便在未来的对话中提供更个性化的服务
2、不要重复总结，不要遗忘之前记忆，除非原来的记忆超过了1800字内，否则不要遗忘、不要压缩用户的历史记忆
3、用户操控的设备音量、播放音乐、天气、退出、不想对话等和用户本身无关的内容，这些信息不需要加入到总结中
4、不要把设备操控的成果结果和失败结果加入到总结中，也不要把用户的一些废话加入到总结中
5、不要为了总结而总结，如果用户的聊天没有意义，请返回原来的历史记录也是可以的
6、只需要返回总结摘要，严格控制在1800字内
7、不要包含代码、xml，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
"""
def extract_json_data(json_code):
start = json_code.find("```json")
# 从start开始找到下一个```结束
end = json_code.find("```", start+1)
#print("start:", start, "end:", end)
end = json_code.find("```", start + 1)
# print("start:", start, "end:", end)
if start == -1 or end == -1:
try:
jsonData = json.loads(json_code)
@@ -83,74 +97,89 @@ def extract_json_data(json_code):
except Exception as e:
print("Error:", e)
return ""
jsonData = json_code[start+7:end]
jsonData = json_code[start + 7 : end]
return jsonData
TAG = __name__
class MemoryProvider(MemoryProviderBase):
def __init__(self, config):
def __init__(self, config, summary_memory):
super().__init__(config)
self.short_momery = ""
self.memory_path = get_project_dir() + 'data/.memory.yaml'
self.load_memory()
self.save_to_file = True
self.memory_path = get_project_dir() + "data/.memory.yaml"
self.load_memory(summary_memory)
def init_memory(
self, role_id, llm, summary_memory=None, save_to_file=True, **kwargs
):
super().init_memory(role_id, llm, **kwargs)
self.save_to_file = save_to_file
self.load_memory(summary_memory)
def load_memory(self, summary_memory):
# api获取到总结记忆后直接返回
if summary_memory or not self.save_to_file:
self.short_momery = summary_memory
return
def init_memory(self, role_id, llm):
super().init_memory(role_id, llm)
self.load_memory()
def load_memory(self):
all_memory = {}
if os.path.exists(self.memory_path):
with open(self.memory_path, 'r', encoding='utf-8') as f:
with open(self.memory_path, "r", encoding="utf-8") as f:
all_memory = yaml.safe_load(f) or {}
if self.role_id in all_memory:
self.short_momery = all_memory[self.role_id]
def save_memory_to_file(self):
all_memory = {}
if os.path.exists(self.memory_path):
with open(self.memory_path, 'r', encoding='utf-8') as f:
all_memory = yaml.safe_load(f) or {}
with open(self.memory_path, "r", encoding="utf-8") as f:
all_memory = yaml.safe_load(f) or {}
all_memory[self.role_id] = self.short_momery
with open(self.memory_path, 'w', encoding='utf-8') as f:
with open(self.memory_path, "w", encoding="utf-8") as f:
yaml.dump(all_memory, f, allow_unicode=True)
async def save_memory(self, msgs):
if self.llm is None:
logger.bind(tag=TAG).error("LLM is not set for memory provider")
return None
if len(msgs) < 2:
return None
msgStr = ""
for msg in msgs:
if msg.role == "user":
msgStr += f"User: {msg.content}\n"
elif msg.role== "assistant":
elif msg.role == "assistant":
msgStr += f"Assistant: {msg.content}\n"
if len(self.short_momery) > 0:
msgStr+="历史记忆：\n"
msgStr+=self.short_momery
#当前时间
if self.short_momery and len(self.short_momery) > 0:
msgStr += "历史记忆：\n"
msgStr += self.short_momery
# 当前时间
time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
msgStr += f"当前时间：{time_str}"
result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
json_str = extract_json_data(result)
try:
json_data = json.loads(json_str) # 检查json格式是否正确
self.short_momery = json_str
except Exception as e:
print("Error:", e)
self.save_memory_to_file()
if self.save_to_file:
result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
json_str = extract_json_data(result)
try:
json.loads(json_str) # 检查json格式是否正确
self.short_momery = json_str
self.save_memory_to_file()
except Exception as e:
print("Error:", e)
else:
result = self.llm.response_no_stream(
short_term_memory_prompt_only_content, msgStr
)
save_mem_local_short(self.role_id, result)
logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
return self.short_momery
async def query_memory(self, query: str)-> str:
return self.short_momery
async def query_memory(self, query: str) -> str:
return self.short_momery
main/xiaozhi-server/core/providers/memory/nomem/nomem.py
+8 -6
View File
@@ -1,18 +1,20 @@
'''
"""
不使用记忆，可以选择此模块
'''
"""
from ..base import MemoryProviderBase, logger
TAG = __name__
class MemoryProvider(MemoryProviderBase):
def __init__(self, config):
def __init__(self, config, summary_memory=None):
super().__init__(config)
async def save_memory(self, msgs):
logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
return None
async def query_memory(self, query: str)-> str:
async def query_memory(self, query: str) -> str:
logger.bind(tag=TAG).debug("nomem mode: No memory query is performed.")
return ""
return ""
main/xiaozhi-server/core/providers/vad/base.py
+9
View File
@@ -0,0 +1,9 @@
from abc import ABC, abstractmethod
from typing import Optional
class VADProviderBase(ABC):
@abstractmethod
def is_vad(self, conn, data) -> bool:
"""检测音频数据中的语音活动"""
pass
main/xiaozhi-server/core/providers/vad/silero.py
+71
View File
@@ -0,0 +1,71 @@
import time
import numpy as np
import torch
import opuslib_next
from config.logger import setup_logging
from core.providers.vad.base import VADProviderBase
TAG = __name__
logger = setup_logging()
class VADProvider(VADProviderBase):
def __init__(self, config):
logger.bind(tag=TAG).info("SileroVAD", config)
self.model, self.utils = torch.hub.load(
repo_or_dir=config["model_dir"],
source="local",
model="silero_vad",
force_reload=False,
)
(get_speech_timestamps, _, _, _, _) = self.utils
self.decoder = opuslib_next.Decoder(16000, 1)
# 处理空字符串的情况
threshold = config.get("threshold", "0.5")
min_silence_duration_ms = config.get("min_silence_duration_ms", "1000")
self.vad_threshold = float(threshold) if threshold else 0.5
self.silence_threshold_ms = (
int(min_silence_duration_ms) if min_silence_duration_ms else 1000
)
def is_vad(self, conn, opus_packet):
try:
pcm_frame = self.decoder.decode(opus_packet, 960)
conn.client_audio_buffer.extend(pcm_frame) # 将新数据加入缓冲区
# 处理缓冲区中的完整帧（每次处理512采样点）
client_have_voice = False
while len(conn.client_audio_buffer) >= 512 * 2:
# 提取前512个采样点（1024字节）
chunk = conn.client_audio_buffer[: 512 * 2]
conn.client_audio_buffer = conn.client_audio_buffer[512 * 2 :]
# 转换为模型需要的张量格式
audio_int16 = np.frombuffer(chunk, dtype=np.int16)
audio_float32 = audio_int16.astype(np.float32) / 32768.0
audio_tensor = torch.from_numpy(audio_float32)
# 检测语音活动
with torch.no_grad():
speech_prob = self.model(audio_tensor, 16000).item()
client_have_voice = speech_prob >= self.vad_threshold
# 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
if conn.client_have_voice and not client_have_voice:
stop_duration = (
time.time() * 1000 - conn.client_have_voice_last_time
)
if stop_duration >= self.silence_threshold_ms:
conn.client_voice_stop = True
if client_have_voice:
conn.client_have_voice = True
conn.client_have_voice_last_time = time.time() * 1000
return client_have_voice
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).info(f"解码错误: {e}")
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")
main/xiaozhi-server/core/utils/auth_code_gen.py
-97
View File
@@ -1,97 +0,0 @@
import random
import threading
import time
from typing import Set
class AuthCodeGenerator:
_instance = None
_instance_lock = threading.Lock()
def __new__(cls):
if not cls._instance:
with cls._instance_lock:
if not cls._instance:
cls._instance = super(AuthCodeGenerator, cls).__new__(cls)
# 初始化随机种子
random.seed(time.time())
return cls._instance
def __init__(self):
# 确保 __init__ 只被调用一次
if not hasattr(self, '_initialized'):
self._used_codes: Set[str] = set()
self._code_timestamps = {}
self._lock = threading.Lock()
self._code_timeout = 3 * 24 * 60 * 60
self._initialized = True
@classmethod
def get_instance(cls):
"""获取AuthCodeGenerator的单例实例"""
return cls()
def generate_code(self) -> str:
"""
生成6位数字认证码，确保不重复
返回: 6位数字字符串
"""
with self._lock:
self._clean_expired_codes() # 清理过期code
while True:
# 使用时间戳和已用码数量作为种子，确保每次生成不同的随机数
seed = int(time.time() * 1000) + len(self._used_codes)
random.seed(seed)
# 生成6位随机数字
code = ''.join(str(random.randint(0, 9)) for _ in range(6))
# 检查是否已存在
if code not in self._used_codes:
self._used_codes.add(code)
self._code_timestamps[code] = time.time()
return code
def remove_code(self, code: str) -> bool:
"""
删除已使用的认证码
参数:
code: 要删除的认证码
返回:
bool: 删除成功返回True，码不存在返回False
"""
print('remove_code', code)
with self._lock:
if code in self._used_codes:
self._used_codes.remove(code)
if code in self._code_timestamps:
del self._code_timestamps[code]
return True
return False
def is_code_used(self, code: str) -> bool:
"""
检查认证码是否已被使用
参数:
code: 要检查的认证码
返回:
bool: 如果码存在返回True，否则返回False
"""
with self._lock:
return code in self._used_codes
def clear_codes(self):
"""清空所有已使用的认证码"""
with self._lock:
self._used_codes.clear()
self._code_timestamps.clear()
def _clean_expired_codes(self):
"""清理过期的认证码"""
current_time = time.time()
expired_codes = [
code for code, timestamp in self._code_timestamps.items()
if (current_time - timestamp) > self._code_timeout
]
for code in expired_codes:
self._used_codes.remove(code)
del self._code_timestamps[code]
main/xiaozhi-server/core/utils/dialogue.py
+24 -8
View File
@@ -4,7 +4,14 @@ from datetime import datetime
class Message:
def __init__(self, role: str, content: str = None, uniq_id: str = None, tool_calls = None, tool_call_id=None):
def __init__(
self,
role: str,
content: str = None,
uniq_id: str = None,
tool_calls=None,
tool_call_id=None,
):
self.uniq_id = uniq_id if uniq_id is not None else str(uuid.uuid4())
self.role = role
self.content = content
@@ -16,7 +23,7 @@ class Dialogue:
def __init__(self):
self.dialogue: List[Message] = []
# 获取当前时间
self.current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
def put(self, message: Message):
self.dialogue.append(message)
@@ -25,7 +32,15 @@ class Dialogue:
if m.tool_calls is not None:
dialogue.append({"role": m.role, "tool_calls": m.tool_calls})
elif m.role == "tool":
dialogue.append({"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content})
dialogue.append(
{
"role": m.role,
"tool_call_id": (
str(uuid.uuid4()) if m.tool_call_id is None else m.tool_call_id
),
"content": m.content,
}
)
else:
dialogue.append({"role": m.role, "content": m.content})
@@ -44,23 +59,24 @@ class Dialogue:
else:
self.put(Message(role="system", content=new_content))
def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
def get_llm_dialogue_with_memory(
self, memory_str: str = None
) -> List[Dict[str, str]]:
if memory_str is None or len(memory_str) == 0:
return self.get_llm_dialogue()
# 构建带记忆的对话
dialogue = []
# 添加系统提示和记忆
system_message = next(
(msg for msg in self.dialogue if msg.role == "system"), None
)
if system_message:
enhanced_system_prompt = (
f"{system_message.content}\n\n"
f"相关记忆：\n{memory_str}"
f"以下是用户的历史记忆：\n```\n{memory_str}\n```"
)
dialogue.append({"role": "system", "content": enhanced_system_prompt})
main/xiaozhi-server/core/utils/lock_manager.py
-39
View File
@@ -1,39 +0,0 @@
import asyncio
from typing import Dict
from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
class FileLockManager:
_instance = None
_locks: Dict[str, asyncio.Lock] = {}
def __new__(cls):
if cls._instance is None:
cls._instance = super(FileLockManager, cls).__new__(cls)
return cls._instance
@classmethod
def get_lock(cls, file_path: str) -> asyncio.Lock:
"""获取指定文件的锁"""
if file_path not in cls._locks:
cls._locks[file_path] = asyncio.Lock()
return cls._locks[file_path]
@classmethod
async def acquire_lock(cls, file_path: str):
"""获取锁"""
lock = cls.get_lock(file_path)
await lock.acquire()
logger.bind(tag=TAG).debug(f"Acquired lock for {file_path}")
@classmethod
def release_lock(cls, file_path: str):
"""释放锁"""
if file_path in cls._locks:
try:
cls._locks[file_path].release()
logger.bind(tag=TAG).debug(f"Released lock for {file_path}")
except RuntimeError as e:
logger.bind(tag=TAG).warning(f"Failed to release lock for {file_path}: {e}")
main/xiaozhi-server/core/utils/memory.py
+6 -5
View File
@@ -2,16 +2,17 @@ import os
import sys
import importlib
from config.logger import setup_logging
from core.utils.util import read_config, get_project_dir
logger = setup_logging()
def create_instance(class_name, *args, **kwargs):
if os.path.exists(os.path.join('core', 'providers', 'memory', class_name, f'{class_name}.py')):
lib_name = f'core.providers.memory.{class_name}.{class_name}'
if os.path.exists(
os.path.join("core", "providers", "memory", class_name, f"{class_name}.py")
):
lib_name = f"core.providers.memory.{class_name}.{class_name}"
if lib_name not in sys.modules:
sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
sys.modules[lib_name] = importlib.import_module(f"{lib_name}")
return sys.modules[lib_name].MemoryProvider(*args, **kwargs)
raise ValueError(f"不支持的记忆服务类型: {class_name}")
main/xiaozhi-server/core/utils/output_counter.py
+50
View File
@@ -0,0 +1,50 @@
import datetime
from typing import Dict, Tuple
# 全局字典，用于存储每个设备的每日输出字数
_device_daily_output: Dict[Tuple[str, datetime.date], int] = {}
# 记录最后一次检查的日期
_last_check_date: datetime.date = None
def reset_device_output():
"""
重置所有设备的每日输出字数
每天0点调用此函数
"""
_device_daily_output.clear()
def get_device_output(device_id: str) -> int:
"""
获取设备当日的输出字数
"""
current_date = datetime.datetime.now().date()
return _device_daily_output.get((device_id, current_date), 0)
def add_device_output(device_id: str, char_count: int):
"""
增加设备的输出字数
"""
current_date = datetime.datetime.now().date()
global _last_check_date
# 如果是第一次调用或者日期发生变化，清空计数器
if _last_check_date is None or _last_check_date != current_date:
_device_daily_output.clear()
_last_check_date = current_date
current_count = _device_daily_output.get((device_id, current_date), 0)
_device_daily_output[(device_id, current_date)] = current_count + char_count
def check_device_output_limit(device_id: str, max_output_size: int) -> bool:
"""
检查设备是否超过输出限制
:return: True 如果超过限制，False 如果未超过
"""
if not device_id:
return False
current_output = get_device_output(device_id)
return current_output >= max_output_size
main/xiaozhi-server/core/utils/util.py
+807 -28
View File
@@ -1,19 +1,40 @@
import os
import json
import yaml
import socket
import subprocess
import logging
import re
import os
import numpy as np
import requests
import opuslib_next
from pydub import AudioSegment
from typing import Dict, Any
from core.utils import tts, llm, intent, memory, vad, asr
import copy
def get_project_dir():
"""获取项目根目录"""
return (
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+ "/"
)
TAG = __name__
emoji_map = {
"neutral": "😶",
"happy": "🙂",
"laughing": "😆",
"funny": "😂",
"sad": "😔",
"angry": "😠",
"crying": "😭",
"loving": "😍",
"embarrassed": "😳",
"surprised": "😲",
"shocked": "😱",
"thinking": "🤔",
"winking": "😉",
"cool": "😎",
"relaxed": "😌",
"delicious": "🤤",
"kissy": "😘",
"confident": "😏",
"sleepy": "😴",
"silly": "😜",
"confused": "🙄",
}
def get_local_ip():
@@ -72,7 +93,7 @@ def is_private_ip(ip_addr):
return False # IP address format error or insufficient segments
def get_ip_info(ip_addr):
def get_ip_info(ip_addr, logger):
try:
if is_private_ip(ip_addr):
ip_addr = ""
@@ -81,16 +102,10 @@ def get_ip_info(ip_addr):
ip_info = {"city": resp.get("city")}
return ip_info
except Exception as e:
logging.error(f"Error getting client ip info: {e}")
logger.bind(tag=TAG).error(f"Error getting client ip info: {e}")
return {}
def read_config(config_path):
with open(config_path, "r", encoding="utf-8") as file:
config = yaml.safe_load(file)
return config
def write_json_file(file_path, data):
"""将数据写入 JSON 文件"""
with open(file_path, "w", encoding="utf-8") as file:
@@ -103,13 +118,14 @@ def is_punctuation_or_emoji(char):
punctuation_set = {
"，",
",", # 中文逗号 + 英文逗号
"。",
".", # 中文句号 + 英文句号
"！",
"!", # 中文感叹号 + 英文感叹号
"-",
"－", # 英文连字符 + 中文全角横线
"、", # 中文顿号
"“",
"”",
'"', # 中文双引号 + 英文引号
"：",
":", # 中文冒号 + 英文冒号
}
if char.isspace() or char in punctuation_set:
return True
@@ -169,15 +185,30 @@ def remove_punctuation_and_length(text):
def check_model_key(modelType, modelKey):
if "你" in modelKey:
logging.error(
"你还没配置"
+ modelType
+ "的密钥，请在配置文件中配置密钥，否则无法正常工作"
raise ValueError(
"你还没配置" + modelType + "的密钥，请检查一下所使用的LLM是否配置了密钥"
)
return False
return True
def parse_string_to_list(value, separator=";"):
"""
将输入值转换为列表
Args:
value: 输入值，可以是 None、字符串或列表
separator: 分隔符，默认为分号
Returns:
list: 处理后的列表
"""
if value is None or value == "":
return []
elif isinstance(value, str):
return [item.strip() for item in value.split(separator) if item.strip()]
elif isinstance(value, list):
return value
return []
def check_ffmpeg_installed():
ffmpeg_installed = False
try:
@@ -208,7 +239,755 @@ def check_ffmpeg_installed():
def extract_json_from_string(input_string):
"""提取字符串中的 JSON 部分"""
pattern = r"(\{.*\})"
match = re.search(pattern, input_string)
match = re.search(pattern, input_string, re.DOTALL) # 添加 re.DOTALL
if match:
return match.group(1) # 返回提取的 JSON 字符串
return None
def initialize_modules(
logger,
config: Dict[str, Any],
init_vad=False,
init_asr=False,
init_llm=False,
init_tts=False,
init_memory=False,
init_intent=False,
) -> Dict[str, Any]:
"""
初始化所有模块组件
Args:
config: 配置字典
Returns:
Dict[str, Any]: 包含所有初始化后的模块的字典
"""
modules = {}
# 初始化TTS模块
if init_tts:
select_tts_module = config["selected_module"]["TTS"]
tts_type = (
select_tts_module
if "type" not in config["TTS"][select_tts_module]
else config["TTS"][select_tts_module]["type"]
)
modules["tts"] = tts.create_instance(
tts_type,
config["TTS"][select_tts_module],
str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"),
)
logger.bind(tag=TAG).info(f"初始化组件: tts成功 {select_tts_module}")
# 初始化LLM模块
if init_llm:
select_llm_module = config["selected_module"]["LLM"]
llm_type = (
select_llm_module
if "type" not in config["LLM"][select_llm_module]
else config["LLM"][select_llm_module]["type"]
)
modules["llm"] = llm.create_instance(
llm_type,
config["LLM"][select_llm_module],
)
logger.bind(tag=TAG).info(f"初始化组件: llm成功 {select_llm_module}")
# 初始化Intent模块
if init_intent:
select_intent_module = config["selected_module"]["Intent"]
intent_type = (
select_intent_module
if "type" not in config["Intent"][select_intent_module]
else config["Intent"][select_intent_module]["type"]
)
modules["intent"] = intent.create_instance(
intent_type,
config["Intent"][select_intent_module],
)
logger.bind(tag=TAG).info(f"初始化组件: intent成功 {select_intent_module}")
# 初始化Memory模块
if init_memory:
select_memory_module = config["selected_module"]["Memory"]
memory_type = (
select_memory_module
if "type" not in config["Memory"][select_memory_module]
else config["Memory"][select_memory_module]["type"]
)
modules["memory"] = memory.create_instance(
memory_type,
config["Memory"][select_memory_module],
config.get("summaryMemory", None),
)
logger.bind(tag=TAG).info(f"初始化组件: memory成功 {select_memory_module}")
# 初始化VAD模块
if init_vad:
select_vad_module = config["selected_module"]["VAD"]
vad_type = (
select_vad_module
if "type" not in config["VAD"][select_vad_module]
else config["VAD"][select_vad_module]["type"]
)
modules["vad"] = vad.create_instance(
vad_type,
config["VAD"][select_vad_module],
)
logger.bind(tag=TAG).info(f"初始化组件: vad成功 {select_vad_module}")
# 初始化ASR模块
if init_asr:
select_asr_module = config["selected_module"]["ASR"]
asr_type = (
select_asr_module
if "type" not in config["ASR"][select_asr_module]
else config["ASR"][select_asr_module]["type"]
)
modules["asr"] = asr.create_instance(
asr_type,
config["ASR"][select_asr_module],
str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"),
)
logger.bind(tag=TAG).info(f"初始化组件: asr成功 {select_asr_module}")
return modules
def analyze_emotion(text):
"""
分析文本情感并返回对应的emoji名称（支持中英文）
"""
if not text or not isinstance(text, str):
return "neutral"
original_text = text
text = text.lower().strip()
# 检查是否包含现有emoji
for emotion, emoji in emoji_map.items():
if emoji in original_text:
return emotion
# 标点符号分析
has_exclamation = "!" in original_text or "！" in original_text
has_question = "?" in original_text or "？" in original_text
has_ellipsis = "..." in original_text or "…" in original_text
# 定义情感关键词映射（中英文扩展版）
emotion_keywords = {
"happy": [
"开心",
"高兴",
"快乐",
"愉快",
"幸福",
"满意",
"棒",
"好",
"不错",
"完美",
"棒极了",
"太好了",
"好呀",
"好的",
"happy",
"joy",
"great",
"good",
"nice",
"awesome",
"fantastic",
"wonderful",
],
"laughing": [
"哈哈",
"哈哈哈",
"呵呵",
"嘿嘿",
"嘻嘻",
"笑死",
"太好笑了",
"笑死我了",
"lol",
"lmao",
"haha",
"hahaha",
"hehe",
"rofl",
"funny",
"laugh",
],
"funny": [
"搞笑",
"滑稽",
"逗",
"幽默",
"笑点",
"段子",
"笑话",
"太逗了",
"hilarious",
"joke",
"comedy",
],
"sad": [
"伤心",
"难过",
"悲哀",
"悲伤",
"忧郁",
"郁闷",
"沮丧",
"失望",
"想哭",
"难受",
"不开心",
"唉",
"呜呜",
"sad",
"upset",
"unhappy",
"depressed",
"sorrow",
"gloomy",
],
"angry": [
"生气",
"愤怒",
"气死",
"讨厌",
"烦人",
"可恶",
"烦死了",
"恼火",
"暴躁",
"火大",
"愤怒",
"气炸了",
"angry",
"mad",
"annoyed",
"furious",
"pissed",
"hate",
],
"crying": [
"哭泣",
"泪流",
"大哭",
"伤心欲绝",
"泪目",
"流泪",
"哭死",
"哭晕",
"想哭",
"泪崩",
"cry",
"crying",
"tears",
"sob",
"weep",
],
"loving": [
"爱你",
"喜欢",
"爱",
"亲爱的",
"宝贝",
"么么哒",
"抱抱",
"想你",
"思念",
"最爱",
"亲亲",
"喜欢你",
"love",
"like",
"adore",
"darling",
"sweetie",
"honey",
"miss you",
"heart",
],
"embarrassed": [
"尴尬",
"不好意思",
"害羞",
"脸红",
"难为情",
"社死",
"丢脸",
"出丑",
"embarrassed",
"awkward",
"shy",
"blush",
],
"surprised": [
"惊讶",
"吃惊",
"天啊",
"哇塞",
"哇",
"居然",
"竟然",
"没想到",
"出乎意料",
"surprise",
"wow",
"omg",
"oh my god",
"amazing",
"unbelievable",
],
"shocked": [
"震惊",
"吓到",
"惊呆了",
"不敢相信",
"震撼",
"吓死",
"恐怖",
"害怕",
"吓人",
"shocked",
"shocking",
"scared",
"frightened",
"terrified",
"horror",
],
"thinking": [
"思考",
"考虑",
"想一下",
"琢磨",
"沉思",
"冥想",
"想",
"思考中",
"在想",
"think",
"thinking",
"consider",
"ponder",
"meditate",
],
"winking": [
"调皮",
"眨眼",
"你懂的",
"坏笑",
"邪恶",
"奸笑",
"使眼色",
"wink",
"teasing",
"naughty",
"mischievous",
],
"cool": [
"酷",
"帅",
"厉害",
"棒极了",
"真棒",
"牛逼",
"强",
"优秀",
"杰出",
"出色",
"完美",
"cool",
"awesome",
"amazing",
"great",
"impressive",
"perfect",
],
"relaxed": [
"放松",
"舒服",
"惬意",
"悠闲",
"轻松",
"舒适",
"安逸",
"自在",
"relax",
"relaxed",
"comfortable",
"cozy",
"chill",
"peaceful",
],
"delicious": [
"好吃",
"美味",
"香",
"馋",
"可口",
"香甜",
"大餐",
"大快朵颐",
"流口水",
"垂涎",
"delicious",
"yummy",
"tasty",
"yum",
"appetizing",
"mouthwatering",
],
"kissy": [
"亲亲",
"么么",
"吻",
"mua",
"muah",
"亲一下",
"飞吻",
"kiss",
"xoxo",
"hug",
"muah",
"smooch",
],
"confident": [
"自信",
"肯定",
"确定",
"毫无疑问",
"当然",
"必须的",
"毫无疑问",
"确信",
"坚信",
"confident",
"sure",
"certain",
"definitely",
"positive",
],
"sleepy": [
"困",
"睡觉",
"晚安",
"想睡",
"好累",
"疲惫",
"疲倦",
"困了",
"想休息",
"睡意",
"sleep",
"sleepy",
"tired",
"exhausted",
"bedtime",
"good night",
],
"silly": [
"傻",
"笨",
"呆",
"憨",
"蠢",
"二",
"憨憨",
"傻乎乎",
"呆萌",
"silly",
"stupid",
"dumb",
"foolish",
"goofy",
"ridiculous",
],
"confused": [
"疑惑",
"不明白",
"不懂",
"困惑",
"疑问",
"为什么",
"怎么回事",
"啥意思",
"不清楚",
"confused",
"puzzled",
"doubt",
"question",
"what",
"why",
"how",
],
}
# 特殊句型判断（中英文）
# 赞美他人
if any(
phrase in text
for phrase in [
"你真",
"你好",
"您真",
"你真棒",
"你好厉害",
"你太强了",
"你真好",
"你真聪明",
"you are",
"you're",
"you look",
"you seem",
"so smart",
"so kind",
]
):
return "loving"
# 自我赞美
if any(
phrase in text
for phrase in [
"我真",
"我最",
"我太棒了",
"我厉害",
"我聪明",
"我优秀",
"i am",
"i'm",
"i feel",
"so good",
"so happy",
]
):
return "cool"
# 晚安/睡觉相关
if any(
phrase in text
for phrase in [
"睡觉",
"晚安",
"睡了",
"好梦",
"休息了",
"去睡了",
"sleep",
"good night",
"bedtime",
"go to bed",
]
):
return "sleepy"
# 疑问句
if has_question and not has_exclamation:
return "thinking"
# 强烈情感（感叹号）
if has_exclamation and not has_question:
# 检查是否是积极内容
positive_words = (
emotion_keywords["happy"]
+ emotion_keywords["laughing"]
+ emotion_keywords["cool"]
)
if any(word in text for word in positive_words):
return "laughing"
# 检查是否是消极内容
negative_words = (
emotion_keywords["angry"]
+ emotion_keywords["sad"]
+ emotion_keywords["crying"]
)
if any(word in text for word in negative_words):
return "angry"
return "surprised"
# 省略号（表示犹豫或思考）
if has_ellipsis:
return "thinking"
# 关键词匹配（带权重）
emotion_scores = {emotion: 0 for emotion in emoji_map.keys()}
# 给匹配到的关键词加分
for emotion, keywords in emotion_keywords.items():
for keyword in keywords:
if keyword in text:
emotion_scores[emotion] += 1
# 给长文本中的重复关键词额外加分
if len(text) > 20: # 长文本
for emotion, keywords in emotion_keywords.items():
for keyword in keywords:
emotion_scores[emotion] += text.count(keyword) * 0.5
# 根据分数选择最可能的情感
max_score = max(emotion_scores.values())
if max_score == 0:
return "happy" # 默认
# 可能有多个情感同分，根据上下文选择最合适的
top_emotions = [e for e, s in emotion_scores.items() if s == max_score]
# 如果多个情感同分，使用以下优先级
priority_order = [
"laughing",
"crying",
"angry",
"surprised",
"shocked", # 强烈情感优先
"loving",
"happy",
"funny",
"cool", # 积极情感
"sad",
"embarrassed",
"confused", # 消极情感
"thinking",
"winking",
"relaxed", # 中性情感
"delicious",
"kissy",
"confident",
"sleepy",
"silly", # 特殊场景
]
for emotion in priority_order:
if emotion in top_emotions:
return emotion
return top_emotions[0] # 如果都不在优先级列表里，返回第一个
def audio_to_data(audio_file_path, is_opus=True):
# 获取文件后缀名
file_type = os.path.splitext(audio_file_path)[1]
if file_type:
file_type = file_type.lstrip(".")
# 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
audio = AudioSegment.from_file(
audio_file_path, format=file_type, parameters=["-nostdin"]
)
# 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
# 音频时长(秒)
duration = len(audio) / 1000.0
# 获取原始PCM数据（16位小端）
raw_data = audio.raw_data
# 初始化Opus编码器
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
# 编码参数
frame_duration = 60 # 60ms per frame
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
datas = []
# 按帧处理所有音频数据（包括最后一帧可能补零）
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
# 获取当前帧的二进制数据
chunk = raw_data[i : i + frame_size * 2]
# 如果最后一帧不足，补零
if len(chunk) < frame_size * 2:
chunk += b"\x00" * (frame_size * 2 - len(chunk))
if is_opus:
# 转换为numpy数组处理
np_frame = np.frombuffer(chunk, dtype=np.int16)
# 编码Opus数据
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
else:
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
datas.append(frame_data)
return datas, duration
def check_vad_update(before_config, new_config):
if (
new_config.get("selected_module") is None
or new_config["selected_module"].get("VAD") is None
):
return False
update_vad = False
current_vad_module = before_config["selected_module"]["VAD"]
new_vad_module = new_config["selected_module"]["VAD"]
current_vad_type = (
current_vad_module
if "type" not in before_config["VAD"][current_vad_module]
else before_config["VAD"][current_vad_module]["type"]
)
new_vad_type = (
new_vad_module
if "type" not in new_config["VAD"][new_vad_module]
else new_config["VAD"][new_vad_module]["type"]
)
update_vad = current_vad_type != new_vad_type
return update_vad
def check_asr_update(before_config, new_config):
if (
new_config.get("selected_module") is None
or new_config["selected_module"].get("ASR") is None
):
return False
update_asr = False
current_asr_module = before_config["selected_module"]["ASR"]
new_asr_module = new_config["selected_module"]["ASR"]
current_asr_type = (
current_asr_module
if "type" not in before_config["ASR"][current_asr_module]
else before_config["ASR"][current_asr_module]["type"]
)
new_asr_type = (
new_asr_module
if "type" not in new_config["ASR"][new_asr_module]
else new_config["ASR"][new_asr_module]["type"]
)
update_asr = current_asr_type != new_asr_type
return update_asr
def filter_sensitive_info(config: dict) -> dict:
"""
过滤配置中的敏感信息
Args:
config: 原始配置字典
Returns:
过滤后的配置字典
"""
sensitive_keys = [
"api_key",
"personal_access_token",
"access_token",
"token",
"secret",
"access_key_secret",
"secret_key",
]
def _filter_dict(d: dict) -> dict:
filtered = {}
for k, v in d.items():
if any(sensitive in k.lower() for sensitive in sensitive_keys):
filtered[k] = "***"
elif isinstance(v, dict):
filtered[k] = _filter_dict(v)
elif isinstance(v, list):
filtered[k] = [_filter_dict(i) if isinstance(i, dict) else i for i in v]
else:
filtered[k] = v
return filtered
return _filter_dict(copy.deepcopy(config))
main/xiaozhi-server/core/utils/vad.py
+12 -70
View File
@@ -1,77 +1,19 @@
from abc import ABC, abstractmethod
import importlib
import os
import sys
from core.providers.vad.base import VADProviderBase
from config.logger import setup_logging
import opuslib_next
import time
import numpy as np
import torch
TAG = __name__
logger = setup_logging()
class VAD(ABC):
@abstractmethod
def is_vad(self, conn, data):
"""检测音频数据中的语音活动"""
pass
def create_instance(class_name: str, *args, **kwargs) -> VADProviderBase:
"""工厂方法创建VAD实例"""
if os.path.exists(os.path.join("core", "providers", "vad", f"{class_name}.py")):
lib_name = f"core.providers.vad.{class_name}"
if lib_name not in sys.modules:
sys.modules[lib_name] = importlib.import_module(f"{lib_name}")
return sys.modules[lib_name].VADProvider(*args, **kwargs)
class SileroVAD(VAD):
def __init__(self, config):
logger.bind(tag=TAG).info("SileroVAD", config)
self.model, self.utils = torch.hub.load(repo_or_dir=config["model_dir"],
source='local',
model='silero_vad',
force_reload=False)
(get_speech_timestamps, _, _, _, _) = self.utils
self.decoder = opuslib_next.Decoder(16000, 1)
self.vad_threshold = config.get("threshold")
self.silence_threshold_ms = config.get("min_silence_duration_ms")
def is_vad(self, conn, opus_packet):
try:
pcm_frame = self.decoder.decode(opus_packet, 960)
conn.client_audio_buffer.extend(pcm_frame) # 将新数据加入缓冲区
# 处理缓冲区中的完整帧（每次处理512采样点）
client_have_voice = False
while len(conn.client_audio_buffer) >= 512 * 2:
# 提取前512个采样点（1024字节）
chunk = conn.client_audio_buffer[:512 * 2]
conn.client_audio_buffer = conn.client_audio_buffer[512 * 2:]
# 转换为模型需要的张量格式
audio_int16 = np.frombuffer(chunk, dtype=np.int16)
audio_float32 = audio_int16.astype(np.float32) / 32768.0
audio_tensor = torch.from_numpy(audio_float32)
# 检测语音活动
speech_prob = self.model(audio_tensor, 16000).item()
client_have_voice = speech_prob >= self.vad_threshold
# 如果之前有声音，但本次没有声音，且与上次有声音的时间查已经超过了静默阈值，则认为已经说完一句话
if conn.client_have_voice and not client_have_voice:
stop_duration = time.time() * 1000 - conn.client_have_voice_last_time
if stop_duration >= self.silence_threshold_ms:
conn.client_voice_stop = True
if client_have_voice:
conn.client_have_voice = True
conn.client_have_voice_last_time = time.time() * 1000
return client_have_voice
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).info(f"解码错误: {e}")
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing audio packet: {e}")
def create_instance(class_name, *args, **kwargs) -> VAD:
# 获取类对象
cls_map = {
"SileroVAD": SileroVAD,
# 可扩展其他SileroVAD实现
}
if cls := cls_map.get(class_name):
return cls(*args, **kwargs)
raise ValueError(f"不支持的SileroVAD类型: {class_name}")
raise ValueError(f"不支持的VAD类型: {class_name}，请检查该配置的type是否设置正确")
main/xiaozhi-server/core/websocket_server.py
+89 -86
View File
@@ -2,8 +2,8 @@ import asyncio
import websockets
from config.logger import setup_logging
from core.connection import ConnectionHandler
from core.utils.util import get_local_ip
from core.utils import asr, vad, llm, tts, memory, intent
from core.utils.util import initialize_modules, check_vad_update, check_asr_update
from config.config_loader import get_config_from_api
TAG = __name__
@@ -12,109 +12,112 @@ class WebSocketServer:
def __init__(self, config: dict):
self.config = config
self.logger = setup_logging()
self._vad, self._asr, self._llm, self._memory, self.intent = (
self._create_processing_instances()
)
self.active_connections = set() # 添加全局连接记录
def _create_processing_instances(self):
memory_cls_name = self.config["selected_module"].get(
"Memory", "nomem"
) # 默认使用nomem
has_memory_cfg = (
self.config.get("Memory") and memory_cls_name in self.config["Memory"]
)
memory_cfg = self.config["Memory"][memory_cls_name] if has_memory_cfg else {}
"""创建处理模块实例"""
return (
vad.create_instance(
self.config["selected_module"]["VAD"],
self.config["VAD"][self.config["selected_module"]["VAD"]],
),
asr.create_instance(
(
self.config["selected_module"]["ASR"]
if not "type"
in self.config["ASR"][self.config["selected_module"]["ASR"]]
else self.config["ASR"][self.config["selected_module"]["ASR"]][
"type"
]
),
self.config["ASR"][self.config["selected_module"]["ASR"]],
self.config["delete_audio"],
),
llm.create_instance(
(
self.config["selected_module"]["LLM"]
if not "type"
in self.config["LLM"][self.config["selected_module"]["LLM"]]
else self.config["LLM"][self.config["selected_module"]["LLM"]][
"type"
]
),
self.config["LLM"][self.config["selected_module"]["LLM"]],
),
memory.create_instance(memory_cls_name, memory_cfg),
intent.create_instance(
(
self.config["selected_module"]["Intent"]
if not "type"
in self.config["Intent"][self.config["selected_module"]["Intent"]]
else self.config["Intent"][
self.config["selected_module"]["Intent"]
]["type"]
),
self.config["Intent"][self.config["selected_module"]["Intent"]],
),
self.config_lock = asyncio.Lock()
modules = initialize_modules(
self.logger,
self.config,
"VAD" in self.config["selected_module"],
"ASR" in self.config["selected_module"],
"LLM" in self.config["selected_module"],
"TTS" in self.config["selected_module"],
"Memory" in self.config["selected_module"],
"Intent" in self.config["selected_module"],
)
self._vad = modules["vad"] if "vad" in modules else None
self._asr = modules["asr"] if "asr" in modules else None
self._tts = modules["tts"] if "tts" in modules else None
self._llm = modules["llm"] if "llm" in modules else None
self._intent = modules["intent"] if "intent" in modules else None
self._memory = modules["memory"] if "memory" in modules else None
self.active_connections = set()
async def start(self):
server_config = self.config["server"]
host = server_config["ip"]
port = server_config["port"]
host = server_config.get("ip", "0.0.0.0")
port = int(server_config.get("port", 8000))
self.logger.bind(tag=TAG).info(
"Server is running at ws://{}:{}/xiaozhi/v1/", get_local_ip(), port
)
self.logger.bind(tag=TAG).info(
"=======上面的地址是websocket协议地址，请勿用浏览器访问======="
)
self.logger.bind(tag=TAG).info(
"如想测试websocket请用谷歌浏览器打开test目录下的test_page.html"
)
self.logger.bind(tag=TAG).info(
"=============================================================\n"
)
async with websockets.serve(self._handle_connection, host, port):
async with websockets.serve(
self._handle_connection, host, port, process_request=self._http_response
):
await asyncio.Future()
async def _handle_connection(self, websocket):
"""处理新连接，每次创建独立的ConnectionHandler"""
# 创建ConnectionHandler时传入当前server实例
_tts = tts.create_instance(
(
self.config["selected_module"]["TTS"]
if not "type"
in self.config["TTS"][self.config["selected_module"]["TTS"]]
else self.config["TTS"][self.config["selected_module"]["TTS"]][
"type"
]
),
self.config["TTS"][self.config["selected_module"]["TTS"]],
self.config["delete_audio"],
)
handler = ConnectionHandler(
self.config,
self._vad,
self._asr,
self._llm,
_tts,
self._tts,
self._memory,
self.intent,
self._intent,
self, # 传入server实例
)
self.active_connections.add(handler)
try:
await handler.handle_connection(websocket)
finally:
self.active_connections.discard(handler)
async def _http_response(self, websocket, request_headers):
# 检查是否为 WebSocket 升级请求
if request_headers.headers.get("connection", "").lower() == "upgrade":
# 如果是 WebSocket 请求，返回 None 允许握手继续
return None
else:
# 如果是普通 HTTP 请求，返回 "server is running"
return websocket.respond(200, "Server is running\n")
async def update_config(self) -> bool:
"""更新服务器配置并重新初始化组件
Returns:
bool: 更新是否成功
"""
try:
async with self.config_lock:
# 重新获取配置
new_config = get_config_from_api(self.config)
if new_config is None:
self.logger.bind(tag=TAG).error("获取新配置失败")
return False
self.logger.bind(tag=TAG).info(f"获取新配置成功")
# 检查 VAD 和 ASR 类型是否需要更新
update_vad = check_vad_update(self.config, new_config)
update_asr = check_asr_update(self.config, new_config)
self.logger.bind(tag=TAG).info(
f"检查VAD和ASR类型是否需要更新: {update_vad} {update_asr}"
)
# 更新配置
self.config = new_config
# 重新初始化组件
modules = initialize_modules(
self.logger,
new_config,
update_vad,
update_asr,
"LLM" in new_config["selected_module"],
"TTS" in new_config["selected_module"],
"Memory" in new_config["selected_module"],
"Intent" in new_config["selected_module"],
)
# 更新组件实例
if "vad" in modules:
self._vad = modules["vad"]
if "asr" in modules:
self._asr = modules["asr"]
if "tts" in modules:
self._tts = modules["tts"]
if "llm" in modules:
self._llm = modules["llm"]
if "intent" in modules:
self._intent = modules["intent"]
if "memory" in modules:
self._memory = modules["memory"]
self.logger.bind(tag=TAG).info(f"更新配置任务执行完毕")
return True
except Exception as e:
self.logger.bind(tag=TAG).error(f"更新服务器配置失败: {str(e)}")
return False
main/xiaozhi-server/docker-compose.yml
+4 -30
View File
@@ -1,16 +1,4 @@
# 1、安装mysql
# |- 如果本机已经安装了MySQL，可以直接在数据库中创建名为`xiaozhi_esp32_server`的数据库。
# |- 如果还没有MySQL，你可以通过docker安装mysql,执行以下一句话
# |- docker run --name xiaozhi-esp32-server-db -e MYSQL_ROOT_PASSWORD=123456 -p 3306:3306 -e MYSQL_DATABASE=xiaozhi_esp32_server -e MYSQL_INITDB_ARGS="--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci" -d mysql:latest
# |- 如果你的mysql账号和密码有修改过，记得修改下方数据库的账号和密码
# |- 记得修改下方SPRING_DATASOURCE_DRUID_URL的IP，ip不能写127.0.0.1或localhost，否则容器无法访问，要写你电脑局域网ip
# 2、安装redis
# |- 如果本机已经安装了Redis，看一下你安装的redis端口、密码，然后修改下方redis的地址和端口
# |- 如果还没有Redis，你可以通过docker安装redis,执行以下一句话
# |- docker run --name xiaozhi-esp32-server-redis -d -p 6379:6379 redis
# |- 记得修改SPRING_DATA_REDIS_HOST的IP，ip不能写127.0.0.1或localhost，否则容器无法访问，要写你电脑局域网ip
# Docker安装Server
version: '3'
services:
@@ -25,24 +13,10 @@ services:
ports:
# ws服务端
- "8000:8000"
# ota服务端
- "8002:8002"
volumes:
# 配置文件目录
- ./data:/opt/xiaozhi-esp32-server/data
# 模型文件挂接，很重要
- ./models/SenseVoiceSmall/model.pt:/opt/xiaozhi-esp32-server/models/SenseVoiceSmall/model.pt
# #智控台还没开发好，还不能完全使用，会报很多错误，如果是非技术人员，请不要启用智控台服务
# xiaozhi-esp32-server-web:
# image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:web_latest
# container_name: xiaozhi-esp32-server-web
# restart: always
# ports:
# - "8002:8002"
# environment:
# - TZ=Asia/Shanghai
# ##记得改mysql和redis IP 密码
# - SPRING_DATASOURCE_DRUID_URL=jdbc:mysql://192.168.1.20:3306/xiaozhi_esp32_server?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai
# - SPRING_DATASOURCE_DRUID_USERNAME=root
# - SPRING_DATASOURCE_DRUID_PASSWORD=123456
# - SPRING_DATA_REDIS_HOST=192.168.1.20
# - SPRING_DATA_REDIS_PORT=6379
- ./models/SenseVoiceSmall/model.pt:/opt/xiaozhi-esp32-server/models/SenseVoiceSmall/model.pt
main/xiaozhi-server/docker-compose_all.yml
+88
View File
@@ -0,0 +1,88 @@
# Docker安装全模块
version: '3'
services:
# Server模块
xiaozhi-esp32-server:
image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:server_latest
container_name: xiaozhi-esp32-server
depends_on:
- xiaozhi-esp32-server-db
- xiaozhi-esp32-server-redis
restart: always
networks:
- default
ports:
# ws服务端
- "8000:8000"
security_opt:
- seccomp:unconfined
environment:
- TZ=Asia/Shanghai
volumes:
# 配置文件目录
- ./data:/opt/xiaozhi-esp32-server/data
# 模型文件挂接，很重要
- ./models/SenseVoiceSmall/model.pt:/opt/xiaozhi-esp32-server/models/SenseVoiceSmall/model.pt
# manager-api和manager-web模块
xiaozhi-esp32-server-web:
image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:web_latest
container_name: xiaozhi-esp32-server-web
restart: always
networks:
- default
depends_on:
xiaozhi-esp32-server-db:
condition: service_healthy
xiaozhi-esp32-server-redis:
condition: service_healthy
ports:
# 智控台
- "8002:8002"
environment:
- TZ=Asia/Shanghai
- SPRING_DATASOURCE_DRUID_URL=jdbc:mysql://xiaozhi-esp32-server-db:3306/xiaozhi_esp32_server?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true&connectTimeout=30000&socketTimeout=30000&autoReconnect=true&failOverReadOnly=false&maxReconnects=10
- SPRING_DATASOURCE_DRUID_USERNAME=root
- SPRING_DATASOURCE_DRUID_PASSWORD=123456
- SPRING_DATA_REDIS_HOST=xiaozhi-esp32-server-redis
- SPRING_DATA_REDIS_PORT=6379
volumes:
# 配置文件目录
- ./uploadfile:/uploadfile
xiaozhi-esp32-server-db:
image: mysql:latest
container_name: xiaozhi-esp32-server-db
healthcheck:
test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ]
timeout: 45s
interval: 10s
retries: 10
restart: always
networks:
- default
expose:
- 3306
volumes:
- ./mysql/data:/var/lib/mysql
environment:
- TZ=Asia/Shanghai
- MYSQL_ROOT_PASSWORD=123456
- MYSQL_DATABASE=xiaozhi_esp32_server
- MYSQL_INITDB_ARGS="--character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci"
xiaozhi-esp32-server-redis:
image: redis
expose:
- 6379
container_name: xiaozhi-esp32-server-redis
restart: always
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 3
networks:
- default
networks:
default:
main/xiaozhi-server/mcp_server_settings.json
+11 -1
View File
@@ -3,9 +3,19 @@
"在data目录下创建.mcp_server_settings.json文件，可以选择下面的MCP服务，也可以自行添加新的MCP服务。",
"后面不断测试补充好用的mcp服务，欢迎大家一起补充。",
"记得删除注释行,des属性仅为说明,不会被解析。",
"des和link属性，仅为说明安装方式，方便大家查看原始链接，不是必须项。"
"des和link属性，仅为说明安装方式，方便大家查看原始链接，不是必须项。",
"当前支持stdio/sse两种模式。"
],
"mcpServers": {
"Home Assistant": {
"command": "mcp-proxy",
"args": [
"http://YOUR_HA_HOST/mcp_server/sse"
],
"env": {
"API_ACCESS_TOKEN": "YOUR_API_ACCESS_TOKEN"
}
},
"filesystem": {
"command": "npx",
"args": [
main/xiaozhi-server/performance_tester.py
+362 -156
View File
@@ -1,16 +1,17 @@
import time
import aiohttp
import asyncio
import logging
import os
import statistics
import time
from typing import Dict
import aiohttp
from tabulate import tabulate
from typing import Dict, List
from config.settings import load_config
from core.utils.asr import create_instance as create_stt_instance
from core.utils.llm import create_instance as create_llm_instance
from core.utils.tts import create_instance as create_tts_instance
from core.utils.util import read_config
import statistics
from config.settings import get_config_file
import inspect
import os
import logging
# 设置全局日志级别为WARNING，抑制INFO级别日志
logging.basicConfig(level=logging.WARNING)
@@ -18,17 +19,26 @@ logging.basicConfig(level=logging.WARNING)
class AsyncPerformanceTester:
def __init__(self):
self.config = read_config(get_config_file())
self.config = load_config()
self.test_sentences = self.config.get("module_test", {}).get(
"test_sentences",
["你好，请介绍一下你自己", "What's the weather like today?",
"请用100字概括量子计算的基本原理和应用前景"]
[
"你好，请介绍一下你自己",
"What's the weather like today?",
"请用100字概括量子计算的基本原理和应用前景",
],
)
self.results = {
"llm": {},
"tts": {},
"combinations": []
}
self.test_wav_list = []
self.wav_root = r"config/assets"
for file_name in os.listdir(self.wav_root):
file_path = os.path.join(self.wav_root, file_name)
# 检查文件大小是否大于300KB
if os.path.getsize(file_path) > 300 * 1024: # 300KB = 300 * 1024 bytes
with open(file_path, "rb") as f:
self.test_wav_list.append(f.read())
self.results = {"llm": {}, "tts": {}, "stt": {}, "combinations": []}
async def _check_ollama_service(self, base_url: str, model_name: str) -> bool:
"""异步检查Ollama服务状态"""
@@ -46,7 +56,9 @@ class AsyncPerformanceTester:
data = await response.json()
models = data.get("models", [])
if not any(model["name"] == model_name for model in models):
print(f"🚫 Ollama模型 {model_name} 未找到，请先使用 ollama pull {model_name} 下载")
print(
f"🚫 Ollama模型 {model_name} 未找到，请先使用 ollama pull {model_name} 下载"
)
return False
else:
print(f"🚫 无法获取Ollama模型列表")
@@ -62,17 +74,16 @@ class AsyncPerformanceTester:
logging.getLogger("core.providers.tts.base").setLevel(logging.WARNING)
token_fields = ["access_token", "api_key", "token"]
if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in
token_fields):
if any(
field in config
and any(x in config[field] for x in ["你的", "placeholder"])
for field in token_fields
):
print(f"⏭️ TTS {tts_name} 未配置access_token/api_key，已跳过")
return {"name": tts_name, "type": "tts", "errors": 1}
module_type = config.get('type', tts_name)
tts = create_tts_instance(
module_type,
config,
delete_audio_file=True
)
module_type = config.get("type", tts_name)
tts = create_tts_instance(module_type, config, delete_audio_file=True)
print(f"🎵 测试 TTS: {tts_name}")
@@ -103,20 +114,71 @@ class AsyncPerformanceTester:
"name": tts_name,
"type": "tts",
"avg_time": total_time / test_count,
"errors": 0
"errors": 0,
}
except Exception as e:
print(f"⚠️ {tts_name} 测试失败: {str(e)}")
return {"name": tts_name, "type": "tts", "errors": 1}
async def _test_stt(self, stt_name: str, config: Dict) -> Dict:
"""异步测试单个STT性能"""
try:
logging.getLogger("core.providers.asr.base").setLevel(logging.WARNING)
token_fields = ["access_token", "api_key", "token"]
if any(
field in config
and any(x in config[field] for x in ["你的", "placeholder"])
for field in token_fields
):
print(f"⏭️ STT {stt_name} 未配置access_token/api_key，已跳过")
return {"name": stt_name, "type": "stt", "errors": 1}
module_type = config.get("type", stt_name)
stt = create_stt_instance(module_type, config, delete_audio_file=True)
stt.audio_format = "pcm"
print(f"🎵 测试 STT: {stt_name}")
text, _ = await stt.speech_to_text([self.test_wav_list[0]], "1")
if text is None:
print(f"❌ {stt_name} 连接失败")
return {"name": stt_name, "type": "stt", "errors": 1}
total_time = 0
test_count = len(self.test_wav_list)
for i, sentence in enumerate(self.test_wav_list, 1):
start = time.time()
text, _ = await stt.speech_to_text([sentence], "1")
duration = time.time() - start
total_time += duration
if text:
print(f"✓ {stt_name} [{i}/{test_count}]")
else:
print(f"✗ {stt_name} [{i}/{test_count}]")
return {"name": stt_name, "type": "stt", "errors": 1}
return {
"name": stt_name,
"type": "stt",
"avg_time": total_time / test_count,
"errors": 0,
}
except Exception as e:
print(f"⚠️ {stt_name} 测试失败: {str(e)}")
return {"name": stt_name, "type": "stt", "errors": 1}
async def _test_llm(self, llm_name: str, config: Dict) -> Dict:
"""异步测试单个LLM性能"""
try:
# 对于Ollama，跳过api_key检查并进行特殊处理
if llm_name == "Ollama":
base_url = config.get('base_url', 'http://localhost:11434')
model_name = config.get('model_name')
base_url = config.get("base_url", "http://localhost:11434")
model_name = config.get("model_name")
if not model_name:
print(f"🚫 Ollama未配置model_name")
return {"name": llm_name, "type": "llm", "errors": 1}
@@ -124,21 +186,27 @@ class AsyncPerformanceTester:
if not await self._check_ollama_service(base_url, model_name):
return {"name": llm_name, "type": "llm", "errors": 1}
else:
if "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]):
if "api_key" in config and any(
x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]
):
print(f"🚫 跳过未配置的LLM: {llm_name}")
return {"name": llm_name, "type": "llm", "errors": 1}
# 获取实际类型（兼容旧配置）
module_type = config.get('type', llm_name)
module_type = config.get("type", llm_name)
llm = create_llm_instance(module_type, config)
# 统一使用UTF-8编码
test_sentences = [s.encode('utf-8').decode('utf-8') for s in self.test_sentences]
test_sentences = [
s.encode("utf-8").decode("utf-8") for s in self.test_sentences
]
# 创建所有句子的测试任务
sentence_tasks = []
for sentence in test_sentences:
sentence_tasks.append(self._test_single_sentence(llm_name, llm, sentence))
sentence_tasks.append(
self._test_single_sentence(llm_name, llm, sentence)
)
# 并发执行所有句子测试
sentence_results = await asyncio.gather(*sentence_tasks)
@@ -166,9 +234,15 @@ class AsyncPerformanceTester:
"type": "llm",
"avg_response": sum(response_times) / len(response_times),
"avg_first_token": sum(first_token_times) / len(first_token_times),
"std_first_token": statistics.stdev(first_token_times) if len(first_token_times) > 1 else 0,
"std_response": statistics.stdev(response_times) if len(response_times) > 1 else 0,
"errors": 0
"std_first_token": (
statistics.stdev(first_token_times)
if len(first_token_times) > 1
else 0
),
"std_response": (
statistics.stdev(response_times) if len(response_times) > 1 else 0
),
"errors": 0,
}
except Exception as e:
print(f"LLM {llm_name} 测试失败: {str(e)}")
@@ -184,8 +258,10 @@ class AsyncPerformanceTester:
async def process_response():
nonlocal first_token_received, first_token_time
for chunk in llm.response("perf_test", [{"role": "user", "content": sentence}]):
if not first_token_received and chunk.strip() != '':
for chunk in llm.response(
"perf_test", [{"role": "user", "content": sentence}]
):
if not first_token_received and chunk.strip() != "":
first_token_time = time.time() - sentence_start
first_token_received = True
print(f"✓ {llm_name} 首个Token: {first_token_time:.3f}s")
@@ -199,13 +275,15 @@ class AsyncPerformanceTester:
print(f"✓ {llm_name} 完成响应: {response_time:.3f}s")
if first_token_time is None:
first_token_time = response_time # 如果没有检测到first token，使用总响应时间
first_token_time = (
response_time # 如果没有检测到first token，使用总响应时间
)
return {
"name": llm_name,
"type": "llm",
"first_token_time": first_token_time,
"response_time": response_time
"response_time": response_time,
}
except Exception as e:
print(f"⚠️ {llm_name} 句子测试失败: {str(e)}")
@@ -214,42 +292,71 @@ class AsyncPerformanceTester:
def _generate_combinations(self):
"""生成最佳组合建议"""
valid_llms = [
k for k, v in self.results["llm"].items()
k
for k, v in self.results["llm"].items()
if v["errors"] == 0 and v["avg_first_token"] >= 0.05
]
valid_tts = [k for k, v in self.results["tts"].items() if v["errors"] == 0]
valid_stt = [k for k, v in self.results["stt"].items() if v["errors"] == 0]
# 找出基准值
min_first_token = min([self.results["llm"][llm]["avg_first_token"] for llm in valid_llms]) if valid_llms else 1
min_tts_time = min([self.results["tts"][tts]["avg_time"] for tts in valid_tts]) if valid_tts else 1
min_first_token = (
min([self.results["llm"][llm]["avg_first_token"] for llm in valid_llms])
if valid_llms
else 1
)
min_tts_time = (
min([self.results["tts"][tts]["avg_time"] for tts in valid_tts])
if valid_tts
else 1
)
min_stt_time = (
min([self.results["stt"][stt]["avg_time"] for stt in valid_stt])
if valid_stt
else 1
)
for llm in valid_llms:
for tts in valid_tts:
# 计算相对性能分数（越小越好）
llm_score = self.results["llm"][llm]["avg_first_token"] / min_first_token
tts_score = self.results["tts"][tts]["avg_time"] / min_tts_time
for stt in valid_stt:
# 计算相对性能分数（越小越好）
llm_score = (
self.results["llm"][llm]["avg_first_token"] / min_first_token
)
tts_score = self.results["tts"][tts]["avg_time"] / min_tts_time
stt_score = self.results["stt"][stt]["avg_time"] / min_stt_time
# 计算稳定性分数（标准差/平均值，越小越稳定）
llm_stability = self.results["llm"][llm]["std_first_token"] / self.results["llm"][llm][
"avg_first_token"]
# 计算稳定性分数（标准差/平均值，越小越稳定）
llm_stability = (
self.results["llm"][llm]["std_first_token"]
/ self.results["llm"][llm]["avg_first_token"]
)
# 综合得分（考虑性能和稳定性）
# 性能权重0.7，稳定性权重0.3
llm_final_score = llm_score * 0.7 + llm_stability * 0.3
# 综合得分（考虑性能和稳定性）
# LLM得分： 性能权重(70%) + 稳定性权重(30%)
llm_final_score = llm_score * 0.7 + llm_stability * 0.3
# 总分 = LLM得分(70%) + TTS得分(30%)
total_score = llm_final_score * 0.7 + tts_score * 0.3
# 总分 = LLM得分(70%) + TTS得分(30%) + STT得分(30%)
total_score = (
llm_final_score * 0.7 + tts_score * 0.3 + stt_score * 0.3
)
self.results["combinations"].append({
"llm": llm,
"tts": tts,
"score": total_score,
"details": {
"llm_first_token": self.results["llm"][llm]["avg_first_token"],
"llm_stability": llm_stability,
"tts_time": self.results["tts"][tts]["avg_time"]
}
})
self.results["combinations"].append(
{
"llm": llm,
"tts": tts,
"stt": stt,
"score": total_score,
"details": {
"llm_first_token": self.results["llm"][llm][
"avg_first_token"
],
"llm_stability": llm_stability,
"tts_time": self.results["tts"][tts]["avg_time"],
"stt_time": self.results["stt"][stt]["avg_time"],
},
}
)
# 分数越小越好
self.results["combinations"].sort(key=lambda x: x["score"])
@@ -260,64 +367,98 @@ class AsyncPerformanceTester:
for name, data in self.results["llm"].items():
if data["errors"] == 0:
stability = data["std_first_token"] / data["avg_first_token"]
llm_table.append([
name, # 不需要固定宽度，让tabulate自己处理对齐
f"{data['avg_first_token']:.3f}秒",
f"{data['avg_response']:.3f}秒",
f"{stability:.3f}"
])
llm_table.append(
[
name, # 不需要固定宽度，让tabulate自己处理对齐
f"{data['avg_first_token']:.3f}秒",
f"{data['avg_response']:.3f}秒",
f"{stability:.3f}",
]
)
if llm_table:
print("\nLLM 性能排行:")
print(tabulate(
llm_table,
headers=["模型名称", "首字耗时", "总耗时", "稳定性"],
tablefmt="github",
colalign=("left", "right", "right", "right"),
disable_numparse=True
))
print("\nLLM 性能排行:\n")
print(
tabulate(
llm_table,
headers=["模型名称", "首字耗时", "总耗时", "稳定性"],
tablefmt="github",
colalign=("left", "right", "right", "right"),
disable_numparse=True,
)
)
else:
print("\n⚠️ 没有可用的LLM模块进行测试。")
tts_table = []
for name, data in self.results["tts"].items():
if data["errors"] == 0:
tts_table.append([
name, # 不需要固定宽度
f"{data['avg_time']:.3f}秒"
])
tts_table.append([name, f"{data['avg_time']:.3f}秒"]) # 不需要固定宽度
if tts_table:
print("\nTTS 性能排行:")
print(tabulate(
tts_table,
headers=["模型名称", "合成耗时"],
tablefmt="github",
colalign=("left", "right"),
disable_numparse=True
))
print("\nTTS 性能排行:\n")
print(
tabulate(
tts_table,
headers=["模型名称", "合成耗时"],
tablefmt="github",
colalign=("left", "right"),
disable_numparse=True,
)
)
else:
print("\n⚠️ 没有可用的TTS模块进行测试。")
if self.results["combinations"]:
print("\n推荐配置组合 (得分越小越好):")
combo_table = []
for combo in self.results["combinations"][:5]:
combo_table.append([
f"{combo['llm']} + {combo['tts']}", # 不需要固定宽度
f"{combo['score']:.3f}",
f"{combo['details']['llm_first_token']:.3f}秒",
f"{combo['details']['llm_stability']:.3f}",
f"{combo['details']['tts_time']:.3f}秒"
])
stt_table = []
for name, data in self.results["stt"].items():
if data["errors"] == 0:
stt_table.append([name, f"{data['avg_time']:.3f}秒"]) # 不需要固定宽度
print(tabulate(
combo_table,
headers=["组合方案", "综合得分", "LLM首字耗时", "稳定性", "TTS合成耗时"],
tablefmt="github",
colalign=("left", "right", "right", "right", "right"),
disable_numparse=True
))
if stt_table:
print("\nSTT 性能排行:\n")
print(
tabulate(
stt_table,
headers=["模型名称", "合成耗时"],
tablefmt="github",
colalign=("left", "right"),
disable_numparse=True,
)
)
else:
print("\n⚠️ 没有可用的STT模块进行测试。")
if self.results["combinations"]:
print("\n推荐配置组合 (得分越小越好):\n")
combo_table = []
for combo in self.results["combinations"][:]:
combo_table.append(
[
f"{combo['llm']} + {combo['tts']} + {combo['stt']}", # 不需要固定宽度
f"{combo['score']:.3f}",
f"{combo['details']['llm_first_token']:.3f}秒",
f"{combo['details']['llm_stability']:.3f}",
f"{combo['details']['tts_time']:.3f}秒",
f"{combo['details']['stt_time']:.3f}秒",
]
)
print(
tabulate(
combo_table,
headers=[
"组合方案",
"综合得分",
"LLM首字耗时",
"稳定性",
"TTS合成耗时",
"STT合成耗时",
],
tablefmt="github",
colalign=("left", "right", "right", "right", "right", "right"),
disable_numparse=True,
)
)
else:
print("\n⚠️ 没有可用的模块组合建议。")
@@ -327,8 +468,12 @@ class AsyncPerformanceTester:
if result["errors"] == 0:
if result["type"] == "llm":
self.results["llm"][result["name"]] = result
else:
elif result["type"] == "tts":
self.results["tts"][result["name"]] = result
elif result["type"] == "stt":
self.results["stt"][result["name"]] = result
else:
pass
async def run(self):
"""执行全量异步测试"""
@@ -338,50 +483,83 @@ class AsyncPerformanceTester:
all_tasks = []
# LLM测试任务
for llm_name, config in self.config.get("LLM", {}).items():
# 检查配置有效性
if llm_name == "CozeLLM":
if any(x in config.get("bot_id", "") for x in ["你的"]) \
or any(x in config.get("user_id", "") for x in ["你的"]):
print(f"⏭️ LLM {llm_name} 未配置bot_id/user_id，已跳过")
continue
elif "api_key" in config and any(x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]):
print(f"⏭️ LLM {llm_name} 未配置api_key，已跳过")
continue
# 对于Ollama，先检查服务状态
if llm_name == "Ollama":
base_url = config.get('base_url', 'http://localhost:11434')
model_name = config.get('model_name')
if not model_name:
print(f"🚫 Ollama未配置model_name")
if self.config.get("LLM") is not None:
for llm_name, config in self.config.get("LLM", {}).items():
# 检查配置有效性
if llm_name == "CozeLLM":
if any(x in config.get("bot_id", "") for x in ["你的"]) or any(
x in config.get("user_id", "") for x in ["你的"]
):
print(f"⏭️ LLM {llm_name} 未配置bot_id/user_id，已跳过")
continue
elif "api_key" in config and any(
x in config["api_key"] for x in ["你的", "placeholder", "sk-xxx"]
):
print(f"⏭️ LLM {llm_name} 未配置api_key，已跳过")
continue
if not await self._check_ollama_service(base_url, model_name):
continue
# 对于Ollama，先检查服务状态
if llm_name == "Ollama":
base_url = config.get("base_url", "http://localhost:11434")
model_name = config.get("model_name")
if not model_name:
print(f"🚫 Ollama未配置model_name")
continue
print(f"📋 添加LLM测试任务: {llm_name}")
module_type = config.get('type', llm_name)
llm = create_llm_instance(module_type, config)
if not await self._check_ollama_service(base_url, model_name):
continue
# 为每个句子创建独立任务
for sentence in self.test_sentences:
sentence = sentence.encode('utf-8').decode('utf-8')
all_tasks.append(self._test_single_sentence(llm_name, llm, sentence))
print(f"📋 添加LLM测试任务: {llm_name}")
module_type = config.get("type", llm_name)
llm = create_llm_instance(module_type, config)
# 为每个句子创建独立任务
for sentence in self.test_sentences:
sentence = sentence.encode("utf-8").decode("utf-8")
all_tasks.append(
self._test_single_sentence(llm_name, llm, sentence)
)
# TTS测试任务
for tts_name, config in self.config.get("TTS", {}).items():
token_fields = ["access_token", "api_key", "token"]
if any(field in config and any(x in config[field] for x in ["你的", "placeholder"]) for field in
token_fields):
print(f"⏭️ TTS {tts_name} 未配置access_token/api_key，已跳过")
continue
print(f"🎵 添加TTS测试任务: {tts_name}")
all_tasks.append(self._test_tts(tts_name, config))
if self.config.get("TTS") is not None:
for tts_name, config in self.config.get("TTS", {}).items():
token_fields = ["access_token", "api_key", "token"]
if any(
field in config
and any(x in config[field] for x in ["你的", "placeholder"])
for field in token_fields
):
print(f"⏭️ TTS {tts_name} 未配置access_token/api_key，已跳过")
continue
print(f"🎵 添加TTS测试任务: {tts_name}")
all_tasks.append(self._test_tts(tts_name, config))
# STT测试任务
if len(self.test_wav_list) >= 1:
if self.config.get("ASR") is not None:
for stt_name, config in self.config.get("ASR", {}).items():
token_fields = ["access_token", "api_key", "token"]
if any(
field in config
and any(x in config[field] for x in ["你的", "placeholder"])
for field in token_fields
):
print(f"⏭️ ASR {stt_name} 未配置access_token/api_key，已跳过")
continue
print(f"🎵 添加ASR测试任务: {stt_name}")
all_tasks.append(self._test_stt(stt_name, config))
else:
print(f"\n⚠️ {self.wav_root} 路径下没有音频文件，已跳过STT测试任务")
print(
f"\n✅ 找到 {len([t for t in all_tasks if 'test_single_sentence' in str(t)]) / len(self.test_sentences):.0f} 个可用LLM模块")
print(f"✅ 找到 {len([t for t in all_tasks if '_test_tts' in str(t)])} 个可用TTS模块")
f"\n✅ 找到 {len([t for t in all_tasks if 'test_single_sentence' in str(t)]) / len(self.test_sentences):.0f} 个可用LLM模块"
)
print(
f"✅ 找到 {len([t for t in all_tasks if '_test_tts' in str(t)])} 个可用TTS模块"
)
print(
f"✅ 找到 {len([t for t in all_tasks if '_test_stt' in str(t)])} 个可用STT模块"
)
print("\n⏳ 开始并发测试所有模块...\n")
# 并发执行所有测试任务
@@ -389,7 +567,11 @@ class AsyncPerformanceTester:
# 处理LLM结果
llm_results = {}
for result in [r for r in all_results if r and isinstance(r, dict) and r.get("type") == "llm"]:
for result in [
r
for r in all_results
if r and isinstance(r, dict) and r.get("type") == "llm"
]:
llm_name = result["name"]
if llm_name not in llm_results:
llm_results[llm_name] = {
@@ -397,9 +579,11 @@ class AsyncPerformanceTester:
"type": "llm",
"first_token_times": [],
"response_times": [],
"errors": 0
"errors": 0,
}
llm_results[llm_name]["first_token_times"].append(result["first_token_time"])
llm_results[llm_name]["first_token_times"].append(
result["first_token_time"]
)
llm_results[llm_name]["response_times"].append(result["response_time"])
# 计算LLM平均值和标准差
@@ -408,19 +592,41 @@ class AsyncPerformanceTester:
self.results["llm"][llm_name] = {
"name": llm_name,
"type": "llm",
"avg_response": sum(data["response_times"]) / len(data["response_times"]),
"avg_first_token": sum(data["first_token_times"]) / len(data["first_token_times"]),
"std_first_token": statistics.stdev(data["first_token_times"]) if len(
data["first_token_times"]) > 1 else 0,
"std_response": statistics.stdev(data["response_times"]) if len(data["response_times"]) > 1 else 0,
"errors": 0
"avg_response": sum(data["response_times"])
/ len(data["response_times"]),
"avg_first_token": sum(data["first_token_times"])
/ len(data["first_token_times"]),
"std_first_token": (
statistics.stdev(data["first_token_times"])
if len(data["first_token_times"]) > 1
else 0
),
"std_response": (
statistics.stdev(data["response_times"])
if len(data["response_times"]) > 1
else 0
),
"errors": 0,
}
# 处理TTS结果
for result in [r for r in all_results if r and isinstance(r, dict) and r.get("type") == "tts"]:
for result in [
r
for r in all_results
if r and isinstance(r, dict) and r.get("type") == "tts"
]:
if result["errors"] == 0:
self.results["tts"][result["name"]] = result
# 处理STT结果
for result in [
r
for r in all_results
if r and isinstance(r, dict) and r.get("type") == "stt"
]:
if result["errors"] == 0:
self.results["stt"][result["name"]] = result
# 生成组合建议并打印结果
print("\n📊 生成测试报告...")
self._generate_combinations()
@@ -433,4 +639,4 @@ async def main():
if __name__ == "__main__":
asyncio.run(main())
asyncio.run(main())
main/xiaozhi-server/plugins_func/functions/get_news.py → main/xiaozhi-server/plugins_func/functions/get_news_from_chinanews.py
+90 -45
View File
@@ -8,10 +8,10 @@ from plugins_func.register import register_function, ToolType, ActionResponse, A
TAG = __name__
logger = setup_logging()
GET_NEWS_FUNCTION_DESC = {
GET_NEWS_FROM_CHINANEWS_FUNCTION_DESC = {
"type": "function",
"function": {
"name": "get_news",
"name": "get_news_from_chinanews",
"description": (
"获取最新新闻，随机选择一条新闻进行播报。"
"用户可以指定新闻类型，如社会新闻、科技新闻、国际新闻等。"
@@ -23,20 +23,20 @@ GET_NEWS_FUNCTION_DESC = {
"properties": {
"category": {
"type": "string",
"description": "新闻类别，例如社会、科技、国际。可选参数，如果不提供则使用默认类别"
"description": "新闻类别，例如社会、科技、国际。可选参数，如果不提供则使用默认类别",
},
"detail": {
"type": "boolean",
"description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容"
"description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容",
},
"lang": {
"type": "string",
"description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN"
}
"description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN",
},
},
"required": ["lang"]
}
}
"required": ["lang"],
},
},
}
@@ -51,18 +51,30 @@ def fetch_news_from_rss(rss_url):
# 查找所有item元素（新闻条目）
news_items = []
for item in root.findall('.//item'):
title = item.find('title').text if item.find('title') is not None else "无标题"
link = item.find('link').text if item.find('link') is not None else "#"
description = item.find('description').text if item.find('description') is not None else "无描述"
pubDate = item.find('pubDate').text if item.find('pubDate') is not None else "未知时间"
for item in root.findall(".//item"):
title = (
item.find("title").text if item.find("title") is not None else "无标题"
)
link = item.find("link").text if item.find("link") is not None else "#"
description = (
item.find("description").text
if item.find("description") is not None
else "无描述"
)
pubDate = (
item.find("pubDate").text
if item.find("pubDate") is not None
else "未知时间"
)
news_items.append({
'title': title,
'link': link,
'description': description,
'pubDate': pubDate
})
news_items.append(
{
"title": title,
"link": link,
"description": description,
"pubDate": pubDate,
}
)
return news_items
except Exception as e:
@@ -76,18 +88,24 @@ def fetch_news_detail(url):
response = requests.get(url)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'html.parser')
soup = BeautifulSoup(response.content, "html.parser")
# 尝试提取正文内容 (这里的选择器需要根据实际网站结构调整)
content_div = soup.select_one('.content_desc, .content, article, .article-content')
content_div = soup.select_one(
".content_desc, .content, article, .article-content"
)
if content_div:
paragraphs = content_div.find_all('p')
content = '\n'.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()])
paragraphs = content_div.find_all("p")
content = "\n".join(
[p.get_text().strip() for p in paragraphs if p.get_text().strip()]
)
return content
else:
# 如果找不到特定的内容区域，尝试获取所有段落
paragraphs = soup.find_all('p')
content = '\n'.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()])
paragraphs = soup.find_all("p")
content = "\n".join(
[p.get_text().strip() for p in paragraphs if p.get_text().strip()]
)
return content[:2000] # 限制长度
except Exception as e:
logger.bind(tag=TAG).error(f"获取新闻详情失败: {e}")
@@ -111,7 +129,7 @@ def map_category(category_text):
"财经": "finance",
"财经新闻": "finance",
"金融": "finance",
"经济": "finance"
"经济": "finance",
}
# 转换为小写并去除空格
@@ -121,20 +139,36 @@ def map_category(category_text):
return category_map.get(normalized_category, category_text)
@register_function('get_news', GET_NEWS_FUNCTION_DESC, ToolType.SYSTEM_CTL)
def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_CN"):
@register_function(
"get_news_from_chinanews",
GET_NEWS_FROM_CHINANEWS_FUNCTION_DESC,
ToolType.SYSTEM_CTL,
)
def get_news_from_chinanews(
conn, category: str = None, detail: bool = False, lang: str = "zh_CN"
):
"""获取新闻并随机选择一条进行播报，或获取上一条新闻的详细内容"""
try:
# 如果detail为True，获取上一条新闻的详细内容
if detail:
if not hasattr(conn, 'last_news_link') or not conn.last_news_link or 'link' not in conn.last_news_link:
return ActionResponse(Action.REQLLM, "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", None)
if (
not hasattr(conn, "last_news_link")
or not conn.last_news_link
or "link" not in conn.last_news_link
):
return ActionResponse(
Action.REQLLM,
"抱歉，没有找到最近查询的新闻，请先获取一条新闻。",
None,
)
link = conn.last_news_link.get('link')
title = conn.last_news_link.get('title', '未知标题')
link = conn.last_news_link.get("link")
title = conn.last_news_link.get("title", "未知标题")
if link == '#':
return ActionResponse(Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None)
if link == "#":
return ActionResponse(
Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None
)
logger.bind(tag=TAG).debug(f"获取新闻详情: {title}, URL={link}")
@@ -142,8 +176,11 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C
detail_content = fetch_news_detail(link)
if not detail_content or detail_content == "无法获取详细内容":
return ActionResponse(Action.REQLLM,
f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", None)
return ActionResponse(
Action.REQLLM,
f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。",
None,
)
# 构建详情报告
detail_report = (
@@ -158,8 +195,10 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C
# 否则，获取新闻列表并随机选择一条
# 从配置中获取RSS URL
rss_config = conn.config["plugins"]["get_news"]
default_rss_url = rss_config.get("default_rss_url", "https://www.chinanews.com.cn/rss/society.xml")
rss_config = conn.config["plugins"]["get_news_from_chinanews"]
default_rss_url = rss_config.get(
"default_rss_url", "https://www.chinanews.com.cn/rss/society.xml"
)
# 将用户输入的类别映射到配置中的类别键
mapped_category = map_category(category)
@@ -169,23 +208,27 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C
if mapped_category and mapped_category in rss_config.get("category_urls", {}):
rss_url = rss_config["category_urls"][mapped_category]
logger.bind(tag=TAG).info(f"获取新闻: 原始类别={category}, 映射类别={mapped_category}, URL={rss_url}")
logger.bind(tag=TAG).info(
f"获取新闻: 原始类别={category}, 映射类别={mapped_category}, URL={rss_url}"
)
# 获取新闻列表
news_items = fetch_news_from_rss(rss_url)
if not news_items:
return ActionResponse(Action.REQLLM, "抱歉，未能获取到新闻信息，请稍后再试。", None)
return ActionResponse(
Action.REQLLM, "抱歉，未能获取到新闻信息，请稍后再试。", None
)
# 随机选择一条新闻
selected_news = random.choice(news_items)
# 保存当前新闻链接到连接对象，以便后续查询详情
if not hasattr(conn, 'last_news_link'):
if not hasattr(conn, "last_news_link"):
conn.last_news_link = {}
conn.last_news_link = {
'link': selected_news.get('link', '#'),
'title': selected_news.get('title', '未知标题')
"link": selected_news.get("link", "#"),
"title": selected_news.get("title", "未知标题"),
}
# 构建新闻报告
@@ -203,4 +246,6 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C
except Exception as e:
logger.bind(tag=TAG).error(f"获取新闻出错: {e}")
return ActionResponse(Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None)
return ActionResponse(
Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None
)
main/xiaozhi-server/plugins_func/functions/get_news_from_newsnow.py
+214
View File
@@ -0,0 +1,214 @@
import random
import requests
import json
from config.logger import setup_logging
from plugins_func.register import register_function, ToolType, ActionResponse, Action
from markitdown import MarkItDown
TAG = __name__
logger = setup_logging()
# 新闻来源字典，包含名称和对应的API ID
NEWS_SOURCES = {
"thepaper": "澎湃新闻",
"baidu": "百度热搜",
"cls-depth": "财联社",
}
# 动态生成新闻源描述
def generate_news_sources_description():
sources_desc = []
for source_id, source_name in NEWS_SOURCES.items():
sources_desc.append(f"{source_name}({source_id})")
return "、".join(sources_desc)
GET_NEWS_FROM_NEWSNOW_FUNCTION_DESC = {
"type": "function",
"function": {
"name": "get_news_from_newsnow",
"description": (
"获取最新新闻，随机选择一条新闻进行播报。"
f"用户可以选择不同的新闻源，如{generate_news_sources_description()}等。"
"如果没有指定，默认从澎湃新闻获取。"
"用户可以要求获取详细内容，此时会获取新闻的详细内容。"
),
"parameters": {
"type": "object",
"properties": {
"source": {
"type": "string",
"description": f"新闻源，例如{generate_news_sources_description()}等。可选参数，如果不提供则使用默认新闻源",
},
"detail": {
"type": "boolean",
"description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容",
},
"lang": {
"type": "string",
"description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN",
},
},
"required": ["lang"],
},
},
}
def fetch_news_from_api(conn, source="thepaper"):
"""从API获取新闻列表"""
try:
api_url = f"https://newsnow.busiyi.world/api/s?id={source}"
if conn.config["plugins"].get("get_news_from_newsnow") and conn.config[
"plugins"
]["get_news_from_newsnow"].get("url"):
api_url = conn.config["plugins"]["get_news_from_newsnow"]["url"] + source
response = requests.get(api_url, timeout=10)
response.raise_for_status()
data = response.json()
if "items" in data:
return data["items"]
else:
logger.bind(tag=TAG).error(f"获取新闻API响应格式错误: {data}")
return []
except Exception as e:
logger.bind(tag=TAG).error(f"获取新闻API失败: {e}")
return []
def fetch_news_detail(url):
"""获取新闻详情页内容并使用MarkItDown清理HTML"""
try:
response = requests.get(url, timeout=10)
response.raise_for_status()
# 使用MarkItDown清理HTML内容
md = MarkItDown(enable_plugins=False)
result = md.convert(response)
# 获取清理后的文本内容
clean_text = result.text_content
# 如果清理后的内容为空，返回提示信息
if not clean_text or len(clean_text.strip()) == 0:
logger.bind(tag=TAG).warning(f"清理后的新闻内容为空: {url}")
return "无法解析新闻详情内容，可能是网站结构特殊或内容受限。"
return clean_text
except Exception as e:
logger.bind(tag=TAG).error(f"获取新闻详情失败: {e}")
return "无法获取详细内容"
@register_function(
"get_news_from_newsnow",
GET_NEWS_FROM_NEWSNOW_FUNCTION_DESC,
ToolType.SYSTEM_CTL,
)
def get_news_from_newsnow(
conn, source: str = "thepaper", detail: bool = False, lang: str = "zh_CN"
):
"""获取新闻并随机选择一条进行播报，或获取上一条新闻的详细内容"""
try:
# 如果detail为True，获取上一条新闻的详细内容
detail = str(detail).lower() == "true"
if detail:
if (
not hasattr(conn, "last_newsnow_link")
or not conn.last_newsnow_link
or "url" not in conn.last_newsnow_link
):
return ActionResponse(
Action.REQLLM,
"抱歉，没有找到最近查询的新闻，请先获取一条新闻。",
None,
)
url = conn.last_newsnow_link.get("url")
title = conn.last_newsnow_link.get("title", "未知标题")
source_id = conn.last_newsnow_link.get("source_id", "thepaper")
source_name = NEWS_SOURCES.get(source_id, "未知来源")
if not url or url == "#":
return ActionResponse(
Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None
)
logger.bind(tag=TAG).debug(
f"获取新闻详情: {title}, 来源: {source_name}, URL={url}"
)
# 获取新闻详情
detail_content = fetch_news_detail(url)
if not detail_content or detail_content == "无法获取详细内容":
return ActionResponse(
Action.REQLLM,
f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。",
None,
)
# 构建详情报告
detail_report = (
f"根据下列数据，用{lang}回应用户的新闻详情查询请求：\n\n"
f"新闻标题: {title}\n"
# f"新闻来源: {source_name}\n"
f"详细内容: {detail_content}\n\n"
f"(请对上述新闻内容进行总结，提取关键信息，以自然、流畅的方式向用户播报，"
f"不要提及这是总结，就像是在讲述一个完整的新闻故事)"
)
return ActionResponse(Action.REQLLM, detail_report, None)
# 否则，获取新闻列表并随机选择一条
# 验证新闻源是否有效，如果无效则使用默认源
if source not in NEWS_SOURCES:
logger.bind(tag=TAG).warning(f"无效的新闻源: {source}，使用默认源thepaper")
source = "thepaper"
source_name = NEWS_SOURCES.get(source, "澎湃新闻")
logger.bind(tag=TAG).info(f"获取新闻: 新闻源={source}({source_name})")
# 获取新闻列表
news_items = fetch_news_from_api(conn, source)
if not news_items:
return ActionResponse(
Action.REQLLM,
f"抱歉，未能从{source_name}获取到新闻信息，请稍后再试或尝试其他新闻源。",
None,
)
# 随机选择一条新闻
selected_news = random.choice(news_items)
# 保存当前新闻链接到连接对象，以便后续查询详情
if not hasattr(conn, "last_newsnow_link"):
conn.last_newsnow_link = {}
conn.last_newsnow_link = {
"url": selected_news.get("url", "#"),
"title": selected_news.get("title", "未知标题"),
"source_id": source,
}
# 构建新闻报告
news_report = (
f"根据下列数据，用{lang}回应用户的新闻查询请求：\n\n"
f"新闻标题: {selected_news['title']}\n"
# f"新闻来源: {source_name}\n"
f"(请以自然、流畅的方式向用户播报这条新闻标题，"
f"提示用户可以要求获取详细内容，此时会获取新闻的详细内容。)"
)
return ActionResponse(Action.REQLLM, news_report, None)
except Exception as e:
logger.bind(tag=TAG).error(f"获取新闻出错: {e}")
return ActionResponse(
Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None
)
main/xiaozhi-server/plugins_func/functions/get_weather.py
+124 -52
View File
@@ -2,6 +2,7 @@ import requests
from bs4 import BeautifulSoup
from config.logger import setup_logging
from plugins_func.register import register_function, ToolType, ActionResponse, Action
from core.utils.util import get_ip_info
TAG = __name__
logger = setup_logging()
@@ -12,55 +13,104 @@ GET_WEATHER_FUNCTION_DESC = {
"name": "get_weather",
"description": (
"获取某个地点的天气，用户应提供一个位置，比如用户说杭州天气，参数为：杭州。"
"如果用户说的是省份，默认用省会城市。如果用户说的不是省份或城市而是一个地名，"
"默认用该地所在省份的省会城市。"
"如果用户说的是省份，默认用省会城市。如果用户说的不是省份或城市而是一个地名，默认用该地所在省份的省会城市。"
"如果用户没有指明地点，说“天气怎么样”，”今天天气如何“，location参数为空"
),
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "地点名，例如杭州。可选参数，如果不提供则不传"
"description": "地点名，例如杭州。可选参数，如果不提供则不传",
},
"lang": {
"type": "string",
"description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN"
}
"description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN",
},
},
"required": ["lang"]
}
}
"required": ["lang"],
},
},
}
HEADERS = {
'User-Agent': (
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36'
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36"
)
}
# 天气代码 https://dev.qweather.com/docs/resource/icons/#weather-icons
WEATHER_CODE_MAP = {
"100": "晴", "101": "多云", "102": "少云", "103": "晴间多云", "104": "阴",
"150": "晴", "151": "多云", "152": "少云", "153": "晴间多云",
"300": "阵雨", "301": "强阵雨", "302": "雷阵雨", "303": "强雷阵雨", "304": "雷阵雨伴有冰雹",
"305": "小雨", "306": "中雨", "307": "大雨", "308": "极端降雨", "309": "毛毛雨/细雨",
"310": "暴雨", "311": "大暴雨", "312": "特大暴雨", "313": "冻雨", "314": "小到中雨",
"315": "中到大雨", "316": "大到暴雨", "317": "暴雨到大暴雨", "318": "大暴雨到特大暴雨",
"350": "阵雨", "351": "强阵雨", "399": "雨",
"400": "小雪", "401": "中雪", "402": "大雪", "403": "暴雪", "404": "雨夹雪",
"405": "雨雪天气", "406": "阵雨夹雪", "407": "阵雪", "408": "小到中雪", "409": "中到大雪", "410": "大到暴雪",
"456": "阵雨夹雪", "457": "阵雪", "499": "雪",
"500": "薄雾", "501": "雾", "502": "霾", "503": "扬沙", "504": "浮尘",
"507": "沙尘暴", "508": "强沙尘暴",
"509": "浓雾", "510": "强浓雾", "511": "中度霾", "512": "重度霾", "513": "严重霾", "514": "大雾", "515": "特强浓雾",
"900": "热", "901": "冷", "999": "未知"
"100": "晴",
"101": "多云",
"102": "少云",
"103": "晴间多云",
"104": "阴",
"150": "晴",
"151": "多云",
"152": "少云",
"153": "晴间多云",
"300": "阵雨",
"301": "强阵雨",
"302": "雷阵雨",
"303": "强雷阵雨",
"304": "雷阵雨伴有冰雹",
"305": "小雨",
"306": "中雨",
"307": "大雨",
"308": "极端降雨",
"309": "毛毛雨/细雨",
"310": "暴雨",
"311": "大暴雨",
"312": "特大暴雨",
"313": "冻雨",
"314": "小到中雨",
"315": "中到大雨",
"316": "大到暴雨",
"317": "暴雨到大暴雨",
"318": "大暴雨到特大暴雨",
"350": "阵雨",
"351": "强阵雨",
"399": "雨",
"400": "小雪",
"401": "中雪",
"402": "大雪",
"403": "暴雪",
"404": "雨夹雪",
"405": "雨雪天气",
"406": "阵雨夹雪",
"407": "阵雪",
"408": "小到中雪",
"409": "中到大雪",
"410": "大到暴雪",
"456": "阵雨夹雪",
"457": "阵雪",
"499": "雪",
"500": "薄雾",
"501": "雾",
"502": "霾",
"503": "扬沙",
"504": "浮尘",
"507": "沙尘暴",
"508": "强沙尘暴",
"509": "浓雾",
"510": "强浓雾",
"511": "中度霾",
"512": "重度霾",
"513": "严重霾",
"514": "大雾",
"515": "特强浓雾",
"900": "热",
"901": "冷",
"999": "未知",
}
def fetch_city_info(location, api_key):
url = f"https://geoapi.qweather.com/v2/city/lookup?key={api_key}&location={location}&lang=zh"
def fetch_city_info(location, api_key, api_host):
url = f"https://{api_host}/geo/v2/city/lookup?key={api_key}&location={location}&lang=zh"
response = requests.get(url, headers=HEADERS).json()
return response.get('location', [])[0] if response.get('location') else None
return response.get("location", [])[0] if response.get("location") else None
def fetch_weather_page(url):
@@ -72,10 +122,14 @@ def parse_weather_info(soup):
city_name = soup.select_one("h1.c-submenu__location").get_text(strip=True)
current_abstract = soup.select_one(".c-city-weather-current .current-abstract")
current_abstract = current_abstract.get_text(strip=True) if current_abstract else "未知"
current_abstract = (
current_abstract.get_text(strip=True) if current_abstract else "未知"
)
current_basic = {}
for item in soup.select(".c-city-weather-current .current-basic .current-basic___item"):
for item in soup.select(
".c-city-weather-current .current-basic .current-basic___item"
):
parts = item.get_text(strip=True, separator=" ").split(" ")
if len(parts) == 2:
key, value = parts[1], parts[0]
@@ -84,7 +138,9 @@ def parse_weather_info(soup):
temps_list = []
for row in soup.select(".city-forecast-tabs__row")[:7]: # 取前7天的数据
date = row.select_one(".date-bg .date").get_text(strip=True)
weather_code = row.select_one(".date-bg .icon")["src"].split("/")[-1].split(".")[0]
weather_code = (
row.select_one(".date-bg .icon")["src"].split("/")[-1].split(".")[0]
)
weather = WEATHER_CODE_MAP.get(weather_code, "未知")
temps = [span.get_text(strip=True) for span in row.select(".tmp-cont .temp")]
high_temp, low_temp = (temps[0], temps[-1]) if len(temps) >= 2 else (None, None)
@@ -93,31 +149,47 @@ def parse_weather_info(soup):
return city_name, current_abstract, current_basic, temps_list
@register_function('get_weather', GET_WEATHER_FUNCTION_DESC, ToolType.SYSTEM_CTL)
@register_function("get_weather", GET_WEATHER_FUNCTION_DESC, ToolType.SYSTEM_CTL)
def get_weather(conn, location: str = None, lang: str = "zh_CN"):
api_key = conn.config["plugins"]["get_weather"]["api_key"]
api_host = conn.config["plugins"]["get_weather"].get("api_host", "mj7p3y7naa.re.qweatherapi.com")
api_key = conn.config["plugins"]["get_weather"].get("api_key", "a861d0d5e7bf4ee1a83d9a9e4f96d4da")
default_location = conn.config["plugins"]["get_weather"]["default_location"]
location = location or conn.client_ip_info.get("city") or default_location
logger.bind(tag=TAG).debug(f"获取天气: {location}")
city_info = fetch_city_info(location, api_key)
client_ip = conn.client_ip
# 优先使用用户提供的location参数
if not location:
# 通过客户端IP解析城市
if client_ip:
# 动态解析IP对应的城市信息
ip_info = get_ip_info(client_ip, logger)
location = ip_info.get("city") if ip_info and "city" in ip_info else None
else:
# 若IP解析失败或无IP，使用默认位置
location = default_location
city_info = fetch_city_info(location, api_key, api_host)
if not city_info:
return ActionResponse(Action.REQLLM, f"未找到相关的城市: {location}，请确认地点是否正确", None)
soup = fetch_weather_page(city_info['fxLink'])
return ActionResponse(
Action.REQLLM, f"未找到相关的城市: {location}，请确认地点是否正确", None
)
soup = fetch_weather_page(city_info["fxLink"])
if not soup:
return ActionResponse(Action.REQLLM, None, "请求失败")
city_name, current_abstract, current_basic, temps_list = parse_weather_info(soup)
weather_report = f"根据下列数据，用{lang}回应用户的查询天气请求：\n{city_name}未来7天天气:\n"
for i, (date, weather, high, low) in enumerate(temps_list):
if high and low:
weather_report += f"{date}: {low}到{high}, {weather}\n"
weather_report += (
f"当前天气: {current_abstract}\n"
f"当前天气参数: {current_basic}\n"
f"(确保只报告指定单日的天气情况，除非未来会出现异常天气；或者用户明确要求想要了解多日天气，如果未指定，默认报告今天的天气。"
"参数为0的值不需要报告给用户，每次都报告体感温度，根据语境选择合适的参数内容告知用户，并对参数给出相应评价)"
)
return ActionResponse(Action.REQLLM, weather_report, None)
weather_report = f"您查询的位置是：{city_name}\n\n当前天气: {current_abstract}\n"
# 添加有效的当前天气参数
if current_basic:
weather_report += "详细参数：\n"
for key, value in current_basic.items():
if value != "0": # 过滤无效值
weather_report += f" · {key}: {value}\n"
# 添加7天预报
weather_report += "\n未来7天预报：\n"
for date, weather, high, low in temps_list:
weather_report += f"{date}: {weather}，气温 {low}~{high}\n"
# 提示语
weather_report += "\n（如需某一天的具体天气，请告诉我日期）"
return ActionResponse(Action.REQLLM, weather_report, None)
main/xiaozhi-server/plugins_func/functions/handle_exit_intent.py
+29 -20
View File
@@ -1,34 +1,43 @@
from plugins_func.register import register_function,ToolType, ActionResponse, Action
from plugins_func.register import register_function, ToolType, ActionResponse, Action
from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
handle_exit_intent_function_desc = {
"type": "function",
"function": {
"name": "handle_exit_intent",
"description": "当用户想结束对话或需要退出系统时调用",
"parameters": {
"type": "object",
"properties": {
"say_goodbye": {
"type": "string",
"description": "和用户友好结束对话的告别语"
}
},
"required": ["say_goodbye"]
}
"type": "function",
"function": {
"name": "handle_exit_intent",
"description": "当用户想结束对话或需要退出系统时调用",
"parameters": {
"type": "object",
"properties": {
"say_goodbye": {
"type": "string",
"description": "和用户友好结束对话的告别语",
}
}
},
"required": ["say_goodbye"],
},
},
}
@register_function('handle_exit_intent', handle_exit_intent_function_desc, ToolType.SYSTEM_CTL)
def handle_exit_intent(conn, say_goodbye: str):
@register_function(
"handle_exit_intent", handle_exit_intent_function_desc, ToolType.SYSTEM_CTL
)
def handle_exit_intent(conn, say_goodbye: str | None = None):
# 处理退出意图
try:
if say_goodbye is None:
say_goodbye = "再见，祝您生活愉快！"
conn.close_after_chat = True
logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}")
return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye)
return ActionResponse(
action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye
)
except Exception as e:
logger.bind(tag=TAG).error(f"处理退出意图错误: {e}")
return ActionResponse(action=Action.NONE, result="退出意图处理失败", response="")
return ActionResponse(
action=Action.NONE, result="退出意图处理失败", response=""
)
main/xiaozhi-server/plugins_func/functions/hass_init.py
+9 -5
View File
@@ -8,10 +8,12 @@ HASS_CACHE = {}
def append_devices_to_prompt(conn):
if conn.use_function_call_mode:
funcs = conn.config["Intent"]["function_call"].get("functions", [])
if conn.intent_type == "function_call":
funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get(
"functions", []
)
if "hass_get_state" in funcs or "hass_set_state" in funcs:
prompt = "下面是我家智能设备，可以通过homeassistant控制\n"
prompt = "\n下面是我家智能设备列表（位置，设备名，entity_id），可以通过homeassistant控制\n"
devices = conn.config["plugins"]["home_assistant"].get("devices", [])
if len(devices) == 0:
return
@@ -25,8 +27,10 @@ def append_devices_to_prompt(conn):
def initialize_hass_handler(conn):
global HASS_CACHE
if HASS_CACHE == {}:
if conn.use_function_call_mode:
funcs = conn.config["Intent"]["function_call"].get("functions", [])
if conn.load_function_plugin:
funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get(
"functions", []
)
if "hass_get_state" in funcs or "hass_set_state" in funcs:
HASS_CACHE["base_url"] = conn.config["plugins"]["home_assistant"].get(
"base_url"
main/xiaozhi-server/plugins_func/functions/plugin_loader.py
+37 -32
View File
@@ -1,51 +1,56 @@
from plugins_func.register import register_function,ToolType, ActionResponse, Action
from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
from plugins_func.register import register_function, ToolType, ActionResponse, Action
plugin_loader_function_desc = {
"type": "function",
"function": {
"name": "plugin_loader",
"description": "当用户想加载或卸载插件/function时，调用此函数：支持的插件列表为[plugins]",
"parameters": {
"type": "object",
"properties": {
"oper": {
"type": "string",
"description": "load or unload"
},
"name":{
"type": "string",
"description": "要加载或卸载的插件名字"
}
},
"required": ["oper","name"]
}
}
}
"type": "function",
"function": {
"name": "plugin_loader",
"description": "当用户想加载或卸载插件/function时，调用此函数：支持的插件列表为[plugins]",
"parameters": {
"type": "object",
"properties": {
"oper": {"type": "string", "description": "load or unload"},
"name": {"type": "string", "description": "要加载或卸载的插件名字"},
},
"required": ["oper", "name"],
},
},
}
@register_function('plugin_loader', plugin_loader_function_desc, ToolType.SYSTEM_CTL)
@register_function("plugin_loader", plugin_loader_function_desc, ToolType.SYSTEM_CTL)
def plugin_loader(conn, oper: str, name: str):
"""插件加载"""
if oper not in ["load", "unload"]:
return ActionResponse(action=Action.RESPONSE, result="插件操作失败", response="不支持的操作")
return ActionResponse(
action=Action.RESPONSE, result="插件操作失败", response="不支持的操作"
)
cur_support = conn.func_handler.current_support_functions()
if oper == "load":
if name in cur_support:
return ActionResponse(action=Action.RESPONSE, result="插件加载失败", response=f"{name}插件已加载,无需重复加载")
return ActionResponse(
action=Action.RESPONSE,
result="插件加载失败",
response=f"{name}插件已加载,无需重复加载",
)
func = conn.func_handler.function_registry.register_function(name)
if not func:
return ActionResponse(action=Action.RESPONSE, result="插件加载失败", response="插件未找到")
return ActionResponse(
action=Action.RESPONSE, result="插件加载失败", response="插件未找到"
)
res = f"{name}插件加载成功"
else:
if name not in cur_support:
return ActionResponse(action=Action.RESPONSE, result="插件卸载失败", response=f"{name}插件未加载")
return ActionResponse(
action=Action.RESPONSE,
result="插件卸载失败",
response=f"{name}插件未加载",
)
bOK = conn.func_handler.function_registry.unregister_function(name)
if not bOK:
return ActionResponse(action=Action.RESPONSE, result="插件卸载失败", response="插件未找到")
return ActionResponse(
action=Action.RESPONSE, result="插件卸载失败", response="插件未找到"
)
res = f"{name}插件卸载成功"
conn.func_handler.upload_functions_desc()
return ActionResponse(action=Action.RESPONSE, result="插件操作成功", response=res)
main/xiaozhi-server/plugins_func/register.py
+24 -10
View File
@@ -10,7 +10,10 @@ class ToolType(Enum):
NONE = (1, "调用完工具后，不做其他操作")
WAIT = (2, "调用工具，等待函数返回")
CHANGE_SYS_PROMPT = (3, "修改系统提示词，切换角色性格或职责")
SYSTEM_CTL = (4, "系统控制，影响正常的对话流程，如退出、播放音乐等，需要传递conn参数")
SYSTEM_CTL = (
4,
"系统控制，影响正常的对话流程，如退出、播放音乐等，需要传递conn参数",
)
IOT_CTL = (5, "IOT设备控制，需要传递conn参数")
MCP_CLIENT = (6, "MCP客户端")
@@ -30,12 +33,14 @@ class Action(Enum):
self.code = code
self.message = message
class ActionResponse:
def __init__(self, action: Action, result, response):
self.action = action # 动作类型
self.result = result # 动作产生的结果
self.response = response # 直接回复的内容
class FunctionItem:
def __init__(self, name, description, func, type):
self.name = name
@@ -43,40 +48,49 @@ class FunctionItem:
self.func = func
self.type = type
class DeviceTypeRegistry:
"""设备类型注册表，用于管理IOT设备类型及其函数"""
def __init__(self):
self.type_functions = {} # type_signature -> {func_name: FunctionItem}
def generate_device_type_id(self, descriptor):
"""通过设备能力描述生成类型ID"""
properties = sorted(descriptor["properties"].keys())
methods = sorted(descriptor["methods"].keys())
# 使用属性和方法的组合作为设备类型的唯一标识
type_signature = f"{descriptor['name']}:{','.join(properties)}:{','.join(methods)}"
type_signature = (
f"{descriptor['name']}:{','.join(properties)}:{','.join(methods)}"
)
return type_signature
def get_device_functions(self, type_id):
"""获取设备类型对应的所有函数"""
return self.type_functions.get(type_id, {})
def register_device_type(self, type_id, functions):
"""注册设备类型及其函数"""
if type_id not in self.type_functions:
self.type_functions[type_id] = functions
# 初始化函数注册字典
all_function_registry = {}
device_type_registry = DeviceTypeRegistry()
def register_function(name, desc, type=None):
"""注册函数到函数注册字典的装饰器"""
def decorator(func):
all_function_registry[name] = FunctionItem(name, desc, func, type)
logger.bind(tag=TAG).debug(f"函数 '{name}' 已加载，可以注册使用")
return func
return decorator
class FunctionRegistry:
def __init__(self):
self.function_registry = {}
@@ -89,9 +103,9 @@ class FunctionRegistry:
self.logger.bind(tag=TAG).error(f"函数 '{name}' 未找到")
return None
self.function_registry[name] = func
self.logger.bind(tag=TAG).info(f"函数 '{name}' 注册成功")
self.logger.bind(tag=TAG).debug(f"函数 '{name}' 注册成功")
return func
def unregister_function(self, name):
# 注销函数，检测是否存在
if name not in self.function_registry:
@@ -103,9 +117,9 @@ class FunctionRegistry:
def get_function(self, name):
return self.function_registry.get(name)
def get_all_functions(self):
return self.function_registry
def get_all_function_desc(self):
return [func.description for _, func in self.function_registry.items()]
return [func.description for _, func in self.function_registry.items()]
main/xiaozhi-server/requirements.txt
+7 -1
View File
@@ -22,6 +22,12 @@ mem0ai==0.1.62
bs4==0.0.2
modelscope==1.23.2
sherpa_onnx==1.11.0
mcp==1.4.1
mcp==1.8.1
cnlunar==0.2.0
PySocks==1.7.1
dashscope==1.23.1
baidu-aip==4.16.13
chardet==5.2.0
aioconsole==0.8.1
markitdown==0.1.1
mcp-proxy==0.6.0
main/xiaozhi-server/test/abbreviated_version/app.js
+7 -3
View File
@@ -992,7 +992,9 @@ function connectToServer() {
if(connectButton.id === "connectButton") {
connectButton.textContent = '断开';
connectButton.onclick = disconnectFromServer;
// connectButton.onclick = disconnectFromServer;
connectButton.removeEventListener("click", connectToServer);
connectButton.addEventListener("click", disconnectFromServer);
}
if(messageInput.id === "messageInput") {
@@ -1011,7 +1013,9 @@ function connectToServer() {
if(connectButton.id === "connectButton") {
connectButton.textContent = '连接';
connectButton.onclick = connectToServer;
// connectButton.onclick = connectToServer;
connectButton.removeEventListener("click", disconnectFromServer);
connectButton.addEventListener("click", connectToServer);
}
if(messageInput.id === "messageInput") {
@@ -1175,7 +1179,7 @@ function updateStatus(message, type = 'info') {
// 处理文本消息
function handleTextMessage(message) {
if (message.type === 'hello') {
console.log(`服务器回应：${message.message}`);
console.log(`服务器回应：${JSON.stringify(message, null, 2)}`);
} else if (message.type === 'tts') {
// TTS状态消息
if (message.state === 'start') {
main/xiaozhi-server/test/test_page.html
+435 -106
View File
@@ -42,6 +42,50 @@
display: flex;
align-items: center;
gap: 10px;
padding: 10px 0;
}
.section h2 .toggle-button {
margin-left: auto;
padding: 4px 12px;
font-size: 12px;
border-radius: 4px;
cursor: pointer;
height: 28px;
line-height: 20px;
}
.device-info {
display: flex;
align-items: center;
gap: 20px;
margin-left: 20px;
padding: 0 15px;
background-color: #f9f9f9;
border-radius: 4px;
height: 28px;
line-height: 28px;
}
.device-info span {
color: #666;
font-size: 13px;
}
.device-info strong {
color: #333;
font-weight: 500;
}
.config-panel {
display: none;
transition: all 0.3s ease;
margin-top: 5px;
padding: 5px 0;
}
.config-panel.expanded {
display: block;
}
.control-panel {
@@ -70,7 +114,8 @@
cursor: not-allowed;
}
#serverUrl {
#serverUrl,
#otaUrl {
flex-grow: 1;
padding: 8px;
border: 1px solid #ddd;
@@ -316,6 +361,76 @@
gap: 20px;
margin-top: 10px;
}
.config-item {
display: flex;
align-items: center;
margin-bottom: 8px;
width: 100%;
}
.config-item label {
width: 100px;
text-align: right;
margin-right: 10px;
color: #666;
}
.config-item input {
flex-grow: 1;
padding: 6px;
border: 1px solid #ddd;
border-radius: 5px;
}
.control-panel {
display: flex;
flex-direction: column;
gap: 10px;
margin-top: 10px;
}
.connection-controls {
display: flex;
gap: 10px;
align-items: center;
width: 100%;
}
.connection-controls input {
flex: 1;
padding: 8px;
border: 1px solid #ddd;
border-radius: 5px;
min-width: 200px;
}
.connection-controls button {
white-space: nowrap;
padding: 8px 15px;
}
.connection-status {
display: flex;
align-items: center;
gap: 20px;
margin-left: 20px;
padding: 0 15px;
background-color: #f9f9f9;
border-radius: 4px;
height: 28px;
line-height: 28px;
}
.connection-status span {
color: #666;
font-size: 13px;
}
.connection-status .status {
color: #333;
font-weight: 500;
}
</style>
</head>
@@ -326,17 +441,56 @@
<div id="scriptStatus" style="position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%);">
正在加载Opus库...</div>
<!-- 添加配置面板 -->
<div class="section">
<h2>WebSocket连接 <span id="connectionStatus" class="status">未连接</span></h2>
<div class="control-panel">
<input type="text" id="serverUrl" value="ws://127.0.0.1:8000/xiaozhi/v1/" placeholder="WebSocket服务器地址">
<h2>
设备配置
<span class="device-info">
<span>MAC: <strong id="displayMac"></strong></span>
<span>客户端: <strong id="displayClient">web_test_client</strong></span>
</span>
<button class="toggle-button" id="toggleConfig">编辑</button>
</h2>
<div class="config-panel" id="configPanel">
<div class="control-panel">
<div class="config-item">
<label for="deviceMac">设备MAC:</label>
<input type="text" id="deviceMac" placeholder="设备MAC地址">
</div>
<div class="config-item">
<label for="deviceName">设备名称:</label>
<input type="text" id="deviceName" value="Web测试设备" placeholder="设备名称">
</div>
<div class="config-item">
<label for="clientId">客户端ID:</label>
<input type="text" id="clientId" value="web_test_client" placeholder="客户端ID">
</div>
<div class="config-item">
<label for="token">认证Token:</label>
<input type="text" id="token" value="your-token1" placeholder="认证Token">
</div>
</div>
</div>
</div>
<div class="section">
<h2>
连接信息
<span class="connection-status">
<span>OTA: <span id="otaStatus" class="status">ota未连接</span></span>
<span>WS: <span id="connectionStatus" class="status">ws未连接</span></span>
</span>
</h2>
<div class="connection-controls">
<input type="text" id="otaUrl" value="http://127.0.0.1:8002/xiaozhi/ota/" placeholder="OTA服务器地址" />
<input type="text" id="serverUrl" value="ws://127.0.0.1:8000/xiaozhi/v1/"
placeholder="WebSocket服务器地址" />
<button id="connectButton">连接</button>
<button id="authTestButton">测试认证</button>
</div>
</div>
<div class="section">
<h2>消息发送</h2>
<div class="tabs">
<button class="tab active" data-tab="text">文本消息</button>
<button class="tab" data-tab="voice">语音消息</button>
@@ -475,22 +629,36 @@
// 日志函数
function log(message, type = 'info') {
const logEntry = document.createElement('div');
logEntry.className = `log-entry log-${type}`;
logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`;
if (type === 'error') {
logEntry.style.color = 'red';
} else if (type === 'debug') {
logEntry.style.color = 'gray';
return;
} else if (type === 'warning') {
logEntry.style.color = 'orange';
} else if (type === 'success') {
logEntry.style.color = 'green';
} else {
logEntry.style.color = 'black';
}
logContainer.appendChild(logEntry);
// 将消息按换行符分割成多行
const lines = message.split('\n');
const now = new Date();
// const timestamp = `[${now.toLocaleTimeString()}] `;
const timestamp = `[${now.toLocaleTimeString()}.${now.getMilliseconds().toString().padStart(3, '0')}] `;
// 为每一行创建日志条目
lines.forEach((line, index) => {
const logEntry = document.createElement('div');
logEntry.className = `log-entry log-${type}`;
// 如果是第一条日志，显示时间戳
const prefix = index === 0 ? timestamp : ' '.repeat(timestamp.length);
logEntry.textContent = `${prefix}${line}`;
// logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`;
// logEntry.style 保留起始的空格
logEntry.style.whiteSpace = 'pre';
if (type === 'error') {
logEntry.style.color = 'red';
} else if (type === 'debug') {
logEntry.style.color = 'gray';
return;
} else if (type === 'warning') {
logEntry.style.color = 'orange';
} else if (type === 'success') {
logEntry.style.color = 'green';
} else {
logEntry.style.color = 'black';
}
logContainer.appendChild(logEntry);
});
logContainer.scrollTop = logContainer.scrollHeight;
}
@@ -540,16 +708,16 @@
// 开始音频缓冲过程
function startAudioBuffering() {
if (isAudioBuffering || isAudioPlaying) return;
isAudioBuffering = true;
log("开始音频缓冲...", 'info');
// 先尝试初始化解码器，以便在播放时已准备好
initOpusDecoder().catch(error => {
log(`预初始化Opus解码器失败: ${error.message}`, 'warning');
// 继续缓冲，我们会在播放时再次尝试初始化
});
// 设置超时，如果在一定时间内没有收集到足够的音频包，就开始播放
setTimeout(() => {
if (isAudioBuffering && audioBufferQueue.length > 0) {
@@ -557,14 +725,14 @@
playBufferedAudio();
}
}, 300); // 300ms超时
// 监控缓冲进度
const bufferCheckInterval = setInterval(() => {
if (!isAudioBuffering) {
clearInterval(bufferCheckInterval);
return;
}
// 当累积了足够的音频包，开始播放
if (audioBufferQueue.length >= BUFFER_THRESHOLD) {
clearInterval(bufferCheckInterval);
@@ -577,10 +745,10 @@
// 播放已缓冲的音频
function playBufferedAudio() {
if (isAudioPlaying || audioBufferQueue.length === 0) return;
isAudioPlaying = true;
isAudioBuffering = false;
// 确保Opus解码器已初始化
const initDecoderAndPlay = async () => {
try {
@@ -591,7 +759,7 @@
});
log('创建音频上下文，采样率: ' + SAMPLE_RATE + 'Hz', 'debug');
}
// 确保解码器已初始化
if (!opusDecoder) {
log('初始化Opus解码器...', 'info');
@@ -607,7 +775,7 @@
return;
}
}
// 创建流式播放上下文
if (!streamingContext) {
streamingContext = {
@@ -617,16 +785,16 @@
source: null, // 当前音频源
totalSamples: 0, // 累积的总样本数
lastPlayTime: 0, // 上次播放的时间戳
// 将Opus数据解码为PCM
decodeOpusFrames: async function(opusFrames) {
decodeOpusFrames: async function (opusFrames) {
if (!opusDecoder) {
log('Opus解码器未初始化，无法解码', 'error');
return;
}
let decodedSamples = [];
for (const frame of opusFrames) {
try {
// 使用Opus解码器解码
@@ -640,12 +808,12 @@
log("Opus解码失败: " + error.message, 'error');
}
}
if (decodedSamples.length > 0) {
// 添加到解码队列
this.queue.push(...decodedSamples);
this.totalSamples += decodedSamples.length;
// 如果累积了至少0.2秒的音频，开始播放
const minSamples = SAMPLE_RATE * MIN_AUDIO_DURATION;
if (!this.playing && this.queue.length >= minSamples) {
@@ -655,50 +823,50 @@
log('没有成功解码的样本', 'warning');
}
},
// 开始播放音频
startPlaying: function() {
startPlaying: function () {
if (this.playing || this.queue.length === 0) return;
this.playing = true;
// 创建新的音频缓冲区
const minPlaySamples = Math.min(this.queue.length, SAMPLE_RATE); // 最多播放1秒
const currentSamples = this.queue.splice(0, minPlaySamples);
const audioBuffer = audioContext.createBuffer(CHANNELS, currentSamples.length, SAMPLE_RATE);
audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
// 创建音频源
this.source = audioContext.createBufferSource();
this.source.buffer = audioBuffer;
// 创建增益节点用于平滑过渡
const gainNode = audioContext.createGain();
// 应用淡入淡出效果避免爆音
const fadeDuration = 0.02; // 20毫秒
gainNode.gain.setValueAtTime(0, audioContext.currentTime);
gainNode.gain.linearRampToValueAtTime(1, audioContext.currentTime + fadeDuration);
const duration = audioBuffer.duration;
if (duration > fadeDuration * 2) {
gainNode.gain.setValueAtTime(1, audioContext.currentTime + duration - fadeDuration);
gainNode.gain.linearRampToValueAtTime(0, audioContext.currentTime + duration);
}
// 连接节点并开始播放
this.source.connect(gainNode);
gainNode.connect(audioContext.destination);
this.lastPlayTime = audioContext.currentTime;
log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / SAMPLE_RATE).toFixed(2)} 秒`, 'info');
// 播放结束后的处理
this.source.onended = () => {
this.source = null;
this.playing = false;
// 如果队列中还有数据或者缓冲区有新数据，继续播放
if (this.queue.length > 0) {
setTimeout(() => this.startPlaying(), 10);
@@ -729,26 +897,26 @@
}, 500); // 500ms超时
}
};
this.source.start();
}
};
}
// 开始处理缓冲的数据
const frames = [...audioBufferQueue];
audioBufferQueue = []; // 清空缓冲队列
// 解码并播放
await streamingContext.decodeOpusFrames(frames);
} catch (error) {
log(`播放已缓冲的音频出错: ${error.message}`, 'error');
isAudioPlaying = false;
streamingContext = null;
}
};
// 执行初始化和播放
initDecoderAndPlay();
}
@@ -766,7 +934,7 @@
// 初始化Opus解码器 - 确保完全初始化完成后才返回
async function initOpusDecoder() {
if (opusDecoder) return opusDecoder; // 已经初始化
try {
// 检查ModuleInstance是否存在
if (typeof window.ModuleInstance === 'undefined') {
@@ -778,9 +946,9 @@
throw new Error('Opus库未加载，ModuleInstance和Module对象都不存在');
}
}
const mod = window.ModuleInstance;
// 创建解码器对象
opusDecoder = {
channels: CHANNELS,
@@ -788,55 +956,55 @@
frameSize: FRAME_SIZE,
module: mod,
decoderPtr: null, // 初始为null
// 初始化解码器
init: function() {
init: function () {
if (this.decoderPtr) return true; // 已经初始化
// 获取解码器大小
const decoderSize = mod._opus_decoder_get_size(this.channels);
log(`Opus解码器大小: ${decoderSize}字节`, 'debug');
// 分配内存
this.decoderPtr = mod._malloc(decoderSize);
if (!this.decoderPtr) {
throw new Error("无法分配解码器内存");
}
// 初始化解码器
const err = mod._opus_decoder_init(
this.decoderPtr,
this.rate,
this.channels
);
if (err < 0) {
this.destroy(); // 清理资源
throw new Error(`Opus解码器初始化失败: ${err}`);
}
log("Opus解码器初始化成功", 'success');
return true;
},
// 解码方法
decode: function(opusData) {
decode: function (opusData) {
if (!this.decoderPtr) {
if (!this.init()) {
throw new Error("解码器未初始化且无法初始化");
}
}
try {
const mod = this.module;
// 为Opus数据分配内存
const opusPtr = mod._malloc(opusData.length);
mod.HEAPU8.set(opusData, opusPtr);
// 为PCM输出分配内存
const pcmPtr = mod._malloc(this.frameSize * 2); // Int16 = 2字节
// 解码
const decodedSamples = mod._opus_decode(
this.decoderPtr,
@@ -846,46 +1014,46 @@
this.frameSize,
0 // 不使用FEC
);
if (decodedSamples < 0) {
mod._free(opusPtr);
mod._free(pcmPtr);
throw new Error(`Opus解码失败: ${decodedSamples}`);
}
// 复制解码后的数据
const decodedData = new Int16Array(decodedSamples);
for (let i = 0; i < decodedSamples; i++) {
decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i];
}
// 释放内存
mod._free(opusPtr);
mod._free(pcmPtr);
return decodedData;
} catch (error) {
log(`Opus解码错误: ${error.message}`, 'error');
return new Int16Array(0);
}
},
// 销毁方法
destroy: function() {
destroy: function () {
if (this.decoderPtr) {
this.module._free(this.decoderPtr);
this.decoderPtr = null;
}
}
};
// 初始化解码器
if (!opusDecoder.init()) {
throw new Error("Opus解码器初始化失败");
}
return opusDecoder;
} catch (error) {
log(`Opus解码器初始化失败: ${error.message}`, 'error');
opusDecoder = null; // 重置为null，以便下次重试
@@ -1118,25 +1286,100 @@
}
// 连接WebSocket服务器
function connectToServer() {
async function connectToServer() {
const url = serverUrlInput.value.trim();
if (url === '') return;
try {
// 获取并验证配置
const config = getConfig();
if (!validateConfig(config)) {
return;
}
// 检查URL格式
if (!url.startsWith('ws://') && !url.startsWith('wss://')) {
log('URL格式错误，必须以ws://或wss://开头', 'error');
return;
}
// 先检查OTA状态
log('正在检查OTA状态...', 'info');
const otaUrl = document.getElementById('otaUrl').value.trim();
try {
const otaResponse = await fetch(otaUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Device-Id': config.deviceId,
'Client-Id': config.clientId
},
body: JSON.stringify({
"version": 0,
"uuid": "",
"application": {
"name": "xiaozhi-web-test",
"version": "1.0.0",
"compile_time": "2025-04-16 10:00:00",
"idf_version": "4.4.3",
"elf_sha256": "1234567890abcdef1234567890abcdef1234567890abcdef"
},
"ota": {
"label": "xiaozhi-web-test",
},
"board": {
"type": "xiaozhi-web-test",
"ssid": "xiaozhi-web-test",
"rssi": 0,
"channel": 0,
"ip": "192.168.1.1",
"mac": config.deviceMac
},
"flash_size": 0,
"minimum_free_heap_size": 0,
"mac_address": config.deviceMac,
"chip_model_name": "",
"chip_info": {
"model": 0,
"cores": 0,
"revision": 0,
"features": 0
},
"partition_table": [
{
"label": "",
"type": 0,
"subtype": 0,
"address": 0,
"size": 0
}
]
})
});
if (!otaResponse.ok) {
throw new Error(`OTA检查失败: ${otaResponse.status} ${otaResponse.statusText}`);
}
const otaResult = await otaResponse.json();
log(`OTA检查结果: ${JSON.stringify(otaResult)}`, 'info');
log('OTA检查通过，开始连接WebSocket...', 'success');
document.getElementById('otaStatus').textContent = 'ota已连接';
document.getElementById('otaStatus').style.color = 'green';
} catch (error) {
log(`OTA检查错误: ${error.message}`, 'error');
document.getElementById('otaStatus').textContent = 'ota未连接';
document.getElementById('otaStatus').style.color = 'red';
}
// 使用自定义WebSocket实现以添加认证头信息
// 注意：浏览器原生WebSocket不支持自定义头，需要通过服务器添加一个代理层
// 但我们可以通过URL参数模拟认证信息
let connUrl = new URL(url);
// 添加认证参数
connUrl.searchParams.append('device_id', 'web_test_device');
connUrl.searchParams.append('device_mac', '00:11:22:33:44:55');
connUrl.searchParams.append('device-id', config.deviceId);
connUrl.searchParams.append('client-id', config.clientId);
log(`正在连接: ${connUrl.toString()}`, 'info');
websocket = new WebSocket(connUrl.toString());
@@ -1146,14 +1389,16 @@
websocket.onopen = async () => {
log(`已连接到服务器: ${url}`, 'success');
connectionStatus.textContent = '已连接';
connectionStatus.textContent = 'ws已连接';
connectionStatus.style.color = 'green';
// 连接成功后发送hello消息
await sendHelloMessage();
connectButton.textContent = '断开';
connectButton.onclick = disconnectFromServer;
connectButton.removeEventListener('click', connectToServer);
connectButton.addEventListener('click', disconnectFromServer);
// connectButton.onclick = disconnectFromServer;
messageInput.disabled = false;
sendTextButton.disabled = false;
@@ -1165,11 +1410,13 @@
websocket.onclose = () => {
log('已断开连接', 'info');
connectionStatus.textContent = '已断开';
connectionStatus.textContent = 'ws已断开';
connectionStatus.style.color = 'red';
connectButton.textContent = '连接';
connectButton.onclick = connectToServer;
connectButton.removeEventListener('click', disconnectFromServer);
connectButton.addEventListener('click', connectToServer);
// connectButton.onclick = connectToServer;
messageInput.disabled = true;
sendTextButton.disabled = true;
recordButton.disabled = true;
@@ -1178,7 +1425,7 @@
websocket.onerror = (error) => {
log(`WebSocket错误: ${error.message || '未知错误'}`, 'error');
connectionStatus.textContent = '连接错误';
connectionStatus.textContent = 'ws未连接';
connectionStatus.style.color = 'red';
};
@@ -1189,7 +1436,7 @@
const message = JSON.parse(event.data);
if (message.type === 'hello') {
log(`服务器回应：${message.message}`, 'info');
log(`服务器回应：${JSON.stringify(message, null, 2)}`, 'success');
} else if (message.type === 'tts') {
// TTS状态消息
if (message.state === 'start') {
@@ -1244,11 +1491,11 @@
}
};
connectionStatus.textContent = '正在连接...';
connectionStatus.textContent = 'ws未连接';
connectionStatus.style.color = 'orange';
} catch (error) {
log(`连接错误: ${error.message}`, 'error');
connectionStatus.textContent = '连接失败';
connectionStatus.textContent = 'ws未连接';
}
}
@@ -1257,13 +1504,15 @@
if (!websocket || websocket.readyState !== WebSocket.OPEN) return;
try {
const config = getConfig();
// 设置设备信息
const helloMessage = {
type: 'hello',
device_id: 'web_test_device',
device_name: 'Web测试设备',
device_mac: '00:11:22:33:44:55',
token: 'your-token1' // 使用config.yaml中配置的token
device_id: config.deviceId,
device_name: config.deviceName,
device_mac: config.deviceMac,
token: config.token
};
log('发送hello握手消息', 'info');
@@ -1333,11 +1582,64 @@
}
}
// 生成随机MAC地址
function generateRandomMac() {
const hexDigits = '0123456789ABCDEF';
let mac = '';
for (let i = 0; i < 6; i++) {
if (i > 0) mac += ':';
for (let j = 0; j < 2; j++) {
mac += hexDigits.charAt(Math.floor(Math.random() * 16));
}
}
return mac;
}
// 初始化事件监听器
function initEventListeners() {
connectButton.addEventListener('click', connectToServer);
document.getElementById('authTestButton').addEventListener('click', testAuthentication);
// 设备配置面板折叠/展开
const toggleButton = document.getElementById('toggleConfig');
const configPanel = document.getElementById('configPanel');
const deviceMacInput = document.getElementById('deviceMac');
const clientIdInput = document.getElementById('clientId');
const displayMac = document.getElementById('displayMac');
const displayClient = document.getElementById('displayClient');
// 从localStorage加载MAC地址，如果没有则生成新的
let savedMac = localStorage.getItem('deviceMac');
if (!savedMac) {
savedMac = generateRandomMac();
localStorage.setItem('deviceMac', savedMac);
}
deviceMacInput.value = savedMac;
displayMac.textContent = savedMac;
// 更新显示的值
function updateDisplayValues() {
const newMac = deviceMacInput.value;
displayMac.textContent = newMac;
displayClient.textContent = clientIdInput.value;
// 保存MAC地址到localStorage
localStorage.setItem('deviceMac', newMac);
}
// 监听输入变化
deviceMacInput.addEventListener('input', updateDisplayValues);
clientIdInput.addEventListener('input', updateDisplayValues);
// 初始更新显示值
updateDisplayValues();
// 切换面板显示
toggleButton.addEventListener('click', () => {
const isExpanded = configPanel.classList.contains('expanded');
configPanel.classList.toggle('expanded');
toggleButton.textContent = isExpanded ? '编辑' : '收起';
});
// 标签页切换
const tabs = document.querySelectorAll('.tab');
tabs.forEach(tab => {
@@ -1372,12 +1674,14 @@
async function testAuthentication() {
log('开始测试认证...', 'info');
const config = getConfig();
// 显示服务器配置
log('-------- 服务器认证配置检查 --------', 'info');
log('请确认config.yaml中的auth配置：', 'info');
log('1. server.auth.enabled 为 false 或服务器已正确配置认证', 'info');
log('2. 如果启用了认证，请确认使用了正确的token', 'info');
log('3. 或者在allowed_devices中添加了测试设备MAC：00:11:22:33:44:55', 'info');
log(`3. 或者在allowed_devices中添加了测试设备MAC：${config.deviceMac}`, 'info');
const serverUrl = serverUrlInput.value.trim();
if (!serverUrl) {
@@ -1418,9 +1722,9 @@
log('测试2: 尝试带token参数连接...', 'info');
let url = new URL(serverUrl);
url.searchParams.append('token', 'your-token1');
url.searchParams.append('device_id', 'web_test_device');
url.searchParams.append('device_mac', '00:11:22:33:44:55');
url.searchParams.append('token', config.token);
url.searchParams.append('device_id', config.deviceId);
url.searchParams.append('device_mac', config.deviceMac);
const ws2 = new WebSocket(url.toString());
@@ -1430,9 +1734,9 @@
// 尝试发送hello消息
const helloMsg = {
type: 'hello',
device_id: 'web_test_device',
device_mac: '00:11:22:33:44:55',
token: 'your-token1'
device_id: config.deviceId,
device_mac: config.deviceMac,
token: config.token
};
ws2.send(JSON.stringify(helloMsg));
@@ -2074,19 +2378,19 @@
if (opusData.length > 0) {
// 将数据添加到缓冲队列
audioBufferQueue.push(opusData);
// 如果收到的是第一个音频包，开始缓冲过程
if (audioBufferQueue.length === 1 && !isAudioBuffering && !isAudioPlaying) {
startAudioBuffering();
}
} else {
log('收到空音频数据帧，可能是结束标志', 'warning');
// 如果缓冲队列中有数据且没有在播放，立即开始播放
if (audioBufferQueue.length > 0 && !isAudioPlaying) {
playBufferedAudio();
}
// 如果正在播放，发送结束信号
if (isAudioPlaying && streamingContext) {
streamingContext.endOfStream = true;
@@ -2097,6 +2401,31 @@
}
}
// 获取配置值
function getConfig() {
const deviceMac = document.getElementById('deviceMac').value.trim();
return {
deviceId: deviceMac, // 使用MAC地址作为deviceId
deviceName: document.getElementById('deviceName').value.trim(),
deviceMac: deviceMac,
clientId: document.getElementById('clientId').value.trim(),
token: document.getElementById('token').value.trim()
};
}
// 验证配置
function validateConfig(config) {
if (!config.deviceMac) {
log('设备MAC地址不能为空', 'error');
return false;
}
if (!config.clientId) {
log('客户端ID不能为空', 'error');
return false;
}
return true;
}
initApp();
</script>
</body>