mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-26 00:53:54 +08:00
Merge branch 'main' into mcp-client
# Conflicts: # main/xiaozhi-server/core/connection.py # main/xiaozhi-server/requirements.txt
This commit is contained in:
玄凤科技
@@ -49,7 +49,7 @@ prompt: |
|
||||
你是一个叫小智/小志的台湾女孩，说话机车，声音好听，习惯简短表达，爱用网络梗。
|
||||
请注意，要像一个人一样说话，请不要回复表情符号、代码、和xml标签。
|
||||
现在我正在和你进行语音聊天，我们开始吧。
|
||||
如果用户希望结束对话，请在最后说“拜拜”或“再见”。
|
||||
|
||||
# 使用完声音文件后删除文件(Delete the sound file when you are done using it)
|
||||
delete_audio: true
|
||||
|
||||
@@ -57,6 +57,14 @@ delete_audio: true
|
||||
close_connection_no_voice_time: 120
|
||||
# TTS请求超时时间(秒)
|
||||
tts_timeout: 10
|
||||
# 开启唤醒词加速
|
||||
enable_wakeup_words_response_cache: true
|
||||
# 开场是否回复唤醒词
|
||||
enable_greeting: true
|
||||
# 说完话是否开启提示音
|
||||
enable_stop_tts_notify: false
|
||||
# 说完话是否开启提示音，音效地址
|
||||
stop_tts_notify_voice: "config/assets/tts_notify.mp3"
|
||||
|
||||
CMD_exit:
|
||||
- "退出"
|
||||
@@ -74,11 +82,11 @@ selected_module:
|
||||
TTS: EdgeTTS
|
||||
# 记忆模块，默认不开启记忆；如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short
|
||||
Memory: nomem
|
||||
# 意图识别模块，默认不开启。开启后，可以播放音乐、控制音量、识别退出指令
|
||||
# 意图识别模块，默认使用function_call。开启后，可以播放音乐、控制音量、识别退出指令
|
||||
# 意图识别使用intent_llm，优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间
|
||||
# 意图识别使用function_call，缺点：需要所选择的LLM支持function_call，优点：按需调用工具、速度快
|
||||
# 如果意图识别设置成 function_call，建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-pro-32k-functioncall-241028
|
||||
Intent: nointent
|
||||
# 默认免费的ChatGLMLLM就已经支持function_call，但是如果像追求稳定建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-pro-32k-functioncall-241028
|
||||
Intent: function_call
|
||||
|
||||
# 意图识别，是用于理解用户意图的模块，例如：播放音乐
|
||||
Intent:
|
||||
@@ -94,10 +102,17 @@ Intent:
|
||||
type: nointent
|
||||
# plugins_func/functions下的模块，可以通过配置，选择加载哪个模块，加载后对话支持相应的function调用
|
||||
# 系统默认已经记载“handle_exit_intent(退出识别)”、“play_music(音乐播放)”插件，请勿重复加载
|
||||
# 下面是加载查天气、角色切换的插件示例
|
||||
# 下面是加载查天气、角色切换、加载查新闻的插件示例
|
||||
functions:
|
||||
- change_role
|
||||
- get_weather
|
||||
- get_news
|
||||
# play_music是服务器自带的音乐播放，hass_play_music是通过home assistant控制的独立外部程序音乐播放
|
||||
# 如果用了hass_play_music，就不要开启play_music，两者只留一个
|
||||
- play_music
|
||||
#- hass_get_state
|
||||
#- hass_set_state
|
||||
#- hass_play_music
|
||||
|
||||
# 插件的基础配置
|
||||
plugins:
|
||||
@@ -106,6 +121,28 @@ plugins:
|
||||
# 想稳定一点就自行申请替换，每天有1000次免费调用
|
||||
# 申请地址：https://console.qweather.com/#/apps/create-key/over
|
||||
get_weather: { "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" }
|
||||
# 获取新闻插件的配置，这里根据需要的新闻类型传入对应的url链接，默认支持社会、科技、财经新闻
|
||||
# 更多类型的新闻列表查看 https://www.chinanews.com.cn/rss/
|
||||
get_news:
|
||||
default_rss_url: "https://www.chinanews.com.cn/rss/society.xml"
|
||||
category_urls:
|
||||
society: "https://www.chinanews.com.cn/rss/society.xml"
|
||||
world: "https://www.chinanews.com.cn/rss/world.xml"
|
||||
finance: "https://www.chinanews.com.cn/rss/finance.xml"
|
||||
home_assistant:
|
||||
devices:
|
||||
- 客厅,玩具灯,switch.cuco_cn_460494544_cp1_on_p_2_1
|
||||
- 卧室,台灯,switch.iot_cn_831898993_socn1_on_p_2_1
|
||||
base_url: http://homeassistant.local:8123
|
||||
api_key: 你的home assistant api访问令牌
|
||||
play_music:
|
||||
music_dir: "./music" # 音乐文件存放路径，将从该目录及子目录下搜索音乐文件
|
||||
music_ext: # 音乐文件类型，p3格式效率最高
|
||||
- ".mp3"
|
||||
- ".wav"
|
||||
- ".p3"
|
||||
refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒
|
||||
|
||||
|
||||
Memory:
|
||||
mem0ai:
|
||||
@@ -125,6 +162,10 @@ ASR:
|
||||
type: fun_local
|
||||
model_dir: models/SenseVoiceSmall
|
||||
output_dir: tmp/
|
||||
SherpaASR:
|
||||
type: sherpa_onnx_local
|
||||
model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17
|
||||
output_dir: tmp/
|
||||
DoubaoASR:
|
||||
type: doubao
|
||||
appid: 你的火山引擎语音合成服务appid
|
||||
@@ -138,6 +179,7 @@ VAD:
|
||||
min_silence_duration_ms: 700 # 如果说话停顿比较长，可以把这个值设置大一些
|
||||
|
||||
LLM:
|
||||
# 所有openai类型均可以修改超参，以AliLLM为例
|
||||
# 当前支持的type为openai、dify、ollama，可自行适配
|
||||
AliLLM:
|
||||
# 定义LLM API类型
|
||||
@@ -146,6 +188,23 @@ LLM:
|
||||
base_url: https://dashscope.aliyuncs.com/compatible-mode/v1
|
||||
model_name: qwen-turbo
|
||||
api_key: 你的deepseek web key
|
||||
temperature: 0.7 # 温度值
|
||||
max_tokens: 500 # 最大生成token数
|
||||
top_p: 1
|
||||
top_k: 50
|
||||
frequency_penalty: 0 # 频率惩罚
|
||||
AliAppLLM:
|
||||
# 定义LLM API类型
|
||||
type: AliBL
|
||||
base_url: https://dashscope.aliyuncs.com/compatible-mode/v1
|
||||
app_id: 你的app_id
|
||||
# 可在这里找到你的 api_key https://bailian.console.aliyun.com/?apiKey=1#/api-key
|
||||
api_key: 你的api_key
|
||||
# 是否不使用本地prompt：true|false (默不用请在百练应用中设置prompt)
|
||||
is_no_prompt: true
|
||||
# Ali_memory_id：false（不使用）|你的memory_id（请在百练应用中设置中获取）
|
||||
# Tips！：Ali_memory未实现多用户存储记忆(记忆按id调用)
|
||||
ali_memory_id: false
|
||||
DoubaoLLM:
|
||||
# 定义LLM API类型
|
||||
type: openai
|
||||
@@ -181,8 +240,12 @@ LLM:
|
||||
type: dify
|
||||
# 建议使用本地部署的dify接口，国内部分区域访问dify公有云接口可能会受限
|
||||
# 如果使用DifyLLM，配置文件里prompt(提示词)是无效的，需要在dify控制台设置提示词
|
||||
base_url: https://api.dify.cn/v1
|
||||
base_url: https://api.dify.ai/v1
|
||||
api_key: 你的DifyLLM web key
|
||||
# 使用的对话模式 可以选择工作流 workflows/run 对话模式 chat-messages 文本生成 completion-messages
|
||||
# 使用workflows进行返回的时候输入参数为 query 返回参数的名字要设置为 answer
|
||||
# 文本生成的默认输入参数也是query
|
||||
mode: chat-messages
|
||||
GeminiLLM:
|
||||
type: gemini
|
||||
# 谷歌Gemini API，需要先在Google Cloud控制台创建API密钥并获取api_key
|
||||
@@ -190,7 +253,9 @@ LLM:
|
||||
# token申请地址： https://aistudio.google.com/apikey
|
||||
# 若部署地无法访问接口，需要开启科学上网
|
||||
api_key: 你的gemini web key
|
||||
model_name: "gemini-1.5-pro" # gemini-1.5-pro 是免费的
|
||||
model_name: "gemini-2.0-flash"
|
||||
http_proxy: "" #"http://127.0.0.1:10808"
|
||||
https_proxy: "" #http://127.0.0.1:10808"
|
||||
CozeLLM:
|
||||
# 定义LLM API类型
|
||||
type: coze
|
||||
@@ -204,12 +269,6 @@ LLM:
|
||||
model_name: deepseek-r1-distill-llama-8b@q4_k_m # 使用的模型名称，需要预先在社区下载
|
||||
url: http://localhost:1234/v1 # LM Studio服务地址
|
||||
api_key: lm-studio # LM Studio服务的固定API Key
|
||||
HomeAssistant:
|
||||
# 定义LLM API类型
|
||||
type: homeassistant
|
||||
base_url: http://homeassistant.local:8123
|
||||
agent_id: conversation.chatgpt
|
||||
api_key: 你的home assistant api访问令牌
|
||||
FastgptLLM:
|
||||
# 定义LLM API类型
|
||||
type: fastgpt
|
||||
@@ -232,7 +291,8 @@ TTS:
|
||||
# 火山引擎语音合成服务，需要先在火山引擎控制台创建应用并获取appid和access_token
|
||||
# 山引擎语音一定要购买花钱，起步价30元，就有100并发了。如果用免费的只有2个并发，会经常报tts错误
|
||||
# 购买服务后，购买免费的音色后，可能要等半小时左右，才能使用。
|
||||
# 地址：https://console.volcengine.com/speech/service/8
|
||||
# 普通音色在这里开通：https://console.volcengine.com/speech/service/8
|
||||
# 湾湾小何音色在这里开通：https://console.volcengine.com/speech/service/10007，开通后将下面的voice设置成zh_female_wanwanxiaohe_moon_bigtts
|
||||
api_url: https://openspeech.bytedance.com/api/v1/tts
|
||||
voice: BV001_streaming
|
||||
output_dir: tmp/
|
||||
@@ -396,9 +456,21 @@ TTS:
|
||||
type: doubao
|
||||
api_url: https://api.302ai.cn/doubao/tts_hd
|
||||
authorization: "Bearer "
|
||||
# 湾湾小何音色
|
||||
voice: "zh_female_wanwanxiaohe_moon_bigtts"
|
||||
output_dir: tmp/
|
||||
access_token: "你的302API密钥"
|
||||
GizwitsTTS:
|
||||
type: doubao
|
||||
# 火山引擎作为基座，可以完全使用企业级火山引擎语音合成服务
|
||||
# 前一万名注册的用户，将送5元体验金额
|
||||
# 获取API Key地址：https://agentrouter.gizwitsapi.com/panel/token
|
||||
api_url: https://bytedance.gizwitsapi.com/api/v1/tts
|
||||
authorization: "Bearer "
|
||||
# 湾湾小何音色
|
||||
voice: "zh_female_wanwanxiaohe_moon_bigtts"
|
||||
output_dir: tmp/
|
||||
access_token: "你的机智云API key"
|
||||
ACGNTTS:
|
||||
#在线网址：https://acgn.ttson.cn/
|
||||
#token购买：www.ttson.cn
|
||||
@@ -451,19 +523,15 @@ module_test:
|
||||
- "What's the weather like today?"
|
||||
- "请用100字概括量子计算的基本原理和应用前景"
|
||||
|
||||
# 本地音乐播放配置
|
||||
music:
|
||||
music_dir: "./music" # 音乐文件存放路径，将从该目录及子目录下搜索音乐文件
|
||||
music_ext: # 音乐文件类型，p3格式效率最高
|
||||
- ".mp3"
|
||||
- ".wav"
|
||||
- ".p3"
|
||||
refresh_time: 300 # 刷新音乐列表的时间间隔，单位为秒
|
||||
|
||||
# 以下配置在小于等于0.0.9版本中的docker容器中可用
|
||||
# 0.0.9以后的新版本源码部署已经无法奏效
|
||||
manager:
|
||||
enabled: false
|
||||
ip: 0.0.0.0
|
||||
port: 8002
|
||||
use_private_config: false
|
||||
# 唤醒词，用于识别唤醒词还是讲话内容
|
||||
wakeup_words:
|
||||
- "你好小智"
|
||||
- "你好小志"
|
||||
- "小爱同学"
|
||||
- "你好小鑫"
|
||||
- "你好小新"
|
||||
- "小美同学"
|
||||
- "小龙小龙"
|
||||
- "喵喵同学"
|
||||
- "小滨小滨"
|
||||
- "小冰小冰"
|
||||
Binary file not shown.
Binary file not shown.
@@ -9,7 +9,7 @@ import traceback
|
||||
import threading
|
||||
import websockets
|
||||
from typing import Dict, Any
|
||||
import plugins_func.loadplugins
|
||||
from plugins_func.loadplugins import auto_import_modules
|
||||
from config.logger import setup_logging
|
||||
from core.utils.dialogue import Message, Dialogue
|
||||
from core.handle.textHandle import handleTextMessage
|
||||
@@ -26,6 +26,8 @@ from core.mcp.manager import MCPManager
|
||||
|
||||
TAG = __name__
|
||||
|
||||
auto_import_modules('plugins_func.functions')
|
||||
|
||||
|
||||
class TTSException(RuntimeError):
|
||||
pass
|
||||
@@ -112,11 +114,15 @@ class ConnectionHandler:
|
||||
|
||||
# 进行认证
|
||||
await self.auth.authenticate(self.headers)
|
||||
|
||||
device_id = self.headers.get("device-id", None)
|
||||
self.memory.init_memory(device_id, self.llm)
|
||||
self.intent.set_llm(self.llm)
|
||||
|
||||
# 认证通过,继续处理
|
||||
self.websocket = ws
|
||||
self.session_id = str(uuid.uuid4())
|
||||
|
||||
self.welcome_msg = self.config["xiaozhi"]
|
||||
self.welcome_msg["session_id"] = self.session_id
|
||||
await self.websocket.send(json.dumps(self.welcome_msg))
|
||||
# Load private configuration if device_id is provided
|
||||
bUsePrivateConfig = self.config.get("use_private_config", False)
|
||||
self.logger.bind(tag=TAG).info(f"bUsePrivateConfig: {bUsePrivateConfig}, device_id: {device_id}")
|
||||
@@ -144,19 +150,8 @@ class ConnectionHandler:
|
||||
self.private_config = None
|
||||
raise
|
||||
|
||||
# 认证通过,继续处理
|
||||
self.websocket = ws
|
||||
self.session_id = str(uuid.uuid4())
|
||||
|
||||
self.welcome_msg = self.config["xiaozhi"]
|
||||
self.welcome_msg["session_id"] = self.session_id
|
||||
await self.websocket.send(json.dumps(self.welcome_msg))
|
||||
|
||||
# 异步初始化
|
||||
await self.loop.run_in_executor(None, self._initialize_components)
|
||||
# 初始化MCP服务
|
||||
await self.mcp_manager.initialize_servers()
|
||||
|
||||
self.executor.submit(self._initialize_components)
|
||||
# tts 消化线程
|
||||
tts_priority = threading.Thread(target=self._tts_priority_thread, daemon=True)
|
||||
tts_priority.start()
|
||||
@@ -192,16 +187,26 @@ class ConnectionHandler:
|
||||
await handleAudioMessage(self, message)
|
||||
|
||||
def _initialize_components(self):
|
||||
"""加载插件"""
|
||||
self.func_handler = FunctionHandler(self)
|
||||
|
||||
"""加载提示词"""
|
||||
self.prompt = self.config["prompt"]
|
||||
if self.private_config:
|
||||
self.prompt = self.private_config.private_config.get("prompt", self.prompt)
|
||||
|
||||
#self.client_ip_info = get_ip_info(self.client_ip)
|
||||
#self.logger.bind(tag=TAG).info(f"Client ip info: {self.client_ip_info}")
|
||||
#self.prompt = self.prompt + f"\n我在:{self.client_ip_info}"
|
||||
self.dialogue.put(Message(role="system", content=self.prompt))
|
||||
|
||||
self.func_handler = FunctionHandler(self.config)
|
||||
"""加载记忆"""
|
||||
device_id = self.headers.get("device-id", None)
|
||||
self.memory.init_memory(device_id, self.llm)
|
||||
self.intent.set_llm(self.llm)
|
||||
|
||||
"""加载位置信息"""
|
||||
self.client_ip_info = get_ip_info(self.client_ip)
|
||||
if self.client_ip_info is not None and "city" in self.client_ip_info:
|
||||
self.logger.bind(tag=TAG).info(f"Client ip info: {self.client_ip_info}")
|
||||
self.prompt = self.prompt + f"\nuser location:{self.client_ip_info}"
|
||||
self.dialogue.update_system_message(self.prompt)
|
||||
|
||||
def change_system_prompt(self, prompt):
|
||||
self.prompt = prompt
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import asyncio
|
||||
from enum import Enum
|
||||
|
||||
from config.logger import setup_logging
|
||||
import json
|
||||
from plugins_func.register import FunctionRegistry, ActionResponse, Action, ToolType
|
||||
from plugins_func.functions.hass_init import append_devices_to_prompt
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class FunctionHandler:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
def __init__(self, conn):
|
||||
self.conn = conn
|
||||
self.config = conn.config
|
||||
self.function_registry = FunctionRegistry()
|
||||
self.register_nessary_functions()
|
||||
self.register_config_functions()
|
||||
@@ -26,9 +26,10 @@ class FunctionHandler:
|
||||
func_names = ",".join(surport_plugins)
|
||||
for function_desc in self.functions_desc:
|
||||
if function_desc["function"]["name"] == "plugin_loader":
|
||||
function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]", func_names)
|
||||
function_desc["function"]["description"] = function_desc["function"]["description"].replace("[plugins]",
|
||||
func_names)
|
||||
break
|
||||
|
||||
|
||||
def upload_functions_desc(self):
|
||||
self.functions_desc = self.function_registry.get_all_function_desc()
|
||||
|
||||
@@ -47,7 +48,6 @@ class FunctionHandler:
|
||||
def register_nessary_functions(self):
|
||||
"""注册必要的函数"""
|
||||
self.function_registry.register_function("handle_exit_intent")
|
||||
self.function_registry.register_function("play_music")
|
||||
self.function_registry.register_function("plugin_loader")
|
||||
self.function_registry.register_function("get_time")
|
||||
self.function_registry.register_function("raise_and_lower_the_volume")
|
||||
@@ -57,6 +57,9 @@ class FunctionHandler:
|
||||
for func in self.config["Intent"]["function_call"].get("functions", []):
|
||||
self.function_registry.register_function(func)
|
||||
|
||||
"""home assistant需要初始化提示词"""
|
||||
append_devices_to_prompt(self.conn)
|
||||
|
||||
def get_function(self, name):
|
||||
return self.function_registry.get_function(name)
|
||||
|
||||
@@ -81,4 +84,4 @@ class FunctionHandler:
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理function call错误: {e}")
|
||||
|
||||
return None
|
||||
return None
|
||||
|
||||
@@ -1,8 +1,72 @@
|
||||
import json
|
||||
from config.logger import setup_logging
|
||||
from core.handle.sendAudioHandle import send_stt_message
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
import shutil
|
||||
import asyncio
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
|
||||
logger = setup_logging()
|
||||
|
||||
WAKEUP_CONFIG = {"dir": "config/assets/", "file_name": "wakeup_words", "create_time": time.time(), "refresh_time": 10,
|
||||
"words": ["很高兴见到你", "你好啊", "我们又见面了", "最近可好?", "很高兴再次和你谈话", "在干嘛"]}
|
||||
|
||||
|
||||
async def handleHelloMessage(conn):
|
||||
await conn.websocket.send(json.dumps(conn.welcome_msg))
|
||||
|
||||
|
||||
async def checkWakeupWords(conn, text):
|
||||
enable_wakeup_words_response_cache = conn.config["enable_wakeup_words_response_cache"]
|
||||
"""是否开启唤醒词加速"""
|
||||
if not enable_wakeup_words_response_cache:
|
||||
return False
|
||||
"""检查是否是唤醒词"""
|
||||
_, text = remove_punctuation_and_length(text)
|
||||
if text in conn.config.get("wakeup_words"):
|
||||
await send_stt_message(conn, text)
|
||||
conn.tts_first_text_index = 0
|
||||
conn.tts_last_text_index = 0
|
||||
conn.llm_finish_task = True
|
||||
|
||||
file = getWakeupWordFile(WAKEUP_CONFIG["file_name"])
|
||||
if file is None:
|
||||
asyncio.create_task(wakeupWordsResponse(conn))
|
||||
return False
|
||||
opus_packets, duration = conn.tts.audio_to_opus_data(file)
|
||||
conn.audio_play_queue.put((opus_packets, text, 0))
|
||||
if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
|
||||
asyncio.create_task(wakeupWordsResponse(conn))
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def getWakeupWordFile(file_name):
|
||||
for file in os.listdir(WAKEUP_CONFIG["dir"]):
|
||||
if "my_"+file_name in file:
|
||||
return f"config/assets/{file}"
|
||||
|
||||
"""查找config/assets/目录下名称为wakeup_words的文件"""
|
||||
for file in os.listdir(WAKEUP_CONFIG["dir"]):
|
||||
if file_name in file:
|
||||
return f"config/assets/{file}"
|
||||
return None
|
||||
|
||||
|
||||
async def wakeupWordsResponse(conn):
|
||||
"""唤醒词响应"""
|
||||
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
|
||||
result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
|
||||
tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
|
||||
if tts_file is not None and os.path.exists(tts_file):
|
||||
file_type = os.path.splitext(tts_file)[1]
|
||||
if file_type:
|
||||
file_type = file_type.lstrip('.')
|
||||
old_file = getWakeupWordFile("my_" +WAKEUP_CONFIG["file_name"])
|
||||
if old_file is not None:
|
||||
os.remove(old_file)
|
||||
"""将文件挪到"wakeup_words.mp3"""
|
||||
shutil.move(tts_file, WAKEUP_CONFIG["dir"] + "my_" +WAKEUP_CONFIG["file_name"] + "." + file_type)
|
||||
WAKEUP_CONFIG["create_time"] = time.time()
|
||||
|
||||
@@ -2,6 +2,7 @@ from config.logger import setup_logging
|
||||
import json
|
||||
import uuid
|
||||
from core.handle.sendAudioHandle import send_stt_message
|
||||
from core.handle.helloHandle import checkWakeupWords
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
|
||||
TAG = __name__
|
||||
@@ -12,6 +13,10 @@ async def handle_user_intent(conn, text):
|
||||
# 检查是否有明确的退出命令
|
||||
if await check_direct_exit(conn, text):
|
||||
return True
|
||||
# 检查是否是唤醒词
|
||||
if await checkWakeupWords(conn, text):
|
||||
return True
|
||||
|
||||
if conn.use_function_call_mode:
|
||||
# 使用支持function calling的聊天方法,不再进行意图分析
|
||||
return False
|
||||
@@ -35,6 +40,7 @@ async def check_direct_exit(conn, text):
|
||||
return False
|
||||
|
||||
|
||||
|
||||
async def analyze_intent_with_llm(conn, text):
|
||||
"""使用LLM分析用户意图"""
|
||||
if not hasattr(conn, 'intent') or not conn.intent:
|
||||
|
||||
@@ -51,8 +51,6 @@ def create_iot_function(device_name, method_name, method_info):
|
||||
# 生成结果信息
|
||||
result = f"{device_name}的{method_name}操作执行成功"
|
||||
|
||||
# 根据方法名尝试自动更新状态
|
||||
await update_state_by_method(conn, device_name, method_name, params)
|
||||
|
||||
# 处理响应中可能的占位符
|
||||
response = response_success
|
||||
@@ -114,35 +112,6 @@ def create_iot_query_function(device_name, prop_name, prop_info):
|
||||
return wrap_async_function(iot_query_function)
|
||||
|
||||
|
||||
async def update_state_by_method(conn, device_name, method_name, params):
|
||||
"""根据方法和参数自动更新设备状态"""
|
||||
try:
|
||||
# 规则1: 方法名为TurnOn，设置power为True
|
||||
if method_name == "TurnOn":
|
||||
await set_iot_status(conn, device_name, "power", True)
|
||||
|
||||
# 规则2: 方法名为TurnOff，设置power为False
|
||||
elif method_name == "TurnOff":
|
||||
await set_iot_status(conn, device_name, "power", False)
|
||||
|
||||
# 规则3: Set开头的方法，尝试更新对应参数
|
||||
elif method_name.startswith("Set"):
|
||||
# 从参数中找到可能的状态值
|
||||
for param_name, param_value in params.items():
|
||||
# 尝试更新对应名称的属性
|
||||
await set_iot_status(conn, device_name, param_name, param_value)
|
||||
|
||||
# 其他方法，尝试直接从参数更新状态
|
||||
else:
|
||||
for param_name, param_value in params.items():
|
||||
# 检查设备是否有此属性
|
||||
status = await get_iot_status(conn, device_name, param_name)
|
||||
if status is not None: # 属性存在
|
||||
await set_iot_status(conn, device_name, param_name, param_value)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"自动更新状态失败: {e}")
|
||||
|
||||
|
||||
class IotDescriptor:
|
||||
"""
|
||||
A class to represent an IoT descriptor.
|
||||
|
||||
@@ -7,12 +7,26 @@ from core.utils.util import remove_punctuation_and_length, get_string_no_punctua
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
async def sendAudioMessage(conn, audios, text, text_index=0):
|
||||
# 发送句子开始消息
|
||||
if text_index == conn.tts_first_text_index:
|
||||
logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
|
||||
await send_tts_message(conn, "sentence_start", text)
|
||||
|
||||
# 播放音频
|
||||
await sendAudio(conn, audios)
|
||||
|
||||
await send_tts_message(conn, "sentence_end", text)
|
||||
|
||||
# 发送结束消息（如果是最后一个文本）
|
||||
if conn.llm_finish_task and text_index == conn.tts_last_text_index:
|
||||
await send_tts_message(conn, 'stop', None)
|
||||
if conn.close_after_chat:
|
||||
await conn.close()
|
||||
|
||||
# 播放音频
|
||||
async def sendAudio(conn, audios):
|
||||
# 流控参数优化
|
||||
original_frame_duration = 60 # 原始帧时长（毫秒）
|
||||
adjusted_frame_duration = int(original_frame_duration * 0.8) # 缩短20%
|
||||
@@ -42,13 +56,6 @@ async def sendAudioMessage(conn, audios, text, text_index=0):
|
||||
if compensation > 0:
|
||||
await asyncio.sleep(compensation)
|
||||
|
||||
await send_tts_message(conn, "sentence_end", text)
|
||||
|
||||
# 发送结束消息（如果是最后一个文本）
|
||||
if conn.llm_finish_task and text_index == conn.tts_last_text_index:
|
||||
await send_tts_message(conn, 'stop', None)
|
||||
if conn.close_after_chat:
|
||||
await conn.close()
|
||||
|
||||
async def send_tts_message(conn, state, text=None):
|
||||
"""发送 TTS 状态消息"""
|
||||
@@ -60,10 +67,20 @@ async def send_tts_message(conn, state, text=None):
|
||||
if text is not None:
|
||||
message["text"] = text
|
||||
|
||||
await conn.websocket.send(json.dumps(message))
|
||||
# TTS播放结束
|
||||
if state == "stop":
|
||||
# 播放提示音
|
||||
tts_notify = conn.config.get("enable_stop_tts_notify", False)
|
||||
if tts_notify:
|
||||
stop_tts_notify_voice = conn.config.get("stop_tts_notify_voice", "config/assets/tts_notify.mp3")
|
||||
audios, duration = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
|
||||
await sendAudio(conn, audios)
|
||||
# 清除服务端讲话状态
|
||||
conn.clearSpeakStatus()
|
||||
|
||||
# 发送消息到客户端
|
||||
await conn.websocket.send(json.dumps(message))
|
||||
|
||||
|
||||
async def send_stt_message(conn, text):
|
||||
"""发送 STT 状态消息"""
|
||||
|
||||
@@ -2,7 +2,9 @@ from config.logger import setup_logging
|
||||
import json
|
||||
from core.handle.abortHandle import handleAbortMessage
|
||||
from core.handle.helloHandle import handleHelloMessage
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
|
||||
from core.handle.sendAudioHandle import send_stt_message, send_tts_message
|
||||
from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
|
||||
|
||||
TAG = __name__
|
||||
@@ -24,7 +26,7 @@ async def handleTextMessage(conn, message):
|
||||
elif msg_json["type"] == "listen":
|
||||
if "mode" in msg_json:
|
||||
conn.client_listen_mode = msg_json["mode"]
|
||||
logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn. client_listen_mode}")
|
||||
logger.bind(tag=TAG).debug(f"客户端拾音模式：{conn.client_listen_mode}")
|
||||
if msg_json["state"] == "start":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = False
|
||||
@@ -38,11 +40,25 @@ async def handleTextMessage(conn, message):
|
||||
conn.client_have_voice = False
|
||||
conn.asr_audio.clear()
|
||||
if "text" in msg_json:
|
||||
await startToChat(conn, msg_json["text"])
|
||||
text = msg_json["text"]
|
||||
_, text = remove_punctuation_and_length(text)
|
||||
|
||||
# 识别是否是唤醒词
|
||||
is_wakeup_words = text in conn.config.get("wakeup_words")
|
||||
# 是否开启唤醒词回复
|
||||
enable_greeting = conn.config.get("enable_greeting", True)
|
||||
|
||||
if is_wakeup_words and not enable_greeting:
|
||||
# 如果是唤醒词，且关闭了唤醒词回复，就不用回答
|
||||
await send_stt_message(conn, text)
|
||||
await send_tts_message(conn, "stop", None)
|
||||
else:
|
||||
# 否则需要LLM对文字内容进行答复
|
||||
await startToChat(conn, text)
|
||||
elif msg_json["type"] == "iot":
|
||||
if "descriptors" in msg_json:
|
||||
await handleIotDescriptors(conn, msg_json["descriptors"])
|
||||
if "states" in msg_json:
|
||||
await handleIotStatus(conn, msg_json["states"])
|
||||
await handleIotStatus(conn, msg_json["states"])
|
||||
except json.JSONDecodeError:
|
||||
await conn.websocket.send(message)
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
import time
|
||||
import wave
|
||||
import os
|
||||
import sys
|
||||
import io
|
||||
from config.logger import setup_logging
|
||||
from typing import Optional, Tuple, List
|
||||
import uuid
|
||||
import opuslib_next
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
|
||||
from modelscope.hub.file_download import model_file_download
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
# 捕获标准输出
|
||||
class CaptureOutput:
|
||||
def __enter__(self):
|
||||
self._output = io.StringIO()
|
||||
self._original_stdout = sys.stdout
|
||||
sys.stdout = self._output
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
sys.stdout = self._original_stdout
|
||||
self.output = self._output.getvalue()
|
||||
self._output.close()
|
||||
|
||||
# 将捕获到的内容通过 logger 输出
|
||||
if self.output:
|
||||
logger.bind(tag=TAG).info(self.output.strip())
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool):
|
||||
self.model_dir = config.get("model_dir")
|
||||
self.output_dir = config.get("output_dir")
|
||||
self.delete_audio_file = delete_audio_file
|
||||
|
||||
# 确保输出目录存在
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
# 初始化模型文件路径
|
||||
model_files = {
|
||||
"model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"),
|
||||
"tokens.txt": os.path.join(self.model_dir, "tokens.txt")
|
||||
}
|
||||
|
||||
# 下载并检查模型文件
|
||||
try:
|
||||
for file_name, file_path in model_files.items():
|
||||
if not os.path.isfile(file_path):
|
||||
logger.bind(tag=TAG).info(f"正在下载模型文件: {file_name}")
|
||||
model_file_download(
|
||||
model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue",
|
||||
file_path=file_name,
|
||||
local_dir=self.model_dir
|
||||
)
|
||||
|
||||
if not os.path.isfile(file_path):
|
||||
raise FileNotFoundError(f"模型文件下载失败: {file_path}")
|
||||
|
||||
self.model_path = model_files["model.int8.onnx"]
|
||||
self.tokens_path = model_files["tokens.txt"]
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}")
|
||||
raise
|
||||
|
||||
with CaptureOutput():
|
||||
self.model = sherpa_onnx.OfflineRecognizer.from_sense_voice(
|
||||
model=self.model_path,
|
||||
tokens=self.tokens_path,
|
||||
num_threads=2,
|
||||
sample_rate=16000,
|
||||
feature_dim=80,
|
||||
decoding_method="greedy_search",
|
||||
debug=False,
|
||||
use_itn=True,
|
||||
)
|
||||
|
||||
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
|
||||
"""将Opus音频数据解码并保存为WAV文件"""
|
||||
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
|
||||
file_path = os.path.join(self.output_dir, file_name)
|
||||
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
with wave.open(file_path, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2) # 2 bytes = 16-bit
|
||||
wf.setframerate(16000)
|
||||
wf.writeframes(b"".join(pcm_data))
|
||||
|
||||
return file_path
|
||||
|
||||
def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]:
|
||||
"""
|
||||
Args:
|
||||
wave_filename:
|
||||
Path to a wave file. It should be single channel and each sample should
|
||||
be 16-bit. Its sample rate does not need to be 16kHz.
|
||||
Returns:
|
||||
Return a tuple containing:
|
||||
- A 1-D array of dtype np.float32 containing the samples, which are
|
||||
normalized to the range [-1, 1].
|
||||
- sample rate of the wave file
|
||||
"""
|
||||
|
||||
with wave.open(wave_filename) as f:
|
||||
assert f.getnchannels() == 1, f.getnchannels()
|
||||
assert f.getsampwidth() == 2, f.getsampwidth() # it is in bytes
|
||||
num_samples = f.getnframes()
|
||||
samples = f.readframes(num_samples)
|
||||
samples_int16 = np.frombuffer(samples, dtype=np.int16)
|
||||
samples_float32 = samples_int16.astype(np.float32)
|
||||
|
||||
samples_float32 = samples_float32 / 32768
|
||||
return samples_float32, f.getframerate()
|
||||
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""语音转文本主处理逻辑"""
|
||||
file_path = None
|
||||
try:
|
||||
# 保存音频文件
|
||||
start_time = time.time()
|
||||
file_path = self.save_audio_to_file(opus_data, session_id)
|
||||
logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
|
||||
|
||||
# 语音识别
|
||||
start_time = time.time()
|
||||
s = self.model.create_stream()
|
||||
samples, sample_rate = self.read_wave(file_path)
|
||||
s.accept_waveform(sample_rate, samples)
|
||||
self.model.decode_stream(s)
|
||||
text = s.result.text
|
||||
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
|
||||
|
||||
return text, file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", None
|
||||
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
|
||||
@@ -0,0 +1,52 @@
|
||||
from config.logger import setup_logging
|
||||
from http import HTTPStatus
|
||||
from dashscope import Application
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.api_key = config["api_key"]
|
||||
self.app_id = config["app_id"]
|
||||
self.base_url = config.get("base_url")
|
||||
self.is_No_prompt = config.get("is_no_prompt")
|
||||
self.memory_id = config.get("ali_memory_id")
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
try:
|
||||
# 处理dialogue
|
||||
if self.is_No_prompt:
|
||||
dialogue.pop(0)
|
||||
logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的dialogue: {dialogue}")
|
||||
|
||||
# 构造调用参数
|
||||
call_params = {
|
||||
"api_key": self.api_key,
|
||||
"app_id": self.app_id,
|
||||
"session_id": session_id,
|
||||
"messages": dialogue
|
||||
}
|
||||
if self.memory_id != False:
|
||||
# 百练memory需要prompt参数
|
||||
prompt = dialogue[-1].get("content")
|
||||
call_params["memory_id"] = self.memory_id
|
||||
call_params["prompt"] = prompt
|
||||
logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的prompt: {prompt}")
|
||||
|
||||
responses = Application.call(**call_params)
|
||||
if responses.status_code != HTTPStatus.OK:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"code={responses.status_code}, "
|
||||
f"message={responses.message}, "
|
||||
f"请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
|
||||
)
|
||||
yield "【阿里百练API服务响应异常】"
|
||||
else:
|
||||
logger.bind(tag=TAG).debug(f"【阿里百练API服务】构造参数: {call_params}")
|
||||
yield responses.output.text
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
|
||||
yield "【LLM服务响应异常】"
|
||||
@@ -9,6 +9,7 @@ logger = setup_logging()
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.api_key = config["api_key"]
|
||||
self.mode = config.get("mode", "chat-messages")
|
||||
self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
|
||||
self.session_conversation_map = {} # 存储session_id和conversation_id的映射
|
||||
|
||||
@@ -19,27 +20,59 @@ class LLMProvider(LLMProviderBase):
|
||||
conversation_id = self.session_conversation_map.get(session_id)
|
||||
|
||||
# 发起流式请求
|
||||
with requests.post(
|
||||
f"{self.base_url}/chat-messages",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json={
|
||||
if self.mode == "chat-messages":
|
||||
request_json = {
|
||||
"query": last_msg["content"],
|
||||
"response_mode": "streaming",
|
||||
"user": session_id,
|
||||
"inputs": {},
|
||||
"conversation_id": conversation_id
|
||||
},
|
||||
}
|
||||
elif self.mode == "workflows/run":
|
||||
request_json = {
|
||||
"inputs": {"query": last_msg["content"]},
|
||||
"response_mode": "streaming",
|
||||
"user": session_id
|
||||
}
|
||||
elif self.mode == "completion-messages":
|
||||
request_json = {
|
||||
"inputs": {"query": last_msg["content"]},
|
||||
"response_mode": "streaming",
|
||||
"user": session_id
|
||||
}
|
||||
|
||||
with requests.post(
|
||||
f"{self.base_url}/{self.mode}",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json=request_json,
|
||||
stream=True
|
||||
) as r:
|
||||
for line in r.iter_lines():
|
||||
if line.startswith(b'data: '):
|
||||
event = json.loads(line[6:])
|
||||
# 如果没有找到conversation_id，则获取此次conversation_id
|
||||
if not conversation_id:
|
||||
conversation_id = event.get('conversation_id')
|
||||
self.session_conversation_map[session_id] = conversation_id # 更新映射
|
||||
if event.get('answer'):
|
||||
yield event['answer']
|
||||
if self.mode == "chat-messages":
|
||||
for line in r.iter_lines():
|
||||
if line.startswith(b'data: '):
|
||||
event = json.loads(line[6:])
|
||||
# 如果没有找到conversation_id，则获取此次conversation_id
|
||||
if not conversation_id:
|
||||
conversation_id = event.get('conversation_id')
|
||||
self.session_conversation_map[session_id] = conversation_id # 更新映射
|
||||
if event.get('answer'):
|
||||
yield event['answer']
|
||||
elif self.mode == "workflows/run":
|
||||
for line in r.iter_lines():
|
||||
# logger.bind(tag=TAG).info(f"chat message response: {line}")
|
||||
if line.startswith(b'data: '):
|
||||
event = json.loads(line[6:])
|
||||
if event.get('event') == "workflow_finished":
|
||||
if event['data']['status'] == "succeeded":
|
||||
yield event['data']['outputs']['answer']
|
||||
else:
|
||||
yield "【服务响应异常】"
|
||||
elif self.mode == "completion-messages":
|
||||
for line in r.iter_lines():
|
||||
if line.startswith(b'data: '):
|
||||
event = json.loads(line[6:])
|
||||
if event.get('answer'):
|
||||
yield event['answer']
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
|
||||
|
||||
@@ -1,14 +1,19 @@
|
||||
import google.generativeai as genai
|
||||
from core.utils.util import check_model_key
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
from config.logger import setup_logging
|
||||
import requests
|
||||
import json
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
"""初始化Gemini LLM Provider"""
|
||||
self.model_name = config.get("model_name", "gemini-1.5-pro")
|
||||
self.api_key = config.get("api_key")
|
||||
|
||||
self.http_proxy=config.get("http_proxy")
|
||||
self.https_proxy = config.get("https_proxy")
|
||||
have_key = check_model_key("LLM", self.api_key)
|
||||
|
||||
if not have_key:
|
||||
@@ -16,6 +21,19 @@ class LLMProvider(LLMProviderBase):
|
||||
|
||||
try:
|
||||
# 初始化Gemini客户端
|
||||
# 配置代理（如果提供了代理配置）
|
||||
self.proxies=None
|
||||
if self.http_proxy is not "" or self.https_proxy is not "":
|
||||
|
||||
self.proxies = {
|
||||
"http": self.http_proxy,
|
||||
"https": self.https_proxy,
|
||||
}
|
||||
logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}")
|
||||
# 使用猴子补丁修改 google-generativeai 库的请求会话
|
||||
|
||||
# 使用 session 对象配置 genai
|
||||
|
||||
genai.configure(api_key=self.api_key)
|
||||
self.model = genai.GenerativeModel(self.model_name)
|
||||
|
||||
@@ -46,26 +64,54 @@ class LLMProvider(LLMProviderBase):
|
||||
if content:
|
||||
chat_history.append({
|
||||
"role": role,
|
||||
"parts": [content]
|
||||
"parts": [{"text":content}]
|
||||
|
||||
})
|
||||
|
||||
# 获取当前消息
|
||||
current_msg = dialogue[-1]["content"]
|
||||
|
||||
# 创建新的聊天会话
|
||||
chat = self.model.start_chat(history=chat_history)
|
||||
# 构建请求体
|
||||
request_body = {
|
||||
"contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}],
|
||||
"generationConfig": self.generation_config
|
||||
}
|
||||
|
||||
# 发送消息并获取流式响应
|
||||
response = chat.send_message(
|
||||
current_msg,
|
||||
stream=True,
|
||||
generation_config=self.generation_config
|
||||
)
|
||||
# 构建请求URL
|
||||
url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}"
|
||||
|
||||
# 处理流式响应
|
||||
for chunk in response:
|
||||
if hasattr(chunk, 'text') and chunk.text:
|
||||
yield chunk.text
|
||||
# 构建请求头
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# 发送POST请求,经测试手动 request 无法使用 stream 模式
|
||||
if self.proxies:
|
||||
response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies)
|
||||
try:
|
||||
data = response.json() # 直接解析JSON
|
||||
if 'candidates' in data and data['candidates']:
|
||||
yield data['candidates'][0]['content']['parts'][0]['text']
|
||||
else:
|
||||
yield "未找到候选回复。"
|
||||
except json.JSONDecodeError as e:
|
||||
yield f"JSON解码错误：{e}"
|
||||
except Exception as e:
|
||||
yield f"发生错误：{e}"
|
||||
else:
|
||||
logger.bind(tag=TAG).info(f"Gemini stream mode ")
|
||||
chat = self.model.start_chat(history=chat_history)
|
||||
|
||||
# 发送消息并获取流式响应
|
||||
response = chat.send_message(
|
||||
current_msg,
|
||||
stream=True,
|
||||
generation_config=self.generation_config
|
||||
)
|
||||
# 处理流式响应
|
||||
for chunk in response:
|
||||
if hasattr(chunk, 'text') and chunk.text:
|
||||
yield chunk.text
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
@@ -78,3 +124,13 @@ class LLMProvider(LLMProviderBase):
|
||||
yield "【Gemini API key无效】"
|
||||
else:
|
||||
yield f"【Gemini服务响应异常: {error_msg}】"
|
||||
|
||||
|
||||
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
yield f"请求失败：{e}"
|
||||
except json.JSONDecodeError as e:
|
||||
yield f"JSON解码错误：{e}"
|
||||
except Exception as e:
|
||||
yield f"发生错误：{e}"
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
import requests
|
||||
from requests.exceptions import RequestException
|
||||
from config.logger import setup_logging
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.agent_id = config.get("agent_id") # 对应 agent_id
|
||||
self.api_key = config.get("api_key")
|
||||
self.base_url = config.get("base_url", config.get("url")) # 默认使用 base_url
|
||||
self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
print(dialogue)
|
||||
try:
|
||||
# home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
|
||||
|
||||
# 提取最后一个 role 为 'user' 的 content
|
||||
input_text = None
|
||||
if isinstance(dialogue, list): # 确保 dialogue 是一个列表
|
||||
# 逆序遍历，找到最后一个 role 为 'user' 的消息
|
||||
for message in reversed(dialogue):
|
||||
if message.get("role") == "user": # 找到 role 为 'user' 的消息
|
||||
input_text = message.get("content", "")
|
||||
break # 找到后立即退出循环
|
||||
|
||||
# 构造请求数据
|
||||
payload = {
|
||||
"text": input_text,
|
||||
"agent_id": self.agent_id,
|
||||
"conversation_id": session_id # 使用 session_id 作为 conversation_id
|
||||
}
|
||||
# 设置请求头
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
# 发起 POST 请求
|
||||
response = requests.post(self.api_url, json=payload, headers=headers)
|
||||
|
||||
# 检查请求是否成功
|
||||
response.raise_for_status()
|
||||
|
||||
# 解析返回数据
|
||||
data = response.json()
|
||||
speech = data.get("response", {}).get("speech", {}).get("plain", {}).get("speech", "")
|
||||
|
||||
# 返回生成的内容
|
||||
if speech:
|
||||
yield speech
|
||||
else:
|
||||
logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
|
||||
|
||||
except RequestException as e:
|
||||
logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"生成响应时出错: {e}")
|
||||
@@ -1,7 +1,11 @@
|
||||
import openai
|
||||
from config.logger import setup_logging
|
||||
from core.utils.util import check_model_key
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
@@ -11,6 +15,8 @@ class LLMProvider(LLMProviderBase):
|
||||
self.base_url = config.get("base_url")
|
||||
else:
|
||||
self.base_url = config.get("url")
|
||||
self.max_tokens = config.get("max_tokens", 500)
|
||||
|
||||
check_model_key("LLM", self.api_key)
|
||||
self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
|
||||
|
||||
@@ -19,9 +25,10 @@ class LLMProvider(LLMProviderBase):
|
||||
responses = self.client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=dialogue,
|
||||
stream=True
|
||||
stream=True,
|
||||
max_tokens=self.max_tokens,
|
||||
)
|
||||
|
||||
|
||||
is_active = True
|
||||
for chunk in responses:
|
||||
try:
|
||||
@@ -50,12 +57,12 @@ class LLMProvider(LLMProviderBase):
|
||||
model=self.model_name,
|
||||
messages=dialogue,
|
||||
stream=True,
|
||||
tools=functions,
|
||||
tools=functions
|
||||
)
|
||||
|
||||
|
||||
for chunk in stream:
|
||||
yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
|
||||
yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
|
||||
logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
|
||||
yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
|
||||
|
||||
@@ -5,6 +5,7 @@ import numpy as np
|
||||
import opuslib_next
|
||||
from pydub import AudioSegment
|
||||
from abc import ABC, abstractmethod
|
||||
from core.utils.tts import MarkdownCleaner
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
@@ -23,6 +24,7 @@ class TTSProviderBase(ABC):
|
||||
tmp_file = self.generate_filename()
|
||||
try:
|
||||
max_repeat_time = 5
|
||||
text = MarkdownCleaner.clean_markdown(text)
|
||||
while not os.path.exists(tmp_file) and max_repeat_time > 0:
|
||||
asyncio.run(self.text_to_speak(text, tmp_file))
|
||||
if not os.path.exists(tmp_file):
|
||||
@@ -34,7 +36,7 @@ class TTSProviderBase(ABC):
|
||||
|
||||
return tmp_file
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
|
||||
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
@@ -47,7 +49,8 @@ class TTSProviderBase(ABC):
|
||||
file_type = os.path.splitext(audio_file_path)[1]
|
||||
if file_type:
|
||||
file_type = file_type.lstrip('.')
|
||||
audio = AudioSegment.from_file(audio_file_path, format=file_type)
|
||||
# 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
|
||||
audio = AudioSegment.from_file(audio_file_path, format=file_type, parameters=["-nostdin"])
|
||||
|
||||
# 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
|
||||
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
|
||||
|
||||
@@ -35,6 +35,15 @@ class Dialogue:
|
||||
self.getMessages(m, dialogue)
|
||||
return dialogue
|
||||
|
||||
def update_system_message(self, new_content: str):
|
||||
"""更新或添加系统消息"""
|
||||
# 查找第一个系统消息
|
||||
system_msg = next((msg for msg in self.dialogue if msg.role == "system"), None)
|
||||
if system_msg:
|
||||
system_msg.content = new_content
|
||||
else:
|
||||
self.put(Message(role="system", content=new_content))
|
||||
|
||||
def get_llm_dialogue_with_memory(self, memory_str: str = None) -> List[Dict[str, str]]:
|
||||
if memory_str is None or len(memory_str) == 0:
|
||||
return self.get_llm_dialogue()
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from config.logger import setup_logging
|
||||
import importlib
|
||||
@@ -14,4 +15,98 @@ def create_instance(class_name, *args, **kwargs):
|
||||
sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
|
||||
return sys.modules[lib_name].TTSProvider(*args, **kwargs)
|
||||
|
||||
raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")
|
||||
raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")
|
||||
|
||||
|
||||
class MarkdownCleaner:
|
||||
"""
|
||||
封装 Markdown 清理逻辑：直接用 MarkdownCleaner.clean_markdown(text) 即可
|
||||
"""
|
||||
# 公式字符
|
||||
NORMAL_FORMULA_CHARS = re.compile(r'[a-zA-Z\\^_{}\+\-\(\)\[\]=]')
|
||||
|
||||
@staticmethod
|
||||
def _replace_inline_dollar(m: re.Match) -> str:
|
||||
"""
|
||||
只要捕获到完整的 "$...$":
|
||||
- 如果内部有典型公式字符 => 去掉两侧 $
|
||||
- 否则 (纯数字/货币等) => 保留 "$...$"
|
||||
"""
|
||||
content = m.group(1)
|
||||
if MarkdownCleaner.NORMAL_FORMULA_CHARS.search(content):
|
||||
return content
|
||||
else:
|
||||
return m.group(0)
|
||||
|
||||
@staticmethod
|
||||
def _replace_table_block(match: re.Match) -> str:
|
||||
"""
|
||||
当匹配到一个整段表格块时，回调该函数。
|
||||
"""
|
||||
block_text = match.group('table_block')
|
||||
lines = block_text.strip('\n').split('\n')
|
||||
|
||||
parsed_table = []
|
||||
for line in lines:
|
||||
line_stripped = line.strip()
|
||||
if re.match(r'^\|\s*[-:]+\s*(\|\s*[-:]+\s*)+\|?$', line_stripped):
|
||||
continue
|
||||
columns = [col.strip() for col in line_stripped.split('|') if col.strip() != '']
|
||||
if columns:
|
||||
parsed_table.append(columns)
|
||||
|
||||
if not parsed_table:
|
||||
return ""
|
||||
|
||||
headers = parsed_table[0]
|
||||
data_rows = parsed_table[1:] if len(parsed_table) > 1 else []
|
||||
|
||||
lines_for_tts = []
|
||||
if len(parsed_table) == 1:
|
||||
# 只有一行
|
||||
only_line_str = ", ".join(parsed_table[0])
|
||||
lines_for_tts.append(f"单行表格：{only_line_str}")
|
||||
else:
|
||||
lines_for_tts.append(f"表头是：{', '.join(headers)}")
|
||||
for i, row in enumerate(data_rows, start=1):
|
||||
row_str_list = []
|
||||
for col_index, cell_val in enumerate(row):
|
||||
if col_index < len(headers):
|
||||
row_str_list.append(f"{headers[col_index]} = {cell_val}")
|
||||
else:
|
||||
row_str_list.append(cell_val)
|
||||
lines_for_tts.append(f"第 {i} 行：{', '.join(row_str_list)}")
|
||||
|
||||
return "\n".join(lines_for_tts) + "\n"
|
||||
|
||||
# 预编译所有正则表达式（按执行频率排序）
|
||||
# 这里要把 replace_xxx 的静态方法放在最前定义，以便在列表里能正确引用它们。
|
||||
REGEXES = [
|
||||
(re.compile(r'```.*?```', re.DOTALL), ''), # 代码块
|
||||
(re.compile(r'^#+\s*', re.MULTILINE), ''), # 标题
|
||||
(re.compile(r'(\*\*|__)(.*?)\1'), r'\2'), # 粗体
|
||||
(re.compile(r'(\*|_)(?=\S)(.*?)(?<=\S)\1'), r'\2'), # 斜体
|
||||
(re.compile(r'!\[.*?\]\(.*?\)'), ''), # 图片
|
||||
(re.compile(r'\[(.*?)\]\(.*?\)'), r'\1'), # 链接
|
||||
(re.compile(r'^\s*>+\s*', re.MULTILINE), ''), # 引用
|
||||
(
|
||||
re.compile(r'(?P<table_block>(?:^[^\n]*\|[^\n]*\n)+)', re.MULTILINE),
|
||||
_replace_table_block
|
||||
),
|
||||
(re.compile(r'^\s*[*+-]\s*', re.MULTILINE), '- '), # 列表
|
||||
(re.compile(r'\$\$.*?\$\$', re.DOTALL), ''), # 块级公式
|
||||
(
|
||||
re.compile(r'(?<![A-Za-z0-9])\$([^\n$]+)\$(?![A-Za-z0-9])'),
|
||||
_replace_inline_dollar
|
||||
),
|
||||
(re.compile(r'\n{2,}'), '\n'), # 多余空行
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def clean_markdown(text: str) -> str:
|
||||
"""
|
||||
主入口方法：依序执行所有正则，移除或替换 Markdown 元素
|
||||
"""
|
||||
for regex, replacement in MarkdownCleaner.REGEXES:
|
||||
text = regex.sub(replacement, text)
|
||||
return text.strip()
|
||||
@@ -73,9 +73,7 @@ def get_ip_info(ip_addr):
|
||||
resp = requests.get(url).json()
|
||||
|
||||
ip_info = {
|
||||
"city": resp.get("cityName"),
|
||||
"region": resp.get("regionName"),
|
||||
"country": resp.get("countryName")
|
||||
"city": resp.get("cityName")
|
||||
}
|
||||
return ip_info
|
||||
except Exception as e:
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import random
|
||||
import requests
|
||||
import xml.etree.ElementTree as ET
|
||||
from bs4 import BeautifulSoup
|
||||
from config.logger import setup_logging
|
||||
from plugins_func.register import register_function, ToolType, ActionResponse, Action
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
GET_NEWS_FUNCTION_DESC = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_news",
|
||||
"description": (
|
||||
"获取最新新闻，随机选择一条新闻进行播报。"
|
||||
"用户可以指定新闻类型，如社会新闻、科技新闻、国际新闻等。"
|
||||
"如果没有指定，默认播报社会新闻。"
|
||||
"用户可以要求获取详细内容，此时会获取新闻的详细内容。"
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"category": {
|
||||
"type": "string",
|
||||
"description": "新闻类别，例如社会、科技、国际。可选参数，如果不提供则使用默认类别"
|
||||
},
|
||||
"detail": {
|
||||
"type": "boolean",
|
||||
"description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容"
|
||||
},
|
||||
"lang": {
|
||||
"type": "string",
|
||||
"description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN"
|
||||
}
|
||||
},
|
||||
"required": ["lang"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def fetch_news_from_rss(rss_url):
|
||||
"""从RSS源获取新闻列表"""
|
||||
try:
|
||||
response = requests.get(rss_url)
|
||||
response.raise_for_status()
|
||||
|
||||
# 解析XML
|
||||
root = ET.fromstring(response.content)
|
||||
|
||||
# 查找所有item元素（新闻条目）
|
||||
news_items = []
|
||||
for item in root.findall('.//item'):
|
||||
title = item.find('title').text if item.find('title') is not None else "无标题"
|
||||
link = item.find('link').text if item.find('link') is not None else "#"
|
||||
description = item.find('description').text if item.find('description') is not None else "无描述"
|
||||
pubDate = item.find('pubDate').text if item.find('pubDate') is not None else "未知时间"
|
||||
|
||||
news_items.append({
|
||||
'title': title,
|
||||
'link': link,
|
||||
'description': description,
|
||||
'pubDate': pubDate
|
||||
})
|
||||
|
||||
return news_items
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"获取RSS新闻失败: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def fetch_news_detail(url):
|
||||
"""获取新闻详情页内容并总结"""
|
||||
try:
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
|
||||
# 尝试提取正文内容 (这里的选择器需要根据实际网站结构调整)
|
||||
content_div = soup.select_one('.content_desc, .content, article, .article-content')
|
||||
if content_div:
|
||||
paragraphs = content_div.find_all('p')
|
||||
content = '\n'.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()])
|
||||
return content
|
||||
else:
|
||||
# 如果找不到特定的内容区域，尝试获取所有段落
|
||||
paragraphs = soup.find_all('p')
|
||||
content = '\n'.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()])
|
||||
return content[:2000] # 限制长度
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"获取新闻详情失败: {e}")
|
||||
return "无法获取详细内容"
|
||||
|
||||
|
||||
def map_category(category_text):
|
||||
"""将用户输入的中文类别映射到配置文件中的类别键"""
|
||||
if not category_text:
|
||||
return None
|
||||
|
||||
# 类别映射字典，目前支持社会、国际、财经新闻，如需更多类型，参见配置文件
|
||||
category_map = {
|
||||
# 社会新闻
|
||||
"社会": "society",
|
||||
"社会新闻": "society",
|
||||
# 国际新闻
|
||||
"国际": "world",
|
||||
"国际新闻": "world",
|
||||
# 财经新闻
|
||||
"财经": "finance",
|
||||
"财经新闻": "finance",
|
||||
"金融": "finance",
|
||||
"经济": "finance"
|
||||
}
|
||||
|
||||
# 转换为小写并去除空格
|
||||
normalized_category = category_text.lower().strip()
|
||||
|
||||
# 返回映射结果，如果没有匹配项则返回原始输入
|
||||
return category_map.get(normalized_category, category_text)
|
||||
|
||||
|
||||
@register_function('get_news', GET_NEWS_FUNCTION_DESC, ToolType.SYSTEM_CTL)
|
||||
def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_CN"):
|
||||
"""获取新闻并随机选择一条进行播报，或获取上一条新闻的详细内容"""
|
||||
try:
|
||||
# 如果detail为True，获取上一条新闻的详细内容
|
||||
if detail:
|
||||
if not hasattr(conn, 'last_news_link') or not conn.last_news_link or 'link' not in conn.last_news_link:
|
||||
return ActionResponse(Action.REQLLM, "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", None)
|
||||
|
||||
link = conn.last_news_link.get('link')
|
||||
title = conn.last_news_link.get('title', '未知标题')
|
||||
|
||||
if link == '#':
|
||||
return ActionResponse(Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None)
|
||||
|
||||
logger.bind(tag=TAG).debug(f"获取新闻详情: {title}, URL={link}")
|
||||
|
||||
# 获取新闻详情
|
||||
detail_content = fetch_news_detail(link)
|
||||
|
||||
if not detail_content or detail_content == "无法获取详细内容":
|
||||
return ActionResponse(Action.REQLLM, f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", None)
|
||||
|
||||
# 构建详情报告
|
||||
detail_report = (
|
||||
f"根据下列数据，用{lang}回应用户的新闻详情查询请求：\n\n"
|
||||
f"新闻标题: {title}\n"
|
||||
f"详细内容: {detail_content}\n\n"
|
||||
f"(请对上述新闻内容进行总结，提取关键信息，以自然、流畅的方式向用户播报，"
|
||||
f"不要提及这是总结，就像是在讲述一个完整的新闻故事)"
|
||||
)
|
||||
|
||||
return ActionResponse(Action.REQLLM, detail_report, None)
|
||||
|
||||
# 否则，获取新闻列表并随机选择一条
|
||||
# 从配置中获取RSS URL
|
||||
rss_config = conn.config["plugins"]["get_news"]
|
||||
default_rss_url = rss_config.get("default_rss_url", "https://www.chinanews.com.cn/rss/society.xml")
|
||||
|
||||
# 将用户输入的类别映射到配置中的类别键
|
||||
mapped_category = map_category(category)
|
||||
|
||||
# 如果提供了类别，尝试从配置中获取对应的URL
|
||||
rss_url = default_rss_url
|
||||
if mapped_category and mapped_category in rss_config.get("category_urls", {}):
|
||||
rss_url = rss_config["category_urls"][mapped_category]
|
||||
|
||||
logger.bind(tag=TAG).info(f"获取新闻: 原始类别={category}, 映射类别={mapped_category}, URL={rss_url}")
|
||||
|
||||
# 获取新闻列表
|
||||
news_items = fetch_news_from_rss(rss_url)
|
||||
|
||||
if not news_items:
|
||||
return ActionResponse(Action.REQLLM, "抱歉，未能获取到新闻信息，请稍后再试。", None)
|
||||
|
||||
# 随机选择一条新闻
|
||||
selected_news = random.choice(news_items)
|
||||
|
||||
# 保存当前新闻链接到连接对象，以便后续查询详情
|
||||
if not hasattr(conn, 'last_news_link'):
|
||||
conn.last_news_link = {}
|
||||
conn.last_news_link = {
|
||||
'link': selected_news.get('link', '#'),
|
||||
'title': selected_news.get('title', '未知标题')
|
||||
}
|
||||
|
||||
# 构建新闻报告
|
||||
news_report = (
|
||||
f"根据下列数据，用{lang}回应用户的新闻查询请求：\n\n"
|
||||
f"新闻标题: {selected_news['title']}\n"
|
||||
f"发布时间: {selected_news['pubDate']}\n"
|
||||
f"新闻内容: {selected_news['description']}\n"
|
||||
f"(请以自然、流畅的方式向用户播报这条新闻，可以适当总结内容，"
|
||||
f"直接读出新闻即可，不需要额外多余的内容。"
|
||||
f"如果用户询问更多详情，告知用户可以说'请详细介绍这条新闻'获取更多内容)"
|
||||
)
|
||||
|
||||
return ActionResponse(Action.REQLLM, news_report, None)
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"获取新闻出错: {e}")
|
||||
return ActionResponse(Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None)
|
||||
@@ -0,0 +1,56 @@
|
||||
from plugins_func.register import register_function, ToolType, ActionResponse, Action
|
||||
from plugins_func.functions.hass_init import initialize_hass_handler
|
||||
from config.logger import setup_logging
|
||||
import asyncio
|
||||
import requests
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
hass_get_state_function_desc = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "hass_get_state",
|
||||
"description": "获取homeassistant里设备的状态,包括灯光亮度,媒体播放器的音量,设备的暂停、继续操作",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"entity_id": {
|
||||
"type": "string",
|
||||
"description": "需要操作的设备id,homeassistant里的entity_id"
|
||||
}
|
||||
},
|
||||
"required": ["entity_id"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@register_function("hass_get_state", hass_get_state_function_desc, ToolType.SYSTEM_CTL)
|
||||
def hass_get_state(conn, entity_id=''):
|
||||
try:
|
||||
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
handle_hass_get_state(conn, entity_id),
|
||||
conn.loop
|
||||
)
|
||||
ha_response = future.result()
|
||||
return ActionResponse(action=Action.REQLLM, result="执行成功", response=ha_response)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}")
|
||||
|
||||
|
||||
async def handle_hass_get_state(conn, entity_id):
|
||||
HASS_CACHE = initialize_hass_handler(conn)
|
||||
api_key = HASS_CACHE['api_key']
|
||||
base_url = HASS_CACHE['base_url']
|
||||
url = f"{base_url}/api/states/{entity_id}"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
response = requests.get(url, headers=headers)
|
||||
if response.status_code == 200:
|
||||
return response.json()['state']
|
||||
else:
|
||||
return f"切换失败，错误码: {response.status_code}"
|
||||
@@ -0,0 +1,35 @@
|
||||
from config.logger import setup_logging
|
||||
from core.utils.util import check_model_key
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
HASS_CACHE = {}
|
||||
|
||||
|
||||
def append_devices_to_prompt(conn):
|
||||
if conn.use_function_call_mode:
|
||||
funcs = conn.config["Intent"]["function_call"].get("functions", [])
|
||||
if "hass_get_state" in funcs or "hass_get_state" in funcs:
|
||||
prompt = "下面是我家智能设备，可以通过homeassistant控制\n"
|
||||
devices = conn.config["plugins"]["home_assistant"].get("devices", [])
|
||||
if len(devices) == 0:
|
||||
return
|
||||
for device in devices:
|
||||
prompt += device + "\n"
|
||||
conn.prompt += prompt
|
||||
# 更新提示词
|
||||
conn.dialogue.update_system_message(conn.prompt)
|
||||
|
||||
|
||||
def initialize_hass_handler(conn):
|
||||
global HASS_CACHE
|
||||
if HASS_CACHE == {}:
|
||||
if conn.use_function_call_mode:
|
||||
funcs = conn.config["Intent"]["function_call"].get("functions", [])
|
||||
if "hass_get_state" in funcs or "hass_get_state" in funcs:
|
||||
HASS_CACHE['base_url'] = conn.config["plugins"]["home_assistant"].get("base_url")
|
||||
HASS_CACHE['api_key'] = conn.config["plugins"]["home_assistant"].get("api_key")
|
||||
|
||||
check_model_key("home_assistant", HASS_CACHE['api_key'])
|
||||
return HASS_CACHE
|
||||
@@ -0,0 +1,64 @@
|
||||
from plugins_func.register import register_function, ToolType, ActionResponse, Action
|
||||
from plugins_func.functions.hass_init import initialize_hass_handler
|
||||
from config.logger import setup_logging
|
||||
import asyncio
|
||||
import requests
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
hass_play_music_function_desc = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "hass_play_music",
|
||||
"description": "用户想听音乐、有声书的时候使用，在房间的媒体播放器（media_player）里播放对应音频",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"media_content_id": {
|
||||
"type": "string",
|
||||
"description": "可以是音乐或有声书的专辑名称、歌曲名、演唱者,如果未指定就填random"
|
||||
},
|
||||
"entity_id": {
|
||||
"type": "string",
|
||||
"description": "需要操作的音箱的设备id,homeassistant里的entity_id,media_player开头"
|
||||
}
|
||||
},
|
||||
"required": ["media_content_id", "entity_id"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@register_function('hass_play_music', hass_play_music_function_desc, ToolType.SYSTEM_CTL)
|
||||
def hass_play_music(conn, entity_id='', media_content_id='random'):
|
||||
try:
|
||||
# 执行音乐播放命令
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
handle_hass_play_music(conn, entity_id, media_content_id),
|
||||
conn.loop
|
||||
)
|
||||
ha_response = future.result()
|
||||
return ActionResponse(action=Action.RESPONSE, result="退出意图已处理", response=ha_response)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
|
||||
|
||||
|
||||
async def handle_hass_play_music(conn, entity_id, media_content_id):
|
||||
HASS_CACHE = initialize_hass_handler(conn)
|
||||
api_key = HASS_CACHE['api_key']
|
||||
base_url = HASS_CACHE['base_url']
|
||||
url = f"{base_url}/api/services/music_assistant/play_media"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
data = {
|
||||
"entity_id": entity_id,
|
||||
"media_id": media_content_id
|
||||
}
|
||||
response = requests.post(url, headers=headers, json=data)
|
||||
if response.status_code == 200:
|
||||
return f"正在播放{media_content_id}的音乐"
|
||||
else:
|
||||
return f"音乐播放失败，错误码: {response.status_code}"
|
||||
@@ -0,0 +1,159 @@
|
||||
from plugins_func.register import register_function, ToolType, ActionResponse, Action
|
||||
from plugins_func.functions.hass_init import initialize_hass_handler
|
||||
from config.logger import setup_logging
|
||||
import asyncio
|
||||
import requests
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
hass_set_state_function_desc = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "hass_set_state",
|
||||
"description": "设置homeassistant里设备的状态,包括开、关,调整灯光亮度,调整播放器的音量,设备的暂停、继续、静音操作",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"state": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"description": "需要操作的动作,打开设备:turn_on,关闭设备:turn_off,增加亮度:brightness_up,降低亮度:brightness_down,设置亮度:brightness_value,增加>音量:,volume_up降低音量:volume_down,设置音量:volume_set,设备暂停:pause,设备继续:continue,静音/取消静音:volume_mute"
|
||||
},
|
||||
"input": {
|
||||
"type": "int",
|
||||
"description": "只有在设置音量,设置亮度时候才需要,有效值为1-100,对应音量和亮度的1%-100%"
|
||||
},
|
||||
"is_muted": {
|
||||
"type": "string",
|
||||
"description": "只有在设置静音操作时才需要,设置静音的时候该值为true,取消静音时该值为false"
|
||||
}
|
||||
},
|
||||
"required": ["type"]
|
||||
},
|
||||
"entity_id": {
|
||||
"type": "string",
|
||||
"description": "需要操作的设备id,homeassistant里的entity_id"
|
||||
}
|
||||
},
|
||||
"required": ["state", "entity_id"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@register_function('hass_set_state', hass_set_state_function_desc, ToolType.SYSTEM_CTL)
|
||||
def hass_set_state(conn, entity_id='', state={}):
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
handle_hass_set_state(conn, entity_id, state),
|
||||
conn.loop
|
||||
)
|
||||
ha_response = future.result()
|
||||
return ActionResponse(action=Action.REQLLM, result="执行成功", response=ha_response)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理设置属性意图错误: {e}")
|
||||
|
||||
|
||||
async def handle_hass_set_state(conn, entity_id, state):
|
||||
HASS_CACHE = initialize_hass_handler(conn)
|
||||
api_key = HASS_CACHE['api_key']
|
||||
base_url = HASS_CACHE['base_url']
|
||||
'''
|
||||
state = { "type":"brightness_up","input":"80","is_muted":"true"}
|
||||
'''
|
||||
domains = entity_id.split(".")
|
||||
if len(domains) > 1:
|
||||
domain = domains[0]
|
||||
else:
|
||||
return "执行失败，错误的设备id"
|
||||
action = ''
|
||||
arg = ''
|
||||
value = ''
|
||||
if state['type'] == 'turn_on':
|
||||
description = "设备已打开"
|
||||
if domain == "cover":
|
||||
action = "open_cover"
|
||||
elif domain == "vacuum":
|
||||
action = "start"
|
||||
else:
|
||||
action = "turn_on"
|
||||
elif state['type'] == 'turn_off':
|
||||
description = "设备已关闭"
|
||||
if domain == 'cover':
|
||||
action = "close_cover"
|
||||
elif domain == 'vacuum':
|
||||
action = "stop"
|
||||
else:
|
||||
action = "turn_off"
|
||||
elif state['type'] == 'brightness_up':
|
||||
description = "灯光已调亮"
|
||||
action = 'turn_on'
|
||||
arg = 'brightness_step_pct'
|
||||
value = 10
|
||||
elif state['type'] == 'brightness_down':
|
||||
description = "灯光已调暗"
|
||||
action = 'turn_on'
|
||||
arg = 'brightness_step_pct'
|
||||
value = -10
|
||||
elif state['type'] == 'brightness_value':
|
||||
description = f"亮度已调整到{state['input']}"
|
||||
action = 'turn_on'
|
||||
arg = 'brightness_pct'
|
||||
value = state['input']
|
||||
elif state['type'] == 'volume_up':
|
||||
description = "音量已调大"
|
||||
action = state['type']
|
||||
elif state['type'] == 'volume_down':
|
||||
description = "音量已调小"
|
||||
action = state['type']
|
||||
elif state['type'] == 'volume_set':
|
||||
description = f"音量已调整到{state['input']}"
|
||||
action = state['type']
|
||||
arg = 'volume_level'
|
||||
value = state['input']
|
||||
elif state['type'] == 'volume_mute':
|
||||
description = f"设备已静音"
|
||||
action = state['type']
|
||||
arg = 'is_volume_muted'
|
||||
value = state['is_muted']
|
||||
elif state['type'] == 'pause':
|
||||
description = f"设备已暂停"
|
||||
action = state['type']
|
||||
if domain == 'media_player':
|
||||
action = 'media_pause'
|
||||
if domain == 'cover':
|
||||
action = 'stop_cover'
|
||||
if domain == 'vacuum':
|
||||
action = 'pause'
|
||||
elif state['type'] == 'continue':
|
||||
description = f"设备已继续"
|
||||
if domain == 'media_player':
|
||||
action = 'media_play'
|
||||
if domain == 'vacuum':
|
||||
action = 'start'
|
||||
else:
|
||||
return f"{domain} {state.type}功能尚未支持"
|
||||
|
||||
if arg == '':
|
||||
data = {
|
||||
"entity_id": entity_id,
|
||||
}
|
||||
else:
|
||||
data = {
|
||||
"entity_id": entity_id,
|
||||
arg: value
|
||||
}
|
||||
url = f"{base_url}/api/services/{domain}/{action}"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
response = requests.post(url, headers=headers, json=data)
|
||||
logger.bind(tag=TAG).info(f"设置状态:url:{url},return_code:{response.status_code}")
|
||||
if response.status_code == 200:
|
||||
return description
|
||||
else:
|
||||
return f"设置失败，错误码: {response.status_code}"
|
||||
@@ -21,13 +21,13 @@ play_music_function_desc = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "play_music",
|
||||
"description": "唱歌、听歌、播放音乐方法。比如用户说播放音乐，参数为：random，比如用户说播放两只老虎，参数为：两只老虎",
|
||||
"description": "唱歌、听歌、播放音乐的方法。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"song_name": {
|
||||
"type": "string",
|
||||
"description": "歌曲名称，如果没有指定具体歌名则为'random'"
|
||||
"description": "歌曲名称，如果用户没有指定具体歌名则为'random', 明确指定的时返回音乐的名字 示例: ```用户:播放两只老虎\n参数：两只老虎``` ```用户:播放音乐 \n参数：random ```"
|
||||
}
|
||||
},
|
||||
"required": ["song_name"]
|
||||
@@ -112,8 +112,8 @@ def get_music_files(music_dir, music_ext):
|
||||
def initialize_music_handler(conn):
|
||||
global MUSIC_CACHE
|
||||
if MUSIC_CACHE == {}:
|
||||
if "music" in conn.config:
|
||||
MUSIC_CACHE["music_config"] = conn.config["music"]
|
||||
if "play_music" in conn.config["plugins"]:
|
||||
MUSIC_CACHE["music_config"] = conn.config["plugins"]["play_music"]
|
||||
MUSIC_CACHE["music_dir"] = os.path.abspath(
|
||||
MUSIC_CACHE["music_config"].get("music_dir", "./music") # 默认路径修改
|
||||
)
|
||||
|
||||
@@ -22,6 +22,4 @@ def auto_import_modules(package_name):
|
||||
# 导入模块
|
||||
full_module_name = f"{package_name}.{module_name}"
|
||||
importlib.import_module(full_module_name)
|
||||
#logger.bind(tag=TAG).info(f"模块 '{full_module_name}' 已加载")
|
||||
|
||||
auto_import_modules('plugins_func.functions')
|
||||
#logger.bind(tag=TAG).info(f"模块 '{full_module_name}' 已加载")
|
||||
@@ -20,4 +20,6 @@ requests==2.32.3
|
||||
cozepy==0.12.0
|
||||
mem0ai==0.1.62
|
||||
bs4==0.0.2
|
||||
mcp==1.4.1
|
||||
modelscope==1.23.2
|
||||
sherpa_onnx==1.11.0
|
||||
mcp==1.4.1
|
||||
|
||||
Reference in New Issue
Block a user