mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-29 05:53:55 +08:00
Merge branch 'test-pr-homeassistant' into main
This commit is contained in:
Huang
@@ -35,10 +35,7 @@ log:
|
||||
log_file: "server.log"
|
||||
# 设置数据文件路径
|
||||
data_dir: data
|
||||
iot:
|
||||
Speaker:
|
||||
# 设置esp32的音量，范围0-100
|
||||
volume: 80
|
||||
|
||||
xiaozhi:
|
||||
type: hello
|
||||
version: 1
|
||||
@@ -85,11 +82,11 @@ selected_module:
|
||||
TTS: EdgeTTS
|
||||
# 记忆模块，默认不开启记忆；如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short
|
||||
Memory: nomem
|
||||
# 意图识别模块，默认不开启。开启后，可以播放音乐、控制音量、识别退出指令
|
||||
# 意图识别模块，默认使用function_call。开启后，可以播放音乐、控制音量、识别退出指令
|
||||
# 意图识别使用intent_llm，优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间
|
||||
# 意图识别使用function_call，缺点：需要所选择的LLM支持function_call，优点：按需调用工具、速度快
|
||||
# 如果意图识别设置成 function_call，建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-pro-32k-functioncall-241028
|
||||
Intent: nointent
|
||||
# 默认免费的ChatGLMLLM就已经支持function_call，但是如果像追求稳定建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-pro-32k-functioncall-241028
|
||||
Intent: function_call
|
||||
|
||||
# 意图识别，是用于理解用户意图的模块，例如：播放音乐
|
||||
Intent:
|
||||
@@ -105,10 +102,11 @@ Intent:
|
||||
type: nointent
|
||||
# plugins_func/functions下的模块，可以通过配置，选择加载哪个模块，加载后对话支持相应的function调用
|
||||
# 系统默认已经记载“handle_exit_intent(退出识别)”、“play_music(音乐播放)”插件，请勿重复加载
|
||||
# 下面是加载查天气、角色切换的插件示例
|
||||
# 下面是加载查天气、角色切换、加载查新闻的插件示例
|
||||
functions:
|
||||
- change_role
|
||||
- get_weather
|
||||
- get_news
|
||||
- play_music
|
||||
#- hass_play_music
|
||||
#- hass_get_state
|
||||
@@ -121,6 +119,14 @@ plugins:
|
||||
# 想稳定一点就自行申请替换，每天有1000次免费调用
|
||||
# 申请地址：https://console.qweather.com/#/apps/create-key/over
|
||||
get_weather: { "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" }
|
||||
# 获取新闻插件的配置，这里根据需要的新闻类型传入对应的url链接，默认支持社会、科技、财经新闻
|
||||
# 更多类型的新闻列表查看 https://www.chinanews.com.cn/rss/
|
||||
get_news:
|
||||
default_rss_url: "https://www.chinanews.com.cn/rss/society.xml"
|
||||
category_urls:
|
||||
society: "https://www.chinanews.com.cn/rss/society.xml"
|
||||
world: "https://www.chinanews.com.cn/rss/world.xml"
|
||||
finance: "https://www.chinanews.com.cn/rss/finance.xml"
|
||||
|
||||
Memory:
|
||||
mem0ai:
|
||||
@@ -129,10 +135,10 @@ Memory:
|
||||
# 每月有1000次免费调用
|
||||
api_key: 你的mem0ai api key
|
||||
nomem:
|
||||
# 不想使用记忆功能，可以使用nomem
|
||||
# 不想使用记忆功能，可以使用nomem
|
||||
type: nomem
|
||||
mem_local_short:
|
||||
# 本地记忆功能，通过selected_module的llm总结，数据保存在本地，不会上传到服务器
|
||||
# 本地记忆功能，通过selected_module的llm总结，数据保存在本地，不会上传到服务器
|
||||
type: mem_local_short
|
||||
|
||||
ASR:
|
||||
@@ -140,6 +146,10 @@ ASR:
|
||||
type: fun_local
|
||||
model_dir: models/SenseVoiceSmall
|
||||
output_dir: tmp/
|
||||
SherpaASR:
|
||||
type: sherpa_onnx_local
|
||||
model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17
|
||||
output_dir: tmp/
|
||||
DoubaoASR:
|
||||
type: doubao
|
||||
appid: 你的火山引擎语音合成服务appid
|
||||
@@ -153,6 +163,7 @@ VAD:
|
||||
min_silence_duration_ms: 700 # 如果说话停顿比较长，可以把这个值设置大一些
|
||||
|
||||
LLM:
|
||||
# 所有openai类型均可以修改超参，以AliLLM为例
|
||||
# 当前支持的type为openai、dify、ollama，可自行适配
|
||||
AliLLM:
|
||||
# 定义LLM API类型
|
||||
@@ -161,6 +172,23 @@ LLM:
|
||||
base_url: https://dashscope.aliyuncs.com/compatible-mode/v1
|
||||
model_name: qwen-turbo
|
||||
api_key: 你的deepseek web key
|
||||
temperature: 0.7 # 温度值
|
||||
max_tokens: 500 # 最大生成token数
|
||||
top_p: 1
|
||||
top_k: 50
|
||||
frequency_penalty: 0 # 频率惩罚
|
||||
AliAppLLM:
|
||||
# 定义LLM API类型
|
||||
type: AliBL
|
||||
base_url: https://dashscope.aliyuncs.com/compatible-mode/v1
|
||||
app_id: 你的app_id
|
||||
# 可在这里找到你的 api_key https://bailian.console.aliyun.com/?apiKey=1#/api-key
|
||||
api_key: 你的api_key
|
||||
# 是否不使用本地prompt：true|false (默不用请在百练应用中设置prompt)
|
||||
is_no_prompt: true
|
||||
# Ali_memory_id：false（不使用）|你的memory_id（请在百练应用中设置中获取）
|
||||
# Tips！：Ali_memory未实现多用户存储记忆(记忆按id调用)
|
||||
ali_memory_id: false
|
||||
DoubaoLLM:
|
||||
# 定义LLM API类型
|
||||
type: openai
|
||||
@@ -196,8 +224,12 @@ LLM:
|
||||
type: dify
|
||||
# 建议使用本地部署的dify接口，国内部分区域访问dify公有云接口可能会受限
|
||||
# 如果使用DifyLLM，配置文件里prompt(提示词)是无效的，需要在dify控制台设置提示词
|
||||
base_url: https://api.dify.cn/v1
|
||||
base_url: https://api.dify.ai/v1
|
||||
api_key: 你的DifyLLM web key
|
||||
# 使用的对话模式 可以选择工作流 workflows/run 对话模式 chat-messages 文本生成 completion-messages
|
||||
# 使用workflows进行返回的时候输入参数为 query 返回参数的名字要设置为 answer
|
||||
# 文本生成的默认输入参数也是query
|
||||
mode: chat-messages
|
||||
GeminiLLM:
|
||||
type: gemini
|
||||
# 谷歌Gemini API，需要先在Google Cloud控制台创建API密钥并获取api_key
|
||||
@@ -205,7 +237,9 @@ LLM:
|
||||
# token申请地址： https://aistudio.google.com/apikey
|
||||
# 若部署地无法访问接口，需要开启科学上网
|
||||
api_key: 你的gemini web key
|
||||
model_name: "gemini-1.5-pro" # gemini-1.5-pro 是免费的
|
||||
model_name: "gemini-2.0-flash"
|
||||
http_proxy: "" #"http://127.0.0.1:10808"
|
||||
https_proxy: "" #http://127.0.0.1:10808"
|
||||
CozeLLM:
|
||||
# 定义LLM API类型
|
||||
type: coze
|
||||
@@ -240,17 +274,18 @@ TTS:
|
||||
# 定义TTS API类型
|
||||
type: edge
|
||||
voice: zh-CN-XiaoxiaoNeural
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
DoubaoTTS:
|
||||
# 定义TTS API类型
|
||||
type: doubao
|
||||
# 火山引擎语音合成服务，需要先在火山引擎控制台创建应用并获取appid和access_token
|
||||
# 山引擎语音一定要购买花钱，起步价30元，就有100并发了。如果用免费的只有2个并发，会经常报tts错误
|
||||
# 购买服务后，购买免费的音色后，可能要等半小时左右，才能使用。
|
||||
# 地址：https://console.volcengine.com/speech/service/8
|
||||
# 普通音色在这里开通：https://console.volcengine.com/speech/service/8
|
||||
# 湾湾小何音色在这里开通：https://console.volcengine.com/speech/service/10007，开通后将下面的voice设置成zh_female_wanwanxiaohe_moon_bigtts
|
||||
api_url: https://openspeech.bytedance.com/api/v1/tts
|
||||
voice: BV001_streaming
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
authorization: "Bearer;"
|
||||
appid: 你的火山引擎语音合成服务appid
|
||||
access_token: 你的火山引擎语音合成服务access_token
|
||||
@@ -261,7 +296,7 @@ TTS:
|
||||
# token申请地址 https://cloud.siliconflow.cn/account/ak
|
||||
model: FunAudioLLM/CosyVoice2-0.5B
|
||||
voice: FunAudioLLM/CosyVoice2-0.5B:alex
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
access_token: 你的硅基流动API密钥
|
||||
response_format: wav
|
||||
CozeCnTTS:
|
||||
@@ -269,7 +304,7 @@ TTS:
|
||||
# COZECN TTS
|
||||
# token申请地址 https://www.coze.cn/open/oauth/pats
|
||||
voice: 7426720361733046281
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
access_token: 你的coze web key
|
||||
response_format: wav
|
||||
FishSpeech:
|
||||
@@ -282,7 +317,7 @@ TTS:
|
||||
#--decoder-config-name firefly_gan_vq
|
||||
#--compile
|
||||
type: fishspeech
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
response_format: wav
|
||||
reference_id: null
|
||||
reference_audio: ["/tmp/test.wav",]
|
||||
@@ -306,7 +341,7 @@ TTS:
|
||||
#python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/caixukun.yaml
|
||||
type: gpt_sovits_v2
|
||||
url: "http://127.0.0.1:9880/tts"
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
text_lang: "auto"
|
||||
ref_audio_path: "caixukun.wav"
|
||||
prompt_text: ""
|
||||
@@ -331,7 +366,7 @@ TTS:
|
||||
#python api.py
|
||||
type: gpt_sovits_v3
|
||||
url: "http://127.0.0.1:9880"
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
text_language: "auto"
|
||||
refer_wav_path: "caixukun.wav"
|
||||
prompt_language: "zh"
|
||||
@@ -352,7 +387,7 @@ TTS:
|
||||
# api_key地址：https://platform.minimaxi.com/user-center/basic-information/interface-key
|
||||
# 定义TTS API类型
|
||||
type: minimax
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
group_id: 你的minimax平台groupID
|
||||
api_key: 你的minimax平台接口密钥
|
||||
model: "speech-01-turbo"
|
||||
@@ -389,7 +424,7 @@ TTS:
|
||||
# token地址：https://nls-portal.console.aliyun.com/overview
|
||||
# 定义TTS API类型
|
||||
type: aliyun
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
appkey: 你的阿里云智能语音交互服务项目Appkey
|
||||
token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret
|
||||
voice: xiaoyun
|
||||
@@ -411,9 +446,21 @@ TTS:
|
||||
type: doubao
|
||||
api_url: https://api.302ai.cn/doubao/tts_hd
|
||||
authorization: "Bearer "
|
||||
# 湾湾小何音色
|
||||
voice: "zh_female_wanwanxiaohe_moon_bigtts"
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
access_token: "你的302API密钥"
|
||||
GizwitsTTS:
|
||||
type: doubao
|
||||
# 火山引擎作为基座，可以完全使用企业级火山引擎语音合成服务
|
||||
# 前一万名注册的用户，将送5元体验金额
|
||||
# 获取API Key地址：https://agentrouter.gizwitsapi.com/panel/token
|
||||
api_url: https://bytedance.gizwitsapi.com/api/v1/tts
|
||||
authorization: "Bearer "
|
||||
# 湾湾小何音色
|
||||
voice: "zh_female_wanwanxiaohe_moon_bigtts"
|
||||
output_dir: tmp/
|
||||
access_token: "你的机智云API key"
|
||||
ACGNTTS:
|
||||
#在线网址：https://acgn.ttson.cn/
|
||||
#token购买：www.ttson.cn
|
||||
@@ -429,7 +476,7 @@ TTS:
|
||||
to_lang: ZH
|
||||
url: https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=
|
||||
format: mp3
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
emotion: 1
|
||||
OpenAITTS:
|
||||
# openai官方文本转语音服务，可支持全球大多数语种
|
||||
@@ -443,7 +490,7 @@ TTS:
|
||||
voice: onyx
|
||||
# 语速范围0.25-4.0
|
||||
speed: 1
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
CustomTTS:
|
||||
# 自定义的TTS接口服务，请求参数可自定义
|
||||
# 要求接口使用GET方式请求，并返回音频文件
|
||||
@@ -458,7 +505,7 @@ TTS:
|
||||
headers: # 自定义请求头
|
||||
# Authorization: Bearer xxxx
|
||||
format: wav # 接口返回的音频格式
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
# 模块测试配置
|
||||
module_test:
|
||||
test_sentences: # 自定义测试语句
|
||||
|
||||
@@ -7,9 +7,48 @@ from core.utils.util import read_config, get_project_dir
|
||||
default_config_file = "config.yaml"
|
||||
|
||||
|
||||
def ensure_directories(config):
|
||||
"""确保所有配置路径存在"""
|
||||
dirs_to_create = set()
|
||||
project_dir = get_project_dir() # 获取项目根目录
|
||||
# 日志文件目录
|
||||
log_dir = config.get('log', {}).get('log_dir', 'tmp')
|
||||
dirs_to_create.add(os.path.join(project_dir, log_dir))
|
||||
|
||||
# ASR/TTS模块输出目录
|
||||
for module in ['ASR', 'TTS']:
|
||||
for provider in config.get(module, {}).values():
|
||||
output_dir = provider.get('output_dir', '')
|
||||
if output_dir:
|
||||
dirs_to_create.add(output_dir)
|
||||
|
||||
# 根据selected_module创建模型目录
|
||||
selected_modules = config.get('selected_module', {})
|
||||
for module_type in ['ASR', 'LLM', 'TTS']:
|
||||
selected_provider = selected_modules.get(module_type)
|
||||
if not selected_provider:
|
||||
continue
|
||||
provider_config = config.get(module_type, {}).get(selected_provider, {})
|
||||
output_dir = provider_config.get('output_dir')
|
||||
if output_dir:
|
||||
full_model_dir = os.path.join(project_dir, output_dir)
|
||||
dirs_to_create.add(full_model_dir)
|
||||
|
||||
# 统一创建目录（保留原data目录创建）
|
||||
for dir_path in dirs_to_create:
|
||||
try:
|
||||
os.makedirs(dir_path, exist_ok=True)
|
||||
except PermissionError:
|
||||
print(f"警告：无法创建目录 {dir_path}，请检查写入权限")
|
||||
|
||||
|
||||
def get_config_file():
|
||||
global default_config_file
|
||||
# 判断是否存在私有的配置文件
|
||||
"""获取配置文件路径，优先使用私有配置文件（若存在）。
|
||||
|
||||
Returns:
|
||||
str: 配置文件路径（相对路径或默认路径）
|
||||
"""
|
||||
config_file = default_config_file
|
||||
if os.path.exists(get_project_dir() + "data/." + default_config_file):
|
||||
config_file = "data/." + default_config_file
|
||||
@@ -20,9 +59,13 @@ def load_config():
|
||||
"""加载配置文件"""
|
||||
parser = argparse.ArgumentParser(description="Server configuration")
|
||||
config_file = get_config_file()
|
||||
|
||||
parser.add_argument("--config_path", type=str, default=config_file)
|
||||
args = parser.parse_args()
|
||||
return read_config(args.config_path)
|
||||
config = read_config(args.config_path)
|
||||
# 初始化目录
|
||||
ensure_directories(config)
|
||||
return config
|
||||
|
||||
|
||||
def update_config(config):
|
||||
@@ -67,7 +110,7 @@ def find_missing_keys(new_config, old_config, parent_key=''):
|
||||
def check_config_file():
|
||||
old_config_file = get_config_file()
|
||||
global default_config_file
|
||||
if not old_config_file.startswith('data'):
|
||||
if not 'data' in old_config_file:
|
||||
return
|
||||
old_config = read_config(get_project_dir() + old_config_file)
|
||||
new_config = read_config(get_project_dir() + default_config_file)
|
||||
|
||||
@@ -197,7 +197,7 @@ class ConnectionHandler:
|
||||
self.dialogue.put(Message(role="system", content=self.prompt))
|
||||
|
||||
self.func_handler = FunctionHandler(self.config)
|
||||
|
||||
|
||||
def change_system_prompt(self, prompt):
|
||||
self.prompt = prompt
|
||||
# 找到原来的role==system，替换原来的系统提示
|
||||
@@ -303,7 +303,7 @@ class ConnectionHandler:
|
||||
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
|
||||
return True
|
||||
|
||||
def chat_with_function_calling(self, query, tool_call = False):
|
||||
def chat_with_function_calling(self, query, tool_call=False):
|
||||
self.logger.bind(tag=TAG).debug(f"Chat with function calling start: {query}")
|
||||
"""Chat with function calling for intent detection using streaming"""
|
||||
if self.isNeedAuth():
|
||||
@@ -311,7 +311,7 @@ class ConnectionHandler:
|
||||
future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
|
||||
future.result()
|
||||
return True
|
||||
|
||||
|
||||
if not tool_call:
|
||||
self.dialogue.put(Message(role="user", content=query))
|
||||
|
||||
@@ -320,7 +320,7 @@ class ConnectionHandler:
|
||||
|
||||
response_message = []
|
||||
processed_chars = 0 # 跟踪已处理的字符位置
|
||||
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
|
||||
@@ -328,7 +328,7 @@ class ConnectionHandler:
|
||||
future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
|
||||
memory_str = future.result()
|
||||
|
||||
#self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}")
|
||||
# self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}")
|
||||
|
||||
# 使用支持functions的streaming接口
|
||||
llm_responses = self.llm.response_with_functions(
|
||||
@@ -351,8 +351,8 @@ class ConnectionHandler:
|
||||
content_arguments = ""
|
||||
for response in llm_responses:
|
||||
content, tools_call = response
|
||||
if content is not None and len(content)>0:
|
||||
if len(response_message)<=0 and (content=="```" or "<tool_call>" in content):
|
||||
if content is not None and len(content) > 0:
|
||||
if len(response_message) <= 0 and (content == "```" or "<tool_call>" in content):
|
||||
tool_call_flag = True
|
||||
|
||||
if tools_call is not None:
|
||||
@@ -366,7 +366,7 @@ class ConnectionHandler:
|
||||
|
||||
if content is not None and len(content) > 0:
|
||||
if tool_call_flag:
|
||||
content_arguments+=content
|
||||
content_arguments += content
|
||||
else:
|
||||
response_message.append(content)
|
||||
|
||||
@@ -422,16 +422,17 @@ class ConnectionHandler:
|
||||
else:
|
||||
function_arguments = json.loads(function_arguments)
|
||||
if not bHasError:
|
||||
self.logger.bind(tag=TAG).info(f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
|
||||
self.logger.bind(tag=TAG).info(
|
||||
f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
|
||||
function_call_data = {
|
||||
"name": function_name,
|
||||
"id": function_id,
|
||||
"arguments": function_arguments
|
||||
}
|
||||
result = self.func_handler.handle_llm_function_call(self, function_call_data)
|
||||
self._handle_function_result(result, function_call_data, text_index+1)
|
||||
self._handle_function_result(result, function_call_data, text_index + 1)
|
||||
|
||||
# 处理最后剩余的文本
|
||||
# 处理最后剩余的文本
|
||||
full_text = "".join(response_message)
|
||||
remaining_text = full_text[processed_chars:]
|
||||
if remaining_text:
|
||||
@@ -443,7 +444,7 @@ class ConnectionHandler:
|
||||
self.tts_queue.put(future)
|
||||
|
||||
# 存储对话内容
|
||||
if len(response_message)>0:
|
||||
if len(response_message) > 0:
|
||||
self.dialogue.put(Message(role="assistant", content="".join(response_message)))
|
||||
|
||||
self.llm_finish_task = True
|
||||
@@ -452,31 +453,40 @@ class ConnectionHandler:
|
||||
return True
|
||||
|
||||
def _handle_function_result(self, result, function_call_data, text_index):
|
||||
if result.action == Action.RESPONSE: # 直接回复前端
|
||||
if result.action == Action.RESPONSE: # 直接回复前端
|
||||
text = result.response
|
||||
self.recode_first_last_text(text, text_index)
|
||||
future = self.executor.submit(self.speak_and_play, text, text_index)
|
||||
self.tts_queue.put(future)
|
||||
self.dialogue.put(Message(role="assistant", content=text))
|
||||
if result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
|
||||
|
||||
elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
|
||||
|
||||
text = result.result
|
||||
if text is not None and len(text) > 0:
|
||||
function_id = function_call_data["id"]
|
||||
function_name = function_call_data["name"]
|
||||
function_arguments = function_call_data["arguments"]
|
||||
self.dialogue.put(Message(role='assistant',
|
||||
tool_calls=[{"id": function_id,
|
||||
"function": {"arguments": function_arguments,"name": function_name},
|
||||
"type": 'function',
|
||||
"index": 0}]))
|
||||
tool_calls=[{"id": function_id,
|
||||
"function": {"arguments": function_arguments,
|
||||
"name": function_name},
|
||||
"type": 'function',
|
||||
"index": 0}]))
|
||||
|
||||
self.dialogue.put(Message(role="tool", tool_call_id=function_id, content=text))
|
||||
self.chat_with_function_calling(text, tool_call=True)
|
||||
if result.action == Action.NOTFOUND:
|
||||
text = result.response
|
||||
|
||||
|
||||
elif result.action == Action.NOTFOUND:
|
||||
text = result.result
|
||||
self.recode_first_last_text(text, text_index)
|
||||
future = self.executor.submit(self.speak_and_play, text, text_index)
|
||||
self.tts_queue.put(future)
|
||||
self.dialogue.put(Message(role="assistant", content=text))
|
||||
else:
|
||||
text = result.result
|
||||
self.recode_first_last_text(text, text_index)
|
||||
future = self.executor.submit(self.speak_and_play, text, text_index)
|
||||
self.tts_queue.put(future)
|
||||
self.dialogue.put(Message(role="assistant", content=text))
|
||||
|
||||
def _tts_priority_thread(self):
|
||||
while not self.stop_event.is_set():
|
||||
|
||||
@@ -50,6 +50,7 @@ class FunctionHandler:
|
||||
#self.function_registry.register_function("play_music")
|
||||
self.function_registry.register_function("plugin_loader")
|
||||
self.function_registry.register_function("get_time")
|
||||
self.function_registry.register_function("raise_and_lower_the_volume")
|
||||
|
||||
def register_config_functions(self):
|
||||
"""注册配置中的函数,可以不同客户端使用不同的配置"""
|
||||
@@ -69,7 +70,7 @@ class FunctionHandler:
|
||||
arguments = function_call_data["arguments"]
|
||||
arguments = json.loads(arguments) if arguments else {}
|
||||
logger.bind(tag=TAG).info(f"调用函数: {function_name}, 参数: {arguments}")
|
||||
if funcItem.type == ToolType.SYSTEM_CTL:
|
||||
if funcItem.type == ToolType.SYSTEM_CTL or funcItem.type == ToolType.IOT_CTL:
|
||||
return func(conn, **arguments)
|
||||
elif funcItem.type == ToolType.WAIT:
|
||||
return func(**arguments)
|
||||
|
||||
@@ -1,24 +1,120 @@
|
||||
import json
|
||||
import asyncio
|
||||
from config.logger import setup_logging
|
||||
from plugins_func.register import device_type_registry, register_function, ActionResponse, Action, ToolType
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
def wrap_async_function(async_func):
|
||||
"""包装异步函数为同步函数"""
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
try:
|
||||
# 获取连接对象（第一个参数）
|
||||
conn = args[0]
|
||||
if not hasattr(conn, 'loop'):
|
||||
logger.bind(tag=TAG).error("Connection对象没有loop属性")
|
||||
return ActionResponse(Action.ERROR, "Connection对象没有loop属性",
|
||||
"执行操作时出错: Connection对象没有loop属性")
|
||||
|
||||
# 使用conn对象中的事件循环
|
||||
loop = conn.loop
|
||||
# 在conn的事件循环中运行异步函数
|
||||
future = asyncio.run_coroutine_threadsafe(async_func(*args, **kwargs), loop)
|
||||
# 等待结果返回
|
||||
return future.result()
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"运行异步函数时出错: {e}")
|
||||
return ActionResponse(Action.ERROR, str(e), f"执行操作时出错: {e}")
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def create_iot_function(device_name, method_name, method_info):
|
||||
"""
|
||||
根据IOT设备描述生成通用的控制函数
|
||||
"""
|
||||
|
||||
async def iot_control_function(conn, response_success=None, response_failure=None, **params):
|
||||
try:
|
||||
# 打印响应参数
|
||||
logger.bind(tag=TAG).info(
|
||||
f"控制函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
|
||||
|
||||
# 发送控制命令
|
||||
await send_iot_conn(conn, device_name, method_name, params)
|
||||
# 等待一小段时间让状态更新
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# 生成结果信息
|
||||
result = f"{device_name}的{method_name}操作执行成功"
|
||||
|
||||
|
||||
# 处理响应中可能的占位符
|
||||
response = response_success
|
||||
# 替换{value}占位符
|
||||
for param_name, param_value in params.items():
|
||||
# 先尝试直接替换参数值
|
||||
if "{" + param_name + "}" in response:
|
||||
response = response.replace("{" + param_name + "}", str(param_value))
|
||||
|
||||
# 如果有{value}占位符，用相关参数替换
|
||||
if "{value}" in response:
|
||||
response = response.replace("{value}", str(param_value))
|
||||
break
|
||||
|
||||
return ActionResponse(Action.RESPONSE, result, response)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"执行{device_name}的{method_name}操作失败: {e}")
|
||||
|
||||
# 操作失败时使用大模型提供的失败响应
|
||||
response = response_failure
|
||||
|
||||
return ActionResponse(Action.ERROR, str(e), response)
|
||||
|
||||
return wrap_async_function(iot_control_function)
|
||||
|
||||
|
||||
def create_iot_query_function(device_name, prop_name, prop_info):
|
||||
"""
|
||||
根据IOT设备属性创建查询函数
|
||||
"""
|
||||
|
||||
async def iot_query_function(conn, response_success=None, response_failure=None):
|
||||
try:
|
||||
# 打印响应参数
|
||||
logger.bind(tag=TAG).info(
|
||||
f"查询函数接收到的响应参数: success='{response_success}', failure='{response_failure}'")
|
||||
|
||||
value = await get_iot_status(conn, device_name, prop_name)
|
||||
|
||||
# 查询成功，生成结果
|
||||
if value is not None:
|
||||
# 使用大模型提供的成功响应，并替换其中的占位符
|
||||
response = response_success.replace("{value}", str(value))
|
||||
|
||||
return ActionResponse(Action.RESPONSE, str(value), response)
|
||||
else:
|
||||
# 查询失败，使用大模型提供的失败响应
|
||||
response = response_failure
|
||||
|
||||
return ActionResponse(Action.ERROR, f"属性{prop_name}不存在", response)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"查询{device_name}的{prop_name}时出错: {e}")
|
||||
|
||||
# 查询出错时使用大模型提供的失败响应
|
||||
response = response_failure
|
||||
|
||||
return ActionResponse(Action.ERROR, str(e), response)
|
||||
|
||||
return wrap_async_function(iot_query_function)
|
||||
|
||||
|
||||
class IotDescriptor:
|
||||
"""
|
||||
A class to represent an IoT descriptor.
|
||||
Attributes:
|
||||
----------
|
||||
name : str
|
||||
The name of the IoT descriptor.
|
||||
description : str
|
||||
A brief description of the IoT descriptor.
|
||||
properties : dict
|
||||
A dictionary containing properties of the IoT descriptor.
|
||||
methods : dict
|
||||
A dictionary containing methods of the IoT descriptor.
|
||||
-------
|
||||
"""
|
||||
|
||||
def __init__(self, name, description, properties, methods):
|
||||
@@ -29,17 +125,7 @@ class IotDescriptor:
|
||||
|
||||
# 根据描述创建属性
|
||||
for key, value in properties.items():
|
||||
# "volume":{"description":"当前音量 值","type":"number"}
|
||||
"""
|
||||
等价于
|
||||
{
|
||||
'name': 名字,
|
||||
'description': 描述,
|
||||
'value': 0
|
||||
}
|
||||
"""
|
||||
# setattr(self, key, {}) # 创建一个空字典, 名字是属性名
|
||||
property_item = globals()[key] = {} # 创建一个空字典, 名字是属性名
|
||||
property_item = globals()[key] = {}
|
||||
property_item['name'] = key
|
||||
property_item["description"] = value["description"]
|
||||
if value["type"] == "number":
|
||||
@@ -52,23 +138,10 @@ class IotDescriptor:
|
||||
|
||||
# 根据描述创建方法
|
||||
for key, value in methods.items():
|
||||
# "SetVolume": {"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}}
|
||||
"""
|
||||
等价于
|
||||
SetVolume = {
|
||||
`description`: 描述,
|
||||
`volume`: {
|
||||
`description`: 描述,
|
||||
`value`: 0
|
||||
}
|
||||
}
|
||||
"""
|
||||
# setattr(self, key, {}) # 创建一个空字典, 名字是方法名
|
||||
method = globals()[key] = {} # 创建一个空字典, 名字是方法名
|
||||
method = globals()[key] = {}
|
||||
method["description"] = value["description"]
|
||||
method['name'] = key
|
||||
for k, v in value["parameters"].items():
|
||||
# 不同的参数解析
|
||||
method[k] = {}
|
||||
method[k]["description"] = v["description"]
|
||||
if v["type"] == "number":
|
||||
@@ -77,58 +150,136 @@ class IotDescriptor:
|
||||
method[k]["value"] = False
|
||||
else:
|
||||
method[k]["value"] = ""
|
||||
|
||||
self.methods.append(method)
|
||||
|
||||
|
||||
async def handleIotDescriptors(conn, descriptors):
|
||||
"""
|
||||
处理物联网描述
|
||||
示例: [{
|
||||
"name":"Speaker",
|
||||
"description":"当前 AI 机器人的扬声器",
|
||||
"properties":{
|
||||
"volume":{"description":"当前音量 值","type":"number"} 可以有boolean, number, string三种类型
|
||||
},
|
||||
"methods":{
|
||||
"SetVolume":{
|
||||
"description":"设置音量","parameters":{"volume":{"description":"0到100之间的整数","type":"number"}}
|
||||
def register_device_type(descriptor):
|
||||
"""注册设备类型及其功能"""
|
||||
device_name = descriptor["name"]
|
||||
type_id = device_type_registry.generate_device_type_id(descriptor)
|
||||
|
||||
# 如果该类型已注册，直接返回类型ID
|
||||
if type_id in device_type_registry.type_functions:
|
||||
return type_id
|
||||
|
||||
functions = {}
|
||||
|
||||
# 为每个属性创建查询函数
|
||||
for prop_name, prop_info in descriptor["properties"].items():
|
||||
func_name = f"get_{device_name.lower()}_{prop_name.lower()}"
|
||||
func_desc = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": func_name,
|
||||
"description": f"查询{descriptor['description']}的{prop_info['description']}",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"response_success": {
|
||||
"type": "string",
|
||||
"description": f"查询成功时的友好回复，必须使用{{value}}作为占位符表示查询到的值"
|
||||
},
|
||||
"response_failure": {
|
||||
"type": "string",
|
||||
"description": f"查询失败时的友好回复，例如：'无法获取{device_name}的{prop_info['description']}'"
|
||||
}
|
||||
},
|
||||
"required": ["response_success", "response_failure"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
descriptors: 描述列表
|
||||
"""
|
||||
query_func = create_iot_query_function(device_name, prop_name, prop_info)
|
||||
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(query_func)
|
||||
functions[func_name] = decorated_func
|
||||
|
||||
# 为每个方法创建控制函数
|
||||
for method_name, method_info in descriptor["methods"].items():
|
||||
func_name = f"{device_name.lower()}_{method_name.lower()}"
|
||||
|
||||
# 创建参数字典，添加原有参数
|
||||
parameters = {
|
||||
param_name: {
|
||||
"type": param_info["type"],
|
||||
"description": param_info["description"]
|
||||
}
|
||||
for param_name, param_info in method_info["parameters"].items()
|
||||
}
|
||||
|
||||
# 添加响应参数
|
||||
parameters.update({
|
||||
"response_success": {
|
||||
"type": "string",
|
||||
"description": "操作成功时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
|
||||
},
|
||||
"response_failure": {
|
||||
"type": "string",
|
||||
"description": "操作失败时的友好回复,关于该设备的操作结果，设备名称尽量使用description中的名称"
|
||||
}
|
||||
})
|
||||
|
||||
# 构建必须参数列表（原有参数 + 响应参数）
|
||||
required_params = list(method_info["parameters"].keys())
|
||||
required_params.extend(["response_success", "response_failure"])
|
||||
|
||||
func_desc = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": func_name,
|
||||
"description": f"{descriptor['description']} - {method_info['description']}",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": parameters,
|
||||
"required": required_params
|
||||
}
|
||||
}
|
||||
}
|
||||
control_func = create_iot_function(device_name, method_name, method_info)
|
||||
decorated_func = register_function(func_name, func_desc, ToolType.IOT_CTL)(control_func)
|
||||
functions[func_name] = decorated_func
|
||||
|
||||
device_type_registry.register_device_type(type_id, functions)
|
||||
return type_id
|
||||
|
||||
|
||||
# 用于接受前端设备推送的搜索iot描述
|
||||
async def handleIotDescriptors(conn, descriptors):
|
||||
"""处理物联网描述"""
|
||||
functions_changed = False
|
||||
|
||||
for descriptor in descriptors:
|
||||
# 创建IOT设备描述符
|
||||
iot_descriptor = IotDescriptor(descriptor["name"], descriptor["description"], descriptor["properties"],
|
||||
descriptor["methods"])
|
||||
conn.iot_descriptors[descriptor["name"]] = iot_descriptor
|
||||
|
||||
# 暂时从配置文件中设置音量，后期通过意图识别控制音量
|
||||
default_iot_volume = 100
|
||||
if "iot" in conn.config:
|
||||
default_iot_volume = conn.config["iot"]["Speaker"]["volume"]
|
||||
logger.bind(tag=TAG).info(f"服务端设置音量为{default_iot_volume}")
|
||||
await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": default_iot_volume})
|
||||
if conn.use_function_call_mode:
|
||||
# 注册或获取设备类型
|
||||
type_id = register_device_type(descriptor)
|
||||
device_functions = device_type_registry.get_device_functions(type_id)
|
||||
|
||||
# 在连接级注册设备函数
|
||||
if hasattr(conn, 'func_handler'):
|
||||
for func_name in device_functions:
|
||||
conn.func_handler.function_registry.register_function(func_name)
|
||||
logger.bind(tag=TAG).info(f"注册IOT函数到function handler: {func_name}")
|
||||
functions_changed = True
|
||||
|
||||
# 如果注册了新函数，更新function描述列表
|
||||
if functions_changed and hasattr(conn, 'func_handler'):
|
||||
conn.func_handler.upload_functions_desc()
|
||||
func_names = conn.func_handler.current_support_functions()
|
||||
logger.bind(tag=TAG).info(f"设备类型: {type_id}")
|
||||
logger.bind(tag=TAG).info(f"更新function描述列表完成，当前支持的函数: {func_names}")
|
||||
|
||||
|
||||
async def handleIotStatus(conn, states):
|
||||
"""
|
||||
处理物联网状态
|
||||
示例: [{
|
||||
"name":"Speaker",
|
||||
"state":{
|
||||
"volume":100
|
||||
}
|
||||
}]
|
||||
states: 状态列表
|
||||
"""
|
||||
"""处理物联网状态"""
|
||||
for state in states:
|
||||
for key, value in conn.iot_descriptors.items():
|
||||
if key == state["name"]:
|
||||
for property_item in value.properties:
|
||||
# properties为字典列表, 记录各种属性
|
||||
for k, v in state["state"].items():
|
||||
# state为字典, 记录各种属性的值, 是需要记录的信息
|
||||
if property_item["name"] == k:
|
||||
# 检查一下属性是不是相同的
|
||||
if type(v) != type(property_item["value"]):
|
||||
logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
|
||||
break
|
||||
@@ -138,41 +289,35 @@ async def handleIotStatus(conn, states):
|
||||
break
|
||||
break
|
||||
|
||||
|
||||
async def get_iot_status(conn, name, property_name):
|
||||
"""
|
||||
获取物联网状态
|
||||
name: 设备名称 "Speaker"
|
||||
property_name: 属性名称 "volume"
|
||||
返回值: 属性值, 实际的属性有int, bool和str三种类型
|
||||
"""
|
||||
"""获取物联网状态"""
|
||||
for key, value in conn.iot_descriptors.items():
|
||||
if key == name:
|
||||
for property_item in value.properties:
|
||||
if property_item["name"] == property_name:
|
||||
return property_item["value"]
|
||||
logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
|
||||
return None
|
||||
|
||||
async def send_iot_conn(conn, name, method_name, parameters):
|
||||
"""
|
||||
发送物联网指令
|
||||
name: 设备名称 "Speaker"
|
||||
method: 方法 "SetVolume"
|
||||
parameters: 参数, 是一个字典 {"volume": 100}
|
||||
发送示例:
|
||||
{
|
||||
"type": "iot",
|
||||
"commands": [
|
||||
{
|
||||
"name" : "Speaker",
|
||||
"method": "SetVolume",
|
||||
"parameters": {
|
||||
"volume": 100
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
async def set_iot_status(conn, name, property_name, value):
|
||||
"""设置物联网状态"""
|
||||
for key, iot_descriptor in conn.iot_descriptors.items():
|
||||
if key == name:
|
||||
for property_item in iot_descriptor.properties:
|
||||
if property_item["name"] == property_name:
|
||||
if type(value) != type(property_item["value"]):
|
||||
logger.bind(tag=TAG).error(f"属性{property_item['name']}的值类型不匹配")
|
||||
return
|
||||
property_item["value"] = value
|
||||
logger.bind(tag=TAG).info(f"物联网状态更新: {name} , {property_name} = {value}")
|
||||
return
|
||||
logger.bind(tag=TAG).warning(f"未找到设备 {name} 的属性 {property_name}")
|
||||
|
||||
|
||||
async def send_iot_conn(conn, name, method_name, parameters):
|
||||
"""发送物联网指令"""
|
||||
for key, value in conn.iot_descriptors.items():
|
||||
if key == name:
|
||||
# 找到了设备
|
||||
|
||||
@@ -2,7 +2,7 @@ from config.logger import setup_logging
|
||||
import json
|
||||
from core.handle.abortHandle import handleAbortMessage
|
||||
from core.handle.helloHandle import handleHelloMessage
|
||||
from core.handle.receiveAudioHandle import startToChat
|
||||
from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
|
||||
from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
|
||||
|
||||
TAG = __name__
|
||||
@@ -31,6 +31,8 @@ async def handleTextMessage(conn, message):
|
||||
elif msg_json["state"] == "stop":
|
||||
conn.client_have_voice = True
|
||||
conn.client_voice_stop = True
|
||||
if len(conn.asr_audio) > 0:
|
||||
await handleAudioMessage(conn, b'')
|
||||
elif msg_json["state"] == "detect":
|
||||
conn.asr_server_receive = False
|
||||
conn.client_have_voice = False
|
||||
@@ -41,6 +43,6 @@ async def handleTextMessage(conn, message):
|
||||
if "descriptors" in msg_json:
|
||||
await handleIotDescriptors(conn, msg_json["descriptors"])
|
||||
if "states" in msg_json:
|
||||
await handleIotStatus(conn, msg_json["states"])
|
||||
await handleIotStatus(conn, msg_json["states"])
|
||||
except json.JSONDecodeError:
|
||||
await conn.websocket.send(message)
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
import time
|
||||
import wave
|
||||
import os
|
||||
import sys
|
||||
import io
|
||||
from config.logger import setup_logging
|
||||
from typing import Optional, Tuple, List
|
||||
import uuid
|
||||
import opuslib_next
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
|
||||
from modelscope.hub.file_download import model_file_download
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
# 捕获标准输出
|
||||
class CaptureOutput:
|
||||
def __enter__(self):
|
||||
self._output = io.StringIO()
|
||||
self._original_stdout = sys.stdout
|
||||
sys.stdout = self._output
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
sys.stdout = self._original_stdout
|
||||
self.output = self._output.getvalue()
|
||||
self._output.close()
|
||||
|
||||
# 将捕获到的内容通过 logger 输出
|
||||
if self.output:
|
||||
logger.bind(tag=TAG).info(self.output.strip())
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool):
|
||||
self.model_dir = config.get("model_dir")
|
||||
self.output_dir = config.get("output_dir")
|
||||
self.delete_audio_file = delete_audio_file
|
||||
|
||||
# 确保输出目录存在
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
# 初始化模型文件路径
|
||||
model_files = {
|
||||
"model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"),
|
||||
"tokens.txt": os.path.join(self.model_dir, "tokens.txt")
|
||||
}
|
||||
|
||||
# 下载并检查模型文件
|
||||
try:
|
||||
for file_name, file_path in model_files.items():
|
||||
if not os.path.isfile(file_path):
|
||||
logger.bind(tag=TAG).info(f"正在下载模型文件: {file_name}")
|
||||
model_file_download(
|
||||
model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue",
|
||||
file_path=file_name,
|
||||
local_dir=self.model_dir
|
||||
)
|
||||
|
||||
if not os.path.isfile(file_path):
|
||||
raise FileNotFoundError(f"模型文件下载失败: {file_path}")
|
||||
|
||||
self.model_path = model_files["model.int8.onnx"]
|
||||
self.tokens_path = model_files["tokens.txt"]
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}")
|
||||
raise
|
||||
|
||||
with CaptureOutput():
|
||||
self.model = sherpa_onnx.OfflineRecognizer.from_sense_voice(
|
||||
model=self.model_path,
|
||||
tokens=self.tokens_path,
|
||||
num_threads=2,
|
||||
sample_rate=16000,
|
||||
feature_dim=80,
|
||||
decoding_method="greedy_search",
|
||||
debug=False,
|
||||
use_itn=True,
|
||||
)
|
||||
|
||||
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
|
||||
"""将Opus音频数据解码并保存为WAV文件"""
|
||||
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
|
||||
file_path = os.path.join(self.output_dir, file_name)
|
||||
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
with wave.open(file_path, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2) # 2 bytes = 16-bit
|
||||
wf.setframerate(16000)
|
||||
wf.writeframes(b"".join(pcm_data))
|
||||
|
||||
return file_path
|
||||
|
||||
def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]:
|
||||
"""
|
||||
Args:
|
||||
wave_filename:
|
||||
Path to a wave file. It should be single channel and each sample should
|
||||
be 16-bit. Its sample rate does not need to be 16kHz.
|
||||
Returns:
|
||||
Return a tuple containing:
|
||||
- A 1-D array of dtype np.float32 containing the samples, which are
|
||||
normalized to the range [-1, 1].
|
||||
- sample rate of the wave file
|
||||
"""
|
||||
|
||||
with wave.open(wave_filename) as f:
|
||||
assert f.getnchannels() == 1, f.getnchannels()
|
||||
assert f.getsampwidth() == 2, f.getsampwidth() # it is in bytes
|
||||
num_samples = f.getnframes()
|
||||
samples = f.readframes(num_samples)
|
||||
samples_int16 = np.frombuffer(samples, dtype=np.int16)
|
||||
samples_float32 = samples_int16.astype(np.float32)
|
||||
|
||||
samples_float32 = samples_float32 / 32768
|
||||
return samples_float32, f.getframerate()
|
||||
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""语音转文本主处理逻辑"""
|
||||
file_path = None
|
||||
try:
|
||||
# 保存音频文件
|
||||
start_time = time.time()
|
||||
file_path = self.save_audio_to_file(opus_data, session_id)
|
||||
logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
|
||||
|
||||
# 语音识别
|
||||
start_time = time.time()
|
||||
s = self.model.create_stream()
|
||||
samples, sample_rate = self.read_wave(file_path)
|
||||
s.accept_waveform(sample_rate, samples)
|
||||
self.model.decode_stream(s)
|
||||
text = s.result.text
|
||||
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
|
||||
|
||||
return text, file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", None
|
||||
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
|
||||
@@ -0,0 +1,52 @@
|
||||
from config.logger import setup_logging
|
||||
from http import HTTPStatus
|
||||
from dashscope import Application
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.api_key = config["api_key"]
|
||||
self.app_id = config["app_id"]
|
||||
self.base_url = config.get("base_url")
|
||||
self.is_No_prompt = config.get("is_no_prompt")
|
||||
self.memory_id = config.get("ali_memory_id")
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
try:
|
||||
# 处理dialogue
|
||||
if self.is_No_prompt:
|
||||
dialogue.pop(0)
|
||||
logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的dialogue: {dialogue}")
|
||||
|
||||
# 构造调用参数
|
||||
call_params = {
|
||||
"api_key": self.api_key,
|
||||
"app_id": self.app_id,
|
||||
"session_id": session_id,
|
||||
"messages": dialogue
|
||||
}
|
||||
if self.memory_id != False:
|
||||
# 百练memory需要prompt参数
|
||||
prompt = dialogue[-1].get("content")
|
||||
call_params["memory_id"] = self.memory_id
|
||||
call_params["prompt"] = prompt
|
||||
logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的prompt: {prompt}")
|
||||
|
||||
responses = Application.call(**call_params)
|
||||
if responses.status_code != HTTPStatus.OK:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"code={responses.status_code}, "
|
||||
f"message={responses.message}, "
|
||||
f"请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
|
||||
)
|
||||
yield "【阿里百练API服务响应异常】"
|
||||
else:
|
||||
logger.bind(tag=TAG).debug(f"【阿里百练API服务】构造参数: {call_params}")
|
||||
yield responses.output.text
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
|
||||
yield "【LLM服务响应异常】"
|
||||
@@ -9,6 +9,7 @@ logger = setup_logging()
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.api_key = config["api_key"]
|
||||
self.mode = config.get("mode", "chat-messages")
|
||||
self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
|
||||
self.session_conversation_map = {} # 存储session_id和conversation_id的映射
|
||||
|
||||
@@ -19,27 +20,59 @@ class LLMProvider(LLMProviderBase):
|
||||
conversation_id = self.session_conversation_map.get(session_id)
|
||||
|
||||
# 发起流式请求
|
||||
with requests.post(
|
||||
f"{self.base_url}/chat-messages",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json={
|
||||
if self.mode == "chat-messages":
|
||||
request_json = {
|
||||
"query": last_msg["content"],
|
||||
"response_mode": "streaming",
|
||||
"user": session_id,
|
||||
"inputs": {},
|
||||
"conversation_id": conversation_id
|
||||
},
|
||||
}
|
||||
elif self.mode == "workflows/run":
|
||||
request_json = {
|
||||
"inputs": {"query": last_msg["content"]},
|
||||
"response_mode": "streaming",
|
||||
"user": session_id
|
||||
}
|
||||
elif self.mode == "completion-messages":
|
||||
request_json = {
|
||||
"inputs": {"query": last_msg["content"]},
|
||||
"response_mode": "streaming",
|
||||
"user": session_id
|
||||
}
|
||||
|
||||
with requests.post(
|
||||
f"{self.base_url}/{self.mode}",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json=request_json,
|
||||
stream=True
|
||||
) as r:
|
||||
for line in r.iter_lines():
|
||||
if line.startswith(b'data: '):
|
||||
event = json.loads(line[6:])
|
||||
# 如果没有找到conversation_id，则获取此次conversation_id
|
||||
if not conversation_id:
|
||||
conversation_id = event.get('conversation_id')
|
||||
self.session_conversation_map[session_id] = conversation_id # 更新映射
|
||||
if event.get('answer'):
|
||||
yield event['answer']
|
||||
if self.mode == "chat-messages":
|
||||
for line in r.iter_lines():
|
||||
if line.startswith(b'data: '):
|
||||
event = json.loads(line[6:])
|
||||
# 如果没有找到conversation_id，则获取此次conversation_id
|
||||
if not conversation_id:
|
||||
conversation_id = event.get('conversation_id')
|
||||
self.session_conversation_map[session_id] = conversation_id # 更新映射
|
||||
if event.get('answer'):
|
||||
yield event['answer']
|
||||
elif self.mode == "workflows/run":
|
||||
for line in r.iter_lines():
|
||||
# logger.bind(tag=TAG).info(f"chat message response: {line}")
|
||||
if line.startswith(b'data: '):
|
||||
event = json.loads(line[6:])
|
||||
if event.get('event') == "workflow_finished":
|
||||
if event['data']['status'] == "succeeded":
|
||||
yield event['data']['outputs']['answer']
|
||||
else:
|
||||
yield "【服务响应异常】"
|
||||
elif self.mode == "completion-messages":
|
||||
for line in r.iter_lines():
|
||||
if line.startswith(b'data: '):
|
||||
event = json.loads(line[6:])
|
||||
if event.get('answer'):
|
||||
yield event['answer']
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
|
||||
|
||||
@@ -1,14 +1,19 @@
|
||||
import google.generativeai as genai
|
||||
from core.utils.util import check_model_key
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
from config.logger import setup_logging
|
||||
import requests
|
||||
import json
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
"""初始化Gemini LLM Provider"""
|
||||
self.model_name = config.get("model_name", "gemini-1.5-pro")
|
||||
self.api_key = config.get("api_key")
|
||||
|
||||
self.http_proxy=config.get("http_proxy")
|
||||
self.https_proxy = config.get("https_proxy")
|
||||
have_key = check_model_key("LLM", self.api_key)
|
||||
|
||||
if not have_key:
|
||||
@@ -16,6 +21,19 @@ class LLMProvider(LLMProviderBase):
|
||||
|
||||
try:
|
||||
# 初始化Gemini客户端
|
||||
# 配置代理（如果提供了代理配置）
|
||||
self.proxies=None
|
||||
if self.http_proxy is not "" or self.https_proxy is not "":
|
||||
|
||||
self.proxies = {
|
||||
"http": self.http_proxy,
|
||||
"https": self.https_proxy,
|
||||
}
|
||||
logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}")
|
||||
# 使用猴子补丁修改 google-generativeai 库的请求会话
|
||||
|
||||
# 使用 session 对象配置 genai
|
||||
|
||||
genai.configure(api_key=self.api_key)
|
||||
self.model = genai.GenerativeModel(self.model_name)
|
||||
|
||||
@@ -46,26 +64,54 @@ class LLMProvider(LLMProviderBase):
|
||||
if content:
|
||||
chat_history.append({
|
||||
"role": role,
|
||||
"parts": [content]
|
||||
"parts": [{"text":content}]
|
||||
|
||||
})
|
||||
|
||||
# 获取当前消息
|
||||
current_msg = dialogue[-1]["content"]
|
||||
|
||||
# 创建新的聊天会话
|
||||
chat = self.model.start_chat(history=chat_history)
|
||||
# 构建请求体
|
||||
request_body = {
|
||||
"contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}],
|
||||
"generationConfig": self.generation_config
|
||||
}
|
||||
|
||||
# 发送消息并获取流式响应
|
||||
response = chat.send_message(
|
||||
current_msg,
|
||||
stream=True,
|
||||
generation_config=self.generation_config
|
||||
)
|
||||
# 构建请求URL
|
||||
url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}"
|
||||
|
||||
# 处理流式响应
|
||||
for chunk in response:
|
||||
if hasattr(chunk, 'text') and chunk.text:
|
||||
yield chunk.text
|
||||
# 构建请求头
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# 发送POST请求,经测试手动 request 无法使用 stream 模式
|
||||
if self.proxies:
|
||||
response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies)
|
||||
try:
|
||||
data = response.json() # 直接解析JSON
|
||||
if 'candidates' in data and data['candidates']:
|
||||
yield data['candidates'][0]['content']['parts'][0]['text']
|
||||
else:
|
||||
yield "未找到候选回复。"
|
||||
except json.JSONDecodeError as e:
|
||||
yield f"JSON解码错误：{e}"
|
||||
except Exception as e:
|
||||
yield f"发生错误：{e}"
|
||||
else:
|
||||
logger.bind(tag=TAG).info(f"Gemini stream mode ")
|
||||
chat = self.model.start_chat(history=chat_history)
|
||||
|
||||
# 发送消息并获取流式响应
|
||||
response = chat.send_message(
|
||||
current_msg,
|
||||
stream=True,
|
||||
generation_config=self.generation_config
|
||||
)
|
||||
# 处理流式响应
|
||||
for chunk in response:
|
||||
if hasattr(chunk, 'text') and chunk.text:
|
||||
yield chunk.text
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
@@ -78,3 +124,13 @@ class LLMProvider(LLMProviderBase):
|
||||
yield "【Gemini API key无效】"
|
||||
else:
|
||||
yield f"【Gemini服务响应异常: {error_msg}】"
|
||||
|
||||
|
||||
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
yield f"请求失败：{e}"
|
||||
except json.JSONDecodeError as e:
|
||||
yield f"JSON解码错误：{e}"
|
||||
except Exception as e:
|
||||
yield f"发生错误：{e}"
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import openai
|
||||
from config.logger import setup_logging
|
||||
from core.utils.util import check_model_key
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
@@ -11,6 +15,8 @@ class LLMProvider(LLMProviderBase):
|
||||
self.base_url = config.get("base_url")
|
||||
else:
|
||||
self.base_url = config.get("url")
|
||||
self.max_tokens = config.get("max_tokens", 500)
|
||||
|
||||
check_model_key("LLM", self.api_key)
|
||||
self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
|
||||
|
||||
@@ -19,9 +25,10 @@ class LLMProvider(LLMProviderBase):
|
||||
responses = self.client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=dialogue,
|
||||
stream=True
|
||||
stream=True,
|
||||
max_tokens=self.max_tokens,
|
||||
)
|
||||
|
||||
|
||||
is_active = True
|
||||
for chunk in responses:
|
||||
try:
|
||||
@@ -50,12 +57,12 @@ class LLMProvider(LLMProviderBase):
|
||||
model=self.model_name,
|
||||
messages=dialogue,
|
||||
stream=True,
|
||||
tools=functions,
|
||||
tools=functions
|
||||
)
|
||||
|
||||
|
||||
for chunk in stream:
|
||||
yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
|
||||
yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
|
||||
logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
|
||||
yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
|
||||
|
||||
@@ -5,6 +5,7 @@ import numpy as np
|
||||
import opuslib_next
|
||||
from pydub import AudioSegment
|
||||
from abc import ABC, abstractmethod
|
||||
from core.utils.tts import MarkdownCleaner
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
@@ -13,7 +14,7 @@ logger = setup_logging()
|
||||
class TTSProviderBase(ABC):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
self.delete_audio_file = delete_audio_file
|
||||
self.output_file = config.get("output_file")
|
||||
self.output_file = config.get("output_dir")
|
||||
|
||||
@abstractmethod
|
||||
def generate_filename(self):
|
||||
@@ -23,6 +24,7 @@ class TTSProviderBase(ABC):
|
||||
tmp_file = self.generate_filename()
|
||||
try:
|
||||
max_repeat_time = 5
|
||||
text = MarkdownCleaner.clean_markdown(text)
|
||||
while not os.path.exists(tmp_file) and max_repeat_time > 0:
|
||||
asyncio.run(self.text_to_speak(text, tmp_file))
|
||||
if not os.path.exists(tmp_file):
|
||||
@@ -47,7 +49,8 @@ class TTSProviderBase(ABC):
|
||||
file_type = os.path.splitext(audio_file_path)[1]
|
||||
if file_type:
|
||||
file_type = file_type.lstrip('.')
|
||||
audio = AudioSegment.from_file(audio_file_path, format=file_type)
|
||||
# 读取音频文件，-nostdin 参数：不要从标准输入读取数据，否则FFmpeg会阻塞
|
||||
audio = AudioSegment.from_file(audio_file_path, format=file_type, parameters=["-nostdin"])
|
||||
|
||||
# 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
|
||||
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
|
||||
|
||||
@@ -15,7 +15,7 @@ class TTSProvider(TTSProviderBase):
|
||||
self.headers = config.get("headers", {})
|
||||
self.params = config.get("params")
|
||||
self.format = config.get("format", "wav")
|
||||
self.output_file = config.get("output_file", "tmp/")
|
||||
self.output_file = config.get("output_dir", "tmp/")
|
||||
|
||||
def generate_filename(self):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}.{self.format}")
|
||||
|
||||
@@ -22,7 +22,7 @@ class TTSProvider(TTSProviderBase):
|
||||
self.cut_punc = config.get("cut_punc","")
|
||||
self.speed = config.get("speed", 1.0)
|
||||
self.inp_refs = config.get("inp_refs",[])
|
||||
self.sample_steps = config.get("inp_refs",32)
|
||||
self.sample_steps = config.get("sample_steps",32)
|
||||
self.if_sr = config.get("if_sr",False)
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ class TTSProvider(TTSProviderBase):
|
||||
self.voice = config.get("voice", "alloy")
|
||||
self.response_format = "wav"
|
||||
self.speed = config.get("speed", 1.0)
|
||||
self.output_file = config.get("output_file", "tmp/")
|
||||
self.output_file = config.get("output_dir", "tmp/")
|
||||
check_model_key("TTS", self.api_key)
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
|
||||
@@ -17,7 +17,7 @@ class TTSProvider(TTSProviderBase):
|
||||
self.volume_change_dB = config.get("volume_change_dB", 0)
|
||||
self.speed_factor = config.get("speed_factor", 1)
|
||||
self.stream = config.get("stream", False)
|
||||
self.output_file = config.get("output_file")
|
||||
self.output_file = config.get("output_dir")
|
||||
self.pitch_factor = config.get("pitch_factor", 0)
|
||||
self.format = config.get("format", "mp3")
|
||||
self.emotion = config.get("emotion", 1)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from config.logger import setup_logging
|
||||
import importlib
|
||||
@@ -14,4 +15,98 @@ def create_instance(class_name, *args, **kwargs):
|
||||
sys.modules[lib_name] = importlib.import_module(f'{lib_name}')
|
||||
return sys.modules[lib_name].TTSProvider(*args, **kwargs)
|
||||
|
||||
raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")
|
||||
raise ValueError(f"不支持的TTS类型: {class_name}，请检查该配置的type是否设置正确")
|
||||
|
||||
|
||||
class MarkdownCleaner:
|
||||
"""
|
||||
封装 Markdown 清理逻辑：直接用 MarkdownCleaner.clean_markdown(text) 即可
|
||||
"""
|
||||
# 公式字符
|
||||
NORMAL_FORMULA_CHARS = re.compile(r'[a-zA-Z\\^_{}\+\-\(\)\[\]=]')
|
||||
|
||||
@staticmethod
|
||||
def _replace_inline_dollar(m: re.Match) -> str:
|
||||
"""
|
||||
只要捕获到完整的 "$...$":
|
||||
- 如果内部有典型公式字符 => 去掉两侧 $
|
||||
- 否则 (纯数字/货币等) => 保留 "$...$"
|
||||
"""
|
||||
content = m.group(1)
|
||||
if MarkdownCleaner.NORMAL_FORMULA_CHARS.search(content):
|
||||
return content
|
||||
else:
|
||||
return m.group(0)
|
||||
|
||||
@staticmethod
|
||||
def _replace_table_block(match: re.Match) -> str:
|
||||
"""
|
||||
当匹配到一个整段表格块时，回调该函数。
|
||||
"""
|
||||
block_text = match.group('table_block')
|
||||
lines = block_text.strip('\n').split('\n')
|
||||
|
||||
parsed_table = []
|
||||
for line in lines:
|
||||
line_stripped = line.strip()
|
||||
if re.match(r'^\|\s*[-:]+\s*(\|\s*[-:]+\s*)+\|?$', line_stripped):
|
||||
continue
|
||||
columns = [col.strip() for col in line_stripped.split('|') if col.strip() != '']
|
||||
if columns:
|
||||
parsed_table.append(columns)
|
||||
|
||||
if not parsed_table:
|
||||
return ""
|
||||
|
||||
headers = parsed_table[0]
|
||||
data_rows = parsed_table[1:] if len(parsed_table) > 1 else []
|
||||
|
||||
lines_for_tts = []
|
||||
if len(parsed_table) == 1:
|
||||
# 只有一行
|
||||
only_line_str = ", ".join(parsed_table[0])
|
||||
lines_for_tts.append(f"单行表格：{only_line_str}")
|
||||
else:
|
||||
lines_for_tts.append(f"表头是：{', '.join(headers)}")
|
||||
for i, row in enumerate(data_rows, start=1):
|
||||
row_str_list = []
|
||||
for col_index, cell_val in enumerate(row):
|
||||
if col_index < len(headers):
|
||||
row_str_list.append(f"{headers[col_index]} = {cell_val}")
|
||||
else:
|
||||
row_str_list.append(cell_val)
|
||||
lines_for_tts.append(f"第 {i} 行：{', '.join(row_str_list)}")
|
||||
|
||||
return "\n".join(lines_for_tts) + "\n"
|
||||
|
||||
# 预编译所有正则表达式（按执行频率排序）
|
||||
# 这里要把 replace_xxx 的静态方法放在最前定义，以便在列表里能正确引用它们。
|
||||
REGEXES = [
|
||||
(re.compile(r'```.*?```', re.DOTALL), ''), # 代码块
|
||||
(re.compile(r'^#+\s*', re.MULTILINE), ''), # 标题
|
||||
(re.compile(r'(\*\*|__)(.*?)\1'), r'\2'), # 粗体
|
||||
(re.compile(r'(\*|_)(?=\S)(.*?)(?<=\S)\1'), r'\2'), # 斜体
|
||||
(re.compile(r'!\[.*?\]\(.*?\)'), ''), # 图片
|
||||
(re.compile(r'\[(.*?)\]\(.*?\)'), r'\1'), # 链接
|
||||
(re.compile(r'^\s*>+\s*', re.MULTILINE), ''), # 引用
|
||||
(
|
||||
re.compile(r'(?P<table_block>(?:^[^\n]*\|[^\n]*\n)+)', re.MULTILINE),
|
||||
_replace_table_block
|
||||
),
|
||||
(re.compile(r'^\s*[*+-]\s*', re.MULTILINE), '- '), # 列表
|
||||
(re.compile(r'\$\$.*?\$\$', re.DOTALL), ''), # 块级公式
|
||||
(
|
||||
re.compile(r'(?<![A-Za-z0-9])\$([^\n$]+)\$(?![A-Za-z0-9])'),
|
||||
_replace_inline_dollar
|
||||
),
|
||||
(re.compile(r'\n{2,}'), '\n'), # 多余空行
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def clean_markdown(text: str) -> str:
|
||||
"""
|
||||
主入口方法：依序执行所有正则，移除或替换 Markdown 元素
|
||||
"""
|
||||
for regex, replacement in MarkdownCleaner.REGEXES:
|
||||
text = regex.sub(replacement, text)
|
||||
return text.strip()
|
||||
Executable → Regular
+1
@@ -21,6 +21,7 @@ services:
|
||||
- ./data:/opt/xiaozhi-esp32-server/data
|
||||
# 模型文件挂接，很重要
|
||||
- ./models/SenseVoiceSmall/model.pt:/opt/xiaozhi-esp32-server/models/SenseVoiceSmall/model.pt
|
||||
|
||||
# #智控台还没开发好，还不能完全使用，会报很多错误，如果是非技术人员，请不要启用智控台服务
|
||||
# xiaozhi-esp32-server-web:
|
||||
# image: ghcr.nju.edu.cn/xinnan-tech/xiaozhi-esp32-server:web_latest
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import random
|
||||
import requests
|
||||
import xml.etree.ElementTree as ET
|
||||
from bs4 import BeautifulSoup
|
||||
from config.logger import setup_logging
|
||||
from plugins_func.register import register_function, ToolType, ActionResponse, Action
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
GET_NEWS_FUNCTION_DESC = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_news",
|
||||
"description": (
|
||||
"获取最新新闻，随机选择一条新闻进行播报。"
|
||||
"用户可以指定新闻类型，如社会新闻、科技新闻、国际新闻等。"
|
||||
"如果没有指定，默认播报社会新闻。"
|
||||
"用户可以要求获取详细内容，此时会获取新闻的详细内容。"
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"category": {
|
||||
"type": "string",
|
||||
"description": "新闻类别，例如社会、科技、国际。可选参数，如果不提供则使用默认类别"
|
||||
},
|
||||
"detail": {
|
||||
"type": "boolean",
|
||||
"description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容"
|
||||
},
|
||||
"lang": {
|
||||
"type": "string",
|
||||
"description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN"
|
||||
}
|
||||
},
|
||||
"required": ["lang"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def fetch_news_from_rss(rss_url):
|
||||
"""从RSS源获取新闻列表"""
|
||||
try:
|
||||
response = requests.get(rss_url)
|
||||
response.raise_for_status()
|
||||
|
||||
# 解析XML
|
||||
root = ET.fromstring(response.content)
|
||||
|
||||
# 查找所有item元素（新闻条目）
|
||||
news_items = []
|
||||
for item in root.findall('.//item'):
|
||||
title = item.find('title').text if item.find('title') is not None else "无标题"
|
||||
link = item.find('link').text if item.find('link') is not None else "#"
|
||||
description = item.find('description').text if item.find('description') is not None else "无描述"
|
||||
pubDate = item.find('pubDate').text if item.find('pubDate') is not None else "未知时间"
|
||||
|
||||
news_items.append({
|
||||
'title': title,
|
||||
'link': link,
|
||||
'description': description,
|
||||
'pubDate': pubDate
|
||||
})
|
||||
|
||||
return news_items
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"获取RSS新闻失败: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def fetch_news_detail(url):
|
||||
"""获取新闻详情页内容并总结"""
|
||||
try:
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
|
||||
# 尝试提取正文内容 (这里的选择器需要根据实际网站结构调整)
|
||||
content_div = soup.select_one('.content_desc, .content, article, .article-content')
|
||||
if content_div:
|
||||
paragraphs = content_div.find_all('p')
|
||||
content = '\n'.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()])
|
||||
return content
|
||||
else:
|
||||
# 如果找不到特定的内容区域，尝试获取所有段落
|
||||
paragraphs = soup.find_all('p')
|
||||
content = '\n'.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()])
|
||||
return content[:2000] # 限制长度
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"获取新闻详情失败: {e}")
|
||||
return "无法获取详细内容"
|
||||
|
||||
|
||||
def map_category(category_text):
|
||||
"""将用户输入的中文类别映射到配置文件中的类别键"""
|
||||
if not category_text:
|
||||
return None
|
||||
|
||||
# 类别映射字典，目前支持社会、国际、财经新闻，如需更多类型，参见配置文件
|
||||
category_map = {
|
||||
# 社会新闻
|
||||
"社会": "society",
|
||||
"社会新闻": "society",
|
||||
# 国际新闻
|
||||
"国际": "world",
|
||||
"国际新闻": "world",
|
||||
# 财经新闻
|
||||
"财经": "finance",
|
||||
"财经新闻": "finance",
|
||||
"金融": "finance",
|
||||
"经济": "finance"
|
||||
}
|
||||
|
||||
# 转换为小写并去除空格
|
||||
normalized_category = category_text.lower().strip()
|
||||
|
||||
# 返回映射结果，如果没有匹配项则返回原始输入
|
||||
return category_map.get(normalized_category, category_text)
|
||||
|
||||
|
||||
@register_function('get_news', GET_NEWS_FUNCTION_DESC, ToolType.SYSTEM_CTL)
|
||||
def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_CN"):
|
||||
"""获取新闻并随机选择一条进行播报，或获取上一条新闻的详细内容"""
|
||||
try:
|
||||
# 如果detail为True，获取上一条新闻的详细内容
|
||||
if detail:
|
||||
if not hasattr(conn, 'last_news_link') or not conn.last_news_link or 'link' not in conn.last_news_link:
|
||||
return ActionResponse(Action.REQLLM, "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", None)
|
||||
|
||||
link = conn.last_news_link.get('link')
|
||||
title = conn.last_news_link.get('title', '未知标题')
|
||||
|
||||
if link == '#':
|
||||
return ActionResponse(Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None)
|
||||
|
||||
logger.bind(tag=TAG).debug(f"获取新闻详情: {title}, URL={link}")
|
||||
|
||||
# 获取新闻详情
|
||||
detail_content = fetch_news_detail(link)
|
||||
|
||||
if not detail_content or detail_content == "无法获取详细内容":
|
||||
return ActionResponse(Action.REQLLM, f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", None)
|
||||
|
||||
# 构建详情报告
|
||||
detail_report = (
|
||||
f"根据下列数据，用{lang}回应用户的新闻详情查询请求：\n\n"
|
||||
f"新闻标题: {title}\n"
|
||||
f"详细内容: {detail_content}\n\n"
|
||||
f"(请对上述新闻内容进行总结，提取关键信息，以自然、流畅的方式向用户播报，"
|
||||
f"不要提及这是总结，就像是在讲述一个完整的新闻故事)"
|
||||
)
|
||||
|
||||
return ActionResponse(Action.REQLLM, detail_report, None)
|
||||
|
||||
# 否则，获取新闻列表并随机选择一条
|
||||
# 从配置中获取RSS URL
|
||||
rss_config = conn.config["plugins"]["get_news"]
|
||||
default_rss_url = rss_config.get("default_rss_url", "https://www.chinanews.com.cn/rss/society.xml")
|
||||
|
||||
# 将用户输入的类别映射到配置中的类别键
|
||||
mapped_category = map_category(category)
|
||||
|
||||
# 如果提供了类别，尝试从配置中获取对应的URL
|
||||
rss_url = default_rss_url
|
||||
if mapped_category and mapped_category in rss_config.get("category_urls", {}):
|
||||
rss_url = rss_config["category_urls"][mapped_category]
|
||||
|
||||
logger.bind(tag=TAG).info(f"获取新闻: 原始类别={category}, 映射类别={mapped_category}, URL={rss_url}")
|
||||
|
||||
# 获取新闻列表
|
||||
news_items = fetch_news_from_rss(rss_url)
|
||||
|
||||
if not news_items:
|
||||
return ActionResponse(Action.REQLLM, "抱歉，未能获取到新闻信息，请稍后再试。", None)
|
||||
|
||||
# 随机选择一条新闻
|
||||
selected_news = random.choice(news_items)
|
||||
|
||||
# 保存当前新闻链接到连接对象，以便后续查询详情
|
||||
if not hasattr(conn, 'last_news_link'):
|
||||
conn.last_news_link = {}
|
||||
conn.last_news_link = {
|
||||
'link': selected_news.get('link', '#'),
|
||||
'title': selected_news.get('title', '未知标题')
|
||||
}
|
||||
|
||||
# 构建新闻报告
|
||||
news_report = (
|
||||
f"根据下列数据，用{lang}回应用户的新闻查询请求：\n\n"
|
||||
f"新闻标题: {selected_news['title']}\n"
|
||||
f"发布时间: {selected_news['pubDate']}\n"
|
||||
f"新闻内容: {selected_news['description']}\n"
|
||||
f"(请以自然、流畅的方式向用户播报这条新闻，可以适当总结内容，"
|
||||
f"直接读出新闻即可，不需要额外多余的内容。"
|
||||
f"如果用户询问更多详情，告知用户可以说'请详细介绍这条新闻'获取更多内容)"
|
||||
)
|
||||
|
||||
return ActionResponse(Action.REQLLM, news_report, None)
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"获取新闻出错: {e}")
|
||||
return ActionResponse(Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None)
|
||||
@@ -6,7 +6,7 @@ get_time_function_desc = {
|
||||
"function": {
|
||||
"name": "get_time",
|
||||
"description": "获取当前时间、日期、星期几",
|
||||
"parameters": {}
|
||||
'parameters': {'type': 'object', 'properties': {}, 'required': []}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -112,7 +112,6 @@ def get_music_files(music_dir, music_ext):
|
||||
def initialize_music_handler(conn):
|
||||
global MUSIC_CACHE
|
||||
if MUSIC_CACHE == {}:
|
||||
logger.bind(tag=TAG).info(f"实例化音乐:")
|
||||
if "music" in conn.config:
|
||||
MUSIC_CACHE["music_config"] = conn.config["music"]
|
||||
MUSIC_CACHE["music_dir"] = os.path.abspath(
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
from config.logger import setup_logging
|
||||
from plugins_func.register import register_function, ToolType, ActionResponse, Action
|
||||
from core.handle.iotHandle import get_iot_status, send_iot_conn
|
||||
import asyncio
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
raise_and_lower_the_volume_function_desc = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "raise_and_lower_the_volume",
|
||||
"description": "用户觉得声音过高或过低，或者用户想提高或降低音量。比如用户说太大声了，参数为：lower，比如用户说提高音量，参数为：raise",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"description": "动作名称，要么是raise，要么是lower"
|
||||
}
|
||||
},
|
||||
"required": ["action"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@register_function('raise_and_lower_the_volume', raise_and_lower_the_volume_function_desc, ToolType.IOT_CTL)
|
||||
def raise_and_lower_the_volume(conn, action: str):
|
||||
"""
|
||||
获取当前设备音量
|
||||
"""
|
||||
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
_raise_and_lower_the_volume(conn, action),
|
||||
conn.loop
|
||||
)
|
||||
|
||||
try:
|
||||
new_volume = future.result() # 同步等待异步操作完成
|
||||
logger.bind(tag=TAG).info(f"音量操作完成: {new_volume}")
|
||||
response = f"音量已调整到{new_volume}"
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"音量操作失败: {e}")
|
||||
response = f"音量调整失败: {e}"
|
||||
|
||||
return ActionResponse(action=Action.RESPONSE, result="指令已接收", response=response)
|
||||
|
||||
|
||||
async def _raise_and_lower_the_volume(conn, action):
|
||||
volume = await get_iot_status(conn, "Speaker", "volume")
|
||||
if volume is None:
|
||||
raise Exception("你的设备不支持音量控制")
|
||||
if action == 'raise':
|
||||
volume += 10
|
||||
elif action == 'lower':
|
||||
volume -= 10
|
||||
# 限制音量范围在0到100之间
|
||||
if volume < 0:
|
||||
volume = 0
|
||||
elif volume > 100:
|
||||
volume = 100
|
||||
await send_iot_conn(conn, "Speaker", "SetVolume", {"volume": volume})
|
||||
return volume
|
||||
@@ -11,6 +11,7 @@ class ToolType(Enum):
|
||||
WAIT = (2, "调用工具，等待函数返回")
|
||||
CHANGE_SYS_PROMPT = (3, "修改系统提示词，切换角色性格或职责")
|
||||
SYSTEM_CTL = (4, "系统控制，影响正常的对话流程，如退出、播放音乐等，需要传递conn参数")
|
||||
IOT_CTL = (5, "IOT设备控制，需要传递conn参数")
|
||||
|
||||
def __init__(self, code, message):
|
||||
self.code = code
|
||||
@@ -18,6 +19,7 @@ class ToolType(Enum):
|
||||
|
||||
|
||||
class Action(Enum):
|
||||
ERROR = (-1, "错误")
|
||||
NOTFOUND = (0, "没有找到函数")
|
||||
NONE = (1, "啥也不干")
|
||||
RESPONSE = (2, "直接回复")
|
||||
@@ -40,8 +42,31 @@ class FunctionItem:
|
||||
self.func = func
|
||||
self.type = type
|
||||
|
||||
class DeviceTypeRegistry:
|
||||
"""设备类型注册表，用于管理IOT设备类型及其函数"""
|
||||
def __init__(self):
|
||||
self.type_functions = {} # type_signature -> {func_name: FunctionItem}
|
||||
|
||||
def generate_device_type_id(self, descriptor):
|
||||
"""通过设备能力描述生成类型ID"""
|
||||
properties = sorted(descriptor["properties"].keys())
|
||||
methods = sorted(descriptor["methods"].keys())
|
||||
# 使用属性和方法的组合作为设备类型的唯一标识
|
||||
type_signature = f"{descriptor['name']}:{','.join(properties)}:{','.join(methods)}"
|
||||
return type_signature
|
||||
|
||||
def get_device_functions(self, type_id):
|
||||
"""获取设备类型对应的所有函数"""
|
||||
return self.type_functions.get(type_id, {})
|
||||
|
||||
def register_device_type(self, type_id, functions):
|
||||
"""注册设备类型及其函数"""
|
||||
if type_id not in self.type_functions:
|
||||
self.type_functions[type_id] = functions
|
||||
|
||||
# 初始化函数注册字典
|
||||
all_function_registry = {}
|
||||
device_type_registry = DeviceTypeRegistry()
|
||||
|
||||
def register_function(name, desc, type=None):
|
||||
"""注册函数到函数注册字典的装饰器"""
|
||||
|
||||
@@ -19,4 +19,6 @@ loguru==0.7.3
|
||||
requests==2.32.3
|
||||
cozepy==0.12.0
|
||||
mem0ai==0.1.62
|
||||
bs4==0.0.2
|
||||
bs4==0.0.2
|
||||
modelscope==1.23.2
|
||||
sherpa_onnx==1.11.0
|
||||
|
||||
Reference in New Issue
Block a user