Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-29 15:13:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

update:合并最新代码

Browse Source
This commit is contained in:
hrz
2025-04-04 00:27:04 +08:00
parent c8a3d378b7
commit 0b4a4df1af
42 changed files with 4559 additions and 532 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/providers/intent/base.py
+11 -3
View File
@@ -10,14 +10,21 @@ class IntentProviderBase(ABC):
def __init__(self, config):
self.config = config
self.intent_options = config.get("intent_options", {
"handle_exit_intent": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候",
"play_music": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图",
"get_weather": "查询天气, 用户希望查询某个地点的天气情况",
"get_news": "查询新闻, 用户希望查询最新新闻或特定类型的新闻",
"get_lunar": "用于获取今天的阴历/农历和黄历信息",
"get_time": "获取今天日期或者当前时间信息",
"continue_chat": "继续聊天",
"end_chat": "结束聊天",
"play_music": "播放音乐"
})
def set_llm(self, llm):
self.llm = llm
logger.bind(tag=TAG).debug("Set LLM for intent provider")
# 获取模型名称和类型信息
model_name = getattr(llm, 'model_name', str(llm.__class__.__name__))
# 记录更详细的日志
logger.bind(tag=TAG).info(f"意图识别设置LLM: {model_name}")
@abstractmethod
async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
@@ -30,5 +37,6 @@ class IntentProviderBase(ABC):
- "继续聊天"
- "结束聊天"
- "播放音乐 歌名" 或 "随机播放音乐"
- "查询天气 地点名" 或 "查询天气 [当前位置]"
"""
pass
main/xiaozhi-server/core/providers/intent/function_call/function_call.py
+20
View File
@@ -0,0 +1,20 @@
from ..base import IntentProviderBase
from typing import List, Dict
from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
class IntentProvider(IntentProviderBase):
async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
"""
默认的意图识别实现，始终返回继续聊天
Args:
dialogue_history: 对话历史记录列表
text: 本次对话记录
Returns:
固定返回"继续聊天"
"""
logger.bind(tag=TAG).debug("Using functionCallProvider, always returning continue chat")
return self.intent_options["continue_chat"]
main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py
+178 -54
View File
@@ -3,6 +3,9 @@ from ..base import IntentProviderBase
from plugins_func.functions.play_music import initialize_music_handler
from config.logger import setup_logging
import re
import json
import hashlib
import time
TAG = __name__
logger = setup_logging()
@@ -13,6 +16,10 @@ class IntentProvider(IntentProviderBase):
super().__init__(config)
self.llm = None
self.promot = self.get_intent_system_prompt()
# 添加缓存管理
self.intent_cache = {} # 缓存意图识别结果
self.cache_expiry = 600 # 缓存有效期10分钟
self.cache_max_size = 100 # 最多缓存100个意图
def get_intent_system_prompt(self) -> str:
"""
@@ -22,62 +29,119 @@ class IntentProvider(IntentProviderBase):
"""
intent_list = []
"""
"continue_chat": "1.继续聊天, 除了播放音乐和结束聊天的时候的选项, 比如日常的聊天和问候, 对话等",
"end_chat": "2.结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候",
"play_music": "3.播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图"
"""
for key, value in self.intent_options.items():
if key == "play_music":
intent_list.append("3.播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图")
elif key == "end_chat":
intent_list.append("2.结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候")
elif key == "continue_chat":
intent_list.append("1.继续聊天, 除了播放音乐和结束聊天的时候的选项, 比如日常的聊天和问候, 对话等")
else:
intent_list.append(value)
# "如果是唱歌、听歌、播放音乐，请指定歌名，格式为'播放音乐 [识别出的歌名]'。\n"
# "如果听不出具体歌名，可以返回'随机播放音乐'。\n"
# "只需要返回意图结果的json，不要解释。"
# "返回格式如下：\n"
prompt = (
"你是一个意图识别助手。你需要根据和用户的对话记录，重点分析用户的最后一句话，判断用户意图属于以下哪一类(使用<start>和<end>标志)：\n"
"你是一个意图识别助手。请分析用户的最后一句话，判断用户意图属于以下哪一类：\n"
"<start>"
f"{', '.join(intent_list)}"
"<end>\n"
"你需要按照以下的步骤处理用户的对话"
"1. 思考出对话的意图是哪一类的"
"2. 属于1和2的意图, 直接返回，返回格式如下：\n"
"{intent: '用户意图'}\n"
"3. 属于3的意图，则继续分析用户希望播放的音乐\n"
"4. 如果无法识别出具体歌名，可以返回'随机播放音乐'\n"
"{intent: '播放音乐 [获取的音乐名字]'}\n"
"下面是几个处理的示例(思考的内容不返回, 只返回json部分, 无额外的内容)\n"
"```"
"处理步骤:"
"1. 思考意图类型，生成function_call格式"
"\n\n"
"返回格式示例：\n"
"1. 播放音乐意图: {\"function_call\": {\"name\": \"play_music\", \"arguments\": {\"song_name\": \"音乐名称\"}}}\n"
"2. 查询天气意图: {\"function_call\": {\"name\": \"get_weather\", \"arguments\": {\"location\": \"地点名称\", \"lang\": \"zh_CN\"}}}\n"
"3. 查询新闻意图: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"category\": \"新闻类别\", \"detail\": false, \"lang\": \"zh_CN\"}}}\n"
"4. 结束对话意图: {\"function_call\": {\"name\": \"handle_exit_intent\", \"arguments\": {\"say_goodbye\": \"goodbye\"}}}\n"
"5. 获取当天日期时间: {\"function_call\": {\"name\": \"get_time\"}}\n"
"6. 获取当前黄历意图: {\"function_call\": {\"name\": \"get_lunar\"}}\n"
"7. 继续聊天意图: {\"function_call\": {\"name\": \"continue_chat\"}}\n"
"\n"
"注意:\n"
"- 播放音乐：无歌名时，song_name设为\"random\"\n"
"- 查询天气：无地点时，location设为null\n"
"- 查询新闻：无类别时，category设为null；查询详情时，detail设为true\n"
"- 如果没有明显的意图，应按照继续聊天意图处理\n"
"- 只返回纯JSON，不要任何其他内容\n"
"\n"
"示例分析:\n"
"```\n"
"用户: 你好小智\n"
"返回: {\"function_call\": {\"name\": \"continue_chat\"}}\n"
"```\n"
"```\n"
"用户: 你今天怎么样?\n"
"思考(不返回): 用户发来的数据是一个问候语，属于继续聊天的意图, 是种类1, 种类1的需求是直接返回\n"
"返回结果: {intent: '继续聊天'}\n"
"```"
"用户: 我今天有点累了, 我们明天再聊吧\n"
"思考(不返回): 用户表达了今天不想继续对话，属于结束聊天的意图, 是种类2, 种类2的需求是直接返回\n"
"返回结果: {intent: '结束聊天'}\n"
"```"
"用户: 我今天有点累了, 我们明天再聊吧\n"
"思考(不返回): 用户表达了今天不想继续对话，属于结束聊天的意图, 是种类2, 种类2的需求是直接返回\n"
"返回结果: {intent: '结束聊天'}\n"
"```"
"用户: 你可以播放一首中秋月给我听吗\n"
"思考(不返回): 用户表达了想听音乐的续签，属于播放音乐的意图, 是种类3, 种类3的需求需要继续判断播放的音乐, 这里用户希望的歌曲名明确给出是中秋月\n"
"返回结果: {intent: '播放音乐 [中秋月]'}\n"
"```"
"你现在可以使用的音乐的名称如下(使用<start>和<end>标志):\n"
"返回: {\"function_call\": {\"name\": \"continue_chat\"}}\n"
"```\n"
"```\n"
"用户: 现在是几号了?现在几点了？\n"
"返回: {\"function_call\": {\"name\": \"get_time\"}}\n"
"```\n"
"```\n"
"用户: 今天农历是多少？\n"
"返回: {\"function_call\": {\"name\": \"get_lunar\"}}\n"
"```\n"
"```\n"
"用户: 我们明天再聊吧\n"
"返回: {\"function_call\": {\"name\": \"handle_exit_intent\"}}\n"
"```\n"
"```\n"
"用户: 播放中秋月\n"
"返回: {\"function_call\": {\"name\": \"play_music\", \"arguments\": {\"song_name\": \"中秋月\"}}}\n"
"```\n"
"```\n"
"用户: 北京天气怎么样\n"
"返回: {\"function_call\": {\"name\": \"get_weather\", \"arguments\": {\"location\": \"北京\", \"lang\": \"zh_CN\"}}}\n"
"```\n"
"```\n"
"用户: 今天天气怎么样\n"
"返回: {\"function_call\": {\"name\": \"get_weather\", \"arguments\": {\"location\": null, \"lang\": \"zh_CN\"}}}\n"
"```\n"
"```\n"
"用户: 播报财经新闻\n"
"返回: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"category\": \"财经\", \"detail\": false, \"lang\": \"zh_CN\"}}}\n"
"```\n"
"```\n"
"用户: 有什么最新新闻\n"
"返回: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"category\": null, \"detail\": false, \"lang\": \"zh_CN\"}}}\n"
"```\n"
"```\n"
"用户: 详细介绍一下这条新闻\n"
"返回: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"detail\": true, \"lang\": \"zh_CN\"}}}\n"
"```\n"
"可用的音乐名称:\n"
)
return prompt
def clean_cache(self):
"""清理过期缓存"""
now = time.time()
# 找出过期键
expired_keys = [k for k, v in self.intent_cache.items() if now - v['timestamp'] > self.cache_expiry]
for key in expired_keys:
del self.intent_cache[key]
# 如果缓存太大，移除最旧的条目
if len(self.intent_cache) > self.cache_max_size:
# 按时间戳排序并保留最新的条目
sorted_items = sorted(self.intent_cache.items(), key=lambda x: x[1]['timestamp'])
for key, _ in sorted_items[:len(sorted_items) - self.cache_max_size]:
del self.intent_cache[key]
async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
if not self.llm:
raise ValueError("LLM provider not set")
# 记录整体开始时间
total_start_time = time.time()
# 打印使用的模型信息
model_info = getattr(self.llm, 'model_name', str(self.llm.__class__.__name__))
logger.bind(tag=TAG).info(f"使用意图识别模型: {model_info}")
# 计算缓存键
cache_key = hashlib.md5(text.encode()).hexdigest()
# 检查缓存
if cache_key in self.intent_cache:
cache_entry = self.intent_cache[cache_key]
# 检查缓存是否过期
if time.time() - cache_entry['timestamp'] <= self.cache_expiry:
cache_time = time.time() - total_start_time
logger.bind(tag=TAG).info(f"使用缓存的意图: {cache_key} -> {cache_entry['intent']}, 耗时: {cache_time:.4f}秒")
return cache_entry['intent']
# 清理缓存
self.clean_cache()
# 构建用户最后一句话的提示
msgStr = ""
@@ -94,18 +158,78 @@ class IntentProvider(IntentProviderBase):
music_file_names = music_config["music_file_names"]
prompt_music = f"{self.promot}\n<start>{music_file_names}\n<end>"
logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
# 记录预处理完成时间
preprocess_time = time.time() - total_start_time
logger.bind(tag=TAG).debug(f"意图识别预处理耗时: {preprocess_time:.4f}秒")
# 使用LLM进行意图识别
llm_start_time = time.time()
logger.bind(tag=TAG).info(f"开始LLM意图识别调用, 模型: {model_info}")
intent = self.llm.response_no_stream(
system_prompt=prompt_music,
user_prompt=user_prompt
)
# 使用正则表达式提取大括号中的内容
# 使用正则表达式提取 {} 中的内容
match = re.search(r'\{.*?\}', intent)
# 记录LLM调用完成时间
llm_time = time.time() - llm_start_time
logger.bind(tag=TAG).info(f"LLM意图识别完成, 模型: {model_info}, 调用耗时: {llm_time:.4f}秒")
# 记录后处理开始时间
postprocess_start_time = time.time()
# 清理和解析响应
intent = intent.strip()
# 尝试提取JSON部分
match = re.search(r'\{.*\}', intent, re.DOTALL)
if match:
result = match.group(0)
intent = result
else:
intent = "{intent: '继续聊天'}"
logger.bind(tag=TAG).info(f"Detected intent: {intent}")
return intent.strip()
intent = match.group(0)
# 记录总处理时间
total_time = time.time() - total_start_time
logger.bind(tag=TAG).info(f"【意图识别性能】模型: {model_info}, 总耗时: {total_time:.4f}秒, LLM调用: {llm_time:.4f}秒, 查询: '{text[:20]}...'")
# 尝试解析为JSON
try:
intent_data = json.loads(intent)
# 如果包含function_call，则格式化为适合处理的格式
if "function_call" in intent_data:
function_data = intent_data["function_call"]
function_name = function_data.get("name")
function_args = function_data.get("arguments", {})
# 记录识别到的function call
logger.bind(tag=TAG).info(f"识别到function call: {function_name}, 参数: {function_args}")
# 添加到缓存
self.intent_cache[cache_key] = {
'intent': intent,
'timestamp': time.time()
}
# 后处理时间
postprocess_time = time.time() - postprocess_start_time
logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
# 确保返回完全序列化的JSON字符串
return intent
else:
# 添加到缓存
self.intent_cache[cache_key] = {
'intent': intent,
'timestamp': time.time()
}
# 后处理时间
postprocess_time = time.time() - postprocess_start_time
logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
# 返回普通意图
return intent
except json.JSONDecodeError:
# 后处理时间
postprocess_time = time.time() - postprocess_start_time
logger.bind(tag=TAG).error(f"无法解析意图JSON: {intent}, 后处理耗时: {postprocess_time:.4f}秒")
# 如果解析失败，默认返回继续聊天意图
return "{\"intent\": \"继续聊天\"}"
main/xiaozhi-server/core/providers/llm/AliBL/AliBL.py
+52
View File
@@ -0,0 +1,52 @@
from config.logger import setup_logging
from http import HTTPStatus
from dashscope import Application
from core.providers.llm.base import LLMProviderBase
TAG = __name__
logger = setup_logging()
class LLMProvider(LLMProviderBase):
def __init__(self, config):
self.api_key = config["api_key"]
self.app_id = config["app_id"]
self.base_url = config.get("base_url")
self.is_No_prompt = config.get("is_no_prompt")
self.memory_id = config.get("ali_memory_id")
def response(self, session_id, dialogue):
try:
# 处理dialogue
if self.is_No_prompt:
dialogue.pop(0)
logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的dialogue: {dialogue}")
# 构造调用参数
call_params = {
"api_key": self.api_key,
"app_id": self.app_id,
"session_id": session_id,
"messages": dialogue
}
if self.memory_id != False:
# 百练memory需要prompt参数
prompt = dialogue[-1].get("content")
call_params["memory_id"] = self.memory_id
call_params["prompt"] = prompt
logger.bind(tag=TAG).debug(f"【阿里百练API服务】处理后的prompt: {prompt}")
responses = Application.call(**call_params)
if responses.status_code != HTTPStatus.OK:
logger.bind(tag=TAG).error(
f"code={responses.status_code}, "
f"message={responses.message}, "
f"请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code"
)
yield "【阿里百练API服务响应异常】"
else:
logger.bind(tag=TAG).debug(f"【阿里百练API服务】构造参数: {call_params}")
yield responses.output.text
except Exception as e:
logger.bind(tag=TAG).error(f"【阿里百练API服务】响应异常: {e}")
yield "【LLM服务响应异常】"
main/xiaozhi-server/core/providers/llm/dify/dify.py
+28 -26
View File
@@ -6,11 +6,12 @@ from core.providers.llm.base import LLMProviderBase
TAG = __name__
logger = setup_logging()
class LLMProvider(LLMProviderBase):
def __init__(self, config):
self.api_key = config["api_key"]
self.mode = config.get("mode", "chat-messages")
self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip("/")
self.session_conversation_map = {} # 存储session_id和conversation_id的映射
def response(self, session_id, dialogue):
@@ -22,57 +23,58 @@ class LLMProvider(LLMProviderBase):
# 发起流式请求
if self.mode == "chat-messages":
request_json = {
"query": last_msg["content"],
"response_mode": "streaming",
"user": session_id,
"inputs": {},
"conversation_id": conversation_id
}
"query": last_msg["content"],
"response_mode": "streaming",
"user": session_id,
"inputs": {},
"conversation_id": conversation_id,
}
elif self.mode == "workflows/run":
request_json = {
"inputs": {"query": last_msg["content"]},
"response_mode": "streaming",
"user": session_id
"user": session_id,
}
elif self.mode == "completion-messages":
request_json = {
"inputs": {"query": last_msg["content"]},
"response_mode": "streaming",
"user": session_id
"user": session_id,
}
with requests.post(
f"{self.base_url}/{self.mode}",
headers={"Authorization": f"Bearer {self.api_key}"},
json=request_json,
stream=True
f"{self.base_url}/{self.mode}",
headers={"Authorization": f"Bearer {self.api_key}"},
json=request_json,
stream=True,
) as r:
if self.mode == "chat-messages":
for line in r.iter_lines():
if line.startswith(b'data: '):
if line.startswith(b"data: "):
event = json.loads(line[6:])
# 如果没有找到conversation_id，则获取此次conversation_id
if not conversation_id:
conversation_id = event.get('conversation_id')
self.session_conversation_map[session_id] = conversation_id # 更新映射
if event.get('answer'):
yield event['answer']
conversation_id = event.get("conversation_id")
self.session_conversation_map[session_id] = (
conversation_id # 更新映射
)
if event.get("answer"):
yield event["answer"]
elif self.mode == "workflows/run":
for line in r.iter_lines():
# logger.bind(tag=TAG).info(f"chat message response: {line}")
if line.startswith(b'data: '):
if line.startswith(b"data: "):
event = json.loads(line[6:])
if event.get('event') == "workflow_finished":
if event['data']['status'] == "succeeded":
yield event['data']['outputs']['answer']
if event.get("event") == "workflow_finished":
if event["data"]["status"] == "succeeded":
yield event["data"]["outputs"]["answer"]
else:
yield "【服务响应异常】"
elif self.mode == "completion-messages":
for line in r.iter_lines():
if line.startswith(b'data: '):
if line.startswith(b"data: "):
event = json.loads(line[6:])
if event.get('answer'):
yield event['answer']
if event.get("answer"):
yield event["answer"]
except Exception as e:
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py
-62
View File
@@ -1,62 +0,0 @@
import requests
from requests.exceptions import RequestException
from config.logger import setup_logging
from core.providers.llm.base import LLMProviderBase
TAG = __name__
logger = setup_logging()
class LLMProvider(LLMProviderBase):
def __init__(self, config):
self.agent_id = config.get("agent_id") # 对应 agent_id
self.api_key = config.get("api_key")
self.base_url = config.get("base_url", config.get("url")) # 默认使用 base_url
self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL
def response(self, session_id, dialogue):
print(dialogue)
try:
# home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
# 提取最后一个 role 为 'user' 的 content
input_text = None
if isinstance(dialogue, list): # 确保 dialogue 是一个列表
# 逆序遍历，找到最后一个 role 为 'user' 的消息
for message in reversed(dialogue):
if message.get("role") == "user": # 找到 role 为 'user' 的消息
input_text = message.get("content", "")
break # 找到后立即退出循环
# 构造请求数据
payload = {
"text": input_text,
"agent_id": self.agent_id,
"conversation_id": session_id # 使用 session_id 作为 conversation_id
}
# 设置请求头
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json"
}
# 发起 POST 请求
response = requests.post(self.api_url, json=payload, headers=headers)
# 检查请求是否成功
response.raise_for_status()
# 解析返回数据
data = response.json()
speech = data.get("response", {}).get("speech", {}).get("plain", {}).get("speech", "")
# 返回生成的内容
if speech:
yield speech
else:
logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
except RequestException as e:
logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
except Exception as e:
logger.bind(tag=TAG).error(f"生成响应时出错: {e}")
main/xiaozhi-server/core/providers/llm/openai/openai.py
+1 -1
View File
@@ -64,5 +64,5 @@ class LLMProvider(LLMProviderBase):
yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
main/xiaozhi-server/core/providers/llm/xinference/xinference.py
+85
View File
@@ -0,0 +1,85 @@
from config.logger import setup_logging
from openai import OpenAI
import json
from core.providers.llm.base import LLMProviderBase
TAG = __name__
logger = setup_logging()
class LLMProvider(LLMProviderBase):
def __init__(self, config):
self.model_name = config.get("model_name")
self.base_url = config.get("base_url", "http://localhost:9997")
# Initialize OpenAI client with Xinference base URL
# 如果没有v1，增加v1
if not self.base_url.endswith("/v1"):
self.base_url = f"{self.base_url}/v1"
logger.bind(tag=TAG).info(f"Initializing Xinference LLM provider with model: {self.model_name}, base_url: {self.base_url}")
try:
self.client = OpenAI(
base_url=self.base_url,
api_key="xinference" # Xinference has a similar setup to Ollama where it doesn't need an actual key
)
logger.bind(tag=TAG).info("Xinference client initialized successfully")
except Exception as e:
logger.bind(tag=TAG).error(f"Error initializing Xinference client: {e}")
raise
def response(self, session_id, dialogue):
try:
logger.bind(tag=TAG).debug(f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}")
responses = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
stream=True
)
is_active=True
for chunk in responses:
try:
delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
content = delta.content if hasattr(delta, 'content') else ''
if content:
if '<think>' in content:
is_active = False
content = content.split('<think>')[0]
if '</think>' in content:
is_active = True
content = content.split('</think>')[-1]
if is_active:
yield content
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
except Exception as e:
logger.bind(tag=TAG).error(f"Error in Xinference response generation: {e}")
yield "【Xinference服务响应异常】"
def response_with_functions(self, session_id, dialogue, functions=None):
try:
logger.bind(tag=TAG).debug(f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}")
if functions:
logger.bind(tag=TAG).debug(f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}")
stream = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
stream=True,
tools=functions,
)
for chunk in stream:
delta = chunk.choices[0].delta
content = delta.content
tool_calls = delta.tool_calls
if content:
yield content, tool_calls
elif tool_calls:
yield None, tool_calls
except Exception as e:
logger.bind(tag=TAG).error(f"Error in Xinference function call: {e}")
yield {"type": "content", "content": f"【Xinference服务响应异常: {str(e)}】"}
main/xiaozhi-server/core/providers/tts/base.py
+9 -4
View File
@@ -121,9 +121,15 @@ class TTSProviderBase(ABC):
async def tts_one_sentence(self, text, u_id=None):
if not u_id:
u_id = str(uuid.uuid4()).replace("-", "")
self.tts.tts_text_queue.put(TTSMessageDTO(u_id=u_id, msg_type=MsgType.START_TTS_REQUEST, content=''))
self.tts.tts_text_queue.put(TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_REQUEST, content=text))
self.tts.tts_text_queue.put(TTSMessageDTO(u_id=u_id, msg_type=MsgType.STOP_TTS_REQUEST, content=text))
self.tts.tts_text_queue.put(
TTSMessageDTO(u_id=u_id, msg_type=MsgType.START_TTS_REQUEST, content="")
)
self.tts.tts_text_queue.put(
TTSMessageDTO(u_id=u_id, msg_type=MsgType.TTS_TEXT_REQUEST, content=text)
)
self.tts.tts_text_queue.put(
TTSMessageDTO(u_id=u_id, msg_type=MsgType.STOP_TTS_REQUEST, content=text)
)
def _enable_two_way_tts(self):
while not self.stop_event.is_set():
@@ -273,7 +279,6 @@ class TTSProviderBase(ABC):
def generate_filename(self):
pass
@abstractmethod
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
pass
main/xiaozhi-server/core/providers/tts/huoshan.py
+151 -92
View File
@@ -9,7 +9,6 @@ import json
import base64
import requests
from datetime import datetime
from mutagen.oggopus import OggOpus
import websockets
@@ -76,14 +75,16 @@ EVENT_TTSResponse = 352
class Header:
def __init__(self,
protocol_version=PROTOCOL_VERSION,
header_size=DEFAULT_HEADER_SIZE,
message_type: int = 0,
message_type_specific_flags: int = 0,
serial_method: int = NO_SERIALIZATION,
compression_type: int = COMPRESSION_NO,
reserved_data=0):
def __init__(
self,
protocol_version=PROTOCOL_VERSION,
header_size=DEFAULT_HEADER_SIZE,
message_type: int = 0,
message_type_specific_flags: int = 0,
serial_method: int = NO_SERIALIZATION,
compression_type: int = COMPRESSION_NO,
reserved_data=0,
):
self.header_size = header_size
self.protocol_version = protocol_version
self.message_type = message_type
@@ -93,16 +94,20 @@ class Header:
self.reserved_data = reserved_data
def as_bytes(self) -> bytes:
return bytes([
(self.protocol_version << 4) | self.header_size,
(self.message_type << 4) | self.message_type_specific_flags,
(self.serial_method << 4) | self.compression_type,
self.reserved_data
])
return bytes(
[
(self.protocol_version << 4) | self.header_size,
(self.message_type << 4) | self.message_type_specific_flags,
(self.serial_method << 4) | self.compression_type,
self.reserved_data,
]
)
class Optional:
def __init__(self, event: int = EVENT_NONE, sessionId: str = None, sequence: int = None):
def __init__(
self, event: int = EVENT_NONE, sessionId: str = None, sequence: int = None
):
self.event = event
self.sessionId = sessionId
self.errorCode: int = 0
@@ -160,45 +165,57 @@ class TTSProvider(TTSProviderBase):
"X-Api-Resource-Id": self.resource_id,
"X-Api-Connect-Id": uuid.uuid4(),
}
self.ws = await websockets.connect(self.ws_url, additional_headers=ws_header, max_size=1000000000)
tts_priority = threading.Thread(target=self._start_monitor_tts_response_thread(), daemon=True)
self.ws = await websockets.connect(
self.ws_url, additional_headers=ws_header, max_size=1000000000
)
tts_priority = threading.Thread(
target=self._start_monitor_tts_response_thread(), daemon=True
)
tts_priority.start()
def generate_filename(self, extension=".wav"):
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
return os.path.join(
self.output_file,
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
)
async def send_event(self, header: bytes, optional: bytes | None = None,
payload: bytes = None):
async def send_event(
self, header: bytes, optional: bytes | None = None, payload: bytes = None
):
full_client_request = bytearray(header)
if optional is not None:
full_client_request.extend(optional)
if payload is not None:
payload_size = len(payload).to_bytes(4, 'big', signed=True)
payload_size = len(payload).to_bytes(4, "big", signed=True)
full_client_request.extend(payload_size)
full_client_request.extend(payload)
await self.ws.send(full_client_request)
async def send_text(self, speaker: str, text: str, session_id):
header = Header(message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON).as_bytes()
header = Header(
message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON,
).as_bytes()
optional = Optional(event=EVENT_TaskRequest, sessionId=session_id).as_bytes()
payload = self.get_payload_bytes(event=EVENT_TaskRequest, text=text, speaker=speaker)
payload = self.get_payload_bytes(
event=EVENT_TaskRequest, text=text, speaker=speaker
)
return await self.send_event(header, optional, payload)
# 读取 res 数组某段 字符串内容
def read_res_content(self, res: bytes, offset: int):
content_size = int.from_bytes(res[offset: offset + 4], "big", signed=True)
content_size = int.from_bytes(res[offset : offset + 4], "big", signed=True)
offset += 4
content = str(res[offset: offset + content_size])
content = str(res[offset : offset + content_size])
offset += content_size
return content, offset
# 读取 payload
def read_res_payload(self, res: bytes, offset: int):
payload_size = int.from_bytes(res[offset: offset + 4], "big", signed=True)
payload_size = int.from_bytes(res[offset : offset + 4], "big", signed=True)
offset += 4
payload = res[offset: offset + payload_size]
payload = res[offset : offset + payload_size]
offset += payload_size
return payload, offset
@@ -211,11 +228,11 @@ class TTSProvider(TTSProviderBase):
header = response.header
num = 0b00001111
header.protocol_version = res[0] >> 4 & num
header.header_size = res[0] & 0x0f
header.header_size = res[0] & 0x0F
header.message_type = (res[1] >> 4) & num
header.message_type_specific_flags = res[1] & 0x0f
header.message_type_specific_flags = res[1] & 0x0F
header.serialization_method = res[2] >> num
header.message_compression = res[2] & 0x0f
header.message_compression = res[2] & 0x0F
header.reserved = res[3]
#
offset = 4
@@ -231,78 +248,101 @@ class TTSProvider(TTSProviderBase):
elif optional.event == EVENT_ConnectionStarted:
optional.connectionId, offset = self.read_res_content(res, offset)
elif optional.event == EVENT_ConnectionFailed:
optional.response_meta_json, offset = self.read_res_content(res, offset)
elif (optional.event == EVENT_SessionStarted
or optional.event == EVENT_SessionFailed
or optional.event == EVENT_SessionFinished):
optional.response_meta_json, offset = self.read_res_content(
res, offset
)
elif (
optional.event == EVENT_SessionStarted
or optional.event == EVENT_SessionFailed
or optional.event == EVENT_SessionFinished
):
optional.sessionId, offset = self.read_res_content(res, offset)
optional.response_meta_json, offset = self.read_res_content(res, offset)
optional.response_meta_json, offset = self.read_res_content(
res, offset
)
else:
optional.sessionId, offset = self.read_res_content(res, offset)
response.payload, offset = self.read_res_payload(res, offset)
elif header.message_type == ERROR_INFORMATION:
optional.errorCode = int.from_bytes(res[offset:offset + 4], "big", signed=True)
optional.errorCode = int.from_bytes(
res[offset : offset + 4], "big", signed=True
)
offset += 4
response.payload, offset = self.read_res_payload(res, offset)
return response
async def start_connection(self):
header = Header(message_type=FULL_CLIENT_REQUEST, message_type_specific_flags=MsgTypeFlagWithEvent).as_bytes()
header = Header(
message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
).as_bytes()
optional = Optional(event=EVENT_Start_Connection).as_bytes()
payload = str.encode("{}")
return await self.send_event(header, optional, payload)
def print_response(self, res, tag_msg: str):
logger.bind(tag=TAG).info(f'===>{tag_msg} header:{res.header.__dict__}')
logger.bind(tag=TAG).info(f'===>{tag_msg} optional:{res.optional.__dict__}')
logger.bind(tag=TAG).info(f"===>{tag_msg} header:{res.header.__dict__}")
logger.bind(tag=TAG).info(f"===>{tag_msg} optional:{res.optional.__dict__}")
def get_payload_bytes(self, uid='1234', event=EVENT_NONE, text='', speaker='', audio_format='pcm',
audio_sample_rate=16000):
return str.encode(json.dumps(
{
"user": {"uid": uid},
"event": event,
"namespace": "BidirectionalTTS",
"req_params": {
"text": text,
"speaker": speaker,
"audio_params": {
"format": audio_format,
"sample_rate": audio_sample_rate
}
def get_payload_bytes(
self,
uid="1234",
event=EVENT_NONE,
text="",
speaker="",
audio_format="pcm",
audio_sample_rate=16000,
):
return str.encode(
json.dumps(
{
"user": {"uid": uid},
"event": event,
"namespace": "BidirectionalTTS",
"req_params": {
"text": text,
"speaker": speaker,
"audio_params": {
"format": audio_format,
"sample_rate": audio_sample_rate,
},
},
}
}
))
)
)
async def finish_connection(self):
header = Header(message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON
).as_bytes()
header = Header(
message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON,
).as_bytes()
optional = Optional(event=EVENT_FinishConnection).as_bytes()
payload = str.encode('{}')
payload = str.encode("{}")
await self.send_event(header, optional, payload)
return
async def start_session(self, session_id):
self.stop_event_response.clear()
header = Header(message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON
).as_bytes()
header = Header(
message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON,
).as_bytes()
optional = Optional(event=EVENT_StartSession, sessionId=session_id).as_bytes()
payload = self.get_payload_bytes(event=EVENT_StartSession, speaker=self.speaker)
await self.send_event(header, optional, payload)
async def finish_session(self, session_id):
self.stop_event_response.set()
header = Header(message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON
).as_bytes()
header = Header(
message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON,
).as_bytes()
optional = Optional(event=EVENT_FinishSession, sessionId=session_id).as_bytes()
payload = str.encode('{}')
payload = str.encode("{}")
await self.send_event(header, optional, payload)
return
@@ -327,59 +367,78 @@ class TTSProvider(TTSProviderBase):
def _start_monitor_tts_response_thread(self):
# 初始化链接
asyncio.run_coroutine_threadsafe(self._start_monitor_tts_response(), loop=self.loop)
asyncio.run_coroutine_threadsafe(
self._start_monitor_tts_response(), loop=self.loop
)
async def _start_monitor_tts_response(self):
chunk_total = b''
chunk_total = b""
while True:
try:
msg = await self.ws.recv() # 确保 `recv()` 运行在同一个 event loop
res = self.parser_response(msg)
self.print_response(res, 'send_text res:')
self.print_response(res, "send_text res:")
if res.optional.event == EVENT_TTSResponse and res.header.message_type == AUDIO_ONLY_RESPONSE:
logger.bind(tag=TAG).info(f'推送数据到队列里面～～')
if (
res.optional.event == EVENT_TTSResponse
and res.header.message_type == AUDIO_ONLY_RESPONSE
):
logger.bind(tag=TAG).info(f"推送数据到队列里面～～")
opus_datas = self.wav_to_opus_data_audio_raw(res.payload)
self.tts_audio_queue.put(
TTSMessageDTO(
u_id=self.u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas,
tts_finish_text="", sentence_type=None, duration=0
u_id=self.u_id,
msg_type=MsgType.TTS_TEXT_RESPONSE,
content=opus_datas,
tts_finish_text="",
sentence_type=None,
duration=0,
)
)
elif res.optional.event == EVENT_TTSSentenceStart:
json_data = json.loads(res.payload.decode('utf-8'))
json_data = json.loads(res.payload.decode("utf-8"))
self.tts_text = json_data.get("text", "")
logger.bind(tag=TAG).info(f'句子开始～～{self.tts_text}')
logger.bind(tag=TAG).info(f"句子开始～～{self.tts_text}")
self.tts_audio_queue.put(
TTSMessageDTO(
u_id=self.u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=[],
u_id=self.u_id,
msg_type=MsgType.TTS_TEXT_RESPONSE,
content=[],
tts_finish_text=self.tts_text,
sentence_type=SentenceType.SENTENCE_START
sentence_type=SentenceType.SENTENCE_START,
)
)
elif res.optional.event == EVENT_TTSSentenceEnd:
logger.bind(tag=TAG).info(f'句子结束～～{self.tts_text}')
logger.bind(tag=TAG).info(f"句子结束～～{self.tts_text}")
self.tts_audio_queue.put(
TTSMessageDTO(
u_id=self.u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=[],
u_id=self.u_id,
msg_type=MsgType.TTS_TEXT_RESPONSE,
content=[],
tts_finish_text=self.tts_text,
sentence_type=SentenceType.SENTENCE_END
sentence_type=SentenceType.SENTENCE_END,
)
)
elif res.optional.event == EVENT_SessionFinished:
logger.bind(tag=TAG).info(f'会话结束～～,最后一句补零')
opus_datas = self.wav_to_opus_data_audio_raw(b'', is_end=True)
logger.bind(tag=TAG).info(f"会话结束～～,最后一句补零")
opus_datas = self.wav_to_opus_data_audio_raw(b"", is_end=True)
self.tts_audio_queue.put(
TTSMessageDTO(
u_id=self.u_id, msg_type=MsgType.TTS_TEXT_RESPONSE, content=opus_datas,
tts_finish_text="", sentence_type=None, duration=0
u_id=self.u_id,
msg_type=MsgType.TTS_TEXT_RESPONSE,
content=opus_datas,
tts_finish_text="",
sentence_type=None,
duration=0,
)
)
self.tts_audio_queue.put(
TTSMessageDTO(
u_id=self.u_id, msg_type=MsgType.STOP_TTS_RESPONSE, content=[],
u_id=self.u_id,
msg_type=MsgType.STOP_TTS_RESPONSE,
content=[],
tts_finish_text=self.tts_text,
sentence_type=SentenceType.SENTENCE_END
sentence_type=SentenceType.SENTENCE_END,
)
)
else:
main/xiaozhi-server/core/providers/tts/tencent.py
+156
View File
@@ -0,0 +1,156 @@
import hashlib
import hmac
import os
import time
import uuid
import json
import base64
import requests
from datetime import datetime, timezone
from core.providers.tts.base import TTSProviderBase
class TTSProvider(TTSProviderBase):
def __init__(self, config, delete_audio_file):
super().__init__(config, delete_audio_file)
self.appid = config.get("appid")
self.secret_id = config.get("secret_id")
self.secret_key = config.get("secret_key")
self.voice = config.get("voice")
self.api_url = "https://tts.tencentcloudapi.com" # 正确的API端点
self.region = config.get("region")
self.output_file = config.get("output_dir")
def _get_auth_headers(self, request_body):
"""生成鉴权请求头"""
# 获取当前UTC时间戳
timestamp = int(time.time())
# 使用UTC时间计算日期
utc_date = datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime('%Y-%m-%d')
# 服务名称必须是 "tts"
service = "tts"
# 拼接凭证范围
credential_scope = f"{utc_date}/{service}/tc3_request"
# 使用TC3-HMAC-SHA256签名方法
algorithm = "TC3-HMAC-SHA256"
# 构建规范请求字符串
http_request_method = "POST"
canonical_uri = "/"
canonical_querystring = ""
# 请求头必须包含host和content-type，且按字典序排列
canonical_headers = (
f"content-type:application/json\n"
f"host:tts.tencentcloudapi.com\n"
)
signed_headers = "content-type;host"
# 请求体哈希值
payload = json.dumps(request_body)
payload_hash = hashlib.sha256(payload.encode('utf-8')).hexdigest()
# 构建规范请求字符串
canonical_request = (
f"{http_request_method}\n"
f"{canonical_uri}\n"
f"{canonical_querystring}\n"
f"{canonical_headers}\n"
f"{signed_headers}\n"
f"{payload_hash}"
)
# 计算规范请求的哈希值
hashed_canonical_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
# 构建待签名字符串
string_to_sign = (
f"{algorithm}\n"
f"{timestamp}\n"
f"{credential_scope}\n"
f"{hashed_canonical_request}"
)
# 计算签名密钥
secret_date = self._hmac_sha256(f"TC3{self.secret_key}".encode('utf-8'), utc_date)
secret_service = self._hmac_sha256(secret_date, service)
secret_signing = self._hmac_sha256(secret_service, "tc3_request")
# 计算签名
signature = hmac.new(
secret_signing,
string_to_sign.encode('utf-8'),
hashlib.sha256
).hexdigest()
# 构建授权头
authorization = (
f"{algorithm} "
f"Credential={self.secret_id}/{credential_scope}, "
f"SignedHeaders={signed_headers}, "
f"Signature={signature}"
)
# 构建请求头
headers = {
"Content-Type": "application/json",
"Host": "tts.tencentcloudapi.com",
"Authorization": authorization,
"X-TC-Action": "TextToVoice",
"X-TC-Timestamp": str(timestamp),
"X-TC-Version": "2019-08-23",
"X-TC-Region": self.region,
"X-TC-Language": "zh-CN"
}
return headers
def _hmac_sha256(self, key, msg):
"""HMAC-SHA256加密"""
if isinstance(msg, str):
msg = msg.encode('utf-8')
return hmac.new(key, msg, hashlib.sha256).digest()
def generate_filename(self, extension=".wav"):
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
async def text_to_speak(self, text, output_file):
# 构建请求体
request_json = {
"Text": text, # 合成语音的源文本
"SessionId": str(uuid.uuid4()), # 会话ID，随机生成
"VoiceType": int(self.voice), # 音色
}
try:
# 获取请求头（每次请求都重新生成，以确保时间戳和签名是最新的）
headers = self._get_auth_headers(request_json)
# 发送请求
resp = requests.post(self.api_url, json.dumps(request_json), headers=headers)
# 检查响应
if resp.status_code == 200:
response_data = resp.json()
# 检查是否成功
if response_data.get("Response", {}).get("Error") is not None:
error_info = response_data["Response"]["Error"]
raise Exception(f"API返回错误: {error_info['Code']}: {error_info['Message']}")
# 提取音频数据
audio_data = response_data["Response"].get("Audio")
if audio_data:
# 解码Base64音频数据并保存
with open(output_file, "wb") as f:
f.write(base64.b64decode(audio_data))
else:
raise Exception(f"{__name__}: 没有返回音频数据: {response_data}")
else:
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
except Exception as e:
raise Exception(f"{__name__} error: {e}")