Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-26 17:13:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: 播放咋音问题

Browse Source
This commit is contained in:
lizhongxiang
2025-04-11 10:25:43 +08:00
parent 8bb888c58e d7aac5e10f
commit 118bb728a9
130 changed files with 8311 additions and 2085 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/providers/intent/base.py
+14 -11
View File
@@ -9,20 +9,23 @@ logger = setup_logging()
class IntentProviderBase(ABC):
def __init__(self, config):
self.config = config
self.intent_options = config.get("intent_options", {
"handle_exit_intent": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候",
"play_music": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图",
"get_weather": "查询天气, 用户希望查询某个地点的天气情况",
"get_news": "查询新闻, 用户希望查询最新新闻或特定类型的新闻",
"get_lunar": "用于获取今天的阴历/农历和黄历信息",
"get_time": "获取今天日期或者当前时间信息",
"continue_chat": "继续聊天",
})
self.intent_options = [
{
"name": "handle_exit_intent",
"desc": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候",
},
{
"name": "play_music",
"desc": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图",
},
{"name": "get_time", "desc": "获取今天日期或者当前时间信息"},
{"name": "continue_chat", "desc": "继续聊天"},
]
def set_llm(self, llm):
self.llm = llm
# 获取模型名称和类型信息
model_name = getattr(llm, 'model_name', str(llm.__class__.__name__))
model_name = getattr(llm, "model_name", str(llm.__class__.__name__))
# 记录更详细的日志
logger.bind(tag=TAG).info(f"意图识别设置LLM: {model_name}")
@@ -35,7 +38,7 @@ class IntentProviderBase(ABC):
Returns:
返回识别出的意图，格式为:
- "继续聊天"
- "结束聊天"
- "结束聊天"
- "播放音乐 歌名" 或 "随机播放音乐"
- "查询天气 地点名" 或 "查询天气 [当前位置]"
"""
main/xiaozhi-server/core/providers/intent/function_call/function_call.py
+4 -2
View File
@@ -16,5 +16,7 @@ class IntentProvider(IntentProviderBase):
Returns:
固定返回"继续聊天"
"""
logger.bind(tag=TAG).debug("Using functionCallProvider, always returning continue chat")
return self.intent_options["continue_chat"]
logger.bind(tag=TAG).debug(
"Using functionCallProvider, always returning continue chat"
)
return '{"function_call": {"name": "continue_chat"}}'
main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py
+68 -86
View File
@@ -27,119 +27,94 @@ class IntentProvider(IntentProviderBase):
Returns:
格式化后的系统提示词
"""
intent_list = []
prompt = (
"你是一个意图识别助手。请分析用户的最后一句话，判断用户意图属于以下哪一类：\n"
"<start>"
f"{', '.join(intent_list)}"
f"{str(self.intent_options)}"
"<end>\n"
"处理步骤:"
"1. 思考意图类型，生成function_call格式"
"\n\n"
"返回格式示例：\n"
"1. 播放音乐意图: {\"function_call\": {\"name\": \"play_music\", \"arguments\": {\"song_name\": \"音乐名称\"}}}\n"
"2. 查询天气意图: {\"function_call\": {\"name\": \"get_weather\", \"arguments\": {\"location\": \"地点名称\", \"lang\": \"zh_CN\"}}}\n"
"3. 查询新闻意图: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"category\": \"新闻类别\", \"detail\": false, \"lang\": \"zh_CN\"}}}\n"
"4. 结束对话意图: {\"function_call\": {\"name\": \"handle_exit_intent\", \"arguments\": {\"say_goodbye\": \"goodbye\"}}}\n"
"5. 获取当天日期时间: {\"function_call\": {\"name\": \"get_time\"}}\n"
"6. 获取当前黄历意图: {\"function_call\": {\"name\": \"get_lunar\"}}\n"
"7. 继续聊天意图: {\"function_call\": {\"name\": \"continue_chat\"}}\n"
'1. 播放音乐意图: {"function_call": {"name": "play_music", "arguments": {"song_name": "音乐名称"}}}\n'
'2. 结束对话意图: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n'
'3. 获取当天日期时间: {"function_call": {"name": "get_time"}}\n'
'4. 继续聊天意图: {"function_call": {"name": "continue_chat"}}\n'
"\n"
"注意:\n"
"- 播放音乐：无歌名时，song_name设为\"random\"\n"
"- 查询天气：无地点时，location设为null\n"
"- 查询新闻：无类别时，category设为null；查询详情时，detail设为true\n"
'- 播放音乐：无歌名时，song_name设为"random"\n'
"- 如果没有明显的意图，应按照继续聊天意图处理\n"
"- 只返回纯JSON，不要任何其他内容\n"
"\n"
"示例分析:\n"
"```\n"
"用户: 你好小智\n"
"返回: {\"function_call\": {\"name\": \"continue_chat\"}}\n"
"```\n"
"```\n"
"用户: 你今天怎么样?\n"
"返回: {\"function_call\": {\"name\": \"continue_chat\"}}\n"
"用户: 你也太搞笑了\n"
'返回: {"function_call": {"name": "continue_chat"}}\n'
"```\n"
"```\n"
"用户: 现在是几号了?现在几点了？\n"
"返回: {\"function_call\": {\"name\": \"get_time\"}}\n"
"```\n"
"```\n"
"用户: 今天农历是多少？\n"
"返回: {\"function_call\": {\"name\": \"get_lunar\"}}\n"
'返回: {"function_call": {"name": "get_time"}}\n'
"```\n"
"```\n"
"用户: 我们明天再聊吧\n"
"返回: {\"function_call\": {\"name\": \"handle_exit_intent\"}}\n"
'返回: {"function_call": {"name": "handle_exit_intent"}}\n'
"```\n"
"```\n"
"用户: 播放中秋月\n"
"返回: {\"function_call\": {\"name\": \"play_music\", \"arguments\": {\"song_name\": \"中秋月\"}}}\n"
'返回: {"function_call": {"name": "play_music", "arguments": {"song_name": "中秋月"}}}\n'
"```\n"
"```\n"
"用户: 北京天气怎么样\n"
"返回: {\"function_call\": {\"name\": \"get_weather\", \"arguments\": {\"location\": \"北京\", \"lang\": \"zh_CN\"}}}\n"
"```\n"
"```\n"
"用户: 今天天气怎么样\n"
"返回: {\"function_call\": {\"name\": \"get_weather\", \"arguments\": {\"location\": null, \"lang\": \"zh_CN\"}}}\n"
"```\n"
"```\n"
"用户: 播报财经新闻\n"
"返回: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"category\": \"财经\", \"detail\": false, \"lang\": \"zh_CN\"}}}\n"
"```\n"
"```\n"
"用户: 有什么最新新闻\n"
"返回: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"category\": null, \"detail\": false, \"lang\": \"zh_CN\"}}}\n"
"```\n"
"```\n"
"用户: 详细介绍一下这条新闻\n"
"返回: {\"function_call\": {\"name\": \"get_news\", \"arguments\": {\"detail\": true, \"lang\": \"zh_CN\"}}}\n"
"```\n"
"可用的音乐名称:\n"
)
return prompt
def clean_cache(self):
"""清理过期缓存"""
now = time.time()
# 找出过期键
expired_keys = [k for k, v in self.intent_cache.items() if now - v['timestamp'] > self.cache_expiry]
expired_keys = [
k
for k, v in self.intent_cache.items()
if now - v["timestamp"] > self.cache_expiry
]
for key in expired_keys:
del self.intent_cache[key]
# 如果缓存太大，移除最旧的条目
if len(self.intent_cache) > self.cache_max_size:
# 按时间戳排序并保留最新的条目
sorted_items = sorted(self.intent_cache.items(), key=lambda x: x[1]['timestamp'])
for key, _ in sorted_items[:len(sorted_items) - self.cache_max_size]:
sorted_items = sorted(
self.intent_cache.items(), key=lambda x: x[1]["timestamp"]
)
for key, _ in sorted_items[: len(sorted_items) - self.cache_max_size]:
del self.intent_cache[key]
async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
if not self.llm:
raise ValueError("LLM provider not set")
# 记录整体开始时间
total_start_time = time.time()
# 打印使用的模型信息
model_info = getattr(self.llm, 'model_name', str(self.llm.__class__.__name__))
logger.bind(tag=TAG).info(f"使用意图识别模型: {model_info}")
model_info = getattr(self.llm, "model_name", str(self.llm.__class__.__name__))
logger.bind(tag=TAG).debug(f"使用意图识别模型: {model_info}")
# 计算缓存键
cache_key = hashlib.md5(text.encode()).hexdigest()
# 检查缓存
if cache_key in self.intent_cache:
cache_entry = self.intent_cache[cache_key]
# 检查缓存是否过期
if time.time() - cache_entry['timestamp'] <= self.cache_expiry:
if time.time() - cache_entry["timestamp"] <= self.cache_expiry:
cache_time = time.time() - total_start_time
logger.bind(tag=TAG).info(f"使用缓存的意图: {cache_key} -> {cache_entry['intent']}, 耗时: {cache_time:.4f}秒")
return cache_entry['intent']
logger.bind(tag=TAG).debug(
f"使用缓存的意图: {cache_key} -> {cache_entry['intent']}, 耗时: {cache_time:.4f}秒"
)
return cache_entry["intent"]
# 清理缓存
self.clean_cache()
@@ -158,38 +133,41 @@ class IntentProvider(IntentProviderBase):
music_file_names = music_config["music_file_names"]
prompt_music = f"{self.promot}\n<start>{music_file_names}\n<end>"
logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}")
# 记录预处理完成时间
preprocess_time = time.time() - total_start_time
logger.bind(tag=TAG).debug(f"意图识别预处理耗时: {preprocess_time:.4f}秒")
# 使用LLM进行意图识别
llm_start_time = time.time()
logger.bind(tag=TAG).info(f"开始LLM意图识别调用, 模型: {model_info}")
logger.bind(tag=TAG).debug(f"开始LLM意图识别调用, 模型: {model_info}")
intent = self.llm.response_no_stream(
system_prompt=prompt_music,
user_prompt=user_prompt
system_prompt=prompt_music, user_prompt=user_prompt
)
# 记录LLM调用完成时间
llm_time = time.time() - llm_start_time
logger.bind(tag=TAG).info(f"LLM意图识别完成, 模型: {model_info}, 调用耗时: {llm_time:.4f}秒")
logger.bind(tag=TAG).debug(
f"LLM意图识别完成, 模型: {model_info}, 调用耗时: {llm_time:.4f}秒"
)
# 记录后处理开始时间
postprocess_start_time = time.time()
# 清理和解析响应
intent = intent.strip()
# 尝试提取JSON部分
match = re.search(r'\{.*\}', intent, re.DOTALL)
match = re.search(r"\{.*\}", intent, re.DOTALL)
if match:
intent = match.group(0)
# 记录总处理时间
total_time = time.time() - total_start_time
logger.bind(tag=TAG).info(f"【意图识别性能】模型: {model_info}, 总耗时: {total_time:.4f}秒, LLM调用: {llm_time:.4f}秒, 查询: '{text[:20]}...'")
logger.bind(tag=TAG).debug(
f"【意图识别性能】模型: {model_info}, 总耗时: {total_time:.4f}秒, LLM调用: {llm_time:.4f}秒, 查询: '{text[:20]}...'"
)
# 尝试解析为JSON
try:
intent_data = json.loads(intent)
@@ -198,38 +176,42 @@ class IntentProvider(IntentProviderBase):
function_data = intent_data["function_call"]
function_name = function_data.get("name")
function_args = function_data.get("arguments", {})
# 记录识别到的function call
logger.bind(tag=TAG).info(f"识别到function call: {function_name}, 参数: {function_args}")
logger.bind(tag=TAG).info(
f"识别到function call: {function_name}, 参数: {function_args}"
)
# 添加到缓存
self.intent_cache[cache_key] = {
'intent': intent,
'timestamp': time.time()
"intent": intent,
"timestamp": time.time(),
}
# 后处理时间
postprocess_time = time.time() - postprocess_start_time
logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
# 确保返回完全序列化的JSON字符串
return intent
else:
# 添加到缓存
self.intent_cache[cache_key] = {
'intent': intent,
'timestamp': time.time()
"intent": intent,
"timestamp": time.time(),
}
# 后处理时间
postprocess_time = time.time() - postprocess_start_time
logger.bind(tag=TAG).debug(f"意图后处理耗时: {postprocess_time:.4f}秒")
# 返回普通意图
return intent
except json.JSONDecodeError:
# 后处理时间
postprocess_time = time.time() - postprocess_start_time
logger.bind(tag=TAG).error(f"无法解析意图JSON: {intent}, 后处理耗时: {postprocess_time:.4f}秒")
logger.bind(tag=TAG).error(
f"无法解析意图JSON: {intent}, 后处理耗时: {postprocess_time:.4f}秒"
)
# 如果解析失败，默认返回继续聊天意图
return "{\"intent\": \"继续聊天\"}"
return '{"intent": "继续聊天"}'
main/xiaozhi-server/core/providers/intent/nointent/nointent.py
+4 -2
View File
@@ -16,5 +16,7 @@ class IntentProvider(IntentProviderBase):
Returns:
固定返回"继续聊天"
"""
logger.bind(tag=TAG).debug("Using NoIntentProvider, always returning continue chat")
return self.intent_options["continue_chat"]
logger.bind(tag=TAG).debug(
"Using NoIntentProvider, always returning continue chat"
)
return '{"function_call": {"name": "continue_chat"}}'
main/xiaozhi-server/core/providers/tts/base.py
+13 -2
View File
@@ -53,6 +53,7 @@ class TTSProviderBase(ABC):
"，",
)
self.tts_request = False
self.tts_stop_request = False
self.processed_chars = 0
self.stream = False
self.last_to_opus_raw = b""
@@ -93,6 +94,9 @@ class TTSProviderBase(ABC):
)
self.processed_chars += len(segment_text_raw) # 更新已处理字符位置
return segment_text
elif self.tts_stop_request and current_text:
segment_text = current_text
return segment_text
else:
return None
@@ -115,8 +119,11 @@ class TTSProviderBase(ABC):
def tts_one_sentence(self, conn, text, u_id=None):
if not u_id:
u_id = str(uuid.uuid4()).replace("-", "")
conn.u_id = u_id
if conn.u_id:
u_id = conn.u_id
else:
u_id = str(uuid.uuid4()).replace("-", "")
conn.u_id = u_id
self.tts_text_queue.put(
TTSMessageDTO(u_id=u_id, msg_type=MsgType.START_TTS_REQUEST, content="")
)
@@ -135,6 +142,7 @@ class TTSProviderBase(ABC):
if msg_type == MsgType.START_TTS_REQUEST:
# 开始传输tts文本
self.tts_request = True
self.tts_stop_request = False
self.u_id = ttsMessageDTO.u_id
# 开启session
future = asyncio.run_coroutine_threadsafe(
@@ -152,6 +160,7 @@ class TTSProviderBase(ABC):
future.result()
elif msg_type == MsgType.STOP_TTS_REQUEST:
self.tts_request = False
self.tts_stop_request = True
future = asyncio.run_coroutine_threadsafe(
self.finish_session(ttsMessageDTO.u_id), loop=self.loop
)
@@ -183,6 +192,7 @@ class TTSProviderBase(ABC):
if msg_type == MsgType.START_TTS_REQUEST:
# 开始传输tts文本
self.tts_request = True
self.tts_stop_request = False
self.processed_chars = 0
self.tts_text_buff = []
elif self.tts_request and msg_type == MsgType.TTS_TEXT_REQUEST:
@@ -190,6 +200,7 @@ class TTSProviderBase(ABC):
elif msg_type == MsgType.STOP_TTS_REQUEST:
# 结束传输tts文本,处理最尾巴的数据
self.tts_request = False
self.tts_stop_request = True
segment_text = self._get_segment_text()
if segment_text:
# 修改部分：创建协程对象