Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix:function call bug (#268)

Browse Source 
* 优化function call消息处理

* fix:用户说话重复记录bug

* 2025-3-10-优化llm intent (#254)

Co-authored-by: 欣南科技 <huangrongzhuang@xin-nan.com>

* update:回复上版提示词，无需再二次识别歌曲名

---------

Co-authored-by: 玄凤科技 <eric230308@gmail.com>
Co-authored-by: hrz <1710360675@qq.com>
Co-authored-by: Jiao Haoyang <108573524+XuSenfeng@users.noreply.github.com>
This commit is contained in:
欣南科技
2025-03-11 00:25:33 +08:00
committed by GitHub
co-authored by 玄凤科技 hrz Jiao Haoyang
parent 4f3fae81c1
commit 2b662d3a14
9 changed files with 106 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/connection.py
+82 -47
View File
@@ -11,7 +11,7 @@ import websockets
from typing import Dict, Any
from core.utils.dialogue import Message, Dialogue
from core.handle.textHandle import handleTextMessage
from core.utils.util import get_string_no_punctuation_or_emoji
from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from core.handle.sendAudioHandle import sendAudioMessage
from core.handle.receiveAudioHandle import handleAudioMessage
@@ -307,8 +307,7 @@ class ConnectionHandler:
response_message = []
processed_chars = 0 # 跟踪已处理的字符位置
function_call_data = None # 存储function call数据
try:
start_time = time.time()
@@ -316,7 +315,7 @@ class ConnectionHandler:
future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
memory_str = future.result()
# self.logger.bind(tag=TAG).info(f"记忆内容: {memory_str}")
#self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}")
# 使用支持functions的streaming接口
llm_responses = self.llm.response_with_functions(
@@ -332,48 +331,61 @@ class ConnectionHandler:
text_index = 0
# 处理流式响应
tool_call_flag = False
function_name = None
function_id = None
function_arguments = ""
content_arguments = ""
for response in llm_responses:
if response["type"] == "content":
content = response["content"]
response_message.append(content)
content, tools_call = response
if content is not None and len(content)>0:
if len(response_message)<=0 and content=="```":
tool_call_flag = True
if self.client_abort:
break
if tools_call is not None:
tool_call_flag = True
if tools_call[0].id is not None:
function_id = tools_call[0].id
if tools_call[0].function.name is not None:
function_name = tools_call[0].function.name
if tools_call[0].function.arguments is not None:
function_arguments += tools_call[0].function.arguments
end_time = time.time()
self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
if content is not None and len(content) > 0:
if tool_call_flag:
content_arguments+=content
else:
response_message.append(content)
# 处理文本分段和TTS逻辑
# 合并当前全部文本并处理未分割部分
full_text = "".join(response_message)
current_text = full_text[processed_chars:] # 从未处理的位置开始
if self.client_abort:
break
# 查找最后一个有效标点
punctuations = ("。", "？", "！", "；", "：")
last_punct_pos = -1
for punct in punctuations:
pos = current_text.rfind(punct)
if pos > last_punct_pos:
last_punct_pos = pos
end_time = time.time()
self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
# 找到分割点则处理
if last_punct_pos != -1:
segment_text_raw = current_text[:last_punct_pos + 1]
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
if segment_text:
text_index += 1
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
self.tts_queue.put(future)
processed_chars += len(segment_text_raw) # 更新已处理字符位置
# 处理文本分段和TTS逻辑
# 合并当前全部文本并处理未分割部分
full_text = "".join(response_message)
current_text = full_text[processed_chars:] # 从未处理的位置开始
elif response["type"] == "function_call":
# Extract function call data
function_call_data = {
"name": response["function_call"]["function"]["name"],
"arguments": response["function_call"]["function"]["arguments"]
}
self.logger.bind(tag=TAG).info(f"Function call detected: {function_call_data}")
# 查找最后一个有效标点
punctuations = ("。", "？", "！", "；", "：")
last_punct_pos = -1
for punct in punctuations:
pos = current_text.rfind(punct)
if pos > last_punct_pos:
last_punct_pos = pos
# 找到分割点则处理
if last_punct_pos != -1:
segment_text_raw = current_text[:last_punct_pos + 1]
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
if segment_text:
text_index += 1
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
self.tts_queue.put(future)
processed_chars += len(segment_text_raw) # 更新已处理字符位置
# 处理最后剩余的文本
full_text = "".join(response_message)
@@ -387,23 +399,46 @@ class ConnectionHandler:
self.tts_queue.put(future)
# 存储对话内容
self.dialogue.put(Message(role="assistant", content="".join(response_message)))
if len(response_message)>0:
self.dialogue.put(Message(role="assistant", content="".join(response_message)))
# 处理function call
if function_call_data:
if tool_call_flag:
if function_id is None:
a = extract_json_from_string(content_arguments)
if a is not None:
content_arguments_json = json.loads(a)
function_name = content_arguments_json["function_name"]
function_arguments = json.dumps(content_arguments_json["args"], ensure_ascii=False)
function_id = str(uuid.uuid4().hex)
else:
return []
function_arguments = json.loads(function_arguments)
self.logger.bind(tag=TAG).info(f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
function_call_data = {
"name": function_name,
"id": function_id,
"arguments": function_arguments
}
result = handle_llm_function_call(self, function_call_data)
if result.action == Action.RESPONSE:
text = result.response
text_index += 1
self.recode_first_last_text(text, text_index)
future = self.executor.submit(self.speak_and_play, text, text_index)
self.tts_queue.put(future)
self._handle_function_result(result, function_call_data, text_index+1)
self.llm_finish_task = True
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
return True
def _handle_function_result(self, result, function_call_data, text_index):
if result.action == Action.RESPONSE: # 直接回复前端
text = result.response
self.recode_first_last_text(text, text_index)
future = self.executor.submit(self.speak_and_play, text, text_index)
self.tts_queue.put(future)
self.dialogue.put(Message(role="assistant", content=text))
if result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
text = result.response
def _tts_priority_thread(self):
while not self.stop_event.is_set():
text = None
main/xiaozhi-server/core/handle/intentHandler.py
+2 -4
View File
@@ -122,12 +122,10 @@ async def analyze_intent_with_llm(conn, text):
logger.bind(tag=TAG).warning("意图识别服务未初始化")
return None
# 创建对话历史记录
# 对话历史记录
dialogue = conn.dialogue
dialogue.put(Message(role="user", content=text))
try:
intent_result = await conn.intent.detect_intent(dialogue.dialogue)
intent_result = await conn.intent.detect_intent(dialogue.dialogue, text)
logger.bind(tag=TAG).info(f"意图识别结果: {intent_result}")
# 尝试解析JSON结果
main/xiaozhi-server/core/handle/receiveAudioHandle.py
+3 -3
View File
@@ -67,9 +67,9 @@ async def no_voice_close_connect(conn):
else:
no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
close_connection_no_voice_time = conn.config.get("close_connection_no_voice_time", 120)
if no_voice_time > 1000 * close_connection_no_voice_time:
if not conn.close_after_chat and no_voice_time > 1000 * close_connection_no_voice_time:
conn.close_after_chat = True
conn.client_abort = False
conn.asr_server_receive = False
conn.close_after_chat = True
prompt = "时间过得真快，我都好久没说话了。请你用十个字左右话跟我告别，以“再见”或“拜拜”为结尾"
prompt = "请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。"
await startToChat(conn, prompt)
main/xiaozhi-server/core/providers/intent/base.py
+3 -2
View File
@@ -5,6 +5,7 @@ from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
class IntentProviderBase(ABC):
def __init__(self, config):
self.config = config
@@ -17,9 +18,9 @@ class IntentProviderBase(ABC):
def set_llm(self, llm):
self.llm = llm
logger.bind(tag=TAG).debug("Set LLM for intent provider")
@abstractmethod
async def detect_intent(self, dialogue_history: List[Dict]) -> str:
async def detect_intent(self, dialogue_history: List[Dict], text: str) -> str:
"""
检测用户最后一句话的意图
Args:
main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py
+2 -3
View File
@@ -37,7 +37,7 @@ class IntentProvider(IntentProviderBase):
)
return prompt
async def detect_intent(self, dialogue_history: List[Dict]) -> str:
async def detect_intent(self, dialogue_history: List[Dict], text:str) -> str:
if not self.llm:
raise ValueError("LLM provider not set")
@@ -48,9 +48,8 @@ class IntentProvider(IntentProviderBase):
msgStr += f"User: {msg.content}\n"
elif msg.role== "assistant":
msgStr += f"Assistant: {msg.content}\n"
msgStr += f"User: {text}\n"
user_prompt = f"请分析用户的意图：\n{msgStr}"
# 使用LLM进行意图识别
intent = self.llm.response_no_stream(
system_prompt=self.promot,
main/xiaozhi-server/core/providers/intent/nointent/nointent.py
+3 -1
View File
@@ -5,12 +5,14 @@ from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
class IntentProvider(IntentProviderBase):
async def detect_intent(self, dialogue_history: List[Dict]) -> str:
async def detect_intent(self, dialogue_history: List[Dict], text: str) -> str:
"""
默认的意图识别实现，始终返回继续聊天
Args:
dialogue_history: 对话历史记录列表
text: 本次对话记录
Returns:
固定返回"继续聊天"
"""
main/xiaozhi-server/core/providers/llm/ollama/ollama.py
+1 -23
View File
@@ -51,30 +51,8 @@ class LLMProvider(LLMProviderBase):
tools=functions,
)
current_function_call = None
current_content = ""
for chunk in stream:
delta = chunk.choices[0].delta
if delta.content:
current_content += delta.content
yield {"type": "content", "content": delta.content}
if delta.tool_calls:
tool_call = delta.tool_calls[0]
# Handle the function call data using proper attribute access
if not current_function_call:
current_function_call = {
"function": {
"name": tool_call.function.name,
"arguments": tool_call.function.arguments
}
}
if current_function_call:
logger.bind(tag=TAG).debug(f"ollama Function call detected: {current_function_call}")
yield {"type": "function_call", "function_call": current_function_call}
yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
except Exception as e:
logger.bind(tag=TAG).error(f"Error in Ollama function call: {e}")
main/xiaozhi-server/core/providers/llm/openai/openai.py
+1 -23
View File
@@ -53,30 +53,8 @@ class LLMProvider(LLMProviderBase):
tools=functions,
)
current_function_call = None
current_content = ""
for chunk in stream:
delta = chunk.choices[0].delta
if delta.content:
current_content += delta.content
yield {"type": "content", "content": delta.content}
if delta.tool_calls:
tool_call = delta.tool_calls[0]
# Handle the function call data using proper attribute access
if not current_function_call:
current_function_call = {
"function": {
"name": tool_call.function.name,
"arguments": tool_call.function.arguments
}
}
if current_function_call:
logger.bind(tag=TAG).debug(f"openai Function call detected: {current_function_call}")
yield {"type": "function_call", "function_call": current_function_call}
yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
except Exception as e:
self.logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
main/xiaozhi-server/core/utils/util.py
+9
View File
@@ -4,6 +4,7 @@ import yaml
import socket
import subprocess
import logging
import re
def get_project_dir():
@@ -119,3 +120,11 @@ def check_ffmpeg_installed():
error_msg += "1、按照项目的安装文档，正确进入conda环境\n"
error_msg += "2、查阅安装文档，如何在conda环境中安装ffmpeg\n"
raise ValueError(error_msg)
def extract_json_from_string(input_string):
"""提取字符串中的 JSON 部分"""
pattern = r'(\{.*\})'
match = re.search(pattern, input_string)
if match:
return match.group(1) # 返回提取的 JSON 字符串
return None