Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-28 01:53:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

本地记忆+意图识别 (#250)

Browse Source 
* 增加本地记忆功能，使用llm总结记忆

* update：增加统一非流式输出输出

* 增加意图识别内容，使用llm进行识别

* 初始化记忆模块

* 完善意图识别处理后的流程

* 通过使用function call实现意图识别

* update:优化意图识别的配置

* update:function call最优设置成doubao-pro-32k-functioncall-241028

---------

Co-authored-by: 玄凤科技 <eric230308@gmail.com>
Co-authored-by: hrz <1710360675@qq.com>
This commit is contained in:
欣南科技
2025-03-09 21:33:45 +08:00
committed by GitHub
co-authored by 玄凤科技 hrz
parent 9c2b2a2dcc
commit 63f34e5a82
19 changed files with 858 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/connection.py
+140 -7
View File
@@ -15,6 +15,7 @@ from core.utils.util import get_string_no_punctuation_or_emoji
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from core.handle.sendAudioHandle import sendAudioMessage
from core.handle.receiveAudioHandle import handleAudioMessage
from core.handle.intentHandler import Action, get_functions, handle_llm_function_call
from config.private_config import PrivateConfig
from core.auth import AuthMiddleware, AuthenticationError
from core.utils.auth_code_gen import AuthCodeGenerator
@@ -27,7 +28,7 @@ class TTSException(RuntimeError):
class ConnectionHandler:
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory):
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory, _intent):
self.config = config
self.logger = setup_logging()
self.auth = AuthMiddleware(config)
@@ -55,6 +56,7 @@ class ConnectionHandler:
self.llm = _llm
self.tts = _tts
self.memory = _memory
self.intent = _intent
# vad相关变量
self.client_audio_buffer = bytes()
@@ -88,6 +90,12 @@ class ConnectionHandler:
self.auth_code_gen = AuthCodeGenerator.get_instance()
self.is_device_verified = False # 添加设备验证状态标志
self.music_handler = _music
self.close_after_chat = False # 是否在聊天结束后关闭连接
self.use_function_call_mode = False
if self.config["selected_module"]["Intent"] == 'function_call':
self.use_function_call_mode = True
self.logger.bind(tag=TAG).info(f"use_function_call_mode:{self.use_function_call_mode}")
async def handle_connection(self, ws):
try:
@@ -101,7 +109,8 @@ class ConnectionHandler:
await self.auth.authenticate(self.headers)
device_id = self.headers.get("device-id", None)
self.memory.set_role_id(device_id)
self.memory.init_memory(device_id, self.llm)
self.intent.set_llm(self.llm)
# Load private configuration if device_id is provided
bUsePrivateConfig = self.config.get("use_private_config", False)
@@ -204,7 +213,6 @@ class ConnectionHandler:
return False
return not self.is_device_verified
def chat(self, query):
if self.isNeedAuth():
self.llm_finish_task = True
@@ -213,6 +221,7 @@ class ConnectionHandler:
return True
self.dialogue.put(Message(role="user", content=query))
response_message = []
processed_chars = 0 # 跟踪已处理的字符位置
try:
@@ -220,10 +229,10 @@ class ConnectionHandler:
# 使用带记忆的对话
future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
memory_str = future.result()
self.logger.bind(tag=TAG).info(f"记忆内容: {memory_str}")
self.logger.bind(tag=TAG).debug(f"记忆内容: {memory_str}")
llm_responses = self.llm.response(
self.session_id,
self.session_id,
self.dialogue.get_llm_dialogue_with_memory(memory_str)
)
except Exception as e:
@@ -245,7 +254,7 @@ class ConnectionHandler:
current_text = full_text[processed_chars:] # 从未处理的位置开始
# 查找最后一个有效标点
punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：")
punctuations = ("。", "？", "！", "；", "：")
last_punct_pos = -1
for punct in punctuations:
pos = current_text.rfind(punct)
@@ -282,6 +291,119 @@ class ConnectionHandler:
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
return True
def chat_with_function_calling(self, query):
self.logger.bind(tag=TAG).debug(f"Chat with function calling start: {query}")
"""Chat with function calling for intent detection using streaming"""
if self.isNeedAuth():
self.llm_finish_task = True
future = asyncio.run_coroutine_threadsafe(self._check_and_broadcast_auth_code(), self.loop)
future.result()
return True
self.dialogue.put(Message(role="user", content=query))
# Define intent functions
functions = get_functions()
response_message = []
processed_chars = 0 # 跟踪已处理的字符位置
function_call_data = None # 存储function call数据
try:
start_time = time.time()
# 使用带记忆的对话
future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
memory_str = future.result()
# self.logger.bind(tag=TAG).info(f"记忆内容: {memory_str}")
# 使用支持functions的streaming接口
llm_responses = self.llm.response_with_functions(
self.session_id,
self.dialogue.get_llm_dialogue_with_memory(memory_str),
functions=functions
)
except Exception as e:
self.logger.bind(tag=TAG).error(f"LLM 处理出错 {query}: {e}")
return None
self.llm_finish_task = False
text_index = 0
# 处理流式响应
for response in llm_responses:
if response["type"] == "content":
content = response["content"]
response_message.append(content)
if self.client_abort:
break
end_time = time.time()
self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
# 处理文本分段和TTS逻辑
# 合并当前全部文本并处理未分割部分
full_text = "".join(response_message)
current_text = full_text[processed_chars:] # 从未处理的位置开始
# 查找最后一个有效标点
punctuations = ("。", "？", "！", "；", "：")
last_punct_pos = -1
for punct in punctuations:
pos = current_text.rfind(punct)
if pos > last_punct_pos:
last_punct_pos = pos
# 找到分割点则处理
if last_punct_pos != -1:
segment_text_raw = current_text[:last_punct_pos + 1]
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
if segment_text:
text_index += 1
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
self.tts_queue.put(future)
processed_chars += len(segment_text_raw) # 更新已处理字符位置
elif response["type"] == "function_call":
# Extract function call data
function_call_data = {
"name": response["function_call"]["function"]["name"],
"arguments": response["function_call"]["function"]["arguments"]
}
self.logger.bind(tag=TAG).info(f"Function call detected: {function_call_data}")
# 处理最后剩余的文本
full_text = "".join(response_message)
remaining_text = full_text[processed_chars:]
if remaining_text:
segment_text = get_string_no_punctuation_or_emoji(remaining_text)
if segment_text:
text_index += 1
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
self.tts_queue.put(future)
# 存储对话内容
self.dialogue.put(Message(role="assistant", content="".join(response_message)))
# 处理function call
if function_call_data:
result = handle_llm_function_call(self, function_call_data)
if result.action == Action.RESPONSE:
text = result.response
text_index += 1
self.recode_first_last_text(text, text_index)
future = self.executor.submit(self.speak_and_play, text, text_index)
self.tts_queue.put(future)
self.llm_finish_task = True
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
return True
def _tts_priority_thread(self):
while not self.stop_event.is_set():
text = None
@@ -372,3 +494,14 @@ class ConnectionHandler:
self.client_have_voice_last_time = 0
self.client_voice_stop = False
self.logger.bind(tag=TAG).debug("VAD states reset.")
def chat_and_close(self, text):
"""Chat with the user and then close the connection"""
try:
# Use the existing chat method
self.chat(text)
# After chat is complete, close the connection
self.close_after_chat = True
except Exception as e:
self.logger.bind(tag=TAG).error(f"Chat and close error: {str(e)}")