Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-28 01:53:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

mergin main，速度提升一下

Browse Source
This commit is contained in:
lizhongxiang
2025-03-20 09:39:25 +08:00
parent a067a6ba87 4aefbe5aa8
commit b153bc11f2
96 changed files with 3438 additions and 1830 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/connection.py
+104 -50
View File
@@ -5,17 +5,20 @@ import time
import queue
import asyncio
import traceback
from config.logger import setup_logging
import threading
import websockets
from typing import Dict, Any
import plugins_func.loadplugins
from config.logger import setup_logging
from core.utils.dialogue import Message, Dialogue
from core.handle.textHandle import handleTextMessage
from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string
from core.utils.util import get_string_no_punctuation_or_emoji, extract_json_from_string, get_ip_info
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from core.handle.sendAudioHandle import sendAudioMessage, sendAudioMessageStream
from core.handle.receiveAudioHandle import handleAudioMessage
from core.handle.intentHandler import Action, get_functions, handle_llm_function_call
from core.handle.functionHandler import FunctionHandler
from plugins_func.register import Action
from config.private_config import PrivateConfig
from core.auth import AuthMiddleware, AuthenticationError
from core.utils.auth_code_gen import AuthCodeGenerator
@@ -28,7 +31,7 @@ class TTSException(RuntimeError):
class ConnectionHandler:
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory, _intent):
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _memory, _intent):
self.config = config
self.logger = setup_logging()
self.auth = AuthMiddleware(config)
@@ -37,6 +40,8 @@ class ConnectionHandler:
self.websocket = None
self.headers = None
self.client_ip = None
self.client_ip_info = {}
self.session_id = None
self.prompt = None
self.welcome_msg = None
@@ -94,7 +99,6 @@ class ConnectionHandler:
self.private_config = None
self.auth_code_gen = AuthCodeGenerator.get_instance()
self.is_device_verified = False # 添加设备验证状态标志
self.music_handler = _music
self.close_after_chat = False # 是否在聊天结束后关闭连接
self.use_function_call_mode = False
if self.config["selected_module"]["Intent"] == 'function_call':
@@ -105,8 +109,8 @@ class ConnectionHandler:
# 获取并验证headers
self.headers = dict(ws.request.headers)
# 获取客户端ip地址
client_ip = ws.remote_address[0]
self.logger.bind(tag=TAG).info(f"{client_ip} conn - Headers: {self.headers}")
self.client_ip = ws.remote_address[0]
self.logger.bind(tag=TAG).info(f"{self.client_ip} conn - Headers: {self.headers}")
# 进行认证
await self.auth.authenticate(self.headers)
@@ -150,6 +154,7 @@ class ConnectionHandler:
self.welcome_msg["session_id"] = self.session_id
await self.websocket.send(json.dumps(self.welcome_msg))
# 异步初始化
await self.loop.run_in_executor(None, self._initialize_components)
# tts 消化线程
@@ -190,12 +195,21 @@ class ConnectionHandler:
self.prompt = self.config["prompt"]
if self.private_config:
self.prompt = self.private_config.private_config.get("prompt", self.prompt)
# 赋予LLM时间观念
if "{date_time}" in self.prompt:
date_time = time.strftime("%Y-%m-%d %H:%M", time.localtime())
self.prompt = self.prompt.replace("{date_time}", date_time)
self.client_ip_info = get_ip_info(self.client_ip)
self.logger.bind(tag=TAG).info(f"Client ip info: {self.client_ip_info}")
self.prompt = self.prompt + f"\n我在:{self.client_ip_info}"
self.dialogue.put(Message(role="system", content=self.prompt))
self.func_handler = FunctionHandler(self.config)
def change_system_prompt(self, prompt):
self.prompt = prompt
# 找到原来的role==system，替换原来的系统提示
for m in self.dialogue.dialogue:
if m.role == "system":
m.content = prompt
async def _check_and_broadcast_auth_code(self):
"""检查设备绑定状态并广播认证码"""
if not self.private_config.get_owner():
@@ -312,7 +326,7 @@ class ConnectionHandler:
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
return True
def chat_with_function_calling(self, query):
def chat_with_function_calling(self, query, tool_call=False):
self.logger.bind(tag=TAG).debug(f"Chat with function calling start: {query}")
"""Chat with function calling for intent detection using streaming"""
if self.isNeedAuth():
@@ -321,10 +335,11 @@ class ConnectionHandler:
future.result()
return True
self.dialogue.put(Message(role="user", content=query))
if not tool_call:
self.dialogue.put(Message(role="user", content=query))
# Define intent functions
functions = get_functions()
functions = self.func_handler.get_functions()
response_message = []
processed_chars = 0 # 跟踪已处理的字符位置
@@ -360,7 +375,7 @@ class ConnectionHandler:
for response in llm_responses:
content, tools_call = response
if content is not None and len(content) > 0:
if len(response_message) <= 0 and content == "```":
if len(response_message) <= 0 and (content == "```" or "<tool_call>" in content):
tool_call_flag = True
if tools_call is not None:
@@ -417,6 +432,38 @@ class ConnectionHandler:
self.tts_queue.put(future)
processed_chars += len(segment_text_raw) # 更新已处理字符位置
# 处理function call
if tool_call_flag:
bHasError = False
if function_id is None:
a = extract_json_from_string(content_arguments)
if a is not None:
try:
content_arguments_json = json.loads(a)
function_name = content_arguments_json["name"]
function_arguments = json.dumps(content_arguments_json["arguments"], ensure_ascii=False)
function_id = str(uuid.uuid4().hex)
except Exception as e:
bHasError = True
response_message.append(a)
else:
bHasError = True
response_message.append(content_arguments)
if bHasError:
self.logger.bind(tag=TAG).error(f"function call error: {content_arguments}")
else:
function_arguments = json.loads(function_arguments)
if not bHasError:
self.logger.bind(tag=TAG).info(
f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
function_call_data = {
"name": function_name,
"id": function_id,
"arguments": function_arguments
}
result = self.func_handler.handle_llm_function_call(self, function_call_data)
self._handle_function_result(result, function_call_data, text_index + 1)
# 处理最后剩余的文本
full_text = "".join(response_message)
remaining_text = full_text[processed_chars:]
@@ -424,6 +471,7 @@ class ConnectionHandler:
segment_text = get_string_no_punctuation_or_emoji(remaining_text)
if segment_text:
text_index += 1
self.recode_first_last_text(segment_text, text_index)
if self.tts_stream:
stream_queue = queue.Queue()
self.executor.submit(self.speak_and_play_stream, segment_text, stream_queue, text_index)
@@ -433,36 +481,13 @@ class ConnectionHandler:
"text_index": text_index
})
else:
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
self.tts_queue.put(future)
self.tts_queue.put(future)
# 存储对话内容
if len(response_message) > 0:
self.dialogue.put(Message(role="assistant", content="".join(response_message)))
# 处理function call
if tool_call_flag:
if function_id is None:
a = extract_json_from_string(content_arguments)
if a is not None:
content_arguments_json = json.loads(a)
function_name = content_arguments_json["function_name"]
function_arguments = json.dumps(content_arguments_json["args"], ensure_ascii=False)
function_id = str(uuid.uuid4().hex)
else:
return []
function_arguments = json.loads(function_arguments)
self.logger.bind(tag=TAG).info(
f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
function_call_data = {
"name": function_name,
"id": function_id,
"arguments": function_arguments
}
result = handle_llm_function_call(self, function_call_data)
self._handle_function_result(result, function_call_data, text_index + 1)
self.llm_finish_task = True
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
@@ -484,10 +509,34 @@ class ConnectionHandler:
future = self.executor.submit(self.speak_and_play, text, text_index)
self.tts_queue.put(future)
self.dialogue.put(Message(role="assistant", content=text))
if result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
text = result.response
if result.action == Action.NOTFOUND:
text = result.response
elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
text = result.result
if text is not None and len(text) > 0:
function_id = function_call_data["id"]
function_name = function_call_data["name"]
function_arguments = function_call_data["arguments"]
self.dialogue.put(Message(role='assistant',
tool_calls=[{"id": function_id,
"function": {"arguments": function_arguments,
"name": function_name},
"type": 'function',
"index": 0}]))
self.dialogue.put(Message(role="tool", tool_call_id=function_id, content=text))
self.chat_with_function_calling(text, tool_call=True)
elif result.action == Action.NOTFOUND:
text = result.result
self.recode_first_last_text(text, text_index)
future = self.executor.submit(self.speak_and_play, text, text_index)
self.tts_queue.put(future)
self.dialogue.put(Message(role="assistant", content=text))
else:
text = result.result
self.recode_first_last_text(text, text_index)
future = self.executor.submit(self.speak_and_play, text, text_index)
self.tts_queue.put(future)
self.dialogue.put(Message(role="assistant", content=text))
def _tts_priority_thread(self):
if self.tts_stream:
@@ -506,7 +555,8 @@ class ConnectionHandler:
opus_datas, text_index, tts_file = [], 0, None
try:
self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
tts_file, text, text_index = future.result(timeout=10)
tts_timeout = self.config.get("tts_timeout", 10)
tts_file, text, text_index = future.result(timeout=tts_timeout)
if text is None or len(text) <= 0:
self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
elif tts_file is None:
@@ -514,7 +564,7 @@ class ConnectionHandler:
else:
self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
if os.path.exists(tts_file):
opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
opus_datas, duration = self.tts.audio_to_opus_data(tts_file)
else:
self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
except TimeoutError:
@@ -596,17 +646,21 @@ class ConnectionHandler:
return tts_file, text, text_index
def speak_and_play_stream(self, text, queue: queue.Queue, text_index=0):
if text is None or len(text) <= 0:
self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
return None, text
self.tts.to_tts_stream(text, queue, text_index)
try:
if text is None or len(text) <= 0:
self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
return None, text
self.tts.to_tts_stream(text, queue, text_index)
except Exception as e:
self.logger.bind(tag=TAG).error(e)
traceback.print_exc()
raise e
def clearSpeakStatus(self):
self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
self.asr_server_receive = True
self.tts_last_text_index = -1
self.tts_first_text_index = -1
self.tts_duration = 0
def recode_first_last_text(self, text, text_index=0):
if self.tts_first_text_index == -1: