Merge branch 'py_test_end' into fix-end

main/xiaozhi-server/core/handle/intentHandler.py

@@ -10,6 +10,7 @@ from core.providers.tts.dto.dto import ContentType
 from core.handle.helloHandle import checkWakeupWords
 from plugins_func.register import Action, ActionResponse
 from core.handle.sendAudioHandle import send_stt_message
+from core.handle.reportHandle import enqueue_tool_report
 from core.utils.util import remove_punctuation_and_length
 from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType
 
@@ -146,6 +147,17 @@ async def process_intent_result(
             await send_stt_message(conn, original_text)
             conn.client_abort = False
 
+            # 准备工具调用参数
+            tool_input = {}
+            if function_args:
+                if isinstance(function_args, str):
+                    tool_input = json.loads(function_args) if function_args else {}
+                elif isinstance(function_args, dict):
+                    tool_input = function_args
+
+            # 上报工具调用
+            enqueue_tool_report(conn, function_name, tool_input)
+
             # 使用executor执行函数调用和结果处理
             def process_function_call():
                 conn.dialogue.put(Message(role="user", content=original_text))
@@ -166,7 +178,10 @@ async def process_intent_result(
                         action=Action.ERROR, result="工具调用超时，请一会再试下哈", response="工具调用超时，请一会再试下哈"
                     )
 
+                # 上报工具调用结果
                 if result:
+                    enqueue_tool_report(conn, function_name, tool_input, str(result.result) if result.result else None, report_tool_call=False)
+
                     if result.action == Action.RESPONSE:  # 直接回复前端
                         text = result.response
                         if text is not None:

main/xiaozhi-server/core/handle/reportHandle.py

@@ -10,6 +10,7 @@ TTS上报功能已集成到ConnectionHandler类中。
 """
 
 import time
+import json
 import opuslib_next
 from typing import TYPE_CHECKING
 
@@ -26,7 +27,7 @@ async def report(conn: "ConnectionHandler", type, text, opus_data, report_time):
 
     Args:
         conn: 连接对象
-        type: 上报类型，1为用户，2为智能体
+        type: 上报类型，1为用户，2为智能体，3为工具调用
         text: 合成文本
         opus_data: opus音频数据
         report_time: 上报时间
@@ -132,6 +133,45 @@ def enqueue_tts_report(conn: "ConnectionHandler", text, opus_data):
         conn.logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}")
 
 
+def enqueue_tool_report(conn: "ConnectionHandler", tool_name: str, tool_input: dict, tool_result: str = None, report_tool_call: bool = True):
+    """将工具调用数据加入上报队列
+
+    Args:
+        conn: 连接对象
+        tool_name: 工具名称
+        tool_input: 工具输入参数
+        tool_result: 工具执行结果（可选）
+        report_tool_call: 是否上报工具调用本身，默认True；仅上报结果时设为False
+    """
+    if not conn.read_config_from_api or conn.need_bind:
+        return
+    if conn.chat_history_conf == 0:
+        return
+
+    try:
+        timestamp = int(time.time())
+
+        # 构建工具调用内容
+        if report_tool_call:
+            tool_text = json.dumps(
+                [
+                    {
+                        "type": "tool",
+                        "text": f"{tool_name}({json.dumps(tool_input, ensure_ascii=False)})",
+                    }
+                ]
+            )
+            conn.report_queue.put((3, tool_text, None, timestamp))
+
+        # 构建工具结果内容
+        if tool_result:
+            result_display = f'{{"result":"{str(tool_result)}"}}'
+            result_content = json.dumps([{"type": "tool_result", "text": result_display}], ensure_ascii=False)
+            conn.report_queue.put((3, result_content, None, timestamp + 1))
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"加入工具上报队列失败: {e}")
+
+
 def enqueue_asr_report(conn: "ConnectionHandler", text, opus_data):
     if not conn.read_config_from_api or conn.need_bind or not conn.report_asr_enable:
         return

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -21,7 +21,6 @@ async def sendAudioMessage(conn: "ConnectionHandler", sentenceType, audios, text
     if conn.tts.tts_audio_first_sentence:
         conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
         conn.tts.tts_audio_first_sentence = False
-        await send_tts_message(conn, "start", None)
 
     if sentenceType == SentenceType.FIRST:
         # 同一句子的后续消息加入流控队列，其他情况立即发送
@@ -204,7 +203,6 @@ def _start_background_sender(conn: "ConnectionHandler", rate_controller, flow_co
 
         conn.last_activity_time = time.time() * 1000
         await _do_send_audio(conn, packet, flow_control)
-        conn.client_is_speaking = True
 
     # 使用 start_sending 启动后台循环
     rate_controller.start_sending(send_callback)
@@ -232,12 +230,10 @@ async def _send_audio_with_rate_control(
         # 预缓冲：前N个包直接发送
         if flow_control["packet_count"] < PRE_BUFFER_COUNT:
             await _do_send_audio(conn, packet, flow_control)
-            conn.client_is_speaking = True
         elif send_delay > 0:
             # 固定延迟模式
             await asyncio.sleep(send_delay)
             await _do_send_audio(conn, packet, flow_control)
-            conn.client_is_speaking = True
         else:
             # 动态流控模式：仅添加到队列，由后台循环负责发送
             rate_controller.add_audio(packet)
@@ -284,6 +280,8 @@ async def send_tts_message(conn: "ConnectionHandler", state, text=None):
             await sendAudio(conn, audios)
         # 等待所有音频包发送完成
         await _wait_for_audio_completion(conn)
+        # 停止音频发送循环
+        conn.audio_rate_controller.stop_sending()
         # 清除服务端讲话状态
         conn.clearSpeakStatus()
 
@@ -318,3 +316,15 @@ async def send_stt_message(conn: "ConnectionHandler", text):
         json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id})
     )
     await send_tts_message(conn, "start")
+    # 发送start消息后客户端状态会处于说话中状态，同步服务端状态
+    conn.client_is_speaking = True
+
+
+async def send_display_message(conn: "ConnectionHandler", text):
+    """发送纯显示消息"""
+    message = {
+        "type": "stt",
+        "text": text,
+        "session_id": conn.session_id
+    }
+    await conn.websocket.send(json.dumps(message))