Merge branch 'xinnan-tech:main' into dev

main/xiaozhi-server/core/handle/intentHandler.py

@@ -1,4 +1,3 @@
-from config.logger import setup_logging
 import json
 import uuid
 from core.handle.sendAudioHandle import send_stt_message
@@ -146,4 +145,3 @@ async def process_intent_result(conn, intent_result, original_text):
 def speak_txt(conn, text):
     conn.tts.tts_one_sentence(conn, ContentType.TEXT, content_detail=text)
     conn.dialogue.put(Message(role="assistant", content=text))
- 

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -1,10 +1,7 @@
-import time
-import copy
-from core.utils.util import remove_punctuation_and_length
 from core.handle.sendAudioHandle import send_stt_message
 from core.handle.intentHandler import handle_user_intent
 from core.utils.output_counter import check_device_output_limit
-from core.handle.reportHandle import enqueue_asr_report
+import time
 from core.handle.sendAudioHandle import SentenceType
 from core.utils.util import audio_to_data
 
@@ -13,46 +10,17 @@ TAG = __name__
 
 async def handleAudioMessage(conn, audio):
     if conn.vad is None:
+        conn.logger.bind(tag=TAG).warning("VAD模块未初始化，继续等待")
         return
-    if not conn.asr_server_receive:
-        conn.logger.bind(tag=TAG).debug(f"前期数据处理中，暂停接收")
+    if conn.asr is None:
+        conn.logger.bind(tag=TAG).warning("ASR模块未初始化，继续等待")
         return
-    if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime":
-        have_voice = conn.vad.is_vad(conn, audio)
-    else:
-        have_voice = conn.client_have_voice
-
-    # 如果本次没有声音，本段也没声音，就把声音丢弃了
-    if have_voice == False and conn.client_have_voice == False:
-        await no_voice_close_connect(conn)
-        conn.asr_audio.append(audio)
-        conn.asr_audio = conn.asr_audio[
-            -10:
-        ]  # 保留最新的10帧音频内容，解决ASR句首丢字问题
-        return
-    conn.client_no_voice_last_time = 0.0
-    conn.asr_audio.append(audio)
-    # 如果本段有声音，且已经停止了
-    if conn.client_voice_stop:
-        conn.client_abort = False
-        conn.asr_server_receive = False
-        # 音频太短了，无法识别
-        if len(conn.asr_audio) < 15:
-            conn.asr_server_receive = True
-        else:
-            raw_text, _ = await conn.asr.speech_to_text(
-                conn.asr_audio, conn.session_id
-            )  # 确保ASR模块返回原始文本
-            conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
-            text_len, _ = remove_punctuation_and_length(raw_text)
-            if text_len > 0:
-                # 使用自定义模块进行上报
-                await startToChat(conn, raw_text)
-                enqueue_asr_report(conn, raw_text, copy.deepcopy(conn.asr_audio))
-            else:
-                conn.asr_server_receive = True
-        conn.asr_audio.clear()
-        conn.reset_vad_states()
+    # 当前片段是否有人说话
+    have_voice = conn.vad.is_vad(conn, audio)
+    # 设备长时间空闲检测，用于say goodbye
+    await no_voice_close_connect(conn, have_voice)
+    # 接收音频
+    await conn.asr.receive_audio(audio, have_voice)
 
 
 async def startToChat(conn, text):
@@ -73,7 +41,6 @@ async def startToChat(conn, text):
 
     if intent_handled:
         # 如果意图已被处理，不再进行聊天
-        conn.asr_server_receive = True
         return
 
     # 意图未被处理，继续常规聊天流程
@@ -81,7 +48,10 @@ async def startToChat(conn, text):
     conn.executor.submit(conn.chat, text)
 
 
-async def no_voice_close_connect(conn):
+async def no_voice_close_connect(conn, have_voice):
+    if have_voice:
+        conn.client_no_voice_last_time = 0.0
+        return
     if conn.client_no_voice_last_time == 0.0:
         conn.client_no_voice_last_time = time.time() * 1000
     else:
@@ -95,7 +65,6 @@ async def no_voice_close_connect(conn):
         ):
             conn.close_after_chat = True
             conn.client_abort = False
-            conn.asr_server_receive = False
             end_prompt = conn.config.get("end_prompt", {})
             if end_prompt and end_prompt.get("enable", True) is False:
                 conn.logger.bind(tag=TAG).info("结束对话，无需发送结束提示语")

main/xiaozhi-server/core/handle/reportHandle.py

@@ -8,6 +8,7 @@ TTS上报功能已集成到ConnectionHandler类中。
 
 具体实现请参考core/connection.py中的相关代码。
 """
+
 import time
 
 import opuslib_next
@@ -145,4 +146,4 @@ def enqueue_asr_report(conn, text, opus_data):
                 f"ASR数据已加入上报队列: {conn.device_id}, 不上报音频"
             )
     except Exception as e:
-        conn.logger.bind(tag=TAG).error(f"加入ASR上报队列失败: {text}, {e}")
+        conn.logger.bind(tag=TAG).debug(f"加入ASR上报队列失败: {text}, {e}")

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -131,6 +131,11 @@ async def send_tts_message(conn, state, text=None):
 
 
 async def send_stt_message(conn, text):
+    end_prompt_str = conn.config.get("end_prompt", {}).get("prompt")
+    if end_prompt_str and end_prompt_str == text:
+        await send_tts_message(conn, "start")
+        return
+
     """发送 STT 状态消息"""
     stt_text = get_string_no_punctuation_or_emoji(text)
     await conn.websocket.send(

main/xiaozhi-server/core/handle/textHandle.py

@@ -42,12 +42,13 @@ async def handleTextMessage(conn, message):
                 if len(conn.asr_audio) > 0:
                     await handleAudioMessage(conn, b"")
             elif msg_json["state"] == "detect":
-                conn.asr_server_receive = False
                 conn.client_have_voice = False
                 conn.asr_audio.clear()
                 if "text" in msg_json:
                     original_text = msg_json["text"]  # 保留原始文本
-                    filtered_len, filtered_text = remove_punctuation_and_length(original_text)
+                    filtered_len, filtered_text = remove_punctuation_and_length(
+                        original_text
+                    )
 
                     # 识别是否是唤醒词
                     is_wakeup_words = filtered_text in conn.config.get("wakeup_words")