update:ASR加入队列

main/xiaozhi-server/core/providers/asr/aliyun.py

@@ -213,7 +213,7 @@ class ASRProvider(ASRProviderBase):
             return None
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str
+        self, opus_data: List[bytes], session_id: str, audio_format="opus"
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if self._is_token_expired():
@@ -223,7 +223,7 @@ class ASRProvider(ASRProviderBase):
         file_path = None
         try:
             # 解码Opus为PCM
-            if self.audio_format == "pcm":
+            if audio_format == "pcm":
                 pcm_data = opus_data
             else:
                 pcm_data = self.decode_opus(opus_data)

main/xiaozhi-server/core/providers/asr/baidu.py

@@ -30,7 +30,7 @@ class ASRProvider(ASRProviderBase):
         os.makedirs(self.output_dir, exist_ok=True)
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str
+        self, opus_data: List[bytes], session_id: str, audio_format="opus"
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if not opus_data:
@@ -45,7 +45,7 @@ class ASRProvider(ASRProviderBase):
                 return None, file_path
 
             # 将Opus音频数据解码为PCM
-            if self.audio_format == "pcm":
+            if audio_format == "pcm":
                 pcm_data = opus_data
             else:
                 pcm_data = self.decode_opus(opus_data)

main/xiaozhi-server/core/providers/asr/base.py

@@ -1,15 +1,19 @@
 import os
-import time
+import wave
 import copy
 import uuid
-import wave
+import queue
+import asyncio
+import traceback
+import threading
 import opuslib_next
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
-from core.utils.util import remove_punctuation_and_length
-from core.handle.reportHandle import enqueue_asr_report
 from core.handle.receiveAudioHandle import startToChat
+from core.handle.reportHandle import enqueue_asr_report
+from core.utils.util import remove_punctuation_and_length
+from core.handle.receiveAudioHandle import handleAudioMessage
 
 TAG = __name__
 logger = setup_logging()
@@ -17,53 +21,71 @@ logger = setup_logging()
 
 class ASRProviderBase(ABC):
     def __init__(self):
-        self.audio_format = "opus"
-        self.conn = None
+        pass
 
     # 打开音频通道
     # 这里默认是非流式的处理方式
     # 流式处理方式请在子类中重写
     async def open_audio_channels(self, conn):
-        self.conn = conn
+        # tts 消化线程
+        conn.asr_priority_thread = threading.Thread(
+            target=self.asr_text_priority_thread, args=(conn,), daemon=True
+        )
+        conn.asr_priority_thread.start()
+
+    # 有序处理ASR音频
+    def asr_text_priority_thread(self, conn):
+        while not conn.stop_event.is_set():
+            try:
+                message = conn.asr_audio_queue.get(timeout=1)
+                future = asyncio.run_coroutine_threadsafe(
+                    handleAudioMessage(conn, message),
+                    conn.loop,
+                )
+                future.result()
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"处理ASR文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
+                )
+                continue
 
     # 接收音频
     # 这里默认是非流式的处理方式
     # 流式处理方式请在子类中重写
-    async def receive_audio(self, audio, audio_have_voice):
-        if (
-            self.conn.client_listen_mode == "auto"
-            or self.conn.client_listen_mode == "realtime"
-        ):
+    async def receive_audio(self, conn, audio, audio_have_voice):
+        if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime":
             have_voice = audio_have_voice
         else:
-            have_voice = self.conn.client_have_voice
+            have_voice = conn.client_have_voice
         # 如果本次没有声音，本段也没声音，就把声音丢弃了
-        self.conn.asr_audio.append(audio)
-        if have_voice == False and self.conn.client_have_voice == False:
-            self.conn.asr_audio = self.conn.asr_audio[-10:]
+        conn.asr_audio.append(audio)
+        if have_voice == False and conn.client_have_voice == False:
+            conn.asr_audio = conn.asr_audio[-10:]
             return
 
         # 如果本段有声音，且已经停止了
-        if self.conn.client_voice_stop:
-            asr_audio_task = copy.deepcopy(self.conn.asr_audio)
-            self.conn.asr_audio.clear()
+        if conn.client_voice_stop:
+            asr_audio_task = copy.deepcopy(conn.asr_audio)
+            conn.asr_audio.clear()
 
             # 音频太短了，无法识别
-            self.conn.reset_vad_states()
+            conn.reset_vad_states()
             if len(asr_audio_task) > 15:
-                await self.handle_voice_stop(asr_audio_task)
+                await self.handle_voice_stop(conn, asr_audio_task)
 
     # 处理语音停止
-    async def handle_voice_stop(self, asr_audio_task):
+    async def handle_voice_stop(self, conn, asr_audio_task):
         raw_text, _ = await self.speech_to_text(
-            asr_audio_task, self.conn.session_id
+            asr_audio_task, conn.session_id, conn.audio_format
         )  # 确保ASR模块返回原始文本
-        self.conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
+        conn.logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
         text_len, _ = remove_punctuation_and_length(raw_text)
         if text_len > 0:
             # 使用自定义模块进行上报
-            await startToChat(self.conn, raw_text)
-            enqueue_asr_report(self.conn, raw_text, asr_audio_task)
+            await startToChat(conn, raw_text)
+            enqueue_asr_report(conn, raw_text, asr_audio_task)
 
     def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
         """PCM数据保存为WAV文件"""
@@ -81,15 +103,11 @@ class ASRProviderBase(ABC):
 
     @abstractmethod
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str
+        self, opus_data: List[bytes], session_id: str, audio_format="opus"
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         pass
 
-    def set_audio_format(self, format: str) -> None:
-        """设置音频格式"""
-        self.audio_format = format
-
     @staticmethod
     def decode_opus(opus_data: List[bytes]) -> bytes:
         """将Opus音频数据解码为PCM数据"""

main/xiaozhi-server/core/providers/asr/doubao.py

@@ -228,14 +228,14 @@ class ASRProvider(ASRProviderBase):
             yield data[offset:data_len], True
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str
+        self, opus_data: List[bytes], session_id: str, audio_format="opus"
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
 
         file_path = None
         try:
             # 合并所有opus数据包
-            if self.audio_format == "pcm":
+            if audio_format == "pcm":
                 pcm_data = opus_data
             else:
                 pcm_data = self.decode_opus(opus_data)

main/xiaozhi-server/core/providers/asr/doubao_stream.py

@@ -190,7 +190,7 @@ class ASRProvider(ASRProviderBase):
                 self.is_reconnecting = False
                 self._session_close_event.set()
 
-    async def receive_audio(self, audio, _):
+    async def receive_audio(self, conn, audio, _):
         if not isinstance(audio, bytes):
             return
 
@@ -411,7 +411,7 @@ class ASRProvider(ASRProviderBase):
                 await asyncio.sleep(self.retry_delay)
                 await self._forward_asr_results()  # 递归重试
 
-    async def speech_to_text(self, opus_data, session_id):
+    async def speech_to_text(self, opus_data, session_id, audio_format):
         result = self.text
         self.text = ""  # 清空text
         return result, None

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -54,7 +54,7 @@ class ASRProvider(ASRProviderBase):
             )
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str
+        self, opus_data: List[bytes], session_id: str, audio_format="opus"
     ) -> Tuple[Optional[str], Optional[str]]:
         """语音转文本主处理逻辑"""
         file_path = None
@@ -63,7 +63,7 @@ class ASRProvider(ASRProviderBase):
         while retry_count < MAX_RETRIES:
             try:
                 # 合并所有opus数据包
-                if self.audio_format == "pcm":
+                if audio_format == "pcm":
                     pcm_data = opus_data
                 else:
                     pcm_data = self.decode_opus(opus_data)

main/xiaozhi-server/core/providers/asr/fun_server.py

@@ -100,7 +100,7 @@ class ASRProvider(ASRProviderBase):
         logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str
+        self, opus_data: List[bytes], session_id: str, audio_format="opus"
     ) -> Tuple[Optional[str], Optional[str]]:
         """
         Convert speech data to text using FunASR.
@@ -109,7 +109,7 @@ class ASRProvider(ASRProviderBase):
         :return: Tuple containing recognized text and optional timestamp.
         """
         file_path = None
-        if self.audio_format == "pcm":
+        if audio_format == "pcm":
             pcm_data = opus_data
         else:
             pcm_data = self.decode_opus(opus_data)

main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py

@@ -109,14 +109,14 @@ class ASRProvider(ASRProviderBase):
             return samples_float32, f.getframerate()
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str
+        self, opus_data: List[bytes], session_id: str, audio_format="opus"
     ) -> Tuple[Optional[str], Optional[str]]:
         """语音转文本主处理逻辑"""
         file_path = None
         try:
             # 保存音频文件
             start_time = time.time()
-            if self.audio_format == "pcm":
+            if audio_format == "pcm":
                 pcm_data = opus_data
             else:
                 pcm_data = self.decode_opus(opus_data)

main/xiaozhi-server/core/providers/asr/tencent.py

@@ -32,7 +32,7 @@ class ASRProvider(ASRProviderBase):
         os.makedirs(self.output_dir, exist_ok=True)
 
     async def speech_to_text(
-        self, opus_data: List[bytes], session_id: str
+        self, opus_data: List[bytes], session_id: str, audio_format="opus"
     ) -> Tuple[Optional[str], Optional[str]]:
         """将语音数据转换为文本"""
         if not opus_data:
@@ -47,7 +47,7 @@ class ASRProvider(ASRProviderBase):
                 return None, file_path
 
             # 将Opus音频数据解码为PCM
-            if self.audio_format == "pcm":
+            if audio_format == "pcm":
                 pcm_data = opus_data
             else:
                 pcm_data = self.decode_opus(opus_data)

main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py

@@ -122,6 +122,8 @@ class IntentProvider(IntentProviderBase):
     async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str:
         if not self.llm:
             raise ValueError("LLM provider not set")
+        if conn.func_handler is None:
+            return '{"function_call": {"name": "continue_chat"}}'
 
         # 记录整体开始时间
         total_start_time = time.time()
@@ -148,9 +150,8 @@ class IntentProvider(IntentProviderBase):
         self.clean_cache()
 
         if self.promot == "":
-            if hasattr(conn, "func_handler"):
-                functions = conn.func_handler.get_functions()
-                self.promot = self.get_intent_system_prompt(functions)
+            functions = conn.func_handler.get_functions()
+            self.promot = self.get_intent_system_prompt(functions)
 
         music_config = initialize_music_handler(conn)
         music_file_names = music_config["music_file_names"]