update: 增加唤醒时声纹处理 ，1秒内发送至大模型 【需优化唤醒锁机制，中途会遭受打断(偶发)，考虑忽略检测】

main/xiaozhi-server/core/providers/asr/base.py

@@ -1,14 +1,14 @@
 import os
+import io
 import wave
 import uuid
+import json
+import time
 import queue
 import asyncio
 import traceback
 import threading
 import opuslib_next
-import json
-import io
-import time
 import concurrent.futures
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
@@ -53,6 +53,10 @@ class ASRProviderBase(ABC):
 
     # 接收音频
     async def receive_audio(self, conn, audio, audio_have_voice):
+        # 检查是否在唤醒处理锁定期内
+        if getattr(conn, 'wakeup_processing_lock', 0) > time.monotonic():
+            return
+            
         if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime":
             have_voice = audio_have_voice
         else:
@@ -62,6 +66,14 @@ class ASRProviderBase(ABC):
         if not have_voice and not conn.client_have_voice:
             conn.asr_audio = conn.asr_audio[-10:]
             return
+        
+        # 检查是否处于唤醒模式
+        if getattr(conn, 'wakeup_mode', False) and len(conn.asr_audio) >= 10:
+            asr_audio_task = conn.asr_audio.copy()
+            conn.reset_vad_states()
+            conn.asr_audio.clear()
+                
+            await self.handle_voice_stop(conn, asr_audio_task)
 
         if conn.client_voice_stop:
             asr_audio_task = conn.asr_audio.copy()
@@ -87,32 +99,13 @@ class ASRProviderBase(ABC):
             
             # 预先准备WAV数据
             wav_data = None
-            # 使用连接的声纹识别提供者
             if conn.voiceprint_provider and combined_pcm_data:
                 wav_data = self._pcm_to_wav(combined_pcm_data)
             
+            # 检查是否处于唤醒模式
+            wakeup_mode = getattr(conn, 'wakeup_mode', False)
             
-            # 定义ASR任务
-            def run_asr():
-                start_time = time.monotonic()
-                try:
-                    loop = asyncio.new_event_loop()
-                    asyncio.set_event_loop(loop)
-                    try:
-                        result = loop.run_until_complete(
-                            self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
-                        )
-                        end_time = time.monotonic()
-                        logger.bind(tag=TAG).info(f"ASR耗时: {end_time - start_time:.3f}s")
-                        return result
-                    finally:
-                        loop.close()
-                except Exception as e:
-                    end_time = time.monotonic()
-                    logger.bind(tag=TAG).error(f"ASR失败: {e}")
-                    return ("", None)
-            
-            # 定义声纹识别任务
+            # 声纹识别任务（公共逻辑）
             def run_voiceprint():
                 if not wav_data:
                     return None
@@ -131,48 +124,83 @@ class ASRProviderBase(ABC):
                     logger.bind(tag=TAG).error(f"声纹识别失败: {e}")
                     return None
             
-            # 使用线程池执行器并行运行
-            with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor:
-                asr_future = thread_executor.submit(run_asr)
+            if wakeup_mode and conn.voiceprint_provider and wav_data:
+                conn.wakeup_mode = False
+                # 设置处理锁，防止后续音频片段重复处理
+                conn.wakeup_processing_lock = time.monotonic() + 3  # 3秒锁定期
                 
-                if conn.voiceprint_provider and wav_data:
+                # 唤醒模式：只执行声纹识别
+                with concurrent.futures.ThreadPoolExecutor(max_workers=1) as thread_executor:
                     voiceprint_future = thread_executor.submit(run_voiceprint)
-                    
-                    # 等待两个线程都完成
-                    asr_result = asr_future.result(timeout=15)
                     voiceprint_result = voiceprint_future.result(timeout=15)
-                    
-                    results = {"asr": asr_result, "voiceprint": voiceprint_result}
-                else:
-                    asr_result = asr_future.result(timeout=15)
-                    results = {"asr": asr_result, "voiceprint": None}
-            
-            
-            # 处理结果
-            raw_text, file_path = results.get("asr", ("", None))
-            speaker_name = results.get("voiceprint", None)
-            
-            # 记录识别结果
-            if raw_text:
-                logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
-            if speaker_name:
+
+                speaker_name = voiceprint_result
                 logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
-            
-            # 性能监控
-            total_time = time.monotonic() - total_start_time
-            logger.bind(tag=TAG).info(f"总处理耗时: {total_time:.3f}s")
-            
-            # 检查文本长度
-            text_len, _ = remove_punctuation_and_length(raw_text)
-            self.stop_ws_connection()
-            
-            if text_len > 0:
-                # 构建包含说话人信息的JSON字符串
-                enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
+
+                fixed_text = "嘿，你好啊"
+                enhanced_text = self._build_enhanced_text(fixed_text, speaker_name)
                 
-                # 使用自定义模块进行上报
+                # 性能监控
+                total_time = time.monotonic() - total_start_time
+                logger.bind(tag=TAG).info(f"唤醒模式总处理耗时: {total_time:.3f}s")
+
                 await startToChat(conn, enhanced_text)
                 enqueue_asr_report(conn, enhanced_text, asr_audio_task)
+            else:
+                # 正常模式：执行声纹识别和ASR
+                def run_asr():
+                    start_time = time.monotonic()
+                    try:
+                        loop = asyncio.new_event_loop()
+                        asyncio.set_event_loop(loop)
+                        try:
+                            result = loop.run_until_complete(
+                                self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
+                            )
+                            end_time = time.monotonic()
+                            logger.bind(tag=TAG).info(f"ASR耗时: {end_time - start_time:.3f}s")
+                            return result
+                        finally:
+                            loop.close()
+                    except Exception as e:
+                        end_time = time.monotonic()
+                        logger.bind(tag=TAG).error(f"ASR失败: {e}")
+                        return ("", None)
+                
+                # 使用线程池执行器并行运行
+                with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor:
+                    asr_future = thread_executor.submit(run_asr)
+                    
+                    if conn.voiceprint_provider and wav_data:
+                        voiceprint_future = thread_executor.submit(run_voiceprint)
+                        asr_result = asr_future.result(timeout=15)
+                        voiceprint_result = voiceprint_future.result(timeout=15)
+                        results = {"asr": asr_result, "voiceprint": voiceprint_result}
+                    else:
+                        asr_result = asr_future.result(timeout=15)
+                        results = {"asr": asr_result, "voiceprint": None}
+                
+                # 处理结果
+                raw_text, _ = results.get("asr", ("", None))
+                speaker_name = results.get("voiceprint", None)
+                
+                if raw_text:
+                    logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
+                if speaker_name:
+                    logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
+                
+                # 性能监控
+                total_time = time.monotonic() - total_start_time
+                logger.bind(tag=TAG).info(f"总处理耗时: {total_time:.3f}s")
+                
+                # 检查文本长度
+                text_len, _ = remove_punctuation_and_length(raw_text)
+                self.stop_ws_connection()
+                
+                if text_len > 0:
+                    enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
+                    await startToChat(conn, enhanced_text)
+                    enqueue_asr_report(conn, enhanced_text, asr_audio_task)
                 
         except Exception as e:
             logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")

main/xiaozhi-server/core/providers/tts/linkerai.py

@@ -1,9 +1,7 @@
 import os
-import time
 import queue
 import aiohttp
 import asyncio
-import requests
 import traceback
 from config.logger import setup_logging
 from core.utils.tts import MarkdownCleaner
@@ -111,10 +109,6 @@ class TTSProvider(TTSProviderBase):
         finally:
             return None
 
-    ###################################################################################
-    # linkerai单流式TTS重写父类的方法--结束
-    ###################################################################################
-
     async def text_to_speak(self, text, is_last):
         """流式处理TTS音频，每句只推送一次音频列表"""
         await self._tts_request(text, is_last)
@@ -201,71 +195,3 @@ class TTSProvider(TTSProviderBase):
         except Exception as e:
             logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
             self.tts_audio_queue.put((SentenceType.LAST, [], None))
-
-    def to_tts(self, text: str) -> list:
-        """非流式TTS处理，用于测试及保存音频文件的场景
-        Args:
-            text: 要转换的文本
-        Returns:
-            list: 返回opus编码后的音频数据列表
-        """
-        start_time = time.time()
-        text = MarkdownCleaner.clean_markdown(text)
-
-        params = {
-            "tts_text": text,
-            "spk_id": self.voice,
-            "frame_duration": 60,
-            "stream": False,
-            "target_sr": 16000,
-            "audio_format": self.audio_format,
-            "instruct_text": "请生成一段自然流畅的语音",
-        }
-        headers = {
-            "Authorization": f"Bearer {self.access_token}",
-            "Content-Type": "application/json",
-        }
-
-        try:
-            with requests.get(
-                self.api_url, params=params, headers=headers, timeout=5
-            ) as response:
-                if response.status_code != 200:
-                    logger.bind(tag=TAG).error(
-                        f"TTS请求失败: {response.status_code}, {response.text}"
-                    )
-                    return []
-
-                logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒")
-
-                # 使用opus编码器处理PCM数据
-                opus_datas = []
-                pcm_data = response.content
-
-                # 计算每帧的字节数
-                frame_bytes = int(
-                    self.opus_encoder.sample_rate
-                    * self.opus_encoder.channels
-                    * self.opus_encoder.frame_size_ms
-                    / 1000
-                    * 2
-                )
-
-                # 分帧处理PCM数据
-                for i in range(0, len(pcm_data), frame_bytes):
-                    frame = pcm_data[i : i + frame_bytes]
-                    if len(frame) < frame_bytes:
-                        # 最后一帧可能不足，用0填充
-                        frame = frame + b"\x00" * (frame_bytes - len(frame))
-
-                    self.opus_encoder.encode_pcm_to_opus_stream(
-                        frame,
-                        end_of_stream=(i + frame_bytes >= len(pcm_data)),
-                        callback=lambda opus: opus_datas.append(opus)
-                    )
-
-                return opus_datas
-
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"TTS请求异常: {e}")
-            return []