From dc7805782351df648f563d9c16e435d029dca250 Mon Sep 17 00:00:00 2001 From: 3030332422 <3030332422@qq.com> Date: Tue, 15 Jul 2025 14:52:56 +0800 Subject: [PATCH 1/4] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8D=E5=A4=9A=E6=A8=A1?= =?UTF-8?q?=E5=9D=97=E9=83=A8=E7=BD=B2=E6=97=B6=E8=BF=9C=E7=A8=8BASR?= =?UTF-8?q?=E5=A3=B0=E7=BA=B9=E8=AF=86=E5=88=AB=E5=8A=9F=E8=83=BD=E6=9C=AA?= =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index ce68e5b2..ac83a656 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -348,6 +348,14 @@ class ConnectionHandler: self.vad = self._vad if self.asr is None: self.asr = self._initialize_asr() + try: + success = initialize_voiceprint(self.asr, self.config) + if success: + self.logger.bind(tag=TAG).info("声纹识别功能已在连接时动态启用") + else: + self.logger.bind(tag=TAG).info("声纹识别功能未启用或配置不完整") + except Exception as e: + self.logger.bind(tag=TAG).warning(f"声纹识别初始化失败: {str(e)}") # 打开语音识别通道 asyncio.run_coroutine_threadsafe( self.asr.open_audio_channels(self), self.loop @@ -416,16 +424,6 @@ class ConnectionHandler: # 因为远程ASR，涉及到websocket连接和接收线程，需要每个连接一个实例 asr = initialize_asr(self.config) - # 动态初始化声纹识别功能 - try: - success = initialize_voiceprint(asr, self.config) - if success: - self.logger.bind(tag=TAG).info("声纹识别功能已在连接时动态启用") - else: - self.logger.bind(tag=TAG).info("声纹识别功能未启用或配置不完整") - except Exception as e: - self.logger.bind(tag=TAG).error(f"动态初始化声纹识别时发生错误: {str(e)}") - return asr def _initialize_private_config(self): From 953a2326545a1fe8ec531a45e33e1409a1def03b Mon Sep 17 00:00:00 2001 From: 3030332422 <3030332422@qq.com> Date: Tue, 15 Jul 2025 18:13:01 +0800 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E8=B1=86=E5=8C=85=E6=B5=81=E5=BC=8FASR=E6=97=B6=EF=BC=8C?= =?UTF-8?q?=E4=B8=8D=E8=83=BD=E4=BD=BF=E7=94=A8=E5=A3=B0=E7=BA=B9=E8=AF=86?= =?UTF-8?q?=E5=88=AB=E5=8A=9F=E8=83=BD=E7=9A=84=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/asr/doubao_stream.py | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/main/xiaozhi-server/core/providers/asr/doubao_stream.py b/main/xiaozhi-server/core/providers/asr/doubao_stream.py index 31704b96..67964075 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao_stream.py +++ b/main/xiaozhi-server/core/providers/asr/doubao_stream.py @@ -56,6 +56,16 @@ class ASRProvider(ASRProviderBase): async def receive_audio(self, conn, audio, audio_have_voice): conn.asr_audio.append(audio) conn.asr_audio = conn.asr_audio[-10:] + + # 存储音频数据 + if not hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + conn.asr_audio_for_voiceprint.append(audio) + + # 当没有音频数据时处理完整语音片段 + if not audio and len(conn.asr_audio_for_voiceprint) > 0: + await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint) + conn.asr_audio_for_voiceprint = [] # 如果本次有声音，且之前没有建立连接 if audio_have_voice and self.asr_ws is None and not self.is_processing: @@ -148,6 +158,8 @@ class ASRProvider(ASRProviderBase): async def _forward_asr_results(self, conn): try: while self.asr_ws and not conn.stop_event.is_set(): + # 获取当前连接的音频数据 + audio_data = getattr(conn, 'asr_audio_for_voiceprint', []) try: response = await self.asr_ws.recv() result = self.parse_response(response) @@ -171,7 +183,8 @@ class ASRProvider(ASRProviderBase): logger.bind(tag=TAG).error(f"识别文本：空") self.text = "" conn.reset_vad_states() - await self.handle_voice_stop(conn, None) + if len(audio_data) > 15: # 确保有足够音频数据 + await self.handle_voice_stop(conn, audio_data) break for utterance in utterances: @@ -181,7 +194,8 @@ class ASRProvider(ASRProviderBase): f"识别到文本: {self.text}" ) conn.reset_vad_states() - await self.handle_voice_stop(conn, None) + if len(audio_data) > 15: # 确保有足够音频数据 + await self.handle_voice_stop(conn, audio_data) break elif "error" in payload: error_msg = payload.get("error", "未知错误") @@ -208,6 +222,13 @@ class ASRProvider(ASRProviderBase): await self.asr_ws.close() self.asr_ws = None self.is_processing = False + if conn: + if hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + if hasattr(conn, 'asr_audio'): + conn.asr_audio = [] + if hasattr(conn, 'has_valid_voice'): + conn.has_valid_voice = False def stop_ws_connection(self): if self.asr_ws: @@ -349,3 +370,12 @@ class ASRProvider(ASRProviderBase): pass self.forward_task = None self.is_processing = False + # 清理所有连接的音频缓冲区 + if hasattr(self, '_connections'): + for conn in self._connections.values(): + if hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + if hasattr(conn, 'asr_audio'): + conn.asr_audio = [] + if hasattr(conn, 'has_valid_voice'): + conn.has_valid_voice = False From 1a411da1335d27dd4ed1c9bb648e21ca08de9a31 Mon Sep 17 00:00:00 2001 From: 3030332422 <3030332422@qq.com> Date: Wed, 16 Jul 2025 15:42:21 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8DFunASR=E5=A4=9A?= =?UTF-8?q?=E8=AE=BE=E5=A4=87=E8=BF=9E=E6=8E=A5=E6=97=B6=E5=A3=B0=E7=BA=B9?= =?UTF-8?q?=E8=AF=86=E5=88=AB=E9=85=8D=E7=BD=AE=E7=9B=B8=E4=BA=92=E8=A6=86?= =?UTF-8?q?=E7=9B=96=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 29 +++++++++++++------ .../xiaozhi-server/core/providers/asr/base.py | 17 ++++------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index ac83a656..98bc87ed 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -24,7 +24,6 @@ from core.utils.modules_initialize import ( initialize_asr, ) from core.handle.reportHandle import report -from core.utils.modules_initialize import initialize_voiceprint from core.providers.tts.default import DefaultTTS from concurrent.futures import ThreadPoolExecutor from core.utils.dialogue import Message, Dialogue @@ -39,6 +38,7 @@ from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType from config.logger import setup_logging, build_module_string, create_connection_logger from config.manage_api_client import DeviceNotFoundException, DeviceBindException from core.utils.prompt_manager import PromptManager +from core.utils.voiceprint_provider import VoiceprintProvider TAG = __name__ @@ -109,6 +109,9 @@ class ConnectionHandler: self.memory = _memory self.intent = _intent + # 为每个连接单独管理声纹识别 + self.voiceprint_provider = None + # vad相关变量 self.client_audio_buffer = bytearray() self.client_have_voice = False @@ -348,14 +351,10 @@ class ConnectionHandler: self.vad = self._vad if self.asr is None: self.asr = self._initialize_asr() - try: - success = initialize_voiceprint(self.asr, self.config) - if success: - self.logger.bind(tag=TAG).info("声纹识别功能已在连接时动态启用") - else: - self.logger.bind(tag=TAG).info("声纹识别功能未启用或配置不完整") - except Exception as e: - self.logger.bind(tag=TAG).warning(f"声纹识别初始化失败: {str(e)}") + + # 初始化声纹识别 + self._initialize_voiceprint() + # 打开语音识别通道 asyncio.run_coroutine_threadsafe( self.asr.open_audio_channels(self), self.loop @@ -426,6 +425,18 @@ class ConnectionHandler: return asr + def _initialize_voiceprint(self): + """为当前连接初始化声纹识别""" + try: + voiceprint_config = self.config.get("voiceprint", {}) + if voiceprint_config: + self.voiceprint_provider = VoiceprintProvider(voiceprint_config) + self.logger.bind(tag=TAG).info("声纹识别功能已在连接时动态启用") + else: + self.logger.bind(tag=TAG).info("声纹识别功能未启用或配置不完整") + except Exception as e: + self.logger.bind(tag=TAG).warning(f"声纹识别初始化失败: {str(e)}") + def _initialize_private_config(self): """如果是从配置文件获取，则进行二次实例化""" if not self.read_config_from_api: diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index ef9fa01e..e34d5c2f 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -17,7 +17,6 @@ from core.handle.receiveAudioHandle import startToChat from core.handle.reportHandle import enqueue_asr_report from core.utils.util import remove_punctuation_and_length from core.handle.receiveAudioHandle import handleAudioMessage -from core.utils.voiceprint_provider import VoiceprintProvider TAG = __name__ logger = setup_logging() @@ -25,13 +24,7 @@ logger = setup_logging() class ASRProviderBase(ABC): def __init__(self): - self.voiceprint_provider = None - - def init_voiceprint(self, voiceprint_config: dict): - """初始化声纹识别""" - if voiceprint_config: - self.voiceprint_provider = VoiceprintProvider(voiceprint_config) - logger.bind(tag=TAG).info("声纹识别模块已初始化") + pass # 将声纹识别从ASR实例分离，移到连接级别管理 # 打开音频通道 async def open_audio_channels(self, conn): @@ -94,7 +87,8 @@ class ASRProviderBase(ABC): # 预先准备WAV数据 wav_data = None - if self.voiceprint_provider and combined_pcm_data: + # 使用连接的声纹识别提供者 + if conn.voiceprint_provider and combined_pcm_data: wav_data = self._pcm_to_wav(combined_pcm_data) @@ -129,8 +123,9 @@ class ASRProviderBase(ABC): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) try: + # 使用连接的声纹识别提供者 result = loop.run_until_complete( - self.voiceprint_provider.identify_speaker(wav_data, conn.session_id) + conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id) ) return result finally: @@ -145,7 +140,7 @@ class ASRProviderBase(ABC): with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor: asr_future = thread_executor.submit(run_asr) - if self.voiceprint_provider and wav_data: + if conn.voiceprint_provider and wav_data: voiceprint_future = thread_executor.submit(run_voiceprint) # 等待两个线程都完成 From 8831ea22b7668fda1199bf7d756597bf743c95c2 Mon Sep 17 00:00:00 2001 From: 3030332422 <3030332422@qq.com> Date: Wed, 16 Jul 2025 16:43:49 +0800 Subject: [PATCH 4/4] =?UTF-8?q?update:=E5=88=A0=E5=8E=BB=E4=B8=80=E4=BA=9B?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E3=80=81=E4=B8=8D=E7=94=A8=E7=9A=84=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/providers/asr/base.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index e34d5c2f..972818d4 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -24,7 +24,7 @@ logger = setup_logging() class ASRProviderBase(ABC): def __init__(self): - pass # 将声纹识别从ASR实例分离，移到连接级别管理 + pass # 打开音频通道 async def open_audio_channels(self, conn): @@ -96,7 +96,6 @@ class ASRProviderBase(ABC): def run_asr(): start_time = time.monotonic() try: - import asyncio loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) try: @@ -117,9 +116,7 @@ class ASRProviderBase(ABC): def run_voiceprint(): if not wav_data: return None - start_time = time.monotonic() try: - import asyncio loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) try: @@ -152,7 +149,6 @@ class ASRProviderBase(ABC): asr_result = asr_future.result(timeout=15) results = {"asr": asr_result, "voiceprint": None} - parallel_execution_time = time.monotonic() - parallel_start_time # 处理结果 raw_text, file_path = results.get("asr", ("", None))