Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-30 05:13:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into py_tts_huoshan

Browse Source
This commit is contained in:
CGD
2025-12-26 14:46:51 +08:00
committed by GitHub
parent 3c46e16494 8f96cf13ac
commit 68b0d9654c
134 changed files with 5839 additions and 1176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/providers/asr/aliyun_stream.py
+79 -70
View File
@@ -8,8 +8,6 @@ import asyncio
import requests
import websockets
import opuslib_next
import random
from typing import Optional, Tuple, List
from urllib import parse
from datetime import datetime
from config.logger import setup_logging
@@ -139,13 +137,13 @@ class ASRProvider(ASRProviderBase):
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-10:]
# 只在有声音且没有连接时建立连接
if audio_have_voice and not self.is_processing:
# 只在有声音且没有连接时建立连接（排除正在停止的情况）
if audio_have_voice and not self.is_processing and not self.asr_ws:
try:
await self._start_recognition(conn)
except Exception as e:
logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}")
await self._cleanup(conn)
await self._cleanup()
return
if self.asr_ws and self.is_processing and self.server_ready:
@@ -185,10 +183,8 @@ class ASRProvider(ASRProviderBase):
"header": {
"namespace": "SpeechTranscriber",
"name": "StartTranscription",
"status": 20000000,
"message_id": uuid.uuid4().hex,
"task_id": self.task_id,
"status_text": "Gateway:SUCCESS:Success.",
"appkey": self.appkey
},
"payload": {
@@ -207,18 +203,21 @@ class ASRProvider(ASRProviderBase):
async def _forward_results(self, conn):
"""转发识别结果"""
try:
while self.asr_ws and not conn.stop_event.is_set():
while not conn.stop_event.is_set():
try:
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
result = json.loads(response)
header = result.get("header", {})
payload = result.get("payload", {})
message_name = header.get("name", "")
status = header.get("status", 0)
if status != 20000000:
if status in [40000004, 40010004]: # 连接超时或客户端断开
if status == 40010004:
logger.bind(tag=TAG).warning(f"请在服务端响应完成后再关闭链接，状态码: {status}")
break
if status in [40000004, 40010003]: # 连接超时或客户端断开
logger.bind(tag=TAG).warning(f"连接问题，状态码: {status}")
break
elif status in [40270002, 40270003]: # 音频问题
@@ -227,12 +226,12 @@ class ASRProvider(ASRProviderBase):
else:
logger.bind(tag=TAG).error(f"识别错误，状态码: {status}, 消息: {header.get('status_text', '')}")
continue
# 收到TranscriptionStarted表示服务器准备好接收音频数据
if message_name == "TranscriptionStarted":
self.server_ready = True
logger.bind(tag=TAG).debug("服务器已准备，开始发送缓存音频...")
# 发送缓存音频
if conn.asr_audio:
for cached_audio in conn.asr_audio[-10:]:
@@ -243,89 +242,89 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}")
break
continue
if message_name == "TranscriptionResultChanged":
# 中间结果
text = payload.get("result", "")
if text:
self.text = text
elif message_name == "SentenceEnd":
# 最终结果
# 句子结束（每个句子都会触发）
text = payload.get("result", "")
if text:
self.text = text
conn.reset_vad_states()
# 传递缓存的音频数据
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
await self.handle_voice_stop(conn, audio_data)
# 清空缓存
conn.asr_audio_for_voiceprint = []
break
elif message_name == "TranscriptionCompleted":
# 识别完成
self.is_processing = False
break
logger.bind(tag=TAG).info(f"识别到文本: {text}")
# 手动模式下累积识别结果
if conn.client_listen_mode == "manual":
if self.text:
self.text += text
else:
self.text = text
# 手动模式下，只有在收到stop信号后才触发处理（仅处理一次）
if conn.client_voice_stop:
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
if len(audio_data) > 0:
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
break
else:
# 自动模式下直接覆盖
self.text = text
conn.reset_vad_states()
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
await self.handle_voice_stop(conn, audio_data)
break
except asyncio.TimeoutError:
continue
except websockets.exceptions.ConnectionClosed:
logger.bind(tag=TAG).error("接收结果超时")
break
except websockets.ConnectionClosed:
logger.bind(tag=TAG).info("ASR服务连接已关闭")
self.is_processing = False
break
except Exception as e:
logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}")
break
except Exception as e:
logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}")
finally:
await self._cleanup(conn)
# 清理连接的音频缓存
await self._cleanup()
if conn:
if hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
async def _cleanup(self, conn):
"""清理资源"""
logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
# 清理连接的音频缓存
if conn and hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
# 判断是否需要发送终止请求
should_stop = self.is_processing or self.server_ready
# 发送停止识别请求
if self.asr_ws and should_stop:
async def _send_stop_request(self):
"""发送停止识别请求（不关闭连接）"""
if self.asr_ws:
try:
# 先停止音频发送
self.is_processing = False
stop_msg = {
"header": {
"namespace": "SpeechTranscriber",
"name": "StopTranscription",
"status": 20000000,
"message_id": uuid.uuid4().hex,
"task_id": self.task_id,
"status_text": "Client:Stop",
"appkey": self.appkey
}
}
logger.bind(tag=TAG).debug("正在发送ASR终止请求")
logger.bind(tag=TAG).debug("停止识别请求已发送")
await self.asr_ws.send(json.dumps(stop_msg, ensure_ascii=False))
await asyncio.sleep(0.1)
logger.bind(tag=TAG).debug("ASR终止请求已发送")
except Exception as e:
logger.bind(tag=TAG).error(f"ASR终止请求发送失败: {e}")
# 状态重置（在终止请求发送后）
logger.bind(tag=TAG).error(f"发送停止识别请求失败: {e}")
async def _cleanup(self):
"""清理资源（关闭连接）"""
logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
# 状态重置
self.is_processing = False
self.server_ready = False
logger.bind(tag=TAG).debug("ASR状态已重置")
# 清理任务
if self.forward_task and not self.forward_task.done():
self.forward_task.cancel()
try:
await asyncio.wait_for(self.forward_task, timeout=1.0)
except Exception as e:
logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}")
finally:
self.forward_task = None
# 关闭连接
if self.asr_ws:
try:
@@ -336,7 +335,10 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
finally:
self.asr_ws = None
# 清理任务引用
self.forward_task = None
logger.bind(tag=TAG).debug("ASR会话清理完成")
async def speech_to_text(self, opus_data, session_id, audio_format):
@@ -347,4 +349,11 @@ class ASRProvider(ASRProviderBase):
async def close(self):
"""关闭资源"""
await self._cleanup()
await self._cleanup(None)
if hasattr(self, 'decoder') and self.decoder is not None:
try:
del self.decoder
self.decoder = None
logger.bind(tag=TAG).debug("Aliyun decoder resources released")
except Exception as e:
logger.bind(tag=TAG).debug(f"释放Aliyun decoder资源时出错: {e}")
main/xiaozhi-server/core/providers/asr/base.py
+57 -83
View File
@@ -9,7 +9,6 @@ import asyncio
import traceback
import threading
import opuslib_next
import concurrent.futures
from abc import ABC, abstractmethod
from config.logger import setup_logging
from typing import Optional, Tuple, List
@@ -53,121 +52,89 @@ class ASRProviderBase(ABC):
# 接收音频
async def receive_audio(self, conn, audio, audio_have_voice):
if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime":
have_voice = audio_have_voice
if conn.client_listen_mode == "manual":
# 手动模式：缓存音频用于ASR识别
conn.asr_audio.append(audio)
else:
have_voice = conn.client_have_voice
conn.asr_audio.append(audio)
if not have_voice and not conn.client_have_voice:
conn.asr_audio = conn.asr_audio[-10:]
return
# 自动/实时模式：使用VAD检测
have_voice = audio_have_voice
if conn.client_voice_stop:
asr_audio_task = conn.asr_audio.copy()
conn.asr_audio.clear()
conn.reset_vad_states()
conn.asr_audio.append(audio)
if not have_voice and not conn.client_have_voice:
conn.asr_audio = conn.asr_audio[-10:]
return
if len(asr_audio_task) > 15:
await self.handle_voice_stop(conn, asr_audio_task)
# 自动模式下通过VAD检测到语音停止时触发识别
if conn.client_voice_stop:
asr_audio_task = conn.asr_audio.copy()
conn.asr_audio.clear()
conn.reset_vad_states()
if len(asr_audio_task) > 15:
await self.handle_voice_stop(conn, asr_audio_task)
# 处理语音停止
async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
"""并行处理ASR和声纹识别"""
try:
total_start_time = time.monotonic()
# 准备音频数据
if conn.audio_format == "pcm":
pcm_data = asr_audio_task
else:
pcm_data = self.decode_opus(asr_audio_task)
combined_pcm_data = b"".join(pcm_data)
# 预先准备WAV数据
wav_data = None
if conn.voiceprint_provider and combined_pcm_data:
wav_data = self._pcm_to_wav(combined_pcm_data)
# 定义ASR任务
def run_asr():
start_time = time.monotonic()
try:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
result = loop.run_until_complete(
self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
)
end_time = time.monotonic()
logger.bind(tag=TAG).debug(f"ASR耗时: {end_time - start_time:.3f}s")
return result
finally:
loop.close()
except Exception as e:
end_time = time.monotonic()
logger.bind(tag=TAG).error(f"ASR失败: {e}")
return ("", None)
# 定义声纹识别任务
def run_voiceprint():
if not wav_data:
return None
try:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
# 使用连接的声纹识别提供者
result = loop.run_until_complete(
conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
)
return result
finally:
loop.close()
except Exception as e:
logger.bind(tag=TAG).error(f"声纹识别失败: {e}")
return None
# 使用线程池执行器并行运行
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as thread_executor:
asr_future = thread_executor.submit(run_asr)
if conn.voiceprint_provider and wav_data:
voiceprint_future = thread_executor.submit(run_voiceprint)
# 等待两个线程都完成
asr_result = asr_future.result(timeout=15)
voiceprint_result = voiceprint_future.result(timeout=15)
results = {"asr": asr_result, "voiceprint": voiceprint_result}
else:
asr_result = asr_future.result(timeout=15)
results = {"asr": asr_result, "voiceprint": None}
# 处理结果
raw_text, _ = results.get("asr", ("", None))
speaker_name = results.get("voiceprint", None)
# 记录识别结果
asr_task = self.speech_to_text(asr_audio_task, conn.session_id, conn.audio_format)
if conn.voiceprint_provider and wav_data:
voiceprint_task = conn.voiceprint_provider.identify_speaker(wav_data, conn.session_id)
# 并发等待两个结果
asr_result, voiceprint_result = await asyncio.gather(
asr_task, voiceprint_task, return_exceptions=True
)
else:
asr_result = await asr_task
voiceprint_result = None
# 记录识别结果 - 检查是否为异常
if isinstance(asr_result, Exception):
logger.bind(tag=TAG).error(f"ASR识别失败: {asr_result}")
raw_text = ""
else:
raw_text, _ = asr_result
if isinstance(voiceprint_result, Exception):
logger.bind(tag=TAG).error(f"声纹识别失败: {voiceprint_result}")
speaker_name = ""
else:
speaker_name = voiceprint_result
if raw_text:
logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
if speaker_name:
logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
# 性能监控
total_time = time.monotonic() - total_start_time
logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s")
# 检查文本长度
text_len, _ = remove_punctuation_and_length(raw_text)
self.stop_ws_connection()
if text_len > 0:
# 构建包含说话人信息的JSON字符串
enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
# 使用自定义模块进行上报
await startToChat(conn, enhanced_text)
enqueue_asr_report(conn, enhanced_text, asr_audio_task)
@@ -241,6 +208,7 @@ class ASRProviderBase(ABC):
@staticmethod
def decode_opus(opus_data: List[bytes]) -> List[bytes]:
"""将Opus音频数据解码为PCM数据"""
decoder = None
try:
decoder = opuslib_next.Decoder(16000, 1)
pcm_data = []
@@ -265,3 +233,9 @@ class ASRProviderBase(ABC):
except Exception as e:
logger.bind(tag=TAG).error(f"音频解码过程发生错误: {e}")
return []
finally:
if decoder is not None:
try:
del decoder
except Exception as e:
logger.bind(tag=TAG).debug(f"释放decoder资源时出错: {e}")
main/xiaozhi-server/core/providers/asr/doubao_stream.py
+64 -15
View File
@@ -18,8 +18,6 @@ class ASRProvider(ASRProviderBase):
self.interface_type = InterfaceType.STREAM
self.config = config
self.text = ""
self.max_retries = 3
self.retry_delay = 2
self.decoder = opuslib_next.Decoder(16000, 1)
self.asr_ws = None
self.forward_task = None
@@ -49,6 +47,8 @@ class ASRProvider(ASRProviderBase):
self.channel = config.get("channel", 1)
self.auth_method = config.get("auth_method", "token")
self.secret = config.get("secret", "access_secret")
end_window_size = config.get("end_window_size")
self.end_window_size = int(end_window_size) if end_window_size else 200
async def open_audio_channels(self, conn):
await super().open_audio_channels(conn)
@@ -56,14 +56,13 @@ class ASRProvider(ASRProviderBase):
async def receive_audio(self, conn, audio, audio_have_voice):
conn.asr_audio.append(audio)
conn.asr_audio = conn.asr_audio[-10:]
# 存储音频数据
if not hasattr(conn, 'asr_audio_for_voiceprint'):
conn.asr_audio_for_voiceprint = []
conn.asr_audio_for_voiceprint.append(audio)
# 当没有音频数据时处理完整语音片段
if not audio and len(conn.asr_audio_for_voiceprint) > 0:
if conn.client_listen_mode != "manual" and not audio and len(conn.asr_audio_for_voiceprint) > 0:
await self.handle_voice_stop(conn, conn.asr_audio_for_voiceprint)
conn.asr_audio_for_voiceprint = []
@@ -179,6 +178,7 @@ class ASRProvider(ASRProviderBase):
payload.get("audio_info", {}).get("duration", 0) > 2000
and not utterances
and not payload["result"].get("text")
and conn.client_listen_mode != "manual"
):
logger.bind(tag=TAG).error(f"识别文本：空")
self.text = ""
@@ -187,15 +187,44 @@ class ASRProvider(ASRProviderBase):
await self.handle_voice_stop(conn, audio_data)
break
# 专门处理没有文本的识别结果（手动模式下可能已经识别完成但是没松按键）
elif not payload["result"].get("text") and not utterances:
if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0:
logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
break
for utterance in utterances:
if utterance.get("definite", False):
self.text = utterance["text"]
current_text = utterance["text"]
logger.bind(tag=TAG).info(
f"识别到文本: {self.text}"
f"识别到文本: {current_text}"
)
conn.reset_vad_states()
if len(audio_data) > 15: # 确保有足够音频数据
await self.handle_voice_stop(conn, audio_data)
# 手动模式下累积识别结果
if conn.client_listen_mode == "manual":
if self.text:
self.text += current_text
else:
self.text = current_text
# 在接收消息中途时收到停止信号
if conn.client_voice_stop and len(audio_data) > 0:
logger.bind(tag=TAG).debug("消息中途收到停止信号，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
break
else:
# 自动模式下直接覆盖
self.text = current_text
conn.reset_vad_states()
if len(audio_data) > 15: # 确保有足够音频数据
await self.handle_voice_stop(conn, audio_data)
break
elif "error" in payload:
error_msg = payload.get("error", "未知错误")
@@ -227,8 +256,6 @@ class ASRProvider(ASRProviderBase):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
if hasattr(conn, 'has_valid_voice'):
conn.has_valid_voice = False
def stop_ws_connection(self):
if self.asr_ws:
@@ -236,6 +263,20 @@ class ASRProvider(ASRProviderBase):
self.asr_ws = None
self.is_processing = False
async def _send_stop_request(self):
"""发送最后一个音频帧以通知服务器结束"""
if self.asr_ws:
try:
# 发送结束标记的音频帧（gzip压缩的空数据）
empty_payload = gzip.compress(b"")
last_audio_request = bytearray(self.generate_last_audio_default_header())
last_audio_request.extend(len(empty_payload).to_bytes(4, "big"))
last_audio_request.extend(empty_payload)
await self.asr_ws.send(last_audio_request)
logger.bind(tag=TAG).debug("已发送结束音频帧")
except Exception as e:
logger.bind(tag=TAG).debug(f"发送结束音频帧时出错: {e}")
def construct_request(self, reqid):
req = {
"app": {
@@ -252,7 +293,7 @@ class ASRProvider(ASRProviderBase):
"sequence": 1,
"boosting_table_name": self.boosting_table_name,
"correct_table_name": self.correct_table_name,
"end_window_size": 200,
"end_window_size": self.end_window_size,
},
"audio": {
"format": self.format,
@@ -370,6 +411,16 @@ class ASRProvider(ASRProviderBase):
pass
self.forward_task = None
self.is_processing = False
# 显式释放decoder资源
if hasattr(self, 'decoder') and self.decoder is not None:
try:
del self.decoder
self.decoder = None
logger.bind(tag=TAG).debug("Doubao decoder resources released")
except Exception as e:
logger.bind(tag=TAG).debug(f"释放Doubao decoder资源时出错: {e}")
# 清理所有连接的音频缓冲区
if hasattr(self, '_connections'):
for conn in self._connections.values():
@@ -377,5 +428,3 @@ class ASRProvider(ASRProviderBase):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, 'asr_audio'):
conn.asr_audio = []
if hasattr(conn, 'has_valid_voice'):
conn.has_valid_voice = False
main/xiaozhi-server/core/providers/asr/fun_local.py
+10 -7
View File
@@ -1,14 +1,16 @@
import time
import os
import sys
import io
import sys
import time
import shutil
import psutil
import asyncio
from config.logger import setup_logging
from typing import Optional, Tuple, List
from core.providers.asr.base import ASRProviderBase
from funasr import AutoModel
from funasr.utils.postprocess_utils import rich_transcription_postprocess
import shutil
from core.providers.asr.base import ASRProviderBase
from core.providers.asr.dto.dto import InterfaceType
TAG = __name__
@@ -90,16 +92,17 @@ class ASRProvider(ASRProviderBase):
else:
file_path = self.save_audio_to_file(pcm_data, session_id)
# 语音识别
# 语音识别 - 使用线程池避免阻塞事件循环
start_time = time.time()
result = self.model.generate(
result = await asyncio.to_thread(
self.model.generate,
input=combined_pcm_data,
cache={},
language="auto",
use_itn=True,
batch_size_s=60,
)
text = rich_transcription_postprocess(result[0]["text"])
text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"])
logger.bind(tag=TAG).debug(
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
)
main/xiaozhi-server/core/providers/asr/qwen3_asr_flash.py
+4 -22
View File
@@ -1,8 +1,5 @@
import os
import json
import asyncio
import tempfile
import difflib
from typing import Optional, Tuple, List
import dashscope
from config.logger import setup_logging
@@ -16,7 +13,8 @@ logger = setup_logging()
class ASRProvider(ASRProviderBase):
def __init__(self, config: dict, delete_audio_file: bool):
super().__init__()
self.interface_type = InterfaceType.STREAM
# 音频文件上传类型，流式文本识别输出
self.interface_type = InterfaceType.NON_STREAM
"""Qwen3-ASR-Flash ASR初始化"""
# 配置参数
@@ -130,27 +128,11 @@ class ASRProvider(ASRProviderBase):
# 处理流式响应
full_text = ""
last_text = "" # 用于存储上一个文本片段
for chunk in response:
try:
text = chunk["output"]["choices"][0]["message"].content[0]["text"]
# 标准化文本片段（去除首尾空格）
normalized_text = text.strip()
# 只有当新文本片段与上一个不同时才处理
if normalized_text != last_text:
# 提取新增的文本部分
# 通过比较当前文本和上一个文本，找到新增的部分
if normalized_text.startswith(last_text):
# 如果当前文本以最后一个文本开头，则新增部分是两者的差集
new_part = normalized_text[len(last_text):]
else:
# 如果不以最后一个文本开头，说明识别结果发生了较大变化，直接使用当前文本
new_part = normalized_text
# 将新增部分添加到完整文本中
full_text += new_part
last_text = normalized_text
# 这里可以实时处理文本片段，例如通过回调函数
# 更新为最新的完整文本
full_text = text.strip()
except:
pass
main/xiaozhi-server/core/providers/asr/xunfei_stream.py
+54 -205
View File
@@ -5,6 +5,7 @@ import hashlib
import asyncio
import websockets
import opuslib_next
import gc
from time import mktime
from datetime import datetime
from urllib.parse import urlencode
@@ -34,9 +35,6 @@ class ASRProvider(ASRProviderBase):
self.forward_task = None
self.is_processing = False
self.server_ready = False
self.last_frame_sent = False # 标记是否已发送最终帧
self.best_text = "" # 保存最佳识别结果
self.has_final_result = False # 标记是否收到最终识别结果
# 讯飞配置
self.app_id = config.get("app_id")
@@ -51,7 +49,6 @@ class ASRProvider(ASRProviderBase):
"domain": config.get("domain", "slm"),
"language": config.get("language", "zh_cn"),
"accent": config.get("accent", "mandarin"),
"dwa": config.get("dwa", "wpgs"),
"result": {"encoding": "utf8", "compress": "raw", "format": "plain"},
}
@@ -115,7 +112,7 @@ class ASRProvider(ASRProviderBase):
await self._start_recognition(conn)
except Exception as e:
logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
await self._cleanup(conn)
await self._cleanup()
return
# 发送当前音频数据
@@ -125,7 +122,7 @@ class ASRProvider(ASRProviderBase):
await self._send_audio_frame(pcm_frame, STATUS_CONTINUE_FRAME)
except Exception as e:
logger.bind(tag=TAG).warning(f"发送音频数据时发生错误: {e}")
await self._cleanup(conn)
await self._cleanup()
async def _start_recognition(self, conn):
"""开始识别会话"""
@@ -135,6 +132,10 @@ class ASRProvider(ASRProviderBase):
ws_url = self.create_url()
logger.bind(tag=TAG).info(f"正在连接ASR服务: {ws_url[:50]}...")
# 如果为手动模式,设置超时时长为一分钟
if conn.client_listen_mode == "manual":
self.iat_params["eos"] = 60000
self.asr_ws = await websockets.connect(
ws_url,
max_size=1000000000,
@@ -145,8 +146,6 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).info("ASR WebSocket连接已建立")
self.server_ready = False
self.last_frame_sent = False
self.best_text = ""
self.forward_task = asyncio.create_task(self._forward_results(conn))
# 发送首帧音频
@@ -195,23 +194,12 @@ class ASRProvider(ASRProviderBase):
await self.asr_ws.send(json.dumps(frame_data, ensure_ascii=False))
# 标记是否发送了最终帧
if status == STATUS_LAST_FRAME:
self.last_frame_sent = True
logger.bind(tag=TAG).info("标记最终帧已发送")
async def _forward_results(self, conn):
"""转发识别结果"""
try:
while self.asr_ws and not conn.stop_event.is_set():
# 获取当前连接的音频数据
audio_data = getattr(conn, "asr_audio_for_voiceprint", [])
while not conn.stop_event.is_set():
try:
# 如果已发送最终帧，增加超时时间等待完整结果
timeout = 3.0 if self.last_frame_sent else 30.0
response = await asyncio.wait_for(
self.asr_ws.recv(), timeout=timeout
)
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=60)
result = json.loads(response)
logger.bind(tag=TAG).debug(f"收到ASR结果: {result}")
@@ -235,144 +223,27 @@ class ASRProvider(ASRProviderBase):
# 解码base64文本
decoded_text = base64.b64decode(text_data).decode("utf-8")
text_json = json.loads(decoded_text)
# 提取文本内容
text_ws = text_json.get("ws", [])
result_text = ""
for i in text_ws:
for j in i.get("cw", []):
w = j.get("w", "")
result_text += w
self.text += w
# 更新识别文本 - 实时更新策略
# 只检查是否为空字符串，不再过滤任何标点符号
# 这样可以确保所有识别到的内容，包括标点符号都能被实时更新
if result_text and result_text.strip():
# 实时更新：正常情况下都更新，提高响应速度
should_update = True
# 保存最佳文本
# 1. 如果是识别完成状态或最终帧后收到的结果，优先保存
# 2. 否则保存最长的有意义文本
# 取消对标点符号的过滤，只检查是否为空
# 这样可以保留所有识别到的内容，包括各种标点符号
is_valid_text = len(result_text.strip()) > 0
if (
self.last_frame_sent or status == 2
) and is_valid_text:
self.best_text = result_text
self.has_final_result = True # 标记已收到最终结果
logger.bind(tag=TAG).debug(
f"保存最终识别结果: {self.best_text}"
)
elif (
len(result_text) > len(self.best_text)
and is_valid_text
and not self.has_final_result
):
self.best_text = result_text
logger.bind(tag=TAG).debug(
f"保存中间最佳文本: {self.best_text}"
)
# 如果已发送最终帧，只过滤空文本
if self.last_frame_sent:
# 只拒绝完全空的结果
if not result_text.strip():
should_update = False
logger.bind(tag=TAG).warning(
f"最终帧后拒绝空文本"
)
if should_update:
# 处理流式识别结果，避免简单替换导致内容丢失
# 1. 如果是中间状态（非最终帧后），可能需要替换为更完整的识别
# 2. 如果是最终帧后收到的结果，可能是对前面文本的补充
if self.last_frame_sent:
# 最终帧后收到的结果可能是标点符号等补充内容
# 检查是否需要合并文本而不是替换
# 如果当前文本是纯标点而前面已有内容，应该追加而不是替换
if len(
self.text
) > 0 and result_text.strip() in [
"。",
".",
"?",
"？",
"!",
"！",
",",
"，",
";",
"；",
]:
# 对于标点符号，追加到现有文本后
self.text = (
self.text.rstrip().rstrip("。.")
+ result_text
)
else:
# 其他情况保持替换逻辑
self.text = result_text
else:
# 中间状态替换为新的识别结果
self.text = result_text
logger.bind(tag=TAG).info(
f"实时更新识别文本: {self.text} (最终帧已发送: {self.last_frame_sent})"
)
# 识别完成，但如果还没发送最终帧，继续等待
if status == 2:
logger.bind(tag=TAG).info(
f"识别完成状态已到达，当前识别文本: {self.text}"
)
# 如果还没发送最终帧，继续等待
if not self.last_frame_sent:
logger.bind(tag=TAG).info(
"识别完成但最终帧未发送，继续等待..."
)
continue
# 已发送最终帧且收到完成状态，使用最佳策略选择最终结果
# 优先使用识别完成状态下的最新结果，而不是仅仅基于长度
if self.best_text:
# 如果当前文本是在最终帧发送后或识别完成状态下收到的，优先使用
if (
self.last_frame_sent or status == 2
) and self.text.strip():
logger.bind(tag=TAG).info(
f"使用完成状态下的最新识别结果: {self.text}"
)
elif len(self.best_text) > len(self.text):
logger.bind(tag=TAG).info(
f"使用更长的最佳文本作为最终结果: {self.text} -> {self.best_text}"
)
self.text = self.best_text
logger.bind(tag=TAG).info(f"获取到最终完整文本: {self.text}")
if conn.client_listen_mode == "manual":
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
if len(audio_data) > 0:
logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
await self.handle_voice_stop(conn, audio_data)
# 清理音频缓存
conn.asr_audio.clear()
conn.reset_vad_states()
if len(audio_data) > 15: # 确保有足够音频数据
# 准备处理结果
pass
break
except asyncio.TimeoutError:
if self.last_frame_sent:
# 超时时也使用最佳文本
if self.best_text and len(self.best_text) > len(self.text):
logger.bind(tag=TAG).info(
f"超时，使用最佳文本: {self.text} -> {self.best_text}"
)
self.text = self.best_text
logger.bind(tag=TAG).info(
f"最终帧后超时，使用结果: {self.text}"
)
break
# 如果还没发送最终帧，继续等待
continue
logger.bind(tag=TAG).error("接收结果超时")
break
except websockets.ConnectionClosed:
logger.bind(tag=TAG).info("ASR服务连接已关闭")
self.is_processing = False
@@ -389,17 +260,15 @@ class ASRProvider(ASRProviderBase):
if hasattr(e, "__cause__") and e.__cause__:
logger.bind(tag=TAG).error(f"错误原因: {str(e.__cause__)}")
finally:
if self.asr_ws:
await self.asr_ws.close()
self.asr_ws = None
self.is_processing = False
# 清理连接资源
await self._cleanup()
# 清理连接的音频缓存
if conn:
if hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, "asr_audio"):
conn.asr_audio = []
if hasattr(conn, "has_valid_voice"):
conn.has_valid_voice = False
async def handle_voice_stop(self, conn, asr_audio_task: List[bytes]):
"""处理语音停止，发送最后一帧并处理识别结果"""
@@ -407,22 +276,13 @@ class ASRProvider(ASRProviderBase):
# 先发送最后一帧表示音频结束
if self.asr_ws and self.is_processing:
try:
# 取最后一个有效的音频帧作为最后一帧数据
last_frame = b""
if asr_audio_task:
last_audio = asr_audio_task[-1]
last_frame = self.decoder.decode(last_audio, 960)
await self._send_audio_frame(last_frame, STATUS_LAST_FRAME)
logger.bind(tag=TAG).info("已发送最后一帧")
await self._send_audio_frame(b"", STATUS_LAST_FRAME)
logger.bind(tag=TAG).debug(f"已发送停止请求")
# 发送最终帧后，给_forward_results适当时间处理最终结果
await asyncio.sleep(0.25)
logger.bind(tag=TAG).info(f"准备处理最终识别结果: {self.text}")
except Exception as e:
logger.bind(tag=TAG).error(f"发送最后一帧失败: {e}")
logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
# 调用父类的handle_voice_stop方法处理识别结果
await super().handle_voice_stop(conn, asr_audio_task)
except Exception as e:
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
@@ -436,40 +296,27 @@ class ASRProvider(ASRProviderBase):
self.asr_ws = None
self.is_processing = False
async def _cleanup(self, conn):
"""清理资源"""
logger.bind(tag=TAG).info(
async def _send_stop_request(self):
"""发送停止识别请求（不关闭连接）"""
if self.asr_ws:
try:
# 先停止音频发送
self.is_processing = False
await self._send_audio_frame(b"", STATUS_LAST_FRAME)
logger.bind(tag=TAG).debug("已发送停止请求")
except Exception as e:
logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
async def _cleanup(self):
"""清理资源（关闭连接）"""
logger.bind(tag=TAG).debug(
f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}"
)
# 发送最后一帧
if self.asr_ws and self.is_processing:
try:
await self._send_audio_frame(b"", STATUS_LAST_FRAME)
await asyncio.sleep(0.1)
logger.bind(tag=TAG).info("已发送最后一帧")
except Exception as e:
logger.bind(tag=TAG).error(f"发送最后一帧失败: {e}")
# 状态重置
self.is_processing = False
self.server_ready = False
self.last_frame_sent = False
self.best_text = ""
self.has_final_result = False
logger.bind(tag=TAG).info("ASR状态已重置")
# 清理任务
if self.forward_task and not self.forward_task.done():
self.forward_task.cancel()
try:
await asyncio.wait_for(self.forward_task, timeout=1.0)
except asyncio.CancelledError:
pass
except Exception as e:
logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}")
finally:
self.forward_task = None
logger.bind(tag=TAG).debug("ASR状态已重置")
# 关闭连接
if self.asr_ws:
@@ -482,16 +329,10 @@ class ASRProvider(ASRProviderBase):
finally:
self.asr_ws = None
# 清理连接的音频缓存
if conn:
if hasattr(conn, "asr_audio_for_voiceprint"):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, "asr_audio"):
conn.asr_audio = []
if hasattr(conn, "has_valid_voice"):
conn.has_valid_voice = False
# 清理任务引用
self.forward_task = None
logger.bind(tag=TAG).info("ASR会话清理完成")
logger.bind(tag=TAG).debug("ASR会话清理完成")
async def speech_to_text(self, opus_data, session_id, audio_format):
"""获取识别结果"""
@@ -512,6 +353,16 @@ class ASRProvider(ASRProviderBase):
pass
self.forward_task = None
self.is_processing = False
# 显式释放decoder资源
if hasattr(self, 'decoder') and self.decoder is not None:
try:
del self.decoder
self.decoder = None
logger.bind(tag=TAG).debug("Xunfei decoder resources released")
except Exception as e:
logger.bind(tag=TAG).debug(f"释放Xunfei decoder资源时出错: {e}")
# 清理所有连接的音频缓冲区
if hasattr(self, "_connections"):
for conn in self._connections.values():
@@ -519,5 +370,3 @@ class ASRProvider(ASRProviderBase):
conn.asr_audio_for_voiceprint = []
if hasattr(conn, "asr_audio"):
conn.asr_audio = []
if hasattr(conn, "has_valid_voice"):
conn.has_valid_voice = False
main/xiaozhi-server/core/providers/llm/openai/openai.py
+1 -4
View File
@@ -24,7 +24,6 @@ class LLMProvider(LLMProviderBase):
"max_tokens": int,
"temperature": lambda x: round(float(x), 1),
"top_p": lambda x: round(float(x), 1),
"top_k": int,
"frequency_penalty": lambda x: round(float(x), 1),
}
@@ -40,7 +39,7 @@ class LLMProvider(LLMProviderBase):
setattr(self, param, None)
logger.debug(
f"意图识别参数初始化: {self.temperature}, {self.max_tokens}, {self.top_p}, {self.top_k}, {self.frequency_penalty}"
f"意图识别参数初始化: {self.temperature}, {self.max_tokens}, {self.top_p}, {self.frequency_penalty}"
)
model_key_msg = check_model_key("LLM", self.api_key)
@@ -71,7 +70,6 @@ class LLMProvider(LLMProviderBase):
"max_tokens": kwargs.get("max_tokens", self.max_tokens),
"temperature": kwargs.get("temperature", self.temperature),
"top_p": kwargs.get("top_p", self.top_p),
"top_k": kwargs.get("top_k", self.top_k),
"frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
}
@@ -116,7 +114,6 @@ class LLMProvider(LLMProviderBase):
"max_tokens": kwargs.get("max_tokens", self.max_tokens),
"temperature": kwargs.get("temperature", self.temperature),
"top_p": kwargs.get("top_p", self.top_p),
"top_k": kwargs.get("top_k", self.top_k),
"frequency_penalty": kwargs.get("frequency_penalty", self.frequency_penalty),
}
main/xiaozhi-server/core/providers/memory/base.py
+1 -1
View File
@@ -14,7 +14,7 @@ class MemoryProviderBase(ABC):
self.llm = llm
@abstractmethod
async def save_memory(self, msgs):
async def save_memory(self, msgs, session_id=None):
"""Save a new memory for specific role and return memory ID"""
print("this is base func", msgs)
main/xiaozhi-server/core/providers/memory/mem0ai/mem0ai.py
+2 -4
View File
@@ -28,7 +28,7 @@ class MemoryProvider(MemoryProviderBase):
logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
self.use_mem0 = False
async def save_memory(self, msgs):
async def save_memory(self, msgs, session_id=None):
if not self.use_mem0:
return None
if len(msgs) < 2:
@@ -41,9 +41,7 @@ class MemoryProvider(MemoryProviderBase):
for message in msgs
if message.role != "system"
]
result = self.client.add(
messages, user_id=self.role_id
)
result = self.client.add(messages, user_id=self.role_id)
logger.bind(tag=TAG).debug(f"Save memory result: {result}")
except Exception as e:
logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py
+9 -22
View File
@@ -4,7 +4,8 @@ import json
import os
import yaml
from config.config_loader import get_project_dir
from config.manage_api_client import save_mem_local_short
from config.manage_api_client import generate_and_save_chat_summary
import asyncio
from core.utils.util import check_model_key
@@ -74,18 +75,6 @@ short_term_memory_prompt = """
```
"""
short_term_memory_prompt_only_content = """
你是一个经验丰富的记忆总结者，擅长将对话内容进行总结摘要，遵循以下规则：
1、总结user的重要信息，以便在未来的对话中提供更个性化的服务
2、不要重复总结，不要遗忘之前记忆，除非原来的记忆超过了1800字内，否则不要遗忘、不要压缩用户的历史记忆
3、用户操控的设备音量、播放音乐、天气、退出、不想对话等和用户本身无关的内容，这些信息不需要加入到总结中
4、聊天内容中的今天的日期时间、今天的天气情况与用户事件无关的数据，这些信息如果当成记忆存储会影响后序对话，这些信息不需要加入到总结中
5、不要把设备操控的成果结果和失败结果加入到总结中，也不要把用户的一些废话加入到总结中
6、不要为了总结而总结，如果用户的聊天没有意义，请返回原来的历史记录也是可以的
7、只需要返回总结摘要，严格控制在1800字内
8、不要包含代码、xml，不需要解释、注释和说明，保存记忆时仅从对话提取信息，不要混入示例内容
"""
def extract_json_data(json_code):
start = json_code.find("```json")
@@ -143,7 +132,7 @@ class MemoryProvider(MemoryProviderBase):
with open(self.memory_path, "w", encoding="utf-8") as f:
yaml.dump(all_memory, f, allow_unicode=True)
async def save_memory(self, msgs):
async def save_memory(self, msgs, session_id=None):
# 打印使用的模型信息
model_info = getattr(self.llm, "model_name", str(self.llm.__class__.__name__))
logger.bind(tag=TAG).debug(f"使用记忆保存模型: {model_info}")
@@ -187,14 +176,12 @@ class MemoryProvider(MemoryProviderBase):
except Exception as e:
print("Error:", e)
else:
result = self.llm.response_no_stream(
short_term_memory_prompt_only_content,
msgStr,
max_tokens=2000,
temperature=0.2,
)
save_mem_local_short(self.role_id, result)
logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
# 当save_to_file为False时，调用Java端的聊天记录总结接口
summary_id = session_id if session_id else self.role_id
await generate_and_save_chat_summary(summary_id)
logger.bind(tag=TAG).info(
f"Save memory successful - Role: {self.role_id}, Session: {session_id}"
)
return self.short_memory
main/xiaozhi-server/core/providers/memory/nomem/nomem.py
+1 -1
View File
@@ -11,7 +11,7 @@ class MemoryProvider(MemoryProviderBase):
def __init__(self, config, summary_memory=None):
super().__init__(config)
async def save_memory(self, msgs):
async def save_memory(self, msgs, session_id=None):
logger.bind(tag=TAG).debug("nomem mode: No memory saving is performed.")
return None
main/xiaozhi-server/core/providers/tools/server_mcp/mcp_manager.py
+36 -18
View File
@@ -3,12 +3,8 @@
import asyncio
import os
import json
from datetime import timedelta
from typing import Dict, Any, List
from mcp import Implementation
from mcp.client.session import SamplingFnT, ElicitationFnT, ListRootsFnT, LoggingFnT, MessageHandlerFnT
from mcp.shared.session import ProgressFnT
from mcp.types import LoggingMessageNotificationParams
from config.config_loader import get_project_dir
@@ -33,6 +29,7 @@ class ServerMCPManager:
)
self.clients: Dict[str, ServerMCPClient] = {}
self.tools = []
self._init_lock = asyncio.Lock()
def load_config(self) -> Dict[str, Any]:
"""加载MCP服务配置"""
@@ -49,29 +46,50 @@ class ServerMCPManager:
)
return {}
async def _init_server(self, name: str, srv_config: Dict[str, Any]):
"""初始化单个MCP服务"""
client = None
try:
# 初始化服务端MCP客户端
logger.bind(tag=TAG).info(f"初始化服务端MCP客户端: {name}")
client = ServerMCPClient(srv_config)
# 设置超时时间10秒
await asyncio.wait_for(client.initialize(logging_callback=self.logging_callback), timeout=10)
# 使用锁保护共享状态的修改
async with self._init_lock:
self.clients[name] = client
client_tools = client.get_available_tools()
self.tools.extend(client_tools)
except asyncio.TimeoutError:
logger.bind(tag=TAG).error(
f"Failed to initialize MCP server {name}: Timeout"
)
if client:
await client.cleanup()
except Exception as e:
logger.bind(tag=TAG).error(
f"Failed to initialize MCP server {name}: {e}"
)
if client:
await client.cleanup()
async def initialize_servers(self) -> None:
"""初始化所有MCP服务"""
config = self.load_config()
tasks = []
for name, srv_config in config.items():
if not srv_config.get("command") and not srv_config.get("url"):
logger.bind(tag=TAG).warning(
f"Skipping server {name}: neither command nor url specified"
)
continue
try:
# 初始化服务端MCP客户端
logger.bind(tag=TAG).info(f"初始化服务端MCP客户端: {name}")
client = ServerMCPClient(srv_config)
await client.initialize(logging_callback=self.logging_callback)
self.clients[name] = client
client_tools = client.get_available_tools()
self.tools.extend(client_tools)
except Exception as e:
logger.bind(tag=TAG).error(
f"Failed to initialize MCP server {name}: {e}"
)
tasks.append(self._init_server(name, srv_config))
if tasks:
await asyncio.gather(*tasks)
# 输出当前支持的服务端MCP工具列表
if hasattr(self.conn, "func_handler") and self.conn.func_handler:
main/xiaozhi-server/core/providers/vad/silero.py
+11
View File
@@ -36,7 +36,18 @@ class VADProvider(VADProviderBase):
# 至少要多少帧才算有语音
self.frame_window_threshold = 3
def __del__(self):
if hasattr(self, 'decoder') and self.decoder is not None:
try:
del self.decoder
except Exception:
pass
def is_vad(self, conn, opus_packet):
# 手动模式：直接返回True，不进行实时VAD检测，所有音频都缓存
if conn.client_listen_mode == "manual":
return True
try:
pcm_frame = self.decoder.decode(opus_packet, 960)
conn.client_audio_buffer.extend(pcm_frame) # 将新数据加入缓冲区