From 244c76793c16919eb11814f3d07a1699a3ac0d77 Mon Sep 17 00:00:00 2001 From: wnk Date: Sun, 27 Apr 2025 11:55:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BD=93=E5=89=8DfunASR=E4=BB=85=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E6=9C=AC=E5=9C=B0=E6=A8=A1=E5=9E=8B=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9EFunASR=E6=9C=8D=E5=8A=A1API=E8=B0=83=E7=94=A8=EF=BC=88?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E5=8F=AF=E6=A0=B9=E6=8D=AE=E5=AE=98=E6=96=B9?= =?UTF-8?q?=E8=AF=B4=E6=98=8E=E8=87=AA=E8=A1=8C=E5=8F=A6=E5=A4=96=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=EF=BC=89=EF=BC=8C=20=E5=A2=9E=E5=8A=A0=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E4=B8=8EAPI=E8=B0=83=E7=94=A8=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202504082211.sql | 1 + .../resources/db/changelog/202504112044.sql | 1 + main/xiaozhi-server/config.yaml | 6 + .../core/providers/asr/fun_server.py | 144 ++++++++++++++++++ 4 files changed, 152 insertions(+) create mode 100644 main/xiaozhi-server/core/providers/asr/fun_server.py diff --git a/main/manager-api/src/main/resources/db/changelog/202504082211.sql b/main/manager-api/src/main/resources/db/changelog/202504082211.sql index 91ce33ff..5aa75bef 100644 --- a/main/manager-api/src/main/resources/db/changelog/202504082211.sql +++ b/main/manager-api/src/main/resources/db/changelog/202504082211.sql @@ -10,6 +10,7 @@ INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `f ('SYSTEM_ASR_FunASR', 'ASR', 'fun_local', 'FunASR语音识别', '[{"key":"model_dir","label":"模型目录","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 1, 1, NOW(), 1, NOW()), ('SYSTEM_ASR_SherpaASR', 'ASR', 'sherpa_onnx_local', 'SherpaASR语音识别', '[{"key":"model_dir","label":"模型目录","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 2, 1, NOW(), 1, NOW()), ('SYSTEM_ASR_DoubaoASR', 'ASR', 'doubao', '火山引擎语音识别', '[{"key":"appid","label":"应用ID","type":"string"},{"key":"access_token","label":"访问令牌","type":"string"},{"key":"cluster","label":"集群","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 3, 1, NOW(), 1, NOW()), +('SYSTEM_ASR_FunASRServer', 'ASR', 'fun_server', 'FunASR服务语音识别', '[{"key":"host","label":"服务地址","type":"string"},{"key":"port","label":"端口号","type":"number"}]', 4, 1, NOW(), 1, NOW()), -- LLM模型供应器 ('SYSTEM_LLM_openai', 'LLM', 'openai', 'OpenAI接口', '[{"key":"base_url","label":"基础URL","type":"string"},{"key":"model_name","label":"模型名称","type":"string"},{"key":"api_key","label":"API密钥","type":"string"},{"key":"temperature","label":"温度","type":"number"},{"key":"max_tokens","label":"最大令牌数","type":"number"},{"key":"top_p","label":"top_p值","type":"number"},{"key":"top_k","label":"top_k值","type":"number"},{"key":"frequency_penalty","label":"频率惩罚","type":"number"}]', 1, 1, NOW(), 1, NOW()), diff --git a/main/manager-api/src/main/resources/db/changelog/202504112044.sql b/main/manager-api/src/main/resources/db/changelog/202504112044.sql index 9fd2257d..3609d235 100644 --- a/main/manager-api/src/main/resources/db/changelog/202504112044.sql +++ b/main/manager-api/src/main/resources/db/changelog/202504112044.sql @@ -11,6 +11,7 @@ INSERT INTO `ai_model_config` VALUES ('ASR_FunASR', 'ASR', 'FunASR', 'FunASR语 INSERT INTO `ai_model_config` VALUES ('ASR_SherpaASR', 'ASR', 'SherpaASR', 'Sherpa语音识别', 0, 1, '{\"type\": \"sherpa_onnx_local\", \"model_dir\": \"models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17\", \"output_dir\": \"tmp/\"}', NULL, NULL, 2, NULL, NULL, NULL, NULL); INSERT INTO `ai_model_config` VALUES ('ASR_DoubaoASR', 'ASR', 'DoubaoASR', '豆包语音识别', 0, 1, '{\"type\": \"doubao\", \"appid\": \"\", \"access_token\": \"\", \"cluster\": \"volcengine_input_common\", \"output_dir\": \"tmp/\"}', NULL, NULL, 3, NULL, NULL, NULL, NULL); INSERT INTO `ai_model_config` VALUES ('ASR_TencentASR', 'ASR', 'TencentASR', '腾讯语音识别', 0, 1, '{\"type\": \"tencent\", \"appid\": \"\", \"secret_id\": \"\", \"secret_key\": \"你的secret_key\", \"output_dir\": \"tmp/\"}', NULL, NULL, 4, NULL, NULL, NULL, NULL); +INSERT INTO `ai_model_config` VALUES ('ASR_FunASRServer', 'ASR', 'FunASRServer', 'FunASR服务语音识别', 0, 1, '{\"type\": \"fun_server\", \"host\": \"127.0.0.1\", \"port\": 10096}', NULL, NULL, 5, NULL, NULL, NULL, NULL); -- LLM模型配置 INSERT INTO `ai_model_config` VALUES ('LLM_ChatGLMLLM', 'LLM', 'ChatGLMLLM', '智谱AI', 1, 1, '{\"type\": \"openai\", \"model_name\": \"glm-4-flash\", \"base_url\": \"https://open.bigmodel.cn/api/paas/v4/\", \"api_key\": \"你的api_key\"}', NULL, NULL, 1, NULL, NULL, NULL, NULL); diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 696c82d0..3ec6b013 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -200,6 +200,12 @@ ASR: type: fun_local model_dir: models/SenseVoiceSmall output_dir: tmp/ + FunASRServer: + # 支持FunASR服务，部署方法：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md + # mode 预置 offline + type: fun_server + host: 127.0.0.1 + port: 10096 SherpaASR: type: sherpa_onnx_local model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17 diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py new file mode 100644 index 00000000..d0701d93 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -0,0 +1,144 @@ +from typing import Optional, Tuple, List +import opuslib_next +from core.providers.asr.base import ASRProviderBase +import ssl +import json +import websockets +from config.logger import setup_logging +import asyncio +TAG = __name__ +logger = setup_logging() + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool): + ''' + Initialize the ASRProvider with server configuration. + :param config: Dictionary containing 'host', 'port', and 'is_ssl'. + :param delete_audio_file: Boolean to indicate whether to delete audio files after processing. + ''' + self.host = config.get('host', 'localhost') + self.port = config.get('port', 10095) + self.is_ssl = config.get('is_ssl', True) + self.delete_audio_file = delete_audio_file + self.uri = f"wss://{self.host}:{self.port}" if self.is_ssl else f"ws://{self.host}:{self.port}" + self.ssl_context = ssl.SSLContext() if self.is_ssl else None + if self.ssl_context: + self.ssl_context.check_hostname = False + self.ssl_context.verify_mode = ssl.CERT_NONE + + def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: + """解码Opus数据并保存为WAV文件""" + pass + + @staticmethod + def decode_opus(opus_data: List[bytes]) -> bytes: + """将Opus音频数据解码为PCM数据""" + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + return b"".join(pcm_data) + + + + async def _receive_responses(self, ws) -> None: + ''' + Asynchronous generator to receive messages from the WebSocket. + Yields each message as it is received. + ''' + text = "" + while True: + try: + response = await asyncio.wait_for(ws.recv(), timeout=5) + response_data = json.loads(response) + logger.bind(tag=TAG).debug(f"Received response: {response_data}") + if response_data.get("is_final", True): + text += response_data.get("text", "") + break + else: + text += response_data.get("text", "") + except asyncio.TimeoutError: + logger.bind(tag=TAG).error("Timeout while waiting for response from WebSocket.") + break + except websockets.exceptions.ConnectionClosed as e: + logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}") + break + return text + async def _send_data(self, ws, pcm_data: bytes, session_id: str) -> tuple: + ''' + Internal method to handle WebSocket communication. + Reuses the persistent WebSocket connection if available. + :param pcm_data: PCM audio data to send. + :param session_id: Unique session identifier. + :return: Tuple containing recognized text and optional timestamp. + ''' + + # Send initial configuration message + config_message = json.dumps({ + "mode": "offline", + "chunk_size": [5, 10, 5], + "chunk_interval": 10, + "wav_name": session_id, + "is_speaking": True, + "itn": False + }) + await ws.send(config_message) + logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}") + + # Send PCM data + await ws.send(pcm_data) + logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes") + + # Indicate end of speech + end_message = json.dumps({"is_speaking": False}) + await ws.send(end_message) + logger.bind(tag=TAG).debug(f"Sent end message: {end_message}") + + + async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + ''' + Convert speech data to text using FunASR. + :param opus_data: List of Opus-encoded audio data chunks. + :param session_id: Unique session identifier. + :return: Tuple containing recognized text and optional timestamp. + ''' + + pcm_data = self.decode_opus(opus_data) + + async with websockets.connect(self.uri, subprotocols=["binary"], ping_interval=None, ssl=self.ssl_context) as ws: + try: + # Use asyncio to handle WebSocket communication + send_task = asyncio.create_task(self._send_data(ws, pcm_data, session_id)) + receive_task = asyncio.create_task(self._receive_responses(ws)) + + # Gather tasks with error handling + done, pending = await asyncio.wait( + [send_task, receive_task], + return_when=asyncio.FIRST_EXCEPTION + ) + + # Cancel any pending tasks + for task in pending: + task.cancel() + + # Check for exceptions in completed tasks + for task in done: + if task.exception(): + raise task.exception() + + # Get the result from the receive task + result = receive_task.result() + return result, None # Return the recognized text and timestamp (if any) + + except websockets.exceptions.ConnectionClosed as e: + logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}") + return "", None + except Exception as e: + logger.bind(tag=TAG).error(f"Error during speech-to-text conversion: {e}", exc_info=True) + return "", None \ No newline at end of file