From 244c76793c16919eb11814f3d07a1699a3ac0d77 Mon Sep 17 00:00:00 2001 From: wnk Date: Sun, 27 Apr 2025 11:55:10 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E5=BD=93=E5=89=8DfunASR=E4=BB=85=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E6=9C=AC=E5=9C=B0=E6=A8=A1=E5=9E=8B=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9EFunASR=E6=9C=8D=E5=8A=A1API=E8=B0=83=E7=94=A8=EF=BC=88?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E5=8F=AF=E6=A0=B9=E6=8D=AE=E5=AE=98=E6=96=B9?= =?UTF-8?q?=E8=AF=B4=E6=98=8E=E8=87=AA=E8=A1=8C=E5=8F=A6=E5=A4=96=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=EF=BC=89=EF=BC=8C=20=E5=A2=9E=E5=8A=A0=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E4=B8=8EAPI=E8=B0=83=E7=94=A8=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202504082211.sql | 1 + .../resources/db/changelog/202504112044.sql | 1 + main/xiaozhi-server/config.yaml | 6 + .../core/providers/asr/fun_server.py | 144 ++++++++++++++++++ 4 files changed, 152 insertions(+) create mode 100644 main/xiaozhi-server/core/providers/asr/fun_server.py diff --git a/main/manager-api/src/main/resources/db/changelog/202504082211.sql b/main/manager-api/src/main/resources/db/changelog/202504082211.sql index 91ce33ff..5aa75bef 100644 --- a/main/manager-api/src/main/resources/db/changelog/202504082211.sql +++ b/main/manager-api/src/main/resources/db/changelog/202504082211.sql @@ -10,6 +10,7 @@ INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `f ('SYSTEM_ASR_FunASR', 'ASR', 'fun_local', 'FunASR语音识别', '[{"key":"model_dir","label":"模型目录","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 1, 1, NOW(), 1, NOW()), ('SYSTEM_ASR_SherpaASR', 'ASR', 'sherpa_onnx_local', 'SherpaASR语音识别', '[{"key":"model_dir","label":"模型目录","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 2, 1, NOW(), 1, NOW()), ('SYSTEM_ASR_DoubaoASR', 'ASR', 'doubao', '火山引擎语音识别', '[{"key":"appid","label":"应用ID","type":"string"},{"key":"access_token","label":"访问令牌","type":"string"},{"key":"cluster","label":"集群","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 3, 1, NOW(), 1, NOW()), +('SYSTEM_ASR_FunASRServer', 'ASR', 'fun_server', 'FunASR服务语音识别', '[{"key":"host","label":"服务地址","type":"string"},{"key":"port","label":"端口号","type":"number"}]', 4, 1, NOW(), 1, NOW()), -- LLM模型供应器 ('SYSTEM_LLM_openai', 'LLM', 'openai', 'OpenAI接口', '[{"key":"base_url","label":"基础URL","type":"string"},{"key":"model_name","label":"模型名称","type":"string"},{"key":"api_key","label":"API密钥","type":"string"},{"key":"temperature","label":"温度","type":"number"},{"key":"max_tokens","label":"最大令牌数","type":"number"},{"key":"top_p","label":"top_p值","type":"number"},{"key":"top_k","label":"top_k值","type":"number"},{"key":"frequency_penalty","label":"频率惩罚","type":"number"}]', 1, 1, NOW(), 1, NOW()), diff --git a/main/manager-api/src/main/resources/db/changelog/202504112044.sql b/main/manager-api/src/main/resources/db/changelog/202504112044.sql index 9fd2257d..3609d235 100644 --- a/main/manager-api/src/main/resources/db/changelog/202504112044.sql +++ b/main/manager-api/src/main/resources/db/changelog/202504112044.sql @@ -11,6 +11,7 @@ INSERT INTO `ai_model_config` VALUES ('ASR_FunASR', 'ASR', 'FunASR', 'FunASR语 INSERT INTO `ai_model_config` VALUES ('ASR_SherpaASR', 'ASR', 'SherpaASR', 'Sherpa语音识别', 0, 1, '{\"type\": \"sherpa_onnx_local\", \"model_dir\": \"models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17\", \"output_dir\": \"tmp/\"}', NULL, NULL, 2, NULL, NULL, NULL, NULL); INSERT INTO `ai_model_config` VALUES ('ASR_DoubaoASR', 'ASR', 'DoubaoASR', '豆包语音识别', 0, 1, '{\"type\": \"doubao\", \"appid\": \"\", \"access_token\": \"\", \"cluster\": \"volcengine_input_common\", \"output_dir\": \"tmp/\"}', NULL, NULL, 3, NULL, NULL, NULL, NULL); INSERT INTO `ai_model_config` VALUES ('ASR_TencentASR', 'ASR', 'TencentASR', '腾讯语音识别', 0, 1, '{\"type\": \"tencent\", \"appid\": \"\", \"secret_id\": \"\", \"secret_key\": \"你的secret_key\", \"output_dir\": \"tmp/\"}', NULL, NULL, 4, NULL, NULL, NULL, NULL); +INSERT INTO `ai_model_config` VALUES ('ASR_FunASRServer', 'ASR', 'FunASRServer', 'FunASR服务语音识别', 0, 1, '{\"type\": \"fun_server\", \"host\": \"127.0.0.1\", \"port\": 10096}', NULL, NULL, 5, NULL, NULL, NULL, NULL); -- LLM模型配置 INSERT INTO `ai_model_config` VALUES ('LLM_ChatGLMLLM', 'LLM', 'ChatGLMLLM', '智谱AI', 1, 1, '{\"type\": \"openai\", \"model_name\": \"glm-4-flash\", \"base_url\": \"https://open.bigmodel.cn/api/paas/v4/\", \"api_key\": \"你的api_key\"}', NULL, NULL, 1, NULL, NULL, NULL, NULL); diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 696c82d0..3ec6b013 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -200,6 +200,12 @@ ASR: type: fun_local model_dir: models/SenseVoiceSmall output_dir: tmp/ + FunASRServer: + # 支持FunASR服务，部署方法：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md + # mode 预置 offline + type: fun_server + host: 127.0.0.1 + port: 10096 SherpaASR: type: sherpa_onnx_local model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17 diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py new file mode 100644 index 00000000..d0701d93 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -0,0 +1,144 @@ +from typing import Optional, Tuple, List +import opuslib_next +from core.providers.asr.base import ASRProviderBase +import ssl +import json +import websockets +from config.logger import setup_logging +import asyncio +TAG = __name__ +logger = setup_logging() + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool): + ''' + Initialize the ASRProvider with server configuration. + :param config: Dictionary containing 'host', 'port', and 'is_ssl'. + :param delete_audio_file: Boolean to indicate whether to delete audio files after processing. + ''' + self.host = config.get('host', 'localhost') + self.port = config.get('port', 10095) + self.is_ssl = config.get('is_ssl', True) + self.delete_audio_file = delete_audio_file + self.uri = f"wss://{self.host}:{self.port}" if self.is_ssl else f"ws://{self.host}:{self.port}" + self.ssl_context = ssl.SSLContext() if self.is_ssl else None + if self.ssl_context: + self.ssl_context.check_hostname = False + self.ssl_context.verify_mode = ssl.CERT_NONE + + def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: + """解码Opus数据并保存为WAV文件""" + pass + + @staticmethod + def decode_opus(opus_data: List[bytes]) -> bytes: + """将Opus音频数据解码为PCM数据""" + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + return b"".join(pcm_data) + + + + async def _receive_responses(self, ws) -> None: + ''' + Asynchronous generator to receive messages from the WebSocket. + Yields each message as it is received. + ''' + text = "" + while True: + try: + response = await asyncio.wait_for(ws.recv(), timeout=5) + response_data = json.loads(response) + logger.bind(tag=TAG).debug(f"Received response: {response_data}") + if response_data.get("is_final", True): + text += response_data.get("text", "") + break + else: + text += response_data.get("text", "") + except asyncio.TimeoutError: + logger.bind(tag=TAG).error("Timeout while waiting for response from WebSocket.") + break + except websockets.exceptions.ConnectionClosed as e: + logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}") + break + return text + async def _send_data(self, ws, pcm_data: bytes, session_id: str) -> tuple: + ''' + Internal method to handle WebSocket communication. + Reuses the persistent WebSocket connection if available. + :param pcm_data: PCM audio data to send. + :param session_id: Unique session identifier. + :return: Tuple containing recognized text and optional timestamp. + ''' + + # Send initial configuration message + config_message = json.dumps({ + "mode": "offline", + "chunk_size": [5, 10, 5], + "chunk_interval": 10, + "wav_name": session_id, + "is_speaking": True, + "itn": False + }) + await ws.send(config_message) + logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}") + + # Send PCM data + await ws.send(pcm_data) + logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes") + + # Indicate end of speech + end_message = json.dumps({"is_speaking": False}) + await ws.send(end_message) + logger.bind(tag=TAG).debug(f"Sent end message: {end_message}") + + + async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + ''' + Convert speech data to text using FunASR. + :param opus_data: List of Opus-encoded audio data chunks. + :param session_id: Unique session identifier. + :return: Tuple containing recognized text and optional timestamp. + ''' + + pcm_data = self.decode_opus(opus_data) + + async with websockets.connect(self.uri, subprotocols=["binary"], ping_interval=None, ssl=self.ssl_context) as ws: + try: + # Use asyncio to handle WebSocket communication + send_task = asyncio.create_task(self._send_data(ws, pcm_data, session_id)) + receive_task = asyncio.create_task(self._receive_responses(ws)) + + # Gather tasks with error handling + done, pending = await asyncio.wait( + [send_task, receive_task], + return_when=asyncio.FIRST_EXCEPTION + ) + + # Cancel any pending tasks + for task in pending: + task.cancel() + + # Check for exceptions in completed tasks + for task in done: + if task.exception(): + raise task.exception() + + # Get the result from the receive task + result = receive_task.result() + return result, None # Return the recognized text and timestamp (if any) + + except websockets.exceptions.ConnectionClosed as e: + logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}") + return "", None + except Exception as e: + logger.bind(tag=TAG).error(f"Error during speech-to-text conversion: {e}", exc_info=True) + return "", None \ No newline at end of file From 16e77b02c21efb6ed74f90e4559f5bd6f9e67f7b Mon Sep 17 00:00:00 2001 From: MakerZorky <1053714527zhq@gmail.com> Date: Tue, 29 Apr 2025 01:28:06 +0800 Subject: [PATCH 2/4] =?UTF-8?q?update:=E6=89=80=E6=9C=89ASR=E5=9D=87?= =?UTF-8?q?=E6=94=B9=E4=B8=BAPCM=E7=9B=B4=E6=8E=A5=E8=AF=86=E5=88=AB?= =?UTF-8?q?=EF=BC=8C=E9=80=9A=E8=BF=87delete=5Faudio=E5=8F=82=E6=95=B0?= =?UTF-8?q?=E5=88=A4=E6=96=AD=E8=A6=81=E4=B8=8D=E8=A6=81=E5=82=A8=E5=AD=98?= =?UTF-8?q?wav?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/providers/asr/base.py | 4 +-- .../core/providers/asr/doubao.py | 21 ++++++------ .../core/providers/asr/fun_local.py | 32 +++++++++++++++---- .../core/providers/asr/sherpa_onnx_local.py | 24 ++++++++------ .../core/providers/asr/tencent.py | 18 ++++------- 5 files changed, 59 insertions(+), 40 deletions(-) diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index d8db22a8..6d9c2c90 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -9,8 +9,8 @@ logger = setup_logging() class ASRProviderBase(ABC): @abstractmethod - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """解码Opus数据并保存为WAV文件""" + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" pass @abstractmethod diff --git a/main/xiaozhi-server/core/providers/asr/doubao.py b/main/xiaozhi-server/core/providers/asr/doubao.py index b19ceb97..6fba472c 100644 --- a/main/xiaozhi-server/core/providers/asr/doubao.py +++ b/main/xiaozhi-server/core/providers/asr/doubao.py @@ -89,6 +89,7 @@ class ASRProvider(ASRProviderBase): self.cluster = config.get("cluster") self.access_token = config.get("access_token") self.output_dir = config.get("output_dir") + self.delete_audio_file = delete_audio_file self.host = "openspeech.bytedance.com" self.ws_url = f"wss://{self.host}/api/v2/asr" @@ -98,21 +99,11 @@ class ASRProvider(ASRProviderBase): # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: + """PCM数据保存为WAV文件""" file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - with wave.open(file_path, "wb") as wf: wf.setnchannels(1) wf.setsampwidth(2) # 2 bytes = 16-bit @@ -274,6 +265,12 @@ class ASRProvider(ASRProviderBase): pcm_data = self.decode_opus(opus_data, session_id) combined_pcm_data = b"".join(pcm_data) + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + self.save_audio_to_file(pcm_data, session_id) + # 直接使用PCM数据 # 计算分段大小 (单声道, 16bit, 16kHz采样率) size_per_sec = 1 * 2 * 16000 # nchannels * sampwidth * framerate diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 82d09ab5..a678780b 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -51,7 +51,7 @@ class ASRProvider(ASRProviderBase): ) def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" + """PCM数据保存为WAV文件""" file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) @@ -72,20 +72,40 @@ class ASRProvider(ASRProviderBase): wf.writeframes(b"".join(pcm_data)) return file_path + + @staticmethod + def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: + + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 + pcm_data = [] + + for opus_packet in opus_data: + try: + pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms + pcm_data.append(pcm_frame) + except opuslib_next.OpusError as e: + logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) + + return pcm_data async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None try: - # 保存音频文件 - start_time = time.time() - file_path = self.save_audio_to_file(opus_data, session_id) - logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}") + # 合并所有opus数据包 + pcm_data = self.decode_opus(opus_data, session_id) + combined_pcm_data = b"".join(pcm_data) + + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + self.save_audio_to_file(pcm_data, session_id) # 语音识别 start_time = time.time() result = self.model.generate( - input=file_path, + input=combined_pcm_data, cache={}, language="auto", use_itn=True, diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index 3dfa3162..b086022b 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -84,10 +84,21 @@ class ASRProvider(ASRProviderBase): ) def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" + """PCM数据保存为WAV文件""" file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) + with wave.open(file_path, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) # 2 bytes = 16-bit + wf.setframerate(16000) + wf.writeframes(b"".join(pcm_data)) + + return file_path + + @staticmethod + def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]: + decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 pcm_data = [] @@ -98,13 +109,7 @@ class ASRProvider(ASRProviderBase): except opuslib_next.OpusError as e: logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - with wave.open(file_path, "wb") as wf: - wf.setnchannels(1) - wf.setsampwidth(2) # 2 bytes = 16-bit - wf.setframerate(16000) - wf.writeframes(b"".join(pcm_data)) - - return file_path + return pcm_data def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]: """ @@ -136,7 +141,8 @@ class ASRProvider(ASRProviderBase): try: # 保存音频文件 start_time = time.time() - file_path = self.save_audio_to_file(opus_data, session_id) + pcm_data = self.decode_opus(opus_data, session_id) + file_path = self.save_audio_to_file(pcm_data, session_id) logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}") # 语音识别 diff --git a/main/xiaozhi-server/core/providers/asr/tencent.py b/main/xiaozhi-server/core/providers/asr/tencent.py index 8221f9df..0ac297a6 100644 --- a/main/xiaozhi-server/core/providers/asr/tencent.py +++ b/main/xiaozhi-server/core/providers/asr/tencent.py @@ -31,21 +31,11 @@ class ASRProvider(ASRProviderBase): os.makedirs(self.output_dir, exist_ok=True) def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: - """将Opus音频数据解码并保存为WAV文件""" + """PCM数据保存为WAV文件""" file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) - decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道 - pcm_data = [] - - for opus_packet in opus_data: - try: - pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms - pcm_data.append(pcm_frame) - except opuslib_next.OpusError as e: - logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True) - with wave.open(file_path, "wb") as wf: wf.setnchannels(1) wf.setsampwidth(2) # 2 bytes = 16-bit @@ -85,6 +75,12 @@ class ASRProvider(ASRProviderBase): # 将Opus音频数据解码为PCM pcm_data = self.decode_opus(opus_data) + + # 判断是否保存为WAV文件 + if self.delete_audio_file: + pass + else: + self.save_audio_to_file(pcm_data, session_id) # 将音频数据转换为Base64编码 base64_audio = base64.b64encode(pcm_data).decode('utf-8') From 6ac894716853392e55e2ae2cb335293933c09629 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 29 Apr 2025 10:14:43 +0800 Subject: [PATCH 3/4] =?UTF-8?q?update:=E6=9B=B4=E6=96=B0issue=E6=A8=A1?= =?UTF-8?q?=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/ISSUE_TEMPLATE/bug_report.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 2d6c173f..deaff830 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -9,26 +9,24 @@ assignees: '' ## 🐛 问题描述 -## 🔍 复现步骤 - +## 🖥️ 环境信息 +- 部署方式: 全模块部署 还是 单Server部署 +- 版本号: 例如 0.3.x + +## 🔍 告诉我们，应该怎么复现这个问题 + 1. 打开 '...' 2. 点击 '...' 3. 滚动到 '...' 4. 看到错误 -## 🤔 预期行为 +## 🤔 你原本希望是怎么样的 -## 😯 截图 +## 😯 提供一些截图 - -## 🖥️ 环境信息 -- 操作系统: [例如 Windows 10] -- 浏览器: [例如 Chrome 89] -- 项目版本: [例如 1.0.0] -- Python版本: [例如 3.9.13] -- Jdk版本：[例如 java 21.0.5 2024-10-15 LTS] -- Nodejs版本：[例如 v20.14.0] +1. 比如日志截图，越多越好 +2. 比如界面反应 ## 📋 其他信息 From ef662e0ccfbde3cb06f7e078e6904a91bd9e87b1 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 29 Apr 2025 18:04:31 +0800 Subject: [PATCH 4/4] =?UTF-8?q?add:=E5=A2=9E=E5=8A=A0=E6=99=BA=E6=8E=A7?= =?UTF-8?q?=E5=8F=B0=E6=A8=A1=E5=9E=8B=E9=85=8D=E7=BD=AE=E5=90=84=E4=B8=AA?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E7=9A=84=E4=BD=BF=E7=94=A8=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202504082211.sql | 1 - .../resources/db/changelog/202504112044.sql | 1 - .../resources/db/changelog/202504291043.sql | 504 ++++++++++++++++++ .../db/changelog/db.changelog-master.yaml | 9 +- main/xiaozhi-server/config.yaml | 22 +- .../core/providers/asr/sherpa_onnx_local.py | 27 +- 6 files changed, 542 insertions(+), 22 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202504291043.sql diff --git a/main/manager-api/src/main/resources/db/changelog/202504082211.sql b/main/manager-api/src/main/resources/db/changelog/202504082211.sql index 5aa75bef..91ce33ff 100644 --- a/main/manager-api/src/main/resources/db/changelog/202504082211.sql +++ b/main/manager-api/src/main/resources/db/changelog/202504082211.sql @@ -10,7 +10,6 @@ INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `f ('SYSTEM_ASR_FunASR', 'ASR', 'fun_local', 'FunASR语音识别', '[{"key":"model_dir","label":"模型目录","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 1, 1, NOW(), 1, NOW()), ('SYSTEM_ASR_SherpaASR', 'ASR', 'sherpa_onnx_local', 'SherpaASR语音识别', '[{"key":"model_dir","label":"模型目录","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 2, 1, NOW(), 1, NOW()), ('SYSTEM_ASR_DoubaoASR', 'ASR', 'doubao', '火山引擎语音识别', '[{"key":"appid","label":"应用ID","type":"string"},{"key":"access_token","label":"访问令牌","type":"string"},{"key":"cluster","label":"集群","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"}]', 3, 1, NOW(), 1, NOW()), -('SYSTEM_ASR_FunASRServer', 'ASR', 'fun_server', 'FunASR服务语音识别', '[{"key":"host","label":"服务地址","type":"string"},{"key":"port","label":"端口号","type":"number"}]', 4, 1, NOW(), 1, NOW()), -- LLM模型供应器 ('SYSTEM_LLM_openai', 'LLM', 'openai', 'OpenAI接口', '[{"key":"base_url","label":"基础URL","type":"string"},{"key":"model_name","label":"模型名称","type":"string"},{"key":"api_key","label":"API密钥","type":"string"},{"key":"temperature","label":"温度","type":"number"},{"key":"max_tokens","label":"最大令牌数","type":"number"},{"key":"top_p","label":"top_p值","type":"number"},{"key":"top_k","label":"top_k值","type":"number"},{"key":"frequency_penalty","label":"频率惩罚","type":"number"}]', 1, 1, NOW(), 1, NOW()), diff --git a/main/manager-api/src/main/resources/db/changelog/202504112044.sql b/main/manager-api/src/main/resources/db/changelog/202504112044.sql index 3609d235..9fd2257d 100644 --- a/main/manager-api/src/main/resources/db/changelog/202504112044.sql +++ b/main/manager-api/src/main/resources/db/changelog/202504112044.sql @@ -11,7 +11,6 @@ INSERT INTO `ai_model_config` VALUES ('ASR_FunASR', 'ASR', 'FunASR', 'FunASR语 INSERT INTO `ai_model_config` VALUES ('ASR_SherpaASR', 'ASR', 'SherpaASR', 'Sherpa语音识别', 0, 1, '{\"type\": \"sherpa_onnx_local\", \"model_dir\": \"models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17\", \"output_dir\": \"tmp/\"}', NULL, NULL, 2, NULL, NULL, NULL, NULL); INSERT INTO `ai_model_config` VALUES ('ASR_DoubaoASR', 'ASR', 'DoubaoASR', '豆包语音识别', 0, 1, '{\"type\": \"doubao\", \"appid\": \"\", \"access_token\": \"\", \"cluster\": \"volcengine_input_common\", \"output_dir\": \"tmp/\"}', NULL, NULL, 3, NULL, NULL, NULL, NULL); INSERT INTO `ai_model_config` VALUES ('ASR_TencentASR', 'ASR', 'TencentASR', '腾讯语音识别', 0, 1, '{\"type\": \"tencent\", \"appid\": \"\", \"secret_id\": \"\", \"secret_key\": \"你的secret_key\", \"output_dir\": \"tmp/\"}', NULL, NULL, 4, NULL, NULL, NULL, NULL); -INSERT INTO `ai_model_config` VALUES ('ASR_FunASRServer', 'ASR', 'FunASRServer', 'FunASR服务语音识别', 0, 1, '{\"type\": \"fun_server\", \"host\": \"127.0.0.1\", \"port\": 10096}', NULL, NULL, 5, NULL, NULL, NULL, NULL); -- LLM模型配置 INSERT INTO `ai_model_config` VALUES ('LLM_ChatGLMLLM', 'LLM', 'ChatGLMLLM', '智谱AI', 1, 1, '{\"type\": \"openai\", \"model_name\": \"glm-4-flash\", \"base_url\": \"https://open.bigmodel.cn/api/paas/v4/\", \"api_key\": \"你的api_key\"}', NULL, NULL, 1, NULL, NULL, NULL, NULL); diff --git a/main/manager-api/src/main/resources/db/changelog/202504291043.sql b/main/manager-api/src/main/resources/db/changelog/202504291043.sql new file mode 100644 index 00000000..555b8cf4 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202504291043.sql @@ -0,0 +1,504 @@ +-- 增加FunASR服务语音识别模型供应器和配置 +DELETE FROM `ai_model_provider` WHERE `id` = 'SYSTEM_ASR_FunASRServer'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_FunASRServer', 'ASR', 'fun_server', 'FunASR服务语音识别', '[{"key":"host","label":"服务地址","type":"string"},{"key":"port","label":"端口号","type":"number"}]', 4, 1, NOW(), 1, NOW()); + +DELETE FROM `ai_model_config` WHERE `id` = 'ASR_FunASRServer'; +INSERT INTO `ai_model_config` VALUES ('ASR_FunASRServer', 'ASR', 'FunASRServer', 'FunASR服务语音识别', 0, 1, '{\"type\": \"fun_server\", \"host\": \"127.0.0.1\", \"port\": 10096}', NULL, NULL, 5, NULL, NULL, NULL, NULL); + +-- 修改ai_model_config表的remark字段类型为TEXT +ALTER TABLE `ai_model_config` MODIFY COLUMN `remark` TEXT COMMENT '备注'; + +-- 更新ASR模型配置的说明文档 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md', +`remark` = '独立部署FunASR，使用FunASR的API服务，只需要五句话 +第一句：mkdir -p ./funasr-runtime-resources/models +第二句：sudo docker run -d -p 10096:10095 --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12 +上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime +不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 & +上一句话执行后会进入到容器，继续第五句：tail -f log.txt +第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了 +以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md' WHERE `id` = 'ASR_FunASRServer'; + +-- 更新FunASR本地模型配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/modelscope/FunASR', +`remark` = 'FunASR本地模型配置说明： +1. 需要下载模型文件到xiaozhi-server/models/SenseVoiceSmall目录 +2. 支持中日韩粤语音识别 +3. 本地推理，无需网络连接 +4. 待识别文件保存在tmp/目录' WHERE `id` = 'ASR_FunASR'; + +-- 更新SherpaASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/k2-fsa/sherpa-onnx', +`remark` = 'SherpaASR配置说明： +1. 运行时自动下载模型文件到models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17目录 +2. 支持中文、英文、日语、韩语、粤语等多种语言 +3. 本地推理，无需网络连接 +4. 输出文件保存在tmp/目录' WHERE `id` = 'ASR_SherpaASR'; + +-- 更新豆包ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.volcengine.com/speech/app', +`remark` = '豆包ASR配置说明： +1. 需要在火山引擎控制台创建应用并获取appid和access_token +2. 支持中文语音识别 +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://console.volcengine.com/speech/app +2. 创建新应用 +3. 获取appid和access_token +4. 填入配置文件中' WHERE `id` = 'ASR_DoubaoASR'; + +-- 更新腾讯ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.cloud.tencent.com/cam/capi', +`remark` = '腾讯ASR配置说明： +1. 需要在腾讯云控制台创建应用并获取appid、secret_id和secret_key +2. 支持中文语音识别 +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://console.cloud.tencent.com/cam/capi 获取密钥 +2. 访问 https://console.cloud.tencent.com/asr/resourcebundle 领取免费资源 +3. 获取appid、secret_id和secret_key +4. 填入配置文件中' WHERE `id` = 'ASR_TencentASR'; + +-- 更新TTS模型配置说明 +-- EdgeTTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/rany2/edge-tts', +`remark` = 'EdgeTTS配置说明： +1. 使用微软Edge TTS服务 +2. 支持多种语言和音色 +3. 免费使用，无需注册 +4. 需要网络连接 +5. 输出文件保存在tmp/目录' WHERE `id` = 'TTS_EdgeTTS'; + +-- 豆包TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.volcengine.com/speech/service/8', +`remark` = '豆包TTS配置说明： +1. 访问 https://console.volcengine.com/speech/service/8 +2. 需要在火山引擎控制台创建应用并获取appid和access_token +3. 山引擎语音一定要购买花钱，起步价30元，就有100并发了。如果用免费的只有2个并发，会经常报tts错误 +4. 购买服务后，购买免费的音色后，可能要等半小时左右，才能使用。 +5. 填入配置文件中' WHERE `id` = 'TTS_DoubaoTTS'; + +-- 硅基流动TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://cloud.siliconflow.cn/account/ak', +`remark` = '硅基流动TTS配置说明： +1. 访问 https://cloud.siliconflow.cn/account/ak +2. 注册并获取API密钥 +3. 填入配置文件中' WHERE `id` = 'TTS_CosyVoiceSiliconflow'; + +-- Coze中文TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://www.coze.cn/open/oauth/pats', +`remark` = 'Coze中文TTS配置说明： +1. 访问 https://www.coze.cn/open/oauth/pats +2. 获取个人令牌 +3. 填入配置文件中' WHERE `id` = 'TTS_CozeCnTTS'; + +-- FishSpeech配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/fishaudio/fish-speech', +`remark` = 'FishSpeech配置说明： +1. 需要本地部署FishSpeech服务 +2. 支持自定义音色 +3. 本地推理，无需网络连接 +4. 输出文件保存在tmp/目录 +5. 运行服务示例命令：python -m tools.api_server --listen 0.0.0.0:8080 --llama-checkpoint-path "checkpoints/fish-speech-1.5" --decoder-checkpoint-path "checkpoints/fish-speech-1.5/firefly-gan-vq-fsq-8x1024-21hz-generator.pth" --decoder-config-name firefly_gan_vq --compile' WHERE `id` = 'TTS_FishSpeech'; + +-- GPT-SoVITS V2配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/RVC-Boss/GPT-SoVITS', +`remark` = 'GPT-SoVITS V2配置说明： +1. 需要本地部署GPT-SoVITS服务 +2. 支持自定义音色克隆 +3. 本地推理，无需网络连接 +4. 输出文件保存在tmp/目录 +部署步骤： +1. 运行服务示例命令：python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/demo.yaml' WHERE `id` = 'TTS_GPT_SOVITS_V2'; + +-- GPT-SoVITS V3配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/RVC-Boss/GPT-SoVITS', +`remark` = 'GPT-SoVITS V3配置说明： +1. 需要本地部署GPT-SoVITS V3服务 +2. 支持自定义音色克隆 +3. 本地推理，无需网络连接 +4. 输出文件保存在tmp/目录' WHERE `id` = 'TTS_GPT_SOVITS_V3'; + +-- MiniMax TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://platform.minimaxi.com/', +`remark` = 'MiniMax TTS配置说明： +1. 需要在MiniMax平台创建账户并充值 +2. 支持多种音色，当前配置使用female-shaonv +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://platform.minimaxi.com/ 注册账号 +2. 访问 https://platform.minimaxi.com/user-center/payment/balance 充值 +3. 访问 https://platform.minimaxi.com/user-center/basic-information 获取group_id +4. 访问 https://platform.minimaxi.com/user-center/basic-information/interface-key 获取api_key +5. 填入配置文件中' WHERE `id` = 'TTS_MinimaxTTS'; + +-- 阿里云TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://nls-portal.console.aliyun.com/', +`remark` = '阿里云TTS配置说明： +1. 需要在阿里云平台开通智能语音交互服务 +2. 支持多种音色，当前配置使用xiaoyun +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://nls-portal.console.aliyun.com/ 开通服务 +2. 访问 https://nls-portal.console.aliyun.com/applist 获取appkey +3. 访问 https://nls-portal.console.aliyun.com/overview 获取token +4. 填入配置文件中 +注意：token是临时的24小时有效，长期使用需要配置access_key_id和access_key_secret' WHERE `id` = 'TTS_AliyunTTS'; + +-- 腾讯TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.cloud.tencent.com/cam/capi', +`remark` = '腾讯TTS配置说明： +1. 需要在腾讯云平台开通智能语音交互服务 +2. 支持多种音色，当前配置使用101001 +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://console.cloud.tencent.com/cam/capi 获取密钥 +2. 访问 https://console.cloud.tencent.com/tts/resourcebundle 领取免费资源 +3. 创建新应用 +4. 获取appid、secret_id和secret_key +5. 填入配置文件中' WHERE `id` = 'TTS_TencentTTS'; + +-- 302AI TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://dash.302.ai/', +`remark` = '302AI TTS配置说明： +1. 需要在302平台创建账户并获取API密钥 +2. 支持多种音色，当前配置使用湾湾小何音色 +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://dash.302.ai/ 注册账号 +2. 访问 https://dash.302.ai/apis/list 获取API密钥 +3. 填入配置文件中 +价格：$35/百万字符' WHERE `id` = 'TTS_TTS302AI'; + +-- 机智云TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://agentrouter.gizwitsapi.com/panel/token', +`remark` = '机智云TTS配置说明： +1. 需要在机智云平台获取API密钥 +2. 支持多种音色，当前配置使用湾湾小何音色 +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://agentrouter.gizwitsapi.com/panel/token 获取API密钥 +2. 填入配置文件中 +注意：前一万名注册的用户，将送5元体验金额' WHERE `id` = 'TTS_GizwitsTTS'; + +-- ACGN TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://acgn.ttson.cn/', +`remark` = 'ACGN TTS配置说明： +1. 需要在ttson平台购买token +2. 支持多种角色音色，当前配置使用角色ID：1695 +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://acgn.ttson.cn/ 查看角色列表 +2. 访问 www.ttson.cn 购买token +3. 填入配置文件中 +开发相关疑问请提交至网站上的qq' WHERE `id` = 'TTS_ACGNTTS'; + +-- OpenAI TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://platform.openai.com/api-keys', +`remark` = 'OpenAI TTS配置说明： +1. 需要在OpenAI平台获取API密钥 +2. 支持多种音色，当前配置使用onyx +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://platform.openai.com/api-keys 获取API密钥 +2. 填入配置文件中 +注意：国内需要使用代理访问' WHERE `id` = 'TTS_OpenAITTS'; + +-- 自定义TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = NULL, +`remark` = '自定义TTS配置说明： +1. 支持自定义TTS接口服务 +2. 使用GET方式请求 +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +配置说明： +1. 在params中配置请求参数 +2. 在headers中配置请求头 +3. 设置返回音频格式' WHERE `id` = 'TTS_CustomTTS'; + +-- 火山引擎边缘大模型网关TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.volcengine.com/vei/aigateway/', +`remark` = '火山引擎边缘大模型网关TTS配置说明： +1. 访问 https://console.volcengine.com/vei/aigateway/ +2. 创建网关访问密钥，搜索并勾选 Doubao-语音合成 +3. 如果需要使用LLM，一并勾选 Doubao-pro-32k-functioncall +4. 访问 https://console.volcengine.com/vei/aigateway/tokens-list 获取密钥 +5. 填入配置文件中 +音色列表参考：https://www.volcengine.com/docs/6561/1257544' WHERE `id` = 'TTS_VolcesAiGatewayTTS'; + +-- 更新LLM模型配置说明 +-- ChatGLM配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://bigmodel.cn/usercenter/proj-mgmt/apikeys', +`remark` = 'ChatGLM配置说明： +1. 访问 https://bigmodel.cn/usercenter/proj-mgmt/apikeys +2. 注册并获取API密钥 +3. 填入配置文件中' WHERE `id` = 'LLM_ChatGLMLLM'; + +-- Ollama配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://ollama.com/', +`remark` = 'Ollama配置说明： +1. 安装Ollama服务 +2. 运行命令：ollama pull qwen2.5 +3. 确保服务运行在http://localhost:11434' WHERE `id` = 'LLM_OllamaLLM'; + +-- 通义千问配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://bailian.console.aliyun.com/?apiKey=1#/api-key', +`remark` = '通义千问配置说明： +1. 访问 https://bailian.console.aliyun.com/?apiKey=1#/api-key +2. 获取API密钥 +3. 填入配置文件中，当前配置使用qwen-turbo模型 +4. 支持自定义参数：temperature=0.7, max_tokens=500, top_p=1, top_k=50' WHERE `id` = 'LLM_AliLLM'; + +-- 通义百炼配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://bailian.console.aliyun.com/?apiKey=1#/api-key', +`remark` = '通义百炼配置说明： +1. 访问 https://bailian.console.aliyun.com/?apiKey=1#/api-key +2. 获取app_id和api_key +3. 填入配置文件中' WHERE `id` = 'LLM_AliAppLLM'; + +-- 豆包大模型配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.volcengine.com/ark/region:ark+cn-beijing/openManagement', +`remark` = '豆包大模型配置说明： +1. 访问 https://console.volcengine.com/ark/region:ark+cn-beijing/openManagement +2. 开通Doubao-1.5-pro服务 +3. 访问 https://console.volcengine.com/ark/region:ark+cn-beijing/apiKey 获取API密钥 +4. 填入配置文件中 +5. 当前建议使用doubao-1-5-pro-32k-250115 +注意：有免费额度500000token' WHERE `id` = 'LLM_DoubaoLLM'; + +-- DeepSeek配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://platform.deepseek.com/', +`remark` = 'DeepSeek配置说明： +1. 访问 https://platform.deepseek.com/ +2. 注册并获取API密钥 +3. 填入配置文件中' WHERE `id` = 'LLM_DeepSeekLLM'; + +-- Dify配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://cloud.dify.ai/', +`remark` = 'Dify配置说明： +1. 访问 https://cloud.dify.ai/ +2. 注册并获取API密钥 +3. 填入配置文件中 +4. 支持多种对话模式：workflows/run, chat-messages, completion-messages +5. 平台设置的角色定义会失效，需要在Dify控制台设置 +注意：建议使用本地部署的Dify接口，国内部分区域访问公有云接口可能受限' WHERE `id` = 'LLM_DifyLLM'; + +-- Gemini配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://aistudio.google.com/apikey', +`remark` = 'Gemini配置说明： +1. 使用谷歌Gemini API服务 +2. 当前配置使用gemini-2.0-flash模型 +3. 需要网络连接 +4. 支持配置代理 +申请步骤： +1. 访问 https://aistudio.google.com/apikey +2. 创建API密钥 +3. 填入配置文件中 +注意：若在中国境内使用，请遵守《生成式人工智能服务管理暂行办法》' WHERE `id` = 'LLM_GeminiLLM'; + +-- Coze配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://www.coze.cn/open/oauth/pats', +`remark` = 'Coze配置说明： +1. 使用Coze平台服务 +2. 需要bot_id、user_id和个人令牌 +3. 需要网络连接 +申请步骤： +1. 访问 https://www.coze.cn/open/oauth/pats +2. 获取个人令牌 +3. 手动计算bot_id和user_id +4. 填入配置文件中' WHERE `id` = 'LLM_CozeLLM'; + +-- LM Studio配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://lmstudio.ai/', +`remark` = 'LM Studio配置说明： +1. 使用本地部署的LM Studio服务 +2. 当前配置使用deepseek-r1-distill-llama-8b@q4_k_m模型 +3. 本地推理，无需网络连接 +4. 需要预先下载模型 +部署步骤： +1. 安装LM Studio +2. 从社区下载模型 +3. 确保服务运行在http://localhost:1234/v1' WHERE `id` = 'LLM_LMStudioLLM'; + +-- FastGPT配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://cloud.tryfastgpt.ai/account/apikey', +`remark` = 'FastGPT配置说明： +1. 使用FastGPT平台服务 +2. 需要网络连接 +3. 配置文件中的prompt无效，需要在FastGPT控制台设置 +4. 支持自定义变量 +申请步骤： +1. 访问 https://cloud.tryfastgpt.ai/account/apikey +2. 获取API密钥 +3. 填入配置文件中' WHERE `id` = 'LLM_FastgptLLM'; + +-- Xinference配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/xorbitsai/inference', +`remark` = 'Xinference配置说明： +1. 使用本地部署的Xinference服务 +2. 当前配置使用qwen2.5:72b-AWQ模型 +3. 本地推理，无需网络连接 +4. 需要预先启动对应模型 +部署步骤： +1. 安装Xinference +2. 启动服务并加载模型 +3. 确保服务运行在http://localhost:9997' WHERE `id` = 'LLM_XinferenceLLM'; + +-- Xinference小模型配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/xorbitsai/inference', +`remark` = 'Xinference小模型配置说明： +1. 使用本地部署的Xinference服务 +2. 当前配置使用qwen2.5:3b-AWQ模型 +3. 本地推理，无需网络连接 +4. 用于意图识别 +部署步骤： +1. 安装Xinference +2. 启动服务并加载模型 +3. 确保服务运行在http://localhost:9997' WHERE `id` = 'LLM_XinferenceSmallLLM'; + +-- 火山引擎边缘大模型网关LLM配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.volcengine.com/vei/aigateway/', +`remark` = '火山引擎边缘大模型网关LLM配置说明： +1. 使用火山引擎边缘大模型网关服务 +2. 需要网关访问密钥 +3. 需要网络连接 +4. 支持function_call功能 +申请步骤： +1. 访问 https://console.volcengine.com/vei/aigateway/ +2. 创建网关访问密钥，搜索并勾选 Doubao-pro-32k-functioncall +3. 如果需要使用语音合成，一并勾选 Doubao-语音合成 +4. 访问 https://console.volcengine.com/vei/aigateway/tokens-list 获取密钥 +5. 填入配置文件中' WHERE `id` = 'LLM_VolcesAiGatewayLLM'; + +-- 更新Memory模型配置说明 +-- 无记忆配置说明 +UPDATE `ai_model_config` SET +`doc_link` = NULL, +`remark` = '无记忆配置说明： +1. 不保存对话历史 +2. 每次对话都是独立的 +3. 无需额外配置 +4. 适合对隐私要求高的场景' WHERE `id` = 'Memory_nomem'; + +-- 本地短期记忆配置说明 +UPDATE `ai_model_config` SET +`doc_link` = NULL, +`remark` = '本地短期记忆配置说明： +1. 使用本地存储保存对话历史 +2. 通过selected_module的llm总结对话内容 +3. 数据保存在本地，不会上传到服务器 +4. 适合注重隐私的场景 +5. 无需额外配置' WHERE `id` = 'Memory_mem_local_short'; + +-- Mem0AI记忆配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://app.mem0.ai/dashboard/api-keys', +`remark` = 'Mem0AI记忆配置说明： +1. 使用Mem0AI服务保存对话历史 +2. 需要API密钥 +3. 需要网络连接 +4. 每月有1000次免费调用 +申请步骤： +1. 访问 https://app.mem0.ai/dashboard/api-keys +2. 获取API密钥 +3. 填入配置文件中' WHERE `id` = 'Memory_mem0ai'; + +-- 更新Intent模型配置说明 +-- 无意图识别配置说明 +UPDATE `ai_model_config` SET +`doc_link` = NULL, +`remark` = '无意图识别配置说明： +1. 不进行意图识别 +2. 所有对话直接传递给LLM处理 +3. 无需额外配置 +4. 适合简单对话场景' WHERE `id` = 'Intent_nointent'; + +-- LLM意图识别配置说明 +UPDATE `ai_model_config` SET +`doc_link` = NULL, +`remark` = 'LLM意图识别配置说明： +1. 使用独立的LLM进行意图识别 +2. 默认使用selected_module.LLM的模型 +3. 可以配置使用独立的LLM（如免费的ChatGLMLLM） +4. 通用性强，但会增加处理时间 +5. 不支持控制音量大小等iot操作 +配置说明： +1. 在llm字段中指定使用的LLM模型 +2. 如果不指定，则使用selected_module.LLM的模型' WHERE `id` = 'Intent_intent_llm'; + +-- 函数调用意图识别配置说明 +UPDATE `ai_model_config` SET +`doc_link` = NULL, +`remark` = '函数调用意图识别配置说明： +1. 使用LLM的function_call功能进行意图识别 +2. 需要所选择的LLM支持function_call +3. 按需调用工具，处理速度快 +4. 支持所有iot指令 +5. 默认已加载以下功能： + - handle_exit_intent（退出识别） + - play_music（音乐播放） + - change_role（角色切换） + - get_weather（天气查询） + - get_news（新闻查询） +配置说明： +1. 在functions字段中配置需要加载的功能模块 +2. 系统默认已加载基础功能，无需重复配置 +3. 可以添加自定义功能模块' WHERE `id` = 'Intent_function_call'; + +-- 更新VAD模型配置说明 +-- SileroVAD配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/snakers4/silero-vad', +`remark` = 'SileroVAD配置说明： +1. 使用SileroVAD模型进行语音活动检测 +2. 本地推理，无需网络连接 +3. 需要下载模型文件到models/snakers4_silero-vad目录 +4. 可配置参数： + - threshold: 0.5（语音检测阈值） + - min_silence_duration_ms: 700（最小静音持续时间，单位毫秒） +5. 如果说话停顿比较长，可以适当增加min_silence_duration_ms的值' WHERE `id` = 'VAD_SileroVAD'; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 1bbddb75..131aaf01 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -78,4 +78,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202504251422.sql \ No newline at end of file + path: classpath:db/changelog/202504251422.sql + - changeSet: + id: 202504291043 + author: jiangkunyin + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202504291043.sql \ No newline at end of file diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index d18b23e9..21266333 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -210,8 +210,14 @@ ASR: model_dir: models/SenseVoiceSmall output_dir: tmp/ FunASRServer: - # 支持FunASR服务，部署方法：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md - # mode 预置 offline + # 独立部署FunASR，使用FunASR的API服务，只需要五句话 + # 第一句：mkdir -p ./funasr-runtime-resources/models + # 第二句：sudo docker run -d -p 10096:10095 --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12 + # 上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime + # 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 & + # 上一句话执行后会进入到容器，继续第五句：tail -f log.txt + # 第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了 + # 以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md type: fun_server host: 127.0.0.1 port: 10096 @@ -272,12 +278,12 @@ LLM: DoubaoLLM: # 定义LLM API类型 type: openai - # 先开通服务，打开以下网址，开通的服务搜索Doubao-pro-32k，开通它 + # 先开通服务，打开以下网址，开通的服务搜索Doubao-1.5-pro，开通它 # 开通改地址：https://console.volcengine.com/ark/region:ark+cn-beijing/openManagement?LLM=%7B%7D&OpenTokenDrawer=false # 免费额度500000token # 开通后，进入这里获取密钥：https://console.volcengine.com/ark/region:ark+cn-beijing/apiKey?apikey=%7B%7D base_url: https://ark.cn-beijing.volces.com/api/v3 - model_name: doubao-pro-32k-functioncall-241028 + model_name: doubao-1-5-pro-32k-250115 api_key: 你的doubao web key DeepSeekLLM: # 定义LLM API类型 @@ -453,12 +459,12 @@ TTS: GPT_SOVITS_V2: # 定义TTS API类型 #启动tts方法： - #python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/caixukun.yaml + #python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/demo.yaml type: gpt_sovits_v2 url: "http://127.0.0.1:9880/tts" output_dir: tmp/ text_lang: "auto" - ref_audio_path: "caixukun.wav" + ref_audio_path: "demo.wav" prompt_text: "" prompt_lang: "zh" top_k: 5 @@ -591,8 +597,8 @@ TTS: ACGNTTS: #在线网址：https://acgn.ttson.cn/ #token购买：www.ttson.cn - #开发相关疑问请提交至3497689533@qq.com - #角色id获取地址：ctrl+f快速检索角色——网站管理者不允许发布,可询问网站管理者：1069379506 + #开发相关疑问请提交至网站上的qq + #角色id获取地址：ctrl+f快速检索角色——网站管理者不允许发布,可询问网站管理者 #各参数意义见开发文档：https://www.yuque.com/alexuh/skmti9/wm6taqislegb02gd?singleDoc# type: ttson token: your_token diff --git a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py index b086022b..cfcbe83b 100644 --- a/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py +++ b/main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py @@ -43,11 +43,11 @@ class ASRProvider(ASRProviderBase): # 确保输出目录存在 os.makedirs(self.output_dir, exist_ok=True) - + # 初始化模型文件路径 model_files = { "model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"), - "tokens.txt": os.path.join(self.model_dir, "tokens.txt") + "tokens.txt": os.path.join(self.model_dir, "tokens.txt"), } # 下载并检查模型文件 @@ -58,15 +58,15 @@ class ASRProvider(ASRProviderBase): model_file_download( model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue", file_path=file_name, - local_dir=self.model_dir + local_dir=self.model_dir, ) - + if not os.path.isfile(file_path): raise FileNotFoundError(f"模型文件下载失败: {file_path}") - + self.model_path = model_files["model.int8.onnx"] self.tokens_path = model_files["tokens.txt"] - + except Exception as e: logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}") raise @@ -83,7 +83,7 @@ class ASRProvider(ASRProviderBase): use_itn=True, ) - def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str: + def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str: """PCM数据保存为WAV文件""" file_name = f"asr_{session_id}_{uuid.uuid4()}.wav" file_path = os.path.join(self.output_dir, file_name) @@ -135,7 +135,9 @@ class ASRProvider(ASRProviderBase): samples_float32 = samples_float32 / 32768 return samples_float32, f.getframerate() - async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]: + async def speech_to_text( + self, opus_data: List[bytes], session_id: str + ) -> Tuple[Optional[str], Optional[str]]: """语音转文本主处理逻辑""" file_path = None try: @@ -143,7 +145,9 @@ class ASRProvider(ASRProviderBase): start_time = time.time() pcm_data = self.decode_opus(opus_data, session_id) file_path = self.save_audio_to_file(pcm_data, session_id) - logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}") + logger.bind(tag=TAG).debug( + f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}" + ) # 语音识别 start_time = time.time() @@ -152,14 +156,15 @@ class ASRProvider(ASRProviderBase): s.accept_waveform(sample_rate, samples) self.model.decode_stream(s) text = s.result.text - logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}") + logger.bind(tag=TAG).debug( + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" + ) return text, file_path except Exception as e: logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True) return "", None - finally: # 文件清理逻辑 if self.delete_audio_file and file_path and os.path.exists(file_path):