From 90a96314501ec8aeae5d50cad17be69edc33a322 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Mon, 30 Jun 2025 14:41:53 +0800 Subject: [PATCH 01/24] =?UTF-8?q?update:=E6=B7=BB=E5=8A=A0=E9=98=BF?= =?UTF-8?q?=E9=87=8C=E4=BA=91=E6=B5=81=E5=BC=8Fasr?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 22 ++ .../core/providers/asr/aliyun_stream.py | 286 ++++++++++++++++++ 2 files changed, 308 insertions(+) create mode 100644 main/xiaozhi-server/core/providers/asr/aliyun_stream.py diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 71c13ec7..bd9f024b 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -298,9 +298,13 @@ ASR: output_dir: tmp/ AliyunASR: # 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息 + # HTTP POST请求，一次性处理完整音频 # 平台地址：https://nls-portal.console.aliyun.com/ # appkey地址：https://nls-portal.console.aliyun.com/applist # token地址：https://nls-portal.console.aliyun.com/overview + # AliyunASR和AliyunStreamASR的区别是：AliyunASR是批量处理场景，AliyunStreamASR是实时交互场景 + # 一般来说非流式ASR更便宜（0.004元/秒，¥0.24/分钟） + # 但是AliyunStreamASR实时性更好（0.005元/秒，¥0.3/分钟） # 定义ASR API类型 type: aliyun appkey: 你的阿里云智能语音交互服务项目Appkey @@ -308,6 +312,24 @@ ASR: access_key_id: 你的阿里云账号access_key_id access_key_secret: 你的阿里云账号access_key_secret output_dir: tmp/ + AliyunStreamASR: + # 阿里云智能语音交互服务 - 实时流式语音识别 + # WebSocket连接，实时处理音频流 + # 平台地址：https://nls-portal.console.aliyun.com/ + # appkey地址：https://nls-portal.console.aliyun.com/applist + # token地址：https://nls-portal.console.aliyun.com/overview + # AliyunASR和AliyunStreamASR的区别是：AliyunASR是批量处理场景，AliyunStreamASR是实时交互场景 + # 一般来说非流式ASR更便宜（0.004元/秒，¥0.24/分钟） + # 但是AliyunStreamASR实时性更好（0.005元/秒，¥0.3/分钟） + # 定义ASR API类型 + type: aliyun + appkey: 你的阿里云智能语音交互服务项目Appkey + token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret + access_key_id: 你的阿里云账号access_key_id + access_key_secret: 你的阿里云账号access_key_secret + # 服务器地域选择，如nls-gateway-cn-hangzhou.aliyuncs.com(杭州)等 + host: 根据自己的地理位置选择最近的服务器地域，不填默认为nls-gateway-cn-shanghai.aliyuncs.com(上海) + output_dir: tmp/ BaiduASR: # 获取AppID、API Key、Secret Key：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list # 查看资源额度：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/overview/resource/list diff --git a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py new file mode 100644 index 00000000..94727798 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py @@ -0,0 +1,286 @@ +import json +import time +import uuid +import hmac +import base64 +import hashlib +import asyncio +import requests +import websockets +import opuslib_next +import random +from typing import Optional, Tuple, List +from urllib import parse +from datetime import datetime +from config.logger import setup_logging +from core.providers.asr.base import ASRProviderBase +from core.providers.asr.dto.dto import InterfaceType + +TAG = __name__ +logger = setup_logging() + + +class AccessToken: + @staticmethod + def _encode_text(text): + encoded_text = parse.quote_plus(text) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def _encode_dict(dic): + keys = dic.keys() + dic_sorted = [(key, dic[key]) for key in sorted(keys)] + encoded_text = parse.urlencode(dic_sorted) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def create_token(access_key_id, access_key_secret): + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } + query_string = AccessToken._encode_dict(parameters) + string_to_sign = ( + "GET" + "&" + AccessToken._encode_text("/") + "&" + AccessToken._encode_text(query_string) + ) + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() + signature = base64.b64encode(secreted_string) + signature = AccessToken._encode_text(signature) + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (signature, query_string) + response = requests.get(full_url) + if response.ok: + root_obj = response.json() + if "Token" in root_obj: + return root_obj["Token"]["Id"], root_obj["Token"]["ExpireTime"] + return None, None + + +class ASRProvider(ASRProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__() + self.interface_type = InterfaceType.STREAM + self.config = config + self.text = "" + self.decoder = opuslib_next.Decoder(16000, 1) + self.asr_ws = None + self.forward_task = None + self.is_processing = False + self.server_ready = False # 服务器准备状态 + + # 基础配置 + self.access_key_id = config.get("access_key_id") + self.access_key_secret = config.get("access_key_secret") + self.appkey = config.get("appkey") + self.token = config.get("token") + self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com") + self.ws_url = f"wss://{self.host}/ws/v1" + self.max_sentence_silence = config.get("max_sentence_silence") + self.output_dir = config.get("output_dir", "./audio_output") + self.delete_audio_file = delete_audio_file + self.expire_time = None + + # Token管理 + if self.access_key_id and self.access_key_secret: + self._refresh_token() + elif not self.token: + raise ValueError("必须提供access_key_id+access_key_secret或者直接提供token") + + def _refresh_token(self): + """刷新Token""" + self.token, expire_time_str = AccessToken.create_token(self.access_key_id, self.access_key_secret) + if not self.token: + raise ValueError("无法获取有效的访问Token") + + try: + expire_str = str(expire_time_str).strip() + if expire_str.isdigit(): + expire_time = datetime.fromtimestamp(int(expire_str)) + else: + expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ") + self.expire_time = expire_time.timestamp() - 60 + except: + self.expire_time = None + + def _is_token_expired(self): + """检查Token是否过期""" + return self.expire_time and time.time() > self.expire_time + + async def open_audio_channels(self, conn): + await super().open_audio_channels(conn) + + async def receive_audio(self, conn, audio, audio_have_voice): + conn.asr_audio.append(audio) + conn.asr_audio = conn.asr_audio[-10:] + + # 参考豆包ASR：只在有声音且没有连接时建立连接 + if audio_have_voice and not self.is_processing: + try: + await self._start_recognition(conn) + except Exception as e: + logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}") + await self._cleanup() + return + + if self.asr_ws and self.is_processing and self.server_ready: + try: + pcm_frame = self.decoder.decode(audio, 960) + await self.asr_ws.send(pcm_frame) + except Exception as e: + logger.bind(tag=TAG).warning(f"发送音频失败: {str(e)}") + await self._cleanup() + + async def _start_recognition(self, conn): + """开始识别会话""" + if self._is_token_expired(): + self._refresh_token() + + # 建立连接 + headers = {"X-NLS-Token": self.token} + self.asr_ws = await websockets.connect( + self.ws_url, + additional_headers=headers, + max_size=1000000000, + ping_interval=None, + ping_timeout=None, + close_timeout=5, + ) + + self.is_processing = True + self.server_ready = False # 重置服务器准备状态 + self.forward_task = asyncio.create_task(self._forward_results(conn)) + + # 发送开始请求 + start_request = { + "header": { + "namespace": "SpeechTranscriber", + "name": "StartTranscription", + "status": 20000000, + "message_id": ''.join(random.choices('0123456789abcdef', k=32)), + "task_id": ''.join(random.choices('0123456789abcdef', k=32)), + "status_text": "Gateway:SUCCESS:Success.", + "appkey": self.appkey + }, + "payload": { + "format": "pcm", + "sample_rate": 16000, + "enable_intermediate_result": True, + "enable_punctuation_prediction": True, + "enable_inverse_text_normalization": True, + "max_sentence_silence": self.max_sentence_silence, + "enable_voice_detection": False, + } + } + await self.asr_ws.send(json.dumps(start_request, ensure_ascii=False)) + logger.bind(tag=TAG).info("已发送开始请求，等待服务器准备...") + + async def _forward_results(self, conn): + """转发识别结果""" + try: + while self.asr_ws and not conn.stop_event.is_set(): + try: + response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0) + result = json.loads(response) + + header = result.get("header", {}) + payload = result.get("payload", {}) + message_name = header.get("name", "") + status = header.get("status", 0) + + if status != 20000000: + if status in [40000004, 40010004]: # 连接超时或客户端断开 + logger.bind(tag=TAG).warning(f"连接问题，状态码: {status}") + break + elif status in [40270002, 40270003]: # 音频问题 + logger.bind(tag=TAG).warning(f"音频处理问题，状态码: {status}") + continue + else: + logger.bind(tag=TAG).error(f"识别错误，状态码: {status}, 消息: {header.get('status_text', '')}") + continue + + # 收到TranscriptionStarted表示服务器准备好接收音频数据 + if message_name == "TranscriptionStarted": + self.server_ready = True + logger.bind(tag=TAG).info("服务器已准备，开始发送缓存音频...") + + # 发送缓存音频 + if conn.asr_audio: + for cached_audio in conn.asr_audio[-10:]: + try: + pcm_frame = self.decoder.decode(cached_audio, 960) + await self.asr_ws.send(pcm_frame) + except Exception as e: + logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}") + break + continue + + if message_name == "TranscriptionResultChanged": + # 中间结果 + text = payload.get("result", "") + if text: + self.text = text + elif message_name == "SentenceEnd": + # 最终结果 + text = payload.get("result", "") + if text: + self.text = text + conn.reset_vad_states() + await self.handle_voice_stop(conn, None) + break + elif message_name == "TranscriptionCompleted": + # 识别完成 + self.is_processing = False + break + + except asyncio.TimeoutError: + continue + except websockets.exceptions.ConnectionClosed: + break + except Exception as e: + logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}") + break + + except Exception as e: + logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}") + finally: + await self._cleanup() + + async def _cleanup(self): + """清理资源""" + self.is_processing = False + self.server_ready = False # 重置服务器准备状态 + + if self.forward_task and not self.forward_task.done(): + self.forward_task.cancel() + try: + await asyncio.wait_for(self.forward_task, timeout=1.0) + except: + pass + self.forward_task = None + + if self.asr_ws: + try: + await asyncio.wait_for(self.asr_ws.close(), timeout=2.0) + except: + pass + self.asr_ws = None + + async def speech_to_text(self, opus_data, session_id, audio_format): + """获取识别结果""" + result = self.text + self.text = "" + return result, None + + async def close(self): + """关闭资源""" + await self._cleanup() From 039badd2650bd11a524fce631bc537d46f5e05f7 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Tue, 1 Jul 2025 18:14:36 +0800 Subject: [PATCH 02/24] =?UTF-8?q?update:=E6=B7=BB=E5=8A=A0=E4=BA=86?= =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91CosyVoice=E6=B5=81=E5=BC=8FTTS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 22 + .../core/providers/tts/aliyun_stream.py | 891 ++++++++++++++++++ 2 files changed, 913 insertions(+) create mode 100644 main/xiaozhi-server/core/providers/tts/aliyun_stream.py diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index bd9f024b..a4b4ccc0 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -690,6 +690,28 @@ TTS: # pitch_rate: 0 # 添加 302.ai TTS 配置 # token申请地址：https://dash.302.ai/ + AliyunStreamTTS: + # 阿里云CosyVoice大模型流式文本语音合成 + # 采用FlowingSpeechSynthesizer接口，支持更低延迟和更自然的语音质量 + # 流式文本语音合成仅提供商用版，不支持试用，详情请参见试用版和商用版。要使用该功能，请开通商用版。 + # 支持龙系列专用音色：longxiaochun、longyu、longchen等 + # 平台地址：https://nls-portal.console.aliyun.com/ + # appkey地址：https://nls-portal.console.aliyun.com/applist + # token地址：https://nls-portal.console.aliyun.com/overview + # 使用三阶段流式交互：StartSynthesis -> RunSynthesis -> StopSynthesis + type: aliyun + output_dir: tmp/ + appkey: 你的阿里云智能语音交互服务项目Appkey + token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret + voice: longxiaochun + access_key_id: 你的阿里云账号access_key_id + access_key_secret: 你的阿里云账号access_key_secret + # 以下可不用设置，使用默认设置 + # format: pcm # 音频格式：pcm、wav、mp3 + # sample_rate: 16000 # 采样率：8000、16000、24000 + # volume: 50 # 音量：0-100 + # speech_rate: 0 # 语速：-500到500 + # pitch_rate: 0 # 语调：-500到500 TencentTTS: # 腾讯云智能语音交互服务，需要先在腾讯云平台开通服务 # appid、secret_id、secret_key申请地址：https://console.cloud.tencent.com/cam/capi diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py new file mode 100644 index 00000000..9aad9614 --- /dev/null +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -0,0 +1,891 @@ +import uuid +import json +import hmac +import hashlib +import base64 +import time +import queue +import asyncio +import traceback +import websockets +import websockets.protocol +import os +import random +import threading +import concurrent.futures +import sys +from datetime import datetime +from urllib import parse +from typing import Optional +from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType +from core.utils.tts import MarkdownCleaner +from core.utils import opus_encoder_utils, textUtils +from config.logger import setup_logging + +TAG = __name__ +logger = setup_logging() + + +class AccessToken: + @staticmethod + def _encode_text(text): + encoded_text = parse.quote_plus(text) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def _encode_dict(dic): + keys = dic.keys() + dic_sorted = [(key, dic[key]) for key in sorted(keys)] + encoded_text = parse.urlencode(dic_sorted) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def create_token(access_key_id, access_key_secret): + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", # 使用上海地域进行Token获取 + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } + + query_string = AccessToken._encode_dict(parameters) + string_to_sign = ( + "GET" + + "&" + + AccessToken._encode_text("/") + + "&" + + AccessToken._encode_text(query_string) + ) + + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() + signature = base64.b64encode(secreted_string) + signature = AccessToken._encode_text(signature) + + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % ( + signature, + query_string, + ) + + import requests + response = requests.get(full_url) + if response.ok: + root_obj = response.json() + key = "Token" + if key in root_obj: + token = root_obj[key]["Id"] + expire_time = root_obj[key]["ExpireTime"] + return token, expire_time + return None, None + + +class TTSProvider(TTSProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__(config, delete_audio_file) + + # 设置为流式接口类型 + self.interface_type = InterfaceType.SINGLE_STREAM + + # 基础配置 + self.access_key_id = config.get("access_key_id") + self.access_key_secret = config.get("access_key_secret") + self.appkey = config.get("appkey") + self.format = config.get("format", "pcm") + self.audio_file_type = config.get("format", "pcm") + + # 采样率配置 + sample_rate = config.get("sample_rate", "16000") + self.sample_rate = int(sample_rate) if sample_rate else 16000 + + # 音色配置 - CosyVoice大模型音色 + if config.get("private_voice"): + self.voice = config.get("private_voice") + else: + self.voice = config.get("voice", "longxiaochun") # CosyVoice默认音色 + + # 音频参数配置 + volume = config.get("volume", "50") + self.volume = int(volume) if volume else 50 + + speech_rate = config.get("speech_rate", "0") + self.speech_rate = int(speech_rate) if speech_rate else 0 + + pitch_rate = config.get("pitch_rate", "0") + self.pitch_rate = int(pitch_rate) if pitch_rate else 0 + + # WebSocket配置 + self.host = config.get("host", "nls-gateway-cn-beijing.aliyuncs.com") + self.ws_url = f"wss://{self.host}/ws/v1" + self.ws = None + + # 流式相关配置 + self.before_stop_play_files = [] + self.segment_count = 0 + + # 创建Opus编码器 + self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( + sample_rate=16000, channels=1, frame_size_ms=60 + ) + + # PCM缓冲区 + self.pcm_buffer = bytearray() + + # Token管理 + if self.access_key_id and self.access_key_secret: + self._refresh_token() + else: + self.token = config.get("token") + self.expire_time = None + + def _refresh_token(self): + """刷新Token并记录过期时间""" + if self.access_key_id and self.access_key_secret: + self.token, expire_time_str = AccessToken.create_token( + self.access_key_id, self.access_key_secret + ) + if not expire_time_str: + raise ValueError("无法获取有效的Token过期时间") + + try: + expire_str = str(expire_time_str).strip() + if expire_str.isdigit(): + expire_time = datetime.fromtimestamp(int(expire_str)) + else: + expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ") + self.expire_time = expire_time.timestamp() - 60 + except Exception as e: + raise ValueError(f"无效的过期时间格式: {expire_str}") from e + else: + self.expire_time = None + + if not self.token: + raise ValueError("无法获取有效的访问Token") + + def _is_token_expired(self): + """检查Token是否过期""" + if not self.expire_time: + return False + return time.time() > self.expire_time + + async def _ensure_connection(self): + """确保WebSocket连接可用""" + # 检查连接状态，兼容不同版本的websockets库 + need_reconnect = False + if self.ws is None: + need_reconnect = True + else: + try: + # 尝试访问closed属性，如果不存在则检查state + if hasattr(self.ws, 'closed'): + need_reconnect = self.ws.closed + elif hasattr(self.ws, 'state'): + # websockets 新版本使用state属性 + need_reconnect = self.ws.state != websockets.protocol.State.OPEN + else: + # 如果都没有，尝试发送ping来检测连接状态 + try: + await asyncio.wait_for(self.ws.ping(), timeout=2.0) + except: + need_reconnect = True + except: + need_reconnect = True + + if need_reconnect: + # 清理旧连接 + if self.ws: + try: + if hasattr(self.ws, 'close'): + if asyncio.iscoroutinefunction(self.ws.close): + await self.ws.close() + else: + self.ws.close() + except: + pass + finally: + self.ws = None + + if self._is_token_expired(): + logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...") + self._refresh_token() + + # 重试连接机制 + max_retries = 3 + retry_delay = 1.0 + + for attempt in range(max_retries): + try: + self.ws = await asyncio.wait_for( + websockets.connect( + self.ws_url, + additional_headers={ + "X-NLS-Token": self.token, + }, + ping_interval=30, + ping_timeout=10, + close_timeout=10, + ), + timeout=10.0 + ) + logger.bind(tag=TAG).info("阿里云CosyVoice流式TTS WebSocket连接建立成功") + return + except Exception as e: + logger.bind(tag=TAG).warning(f"WebSocket连接失败 (尝试 {attempt + 1}/{max_retries}): {e}") + if attempt < max_retries - 1: + await asyncio.sleep(retry_delay * (attempt + 1)) + else: + logger.bind(tag=TAG).error(f"WebSocket连接最终失败: {e}") + raise + + def tts_text_priority_thread(self): + """流式文本处理线程""" + while not self.conn.stop_event.is_set(): + try: + message = self.tts_text_queue.get(timeout=1) + if message.sentence_type == SentenceType.FIRST: + # 初始化参数 + self.tts_stop_request = False + self.processed_chars = 0 + self.tts_text_buff = [] + self.segment_count = 0 + self.tts_audio_first_sentence = True + self.before_stop_play_files.clear() + elif ContentType.TEXT == message.content_type: + self.tts_text_buff.append(message.content_detail) + segment_text = self._get_segment_text() + if segment_text: + self.to_tts_single_stream(segment_text) + + elif ContentType.FILE == message.content_type: + logger.bind(tag=TAG).info( + f"添加音频文件到待播放列表: {message.content_file}" + ) + self.before_stop_play_files.append( + (message.content_file, message.content_detail) + ) + + if message.sentence_type == SentenceType.LAST: + # 处理剩余的文本 + self._process_remaining_text(True) + + except queue.Empty: + continue + except Exception as e: + logger.bind(tag=TAG).error( + f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" + ) + + def _process_remaining_text(self, is_last=False): + """处理剩余的文本并生成语音""" + full_text = "".join(self.tts_text_buff) + remaining_text = full_text[self.processed_chars:] + if remaining_text: + segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) + if segment_text: + self.to_tts_single_stream(segment_text, is_last) + self.processed_chars += len(full_text) + else: + self._process_before_stop_play_files() + else: + self._process_before_stop_play_files() + + def to_tts_single_stream(self, text, is_last=False): + """流式TTS处理 - 使用线程池执行异步任务""" + try: + text = MarkdownCleaner.clean_markdown(text) + + # 使用线程池来执行异步任务，避免事件循环冲突 + def run_async_task(): + """在新线程中运行异步任务""" + try: + # 创建新的事件循环用于这个线程 + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # Windows下设置事件循环策略 + if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + + try: + # 运行TTS任务 + loop.run_until_complete(self._tts_request_unified(text, is_last)) + return True + finally: + # 安全关闭事件循环 + try: + # 取消所有未完成的任务 + pending = asyncio.all_tasks(loop) + if pending: + for task in pending: + task.cancel() + # 等待任务取消完成 + loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + except Exception as cleanup_error: + logger.bind(tag=TAG).debug(f"清理事件循环异常: {cleanup_error}") + finally: + loop.close() + + except Exception as e: + logger.bind(tag=TAG).error(f"异步任务执行失败: {e}") + return False + + # 使用线程池执行异步任务 + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(run_async_task) + success = future.result(timeout=30) # 30秒超时 + + if success: + logger.bind(tag=TAG).info(f"语音生成成功: {text}") + else: + logger.bind(tag=TAG).error(f"语音生成失败: {text}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + + except concurrent.futures.TimeoutError: + logger.bind(tag=TAG).error(f"TTS任务超时: {text}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + except Exception as e: + logger.bind(tag=TAG).error(f"TTS处理异常: {text}, 错误: {e}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + + return None + + async def text_to_speak(self, text, is_last=False): + """流式处理TTS音频""" + try: + # 确保连接可用 + await self._ensure_connection() + ws_connection = self.ws + + # 确保Token有效 + if self._is_token_expired(): + self._refresh_token() + + # 生成task_id和message_id + task_id = str(uuid.uuid4()).replace('-', '') + + # 第一阶段：发送StartSynthesis指令（设置参数） + start_message_id = str(uuid.uuid4()).replace('-', '') + start_request = { + "header": { + "message_id": start_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StartSynthesis", + "appkey": self.appkey, + }, + "payload": { + "voice": self.voice, + "format": self.format, + "sample_rate": self.sample_rate, + "volume": self.volume, + "speech_rate": self.speech_rate, + "pitch_rate": self.pitch_rate, + } + } + + await ws_connection.send(json.dumps(start_request)) + logger.bind(tag=TAG).debug(f"发送StartSynthesis指令: {start_message_id}") + + # 第二阶段：发送RunSynthesis指令（发送文本） + run_message_id = str(uuid.uuid4()).replace('-', '') + run_request = { + "header": { + "message_id": run_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": self.appkey, + }, + "payload": { + "text": text + } + } + + await ws_connection.send(json.dumps(run_request)) + logger.bind(tag=TAG).debug(f"发送RunSynthesis指令: {text} (message_id: {run_message_id})") + + # 立即发送StopSynthesis指令，避免IDLE_TIMEOUT + stop_message_id = str(uuid.uuid4()).replace('-', '') + stop_request = { + "header": { + "message_id": stop_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StopSynthesis", + "appkey": self.appkey, + } + } + + await ws_connection.send(json.dumps(stop_request)) + logger.bind(tag=TAG).debug(f"发送StopSynthesis指令: {stop_message_id}") + + # 初始化缓冲区和计数器 + self.pcm_buffer.clear() + self.segment_count = 0 # 重置分段计数器 + opus_datas_cache = [] + + # 发送第一个音频包 + self.tts_audio_queue.put((SentenceType.FIRST, [], text)) + + # 标记是否需要发送StopSynthesis + synthesis_completed = False + + # 处理响应 - 设置超时避免长时间等待 + try: + async def process_messages(): + nonlocal synthesis_completed + async for message in ws_connection: + try: + if isinstance(message, str): + # 处理JSON消息 + data = json.loads(message) + header = data.get("header", {}) + event_name = header.get("name") + + if event_name == "SynthesisStarted": + logger.bind(tag=TAG).debug(f"TTS合成已启动: {task_id}") + + elif event_name == "SynthesisCompleted": + logger.bind(tag=TAG).debug(f"TTS合成完成: {text}") + synthesis_completed = True + break + + elif event_name == "TaskFailed": + error_msg = header.get("status_text", "未知错误") + logger.bind(tag=TAG).error(f"TTS合成失败: {error_msg}") + synthesis_completed = True + break + + elif isinstance(message, bytes): + # 处理二进制音频数据 + self.pcm_buffer.extend(message) + + # 计算每帧的字节数 + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels + * self.opus_encoder.frame_size_ms + / 1000 + * 2 + ) + + # 分帧处理PCM数据 + while len(self.pcm_buffer) >= frame_bytes: + frame = bytes(self.pcm_buffer[:frame_bytes]) + del self.pcm_buffer[:frame_bytes] + + # 编码为Opus + opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) + if opus_packets: + if self.segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_packets, None) + ) + self.segment_count += 1 + else: + opus_datas_cache.extend(opus_packets) + + except json.JSONDecodeError: + logger.bind(tag=TAG).warning("收到无效的JSON消息") + except Exception as e: + logger.bind(tag=TAG).error(f"处理响应消息失败: {e}") + + await asyncio.wait_for(process_messages(), timeout=15) # 15秒超时 + + except asyncio.TimeoutError: + logger.bind(tag=TAG).warning(f"TTS请求超时，但可能已获取部分音频数据: {text}") + except websockets.ConnectionClosed: + logger.bind(tag=TAG).debug("WebSocket连接已正常关闭") + except Exception as e: + logger.bind(tag=TAG).error(f"处理WebSocket消息失败: {e}") + + # 因为已经提前发送了StopSynthesis，这里不需要再次发送 + # 直接处理剩余的PCM数据 + if self.pcm_buffer: + opus_packets = self.opus_encoder.encode_pcm_to_opus( + bytes(self.pcm_buffer), end_of_stream=True + ) + if opus_packets: + if self.segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_packets, None) + ) + self.segment_count += 1 + else: + opus_datas_cache.extend(opus_packets) + self.pcm_buffer.clear() + + # 发送缓存的数据 + if self.segment_count >= 10 and opus_datas_cache: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas_cache, None) + ) + + # 如果是最后一段，处理待播放文件 + if is_last: + self._process_before_stop_play_files() + + except Exception as e: + logger.bind(tag=TAG).error(f"TTS异步请求异常: {e}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + + async def _tts_request_unified(self, text: str, is_last: bool) -> None: + """统一的TTS请求方法""" + ws_connection = None + + try: + # 确保Token有效 + if self._is_token_expired(): + self._refresh_token() + + # 总是创建独立连接，避免与其他线程的事件循环冲突 + ws_url = f"wss://{self.host}/ws/v1" + ws_connection = await asyncio.wait_for( + websockets.connect( + ws_url, + additional_headers={ + "X-NLS-Token": self.token, + }, + ping_interval=15, # 每15秒发送ping，保持连接活跃 + ping_timeout=5, # ping超时时间5秒 + close_timeout=3, # 关闭超时时间3秒 + ), + timeout=8.0 # 连接超时时间8秒 + ) + logger.bind(tag=TAG).debug(f"建立独立WebSocket连接: {text}") + + # 生成task_id + task_id = str(uuid.uuid4()).replace('-', '') + + # 第一阶段：发送StartSynthesis指令（设置参数） + start_message_id = str(uuid.uuid4()).replace('-', '') + start_request = { + "header": { + "message_id": start_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StartSynthesis", + "appkey": self.appkey, + }, + "payload": { + "voice": self.voice, + "format": self.format, + "sample_rate": self.sample_rate, + "volume": self.volume, + "speech_rate": self.speech_rate, + "pitch_rate": self.pitch_rate, + } + } + + await ws_connection.send(json.dumps(start_request)) + logger.bind(tag=TAG).debug(f"发送StartSynthesis指令: {start_message_id}") + + # 第二阶段：发送RunSynthesis指令（发送文本） + run_message_id = str(uuid.uuid4()).replace('-', '') + run_request = { + "header": { + "message_id": run_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": self.appkey, + }, + "payload": { + "text": text + } + } + + await ws_connection.send(json.dumps(run_request)) + logger.bind(tag=TAG).debug(f"发送RunSynthesis指令: {text} (message_id: {run_message_id})") + + # 立即发送StopSynthesis指令，避免IDLE_TIMEOUT + stop_message_id = str(uuid.uuid4()).replace('-', '') + stop_request = { + "header": { + "message_id": stop_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StopSynthesis", + "appkey": self.appkey, + } + } + + await ws_connection.send(json.dumps(stop_request)) + logger.bind(tag=TAG).debug(f"发送StopSynthesis指令: {stop_message_id}") + + # 初始化处理参数 - 使用独立缓冲区 + pcm_buffer = bytearray() + opus_datas_cache = [] + segment_count = 0 + self.segment_count = 0 # 同时重置实例变量 + synthesis_completed = False + + # 发送第一个音频包 + self.tts_audio_queue.put((SentenceType.FIRST, [], text)) + + # 处理响应 - 设置超时时间避免长时间等待 + timeout_duration = 15 # 15秒超时 + try: + # 使用asyncio.wait_for替代asyncio.timeout以保证兼容性 + async def process_messages(): + nonlocal synthesis_completed, segment_count, pcm_buffer, opus_datas_cache # 声明使用外层变量 + async for message in ws_connection: + try: + if isinstance(message, str): + # 处理JSON消息 + data = json.loads(message) + header = data.get("header", {}) + event_name = header.get("name") + + if event_name == "SynthesisStarted": + logger.bind(tag=TAG).debug(f"TTS合成已启动: {task_id}") + + elif event_name == "SynthesisCompleted": + logger.bind(tag=TAG).debug(f"TTS合成完成: {text}") + synthesis_completed = True + break + + elif event_name == "TaskFailed": + error_msg = header.get("status_text", "未知错误") + logger.bind(tag=TAG).error(f"TTS合成失败: {error_msg}") + synthesis_completed = True + break + + elif isinstance(message, bytes): + # 处理二进制音频数据 + pcm_buffer.extend(message) + + # 计算每帧的字节数 + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels + * self.opus_encoder.frame_size_ms + / 1000 + * 2 + ) + + # 分帧处理PCM数据 + while len(pcm_buffer) >= frame_bytes: + frame = bytes(pcm_buffer[:frame_bytes]) + del pcm_buffer[:frame_bytes] # 清除已处理的数据 + + # 编码为Opus + opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) + if opus_packets: + if segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_packets, None) + ) + segment_count += 1 + else: + opus_datas_cache.extend(opus_packets) + + except json.JSONDecodeError: + logger.bind(tag=TAG).warning("收到无效的JSON消息") + except Exception as e: + logger.bind(tag=TAG).error(f"处理响应消息失败: {e}") + + await asyncio.wait_for(process_messages(), timeout=timeout_duration) + + except asyncio.TimeoutError: + logger.bind(tag=TAG).warning(f"TTS请求超时，但可能已获取部分音频数据: {text}") + except websockets.ConnectionClosed: + logger.bind(tag=TAG).debug("WebSocket连接已正常关闭") + except Exception as e: + logger.bind(tag=TAG).error(f"处理WebSocket消息失败: {e}") + + # 因为已经提前发送了StopSynthesis，这里不需要再次发送 + # 直接处理剩余的PCM数据 + if pcm_buffer: + opus_packets = self.opus_encoder.encode_pcm_to_opus( + bytes(pcm_buffer), end_of_stream=True + ) + if opus_packets: + if segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_packets, None) + ) + segment_count += 1 + else: + opus_datas_cache.extend(opus_packets) + + # 发送缓存的数据 + if segment_count >= 10 and opus_datas_cache: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas_cache, None) + ) + + # 如果是最后一段，处理待播放文件 + if is_last: + self._process_before_stop_play_files() + + except Exception as e: + logger.bind(tag=TAG).error(f"TTS请求异常: {e}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + finally: + # 确保WebSocket连接被关闭 + if ws_connection: + try: + if hasattr(ws_connection, 'close'): + if asyncio.iscoroutinefunction(ws_connection.close): + await ws_connection.close() + else: + ws_connection.close() + except Exception as e: + logger.bind(tag=TAG).debug(f"关闭WebSocket连接时出现异常: {e}") + async def close(self): + """资源清理""" + if self.ws: + try: + # 兼容不同版本的websockets库关闭方式 + if hasattr(self.ws, 'close'): + if asyncio.iscoroutinefunction(self.ws.close): + await self.ws.close() + else: + self.ws.close() + elif hasattr(self.ws, 'close_connection'): + await self.ws.close_connection() + except Exception as e: + logger.bind(tag=TAG).debug(f"关闭WebSocket连接时出现异常: {e}") + finally: + self.ws = None + + if hasattr(self, "opus_encoder"): + self.opus_encoder.close() + + await super().close() + + def _process_before_stop_play_files(self): + """处理停止前的待播放文件""" + for tts_file, text in self.before_stop_play_files: + if tts_file and os.path.exists(tts_file): + audio_datas = self._process_audio_file(tts_file) + self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text)) + self.before_stop_play_files.clear() + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + + def _process_audio_file(self, tts_file): + """处理音频文件并转换为指定格式""" + audio_datas = [] + if self.conn.audio_format == "pcm": + audio_datas, _ = self.audio_to_pcm_data(tts_file) + else: + audio_datas, _ = self.audio_to_opus_data(tts_file) + + if ( + self.delete_audio_file + and tts_file is not None + and os.path.exists(tts_file) + and tts_file.startswith(self.output_file) + ): + os.remove(tts_file) + return audio_datas + + def to_tts(self, text: str) -> list: + """非流式TTS处理，用于测试及保存音频文件的场景""" + start_time = time.time() + text = MarkdownCleaner.clean_markdown(text) + + try: + # 使用同步方式进行TTS转换 + if self._is_token_expired(): + self._refresh_token() + + # 构造请求数据 + request_json = { + "appkey": self.appkey, + "token": self.token, + "text": text, + "format": "pcm", + "sample_rate": self.sample_rate, + "voice": self.voice, + "volume": self.volume, + "speech_rate": self.speech_rate, + "pitch_rate": self.pitch_rate, + } + + # 使用HTTP接口进行同步请求 + import requests + api_url = f"https://{self.host}/stream/v1/tts" + headers = {"Content-Type": "application/json"} + + resp = requests.post(api_url, json=request_json, headers=headers) + + if resp.status_code == 401: # Token过期特殊处理 + self._refresh_token() + resp = requests.post(api_url, json=request_json, headers=headers) + + if resp.headers["Content-Type"].startswith("audio/"): + pcm_data = resp.content + + # 使用opus编码器处理PCM数据 + opus_datas = [] + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels + * self.opus_encoder.frame_size_ms + / 1000 + * 2 + ) + + # 分帧处理PCM数据 + for i in range(0, len(pcm_data), frame_bytes): + frame = pcm_data[i:i + frame_bytes] + if len(frame) == frame_bytes: + opus = self.opus_encoder.encode_pcm_to_opus(frame, False) + if opus: + opus_datas.extend(opus) + + logger.bind(tag=TAG).info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") + return opus_datas + else: + logger.bind(tag=TAG).error(f"TTS请求失败: {resp.content}") + return [] + + except Exception as e: + logger.bind(tag=TAG).error(f"TTS请求异常: {e}") + return [] + + def _get_segment_text(self): + """获取当前可以处理的文本段""" + if not self.tts_text_buff: + return None + + full_text = "".join(self.tts_text_buff) + if len(full_text) <= self.processed_chars: + return None + + # 获取未处理的文本 + remaining_text = full_text[self.processed_chars:] + + # 如果文本较短或者到达了句子结尾标点，直接处理 + sentence_endings = ['。', '！', '？', '.', '!', '?', '\n'] + if len(remaining_text) < 20: + return None + + # 查找句子结尾 + for i, char in enumerate(remaining_text): + if char in sentence_endings and i > 10: # 至少10个字符 + segment = remaining_text[:i+1] + segment_text = textUtils.get_string_no_punctuation_or_emoji(segment) + if segment_text: + self.processed_chars += i + 1 + return segment_text + + # 如果没有找到句子结尾，但文本足够长，按长度分段 + if len(remaining_text) > 50: + segment = remaining_text[:30] + segment_text = textUtils.get_string_no_punctuation_or_emoji(segment) + if segment_text: + self.processed_chars += 30 + return segment_text + + return None From 155ba92ba68bcde8d2bca8b472f4d1f228689cf3 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Mon, 7 Jul 2025 15:29:13 +0800 Subject: [PATCH 03/24] =?UTF-8?q?update:=E5=A2=9E=E5=8A=A0=E4=BA=86?= =?UTF-8?q?=E6=99=BA=E6=8E=A7=E5=8F=B0=E7=9A=84sql=E8=AF=AD=E5=8F=A5?= =?UTF-8?q?=EF=BC=8C=E4=BC=98=E5=8C=96=E4=BA=86=E9=83=A8=E5=88=86=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{202505081146.sql => 202505081147.sql} | 2 +- .../resources/db/changelog/202507071130.sql | 26 +++++++++ .../resources/db/changelog/202507071530.sql | 56 +++++++++++++++++++ .../db/changelog/db.changelog-master.yaml | 20 ++++++- main/xiaozhi-server/config.yaml | 10 ++-- .../core/providers/asr/aliyun_stream.py | 2 +- 6 files changed, 107 insertions(+), 9 deletions(-) rename main/manager-api/src/main/resources/db/changelog/{202505081146.sql => 202505081147.sql} (97%) create mode 100644 main/manager-api/src/main/resources/db/changelog/202507071130.sql create mode 100644 main/manager-api/src/main/resources/db/changelog/202507071530.sql diff --git a/main/manager-api/src/main/resources/db/changelog/202505081146.sql b/main/manager-api/src/main/resources/db/changelog/202505081147.sql similarity index 97% rename from main/manager-api/src/main/resources/db/changelog/202505081146.sql rename to main/manager-api/src/main/resources/db/changelog/202505081147.sql index fe7582f3..1a561bda 100644 --- a/main/manager-api/src/main/resources/db/changelog/202505081146.sql +++ b/main/manager-api/src/main/resources/db/changelog/202505081147.sql @@ -1,6 +1,6 @@ -- 添加百度ASR模型配置 delete from `ai_model_config` where `id` = 'ASR_BaiduASR'; -INSERT INTO `ai_model_config` VALUES ('ASR_BaiduASR', 'ASR', 'BaiduASR', '百度语音识别', 0, 1, '{\"type\": \"baidu\", \"app_id\": \"\", \"api_key\": \"\", \"secret_key\": \"\", \"dev_pid\": 1537, \"output_dir\": \"tmp/\"}', NULL, NULL, 7, NULL, NULL, NULL, NULL); +INSERT INTO `ai_model_config` VALUES ('ASR_BaiduASR', 'ASR', 'BaiduASR', '百度语音识别', 0, 1, '{\"type\": \"baidu\", \"app_id\": \"\", \"api_key\": \"\", \"secret_key\": \"\", \"dev_pid\": 1537, \"output_dir\": \"tmp/\"}', NULL, NULL, 8, NULL, NULL, NULL, NULL); -- 添加百度ASR供应器 diff --git a/main/manager-api/src/main/resources/db/changelog/202507071130.sql b/main/manager-api/src/main/resources/db/changelog/202507071130.sql new file mode 100644 index 00000000..008d624e --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507071130.sql @@ -0,0 +1,26 @@ +-- 添加阿里云流式ASR供应器 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_AliyunStreamASR'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_AliyunStreamASR', 'ASR', 'aliyun_stream', '阿里云语音识别(流式)', '[{"key":"appkey","label":"应用AppKey","type":"string"},{"key":"token","label":"临时Token","type":"string"},{"key":"access_key_id","label":"AccessKey ID","type":"string"},{"key":"access_key_secret","label":"AccessKey Secret","type":"string"},{"key":"host","label":"服务地址","type":"string"},{"key":"max_sentence_silence","label":"断句检测时间","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 6, 1, NOW(), 1, NOW()); + +-- 添加阿里云流式ASR模型配置 +delete from `ai_model_config` where id = 'ASR_AliyunStreamASR'; +INSERT INTO `ai_model_config` VALUES ('ASR_AliyunStreamASR', 'ASR', 'AliyunStreamASR', '阿里云语音识别(流式)', 0, 1, '{\"type\": \"aliyun_stream\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"host\": \"nls-gateway-cn-shanghai.aliyuncs.com\", \"max_sentence_silence\": 800, \"output_dir\": \"tmp/\"}', NULL, NULL, 7, NULL, NULL, NULL, NULL); + +-- 更新阿里云流式ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://nls-portal.console.aliyun.com/', +`remark` = '阿里云流式ASR配置说明： +1. 阿里云ASR和阿里云(流式)ASR的区别是：阿里云ASR是一次性识别，阿里云(流式)ASR是实时流式识别 +2. 流式ASR具有更低的延迟和更好的实时性，适合语音交互场景 +3. 需要在阿里云智能语音交互控制台创建应用并获取认证信息 +4. 支持中文实时语音识别，支持标点符号预测和逆文本规范化 +5. 需要网络连接，输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://nls-portal.console.aliyun.com/ 开通智能语音交互服务 +2. 访问 https://nls-portal.console.aliyun.com/applist 创建项目并获取appkey +3. 访问 https://nls-portal.console.aliyun.com/overview 获取临时token（或配置access_key_id和access_key_secret自动获取） +4. 如需动态token管理，建议配置access_key_id和access_key_secret +5. max_sentence_silence参数控制断句检测时间（毫秒），默认800ms +如需了解更多参数配置，请参考：https://help.aliyun.com/zh/isi/developer-reference/real-time-speech-recognition +' WHERE `id` = 'ASR_AliyunStreamASR'; diff --git a/main/manager-api/src/main/resources/db/changelog/202507071530.sql b/main/manager-api/src/main/resources/db/changelog/202507071530.sql new file mode 100644 index 00000000..ed3dcf49 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507071530.sql @@ -0,0 +1,56 @@ +-- 添加阿里云流式TTS供应器 +delete from `ai_model_provider` where id = 'SYSTEM_TTS_AliyunStreamTTS'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_TTS_AliyunStreamTTS', 'TTS', 'aliyun_stream', '阿里云语音合成(流式)', '[{"key":"appkey","label":"应用AppKey","type":"string"},{"key":"token","label":"临时Token","type":"string"},{"key":"access_key_id","label":"AccessKey ID","type":"string"},{"key":"access_key_secret","label":"AccessKey Secret","type":"string"},{"key":"host","label":"服务地址","type":"string"},{"key":"voice","label":"默认音色","type":"string"},{"key":"format","label":"音频格式","type":"string"},{"key":"sample_rate","label":"采样率","type":"number"},{"key":"volume","label":"音量","type":"number"},{"key":"speech_rate","label":"语速","type":"number"},{"key":"pitch_rate","label":"音调","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 15, 1, NOW(), 1, NOW()); + +-- 添加阿里云流式TTS模型配置 +delete from `ai_model_config` where id = 'TTS_AliyunStreamTTS'; +INSERT INTO `ai_model_config` VALUES ('TTS_AliyunStreamTTS', 'TTS', 'AliyunStreamTTS', '阿里云语音合成(流式)', 0, 1, '{\"type\": \"aliyun_stream\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"host\": \"nls-gateway-cn-beijing.aliyuncs.com\", \"voice\": \"longxiaochun\", \"format\": \"pcm\", \"sample_rate\": 16000, \"volume\": 50, \"speech_rate\": 0, \"pitch_rate\": 0, \"output_dir\": \"tmp/\"}', NULL, NULL, 18, NULL, NULL, NULL, NULL); + +-- 更新阿里云流式TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://nls-portal.console.aliyun.com/', +`remark` = '阿里云流式TTS配置说明： +1. 阿里云TTS和阿里云(流式)TTS的区别是：阿里云TTS是一次性合成，阿里云(流式)TTS是实时流式合成 +2. 流式TTS具有更低的延迟和更好的实时性，适合语音交互场景 +3. 需要在阿里云智能语音交互控制台创建应用并获取认证信息 +4. 支持CosyVoice大模型音色，音质更加自然 +5. 支持实时调节音量、语速、音调等参数 +申请步骤： +1. 访问 https://nls-portal.console.aliyun.com/ 开通智能语音交互服务 +2. 访问 https://nls-portal.console.aliyun.com/applist 创建项目并获取appkey +3. 访问 https://nls-portal.console.aliyun.com/overview 获取临时token（或配置access_key_id和access_key_secret自动获取） +4. 如需动态token管理，建议配置access_key_id和access_key_secret +5. 可选择北京、上海等不同地域的服务器以优化延迟 +6. voice参数支持CosyVoice大模型音色，如longxiaochun、longyueyue等 +如需了解更多参数配置，请参考：https://help.aliyun.com/zh/isi/developer-reference/real-time-speech-synthesis +' WHERE `id` = 'TTS_AliyunStreamTTS'; + +-- 添加阿里云流式TTS音色 +delete from `ai_tts_voice` where tts_model_id = 'TTS_AliyunStreamTTS'; +-- 温柔女声系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0001', 'TTS_AliyunStreamTTS', '龙小淳-温柔姐姐', 'longxiaochun', '中文及中英文混合', NULL, NULL, 1, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0002', 'TTS_AliyunStreamTTS', '龙小夏-温柔女声', 'longxiaoxia', '中文及中英文混合', NULL, NULL, 2, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0003', 'TTS_AliyunStreamTTS', '龙玫-温柔女声', 'longmei', '中文及中英文混合', NULL, NULL, 3, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0004', 'TTS_AliyunStreamTTS', '龙瑰-温柔女声', 'longgui', '中文及中英文混合', NULL, NULL, 4, NULL, NULL, NULL, NULL); +-- 御姐女声系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0005', 'TTS_AliyunStreamTTS', '龙玉-御姐女声', 'longyu', '中文及中英文混合', NULL, NULL, 5, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0006', 'TTS_AliyunStreamTTS', '龙娇-御姐女声', 'longjiao', '中文及中英文混合', NULL, NULL, 6, NULL, NULL, NULL, NULL); +-- 男声系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0007', 'TTS_AliyunStreamTTS', '龙臣-译制片男声', 'longchen', '中文及中英文混合', NULL, NULL, 7, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0008', 'TTS_AliyunStreamTTS', '龙修-青年男声', 'longxiu', '中文及中英文混合', NULL, NULL, 8, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0009', 'TTS_AliyunStreamTTS', '龙橙-阳光男声', 'longcheng', '中文及中英文混合', NULL, NULL, 9, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0010', 'TTS_AliyunStreamTTS', '龙哲-成熟男声', 'longzhe', '中文及中英文混合', NULL, NULL, 10, NULL, NULL, NULL, NULL); +-- 专业播报系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0011', 'TTS_AliyunStreamTTS', 'Bella2.0-新闻女声', 'loongbella', '中文及中英文混合', NULL, NULL, 11, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0012', 'TTS_AliyunStreamTTS', 'Stella2.0-飒爽女声', 'loongstella', '中文及中英文混合', NULL, NULL, 12, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0013', 'TTS_AliyunStreamTTS', '龙书-新闻男声', 'longshu', '中文及中英文混合', NULL, NULL, 13, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0014', 'TTS_AliyunStreamTTS', '龙婧-严肃女声', 'longjing', '中文及中英文混合', NULL, NULL, 14, NULL, NULL, NULL, NULL); +-- 特色音色系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0015', 'TTS_AliyunStreamTTS', '龙奇-活泼童声', 'longqi', '中文及中英文混合', NULL, NULL, 15, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0016', 'TTS_AliyunStreamTTS', '龙华-活泼女童', 'longhua', '中文及中英文混合', NULL, NULL, 16, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0017', 'TTS_AliyunStreamTTS', '龙无-无厘头男声', 'longwu', '中文及中英文混合', NULL, NULL, 17, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0018', 'TTS_AliyunStreamTTS', '龙大锤-幽默男声', 'longdachui', '中文及中英文混合', NULL, NULL, 18, NULL, NULL, NULL, NULL); +-- 粤语系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0019', 'TTS_AliyunStreamTTS', '龙嘉怡-粤语女声', 'longjiayi', '粤语及粤英混合', NULL, NULL, 19, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0020', 'TTS_AliyunStreamTTS', '龙桃-粤语女声', 'longtao', '粤语及粤英混合', NULL, NULL, 20, NULL, NULL, NULL, NULL); diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index b60094ef..3852c545 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -101,12 +101,12 @@ databaseChangeLog: encoding: utf8 path: classpath:db/changelog/202505022134.sql - changeSet: - id: 202505081146 + id: 202505081147 author: hrz changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202505081146.sql + path: classpath:db/changelog/202505081147.sql - changeSet: id: 202505091555 author: whosmyqueen @@ -225,4 +225,18 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202506261637.sql \ No newline at end of file + path: classpath:db/changelog/202506261637.sql + - changeSet: + id: 202507071130 + author: cgd + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202507071130.sql + - changeSet: + id: 202507071530 + author: cgd + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202507071530.sql diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index a4b4ccc0..be052dc4 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -322,13 +322,15 @@ ASR: # 一般来说非流式ASR更便宜（0.004元/秒，¥0.24/分钟） # 但是AliyunStreamASR实时性更好（0.005元/秒，¥0.3/分钟） # 定义ASR API类型 - type: aliyun + type: aliyun_stream appkey: 你的阿里云智能语音交互服务项目Appkey token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret access_key_id: 你的阿里云账号access_key_id access_key_secret: 你的阿里云账号access_key_secret - # 服务器地域选择，如nls-gateway-cn-hangzhou.aliyuncs.com(杭州)等 - host: 根据自己的地理位置选择最近的服务器地域，不填默认为nls-gateway-cn-shanghai.aliyuncs.com(上海) + # 服务器地域选择，可选择距离更近的服务器以减少延迟，如nls-gateway-cn-hangzhou.aliyuncs.com(杭州)等 + host: nls-gateway-cn-shanghai.aliyuncs.com + # 断句检测时间(毫秒)，控制静音多长时间后进行断句，默认800毫秒 + max_sentence_silence: 800 output_dir: tmp/ BaiduASR: # 获取AppID、API Key、Secret Key：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list @@ -699,7 +701,7 @@ TTS: # appkey地址：https://nls-portal.console.aliyun.com/applist # token地址：https://nls-portal.console.aliyun.com/overview # 使用三阶段流式交互：StartSynthesis -> RunSynthesis -> StopSynthesis - type: aliyun + type: aliyun_stream output_dir: tmp/ appkey: 你的阿里云智能语音交互服务项目Appkey token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret diff --git a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py index 94727798..3a41b349 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py @@ -123,7 +123,7 @@ class ASRProvider(ASRProviderBase): conn.asr_audio.append(audio) conn.asr_audio = conn.asr_audio[-10:] - # 参考豆包ASR：只在有声音且没有连接时建立连接 + # 只在有声音且没有连接时建立连接 if audio_have_voice and not self.is_processing: try: await self._start_recognition(conn) From d533fb83eb6bb81d6afc6023ff114bf2620f5524 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 10 Jul 2025 09:52:54 +0800 Subject: [PATCH 04/24] =?UTF-8?q?fix:=20=E6=8F=92=E5=85=A5=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E4=BF=AE=E5=A4=8D=EF=BC=8C=E8=B7=9F=E9=9A=8F=E4=B8=BB?= =?UTF-8?q?=E5=88=86=E6=94=AF=E6=96=B0=E5=A2=9E=E5=88=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202507071530.sql | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/main/manager-api/src/main/resources/db/changelog/202507071530.sql b/main/manager-api/src/main/resources/db/changelog/202507071530.sql index ed3dcf49..f1cf7a9b 100644 --- a/main/manager-api/src/main/resources/db/changelog/202507071530.sql +++ b/main/manager-api/src/main/resources/db/changelog/202507071530.sql @@ -29,28 +29,28 @@ UPDATE `ai_model_config` SET -- 添加阿里云流式TTS音色 delete from `ai_tts_voice` where tts_model_id = 'TTS_AliyunStreamTTS'; -- 温柔女声系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0001', 'TTS_AliyunStreamTTS', '龙小淳-温柔姐姐', 'longxiaochun', '中文及中英文混合', NULL, NULL, 1, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0002', 'TTS_AliyunStreamTTS', '龙小夏-温柔女声', 'longxiaoxia', '中文及中英文混合', NULL, NULL, 2, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0003', 'TTS_AliyunStreamTTS', '龙玫-温柔女声', 'longmei', '中文及中英文混合', NULL, NULL, 3, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0004', 'TTS_AliyunStreamTTS', '龙瑰-温柔女声', 'longgui', '中文及中英文混合', NULL, NULL, 4, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0001', 'TTS_AliyunStreamTTS', '龙小淳-温柔姐姐', 'longxiaochun', '中文及中英文混合', NULL, NULL, NULL, NULL, 1, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0002', 'TTS_AliyunStreamTTS', '龙小夏-温柔女声', 'longxiaoxia', '中文及中英文混合', NULL, NULL, NULL, NULL, 2, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0003', 'TTS_AliyunStreamTTS', '龙玫-温柔女声', 'longmei', '中文及中英文混合', NULL, NULL, NULL, NULL, 3, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0004', 'TTS_AliyunStreamTTS', '龙瑰-温柔女声', 'longgui', '中文及中英文混合', NULL, NULL, NULL, NULL, 4, NULL, NULL, NULL, NULL); -- 御姐女声系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0005', 'TTS_AliyunStreamTTS', '龙玉-御姐女声', 'longyu', '中文及中英文混合', NULL, NULL, 5, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0006', 'TTS_AliyunStreamTTS', '龙娇-御姐女声', 'longjiao', '中文及中英文混合', NULL, NULL, 6, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0005', 'TTS_AliyunStreamTTS', '龙玉-御姐女声', 'longyu', '中文及中英文混合', NULL, NULL, NULL, NULL, 5, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0006', 'TTS_AliyunStreamTTS', '龙娇-御姐女声', 'longjiao', '中文及中英文混合', NULL, NULL, NULL, NULL, 6, NULL, NULL, NULL, NULL); -- 男声系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0007', 'TTS_AliyunStreamTTS', '龙臣-译制片男声', 'longchen', '中文及中英文混合', NULL, NULL, 7, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0008', 'TTS_AliyunStreamTTS', '龙修-青年男声', 'longxiu', '中文及中英文混合', NULL, NULL, 8, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0009', 'TTS_AliyunStreamTTS', '龙橙-阳光男声', 'longcheng', '中文及中英文混合', NULL, NULL, 9, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0010', 'TTS_AliyunStreamTTS', '龙哲-成熟男声', 'longzhe', '中文及中英文混合', NULL, NULL, 10, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0007', 'TTS_AliyunStreamTTS', '龙臣-译制片男声', 'longchen', '中文及中英文混合', NULL, NULL, NULL, NULL, 7, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0008', 'TTS_AliyunStreamTTS', '龙修-青年男声', 'longxiu', '中文及中英文混合', NULL, NULL, NULL, NULL, 8, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0009', 'TTS_AliyunStreamTTS', '龙橙-阳光男声', 'longcheng', '中文及中英文混合', NULL, NULL, NULL, NULL, 9, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0010', 'TTS_AliyunStreamTTS', '龙哲-成熟男声', 'longzhe', '中文及中英文混合', NULL, NULL, NULL, NULL, 10, NULL, NULL, NULL, NULL); -- 专业播报系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0011', 'TTS_AliyunStreamTTS', 'Bella2.0-新闻女声', 'loongbella', '中文及中英文混合', NULL, NULL, 11, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0012', 'TTS_AliyunStreamTTS', 'Stella2.0-飒爽女声', 'loongstella', '中文及中英文混合', NULL, NULL, 12, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0013', 'TTS_AliyunStreamTTS', '龙书-新闻男声', 'longshu', '中文及中英文混合', NULL, NULL, 13, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0014', 'TTS_AliyunStreamTTS', '龙婧-严肃女声', 'longjing', '中文及中英文混合', NULL, NULL, 14, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0011', 'TTS_AliyunStreamTTS', 'Bella2.0-新闻女声', 'loongbella', '中文及中英文混合', NULL, NULL, NULL, NULL, 11, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0012', 'TTS_AliyunStreamTTS', 'Stella2.0-飒爽女声', 'loongstella', '中文及中英文混合', NULL, NULL, NULL, NULL, 12, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0013', 'TTS_AliyunStreamTTS', '龙书-新闻男声', 'longshu', '中文及中英文混合', NULL, NULL, NULL, NULL, 13, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0014', 'TTS_AliyunStreamTTS', '龙婧-严肃女声', 'longjing', '中文及中英文混合', NULL, NULL, NULL, NULL, 14, NULL, NULL, NULL, NULL); -- 特色音色系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0015', 'TTS_AliyunStreamTTS', '龙奇-活泼童声', 'longqi', '中文及中英文混合', NULL, NULL, 15, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0016', 'TTS_AliyunStreamTTS', '龙华-活泼女童', 'longhua', '中文及中英文混合', NULL, NULL, 16, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0017', 'TTS_AliyunStreamTTS', '龙无-无厘头男声', 'longwu', '中文及中英文混合', NULL, NULL, 17, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0018', 'TTS_AliyunStreamTTS', '龙大锤-幽默男声', 'longdachui', '中文及中英文混合', NULL, NULL, 18, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0015', 'TTS_AliyunStreamTTS', '龙奇-活泼童声', 'longqi', '中文及中英文混合', NULL, NULL, NULL, NULL, 15, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0016', 'TTS_AliyunStreamTTS', '龙华-活泼女童', 'longhua', '中文及中英文混合', NULL, NULL, NULL, NULL, 16, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0017', 'TTS_AliyunStreamTTS', '龙无-无厘头男声', 'longwu', '中文及中英文混合', NULL, NULL, NULL, NULL, 17, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0018', 'TTS_AliyunStreamTTS', '龙大锤-幽默男声', 'longdachui', '中文及中英文混合', NULL, NULL, NULL, NULL, 18, NULL, NULL, NULL, NULL); -- 粤语系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0019', 'TTS_AliyunStreamTTS', '龙嘉怡-粤语女声', 'longjiayi', '粤语及粤英混合', NULL, NULL, 19, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0020', 'TTS_AliyunStreamTTS', '龙桃-粤语女声', 'longtao', '粤语及粤英混合', NULL, NULL, 20, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0019', 'TTS_AliyunStreamTTS', '龙嘉怡-粤语女声', 'longjiayi', '粤语及粤英混合', NULL, NULL, NULL, NULL, 19, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0020', 'TTS_AliyunStreamTTS', '龙桃-粤语女声', 'longtao', '粤语及粤英混合', NULL, NULL, NULL, NULL, 20, NULL, NULL, NULL, NULL); From ce358dcd651debd7cebe7da71c1d3d826ed849ec Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 10 Jul 2025 10:17:07 +0800 Subject: [PATCH 05/24] =?UTF-8?q?fix:=20=E9=9F=B3=E9=A2=91=E6=92=AD?= =?UTF-8?q?=E6=94=BE=E6=96=B9=E6=B3=95=E8=B7=9F=E9=9A=8F=E4=B8=BB=E5=88=86?= =?UTF-8?q?=E6=94=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun_stream.py | 39 ++++--------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 9aad9614..daecc924 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -10,13 +10,10 @@ import traceback import websockets import websockets.protocol import os -import random -import threading import concurrent.futures import sys from datetime import datetime from urllib import parse -from typing import Optional from core.providers.tts.base import TTSProviderBase from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType from core.utils.tts import MarkdownCleaner @@ -268,9 +265,12 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).info( f"添加音频文件到待播放列表: {message.content_file}" ) - self.before_stop_play_files.append( - (message.content_file, message.content_detail) - ) + if message.content_file and os.path.exists(message.content_file): + # 先处理文件音频数据 + file_audio = self._process_before_stop_play_files(message.content_file) + self.before_stop_play_files.append( + (file_audio, message.content_detail) + ) if message.sentence_type == SentenceType.LAST: # 处理剩余的文本 @@ -741,6 +741,7 @@ class TTSProvider(TTSProviderBase): ws_connection.close() except Exception as e: logger.bind(tag=TAG).debug(f"关闭WebSocket连接时出现异常: {e}") + async def close(self): """资源清理""" if self.ws: @@ -763,32 +764,6 @@ class TTSProvider(TTSProviderBase): await super().close() - def _process_before_stop_play_files(self): - """处理停止前的待播放文件""" - for tts_file, text in self.before_stop_play_files: - if tts_file and os.path.exists(tts_file): - audio_datas = self._process_audio_file(tts_file) - self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text)) - self.before_stop_play_files.clear() - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - - def _process_audio_file(self, tts_file): - """处理音频文件并转换为指定格式""" - audio_datas = [] - if self.conn.audio_format == "pcm": - audio_datas, _ = self.audio_to_pcm_data(tts_file) - else: - audio_datas, _ = self.audio_to_opus_data(tts_file) - - if ( - self.delete_audio_file - and tts_file is not None - and os.path.exists(tts_file) - and tts_file.startswith(self.output_file) - ): - os.remove(tts_file) - return audio_datas - def to_tts(self, text: str) -> list: """非流式TTS处理，用于测试及保存音频文件的场景""" start_time = time.time() From dede73e9a50320dc343adb64be6a85a2d2d4cefc Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 10 Jul 2025 11:34:31 +0800 Subject: [PATCH 06/24] =?UTF-8?q?fix:=20id=E4=B8=80=E8=87=B4=20=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun_stream.py | 203 ++---------------- 1 file changed, 13 insertions(+), 190 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index daecc924..57d9ffcc 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -267,7 +267,7 @@ class TTSProvider(TTSProviderBase): ) if message.content_file and os.path.exists(message.content_file): # 先处理文件音频数据 - file_audio = self._process_before_stop_play_files(message.content_file) + file_audio = self._process_audio_file(message.content_file) self.before_stop_play_files.append( (file_audio, message.content_detail) ) @@ -316,7 +316,7 @@ class TTSProvider(TTSProviderBase): try: # 运行TTS任务 - loop.run_until_complete(self._tts_request_unified(text, is_last)) + loop.run_until_complete(self.text_to_speak(text, is_last)) return True finally: # 安全关闭事件循环 @@ -357,185 +357,9 @@ class TTSProvider(TTSProviderBase): return None - async def text_to_speak(self, text, is_last=False): + async def text_to_speak(self, text, is_last): """流式处理TTS音频""" - try: - # 确保连接可用 - await self._ensure_connection() - ws_connection = self.ws - - # 确保Token有效 - if self._is_token_expired(): - self._refresh_token() - - # 生成task_id和message_id - task_id = str(uuid.uuid4()).replace('-', '') - - # 第一阶段：发送StartSynthesis指令（设置参数） - start_message_id = str(uuid.uuid4()).replace('-', '') - start_request = { - "header": { - "message_id": start_message_id, - "task_id": task_id, - "namespace": "FlowingSpeechSynthesizer", - "name": "StartSynthesis", - "appkey": self.appkey, - }, - "payload": { - "voice": self.voice, - "format": self.format, - "sample_rate": self.sample_rate, - "volume": self.volume, - "speech_rate": self.speech_rate, - "pitch_rate": self.pitch_rate, - } - } - - await ws_connection.send(json.dumps(start_request)) - logger.bind(tag=TAG).debug(f"发送StartSynthesis指令: {start_message_id}") - - # 第二阶段：发送RunSynthesis指令（发送文本） - run_message_id = str(uuid.uuid4()).replace('-', '') - run_request = { - "header": { - "message_id": run_message_id, - "task_id": task_id, - "namespace": "FlowingSpeechSynthesizer", - "name": "RunSynthesis", - "appkey": self.appkey, - }, - "payload": { - "text": text - } - } - - await ws_connection.send(json.dumps(run_request)) - logger.bind(tag=TAG).debug(f"发送RunSynthesis指令: {text} (message_id: {run_message_id})") - - # 立即发送StopSynthesis指令，避免IDLE_TIMEOUT - stop_message_id = str(uuid.uuid4()).replace('-', '') - stop_request = { - "header": { - "message_id": stop_message_id, - "task_id": task_id, - "namespace": "FlowingSpeechSynthesizer", - "name": "StopSynthesis", - "appkey": self.appkey, - } - } - - await ws_connection.send(json.dumps(stop_request)) - logger.bind(tag=TAG).debug(f"发送StopSynthesis指令: {stop_message_id}") - - # 初始化缓冲区和计数器 - self.pcm_buffer.clear() - self.segment_count = 0 # 重置分段计数器 - opus_datas_cache = [] - - # 发送第一个音频包 - self.tts_audio_queue.put((SentenceType.FIRST, [], text)) - - # 标记是否需要发送StopSynthesis - synthesis_completed = False - - # 处理响应 - 设置超时避免长时间等待 - try: - async def process_messages(): - nonlocal synthesis_completed - async for message in ws_connection: - try: - if isinstance(message, str): - # 处理JSON消息 - data = json.loads(message) - header = data.get("header", {}) - event_name = header.get("name") - - if event_name == "SynthesisStarted": - logger.bind(tag=TAG).debug(f"TTS合成已启动: {task_id}") - - elif event_name == "SynthesisCompleted": - logger.bind(tag=TAG).debug(f"TTS合成完成: {text}") - synthesis_completed = True - break - - elif event_name == "TaskFailed": - error_msg = header.get("status_text", "未知错误") - logger.bind(tag=TAG).error(f"TTS合成失败: {error_msg}") - synthesis_completed = True - break - - elif isinstance(message, bytes): - # 处理二进制音频数据 - self.pcm_buffer.extend(message) - - # 计算每帧的字节数 - frame_bytes = int( - self.opus_encoder.sample_rate - * self.opus_encoder.channels - * self.opus_encoder.frame_size_ms - / 1000 - * 2 - ) - - # 分帧处理PCM数据 - while len(self.pcm_buffer) >= frame_bytes: - frame = bytes(self.pcm_buffer[:frame_bytes]) - del self.pcm_buffer[:frame_bytes] - - # 编码为Opus - opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) - if opus_packets: - if self.segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_packets, None) - ) - self.segment_count += 1 - else: - opus_datas_cache.extend(opus_packets) - - except json.JSONDecodeError: - logger.bind(tag=TAG).warning("收到无效的JSON消息") - except Exception as e: - logger.bind(tag=TAG).error(f"处理响应消息失败: {e}") - - await asyncio.wait_for(process_messages(), timeout=15) # 15秒超时 - - except asyncio.TimeoutError: - logger.bind(tag=TAG).warning(f"TTS请求超时，但可能已获取部分音频数据: {text}") - except websockets.ConnectionClosed: - logger.bind(tag=TAG).debug("WebSocket连接已正常关闭") - except Exception as e: - logger.bind(tag=TAG).error(f"处理WebSocket消息失败: {e}") - - # 因为已经提前发送了StopSynthesis，这里不需要再次发送 - # 直接处理剩余的PCM数据 - if self.pcm_buffer: - opus_packets = self.opus_encoder.encode_pcm_to_opus( - bytes(self.pcm_buffer), end_of_stream=True - ) - if opus_packets: - if self.segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_packets, None) - ) - self.segment_count += 1 - else: - opus_datas_cache.extend(opus_packets) - self.pcm_buffer.clear() - - # 发送缓存的数据 - if self.segment_count >= 10 and opus_datas_cache: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_datas_cache, None) - ) - - # 如果是最后一段，处理待播放文件 - if is_last: - self._process_before_stop_play_files() - - except Exception as e: - logger.bind(tag=TAG).error(f"TTS异步请求异常: {e}") - self.tts_audio_queue.put((SentenceType.LAST, [], None)) + await self._tts_request_unified(text, is_last) async def _tts_request_unified(self, text: str, is_last: bool) -> None: """统一的TTS请求方法""" @@ -564,12 +388,12 @@ class TTSProvider(TTSProviderBase): # 生成task_id task_id = str(uuid.uuid4()).replace('-', '') - + message_id = str(uuid.uuid4()).replace('-', '') # 第一阶段：发送StartSynthesis指令（设置参数） - start_message_id = str(uuid.uuid4()).replace('-', '') + start_request = { "header": { - "message_id": start_message_id, + "message_id": message_id, "task_id": task_id, "namespace": "FlowingSpeechSynthesizer", "name": "StartSynthesis", @@ -586,13 +410,12 @@ class TTSProvider(TTSProviderBase): } await ws_connection.send(json.dumps(start_request)) - logger.bind(tag=TAG).debug(f"发送StartSynthesis指令: {start_message_id}") + logger.bind(tag=TAG).debug(f"发送StartSynthesis指令 会话id：{task_id}") # 第二阶段：发送RunSynthesis指令（发送文本） - run_message_id = str(uuid.uuid4()).replace('-', '') run_request = { "header": { - "message_id": run_message_id, + "message_id": message_id, "task_id": task_id, "namespace": "FlowingSpeechSynthesizer", "name": "RunSynthesis", @@ -604,13 +427,13 @@ class TTSProvider(TTSProviderBase): } await ws_connection.send(json.dumps(run_request)) - logger.bind(tag=TAG).debug(f"发送RunSynthesis指令: {text} (message_id: {run_message_id})") + logger.bind(tag=TAG).debug(f"发送RunSynthesis指令 {text} 会话id： {task_id})") # 立即发送StopSynthesis指令，避免IDLE_TIMEOUT - stop_message_id = str(uuid.uuid4()).replace('-', '') + stop_request = { "header": { - "message_id": stop_message_id, + "message_id": message_id, "task_id": task_id, "namespace": "FlowingSpeechSynthesizer", "name": "StopSynthesis", @@ -619,7 +442,7 @@ class TTSProvider(TTSProviderBase): } await ws_connection.send(json.dumps(stop_request)) - logger.bind(tag=TAG).debug(f"发送StopSynthesis指令: {stop_message_id}") + logger.bind(tag=TAG).debug(f"发送StopSynthesis指令 会话id：{task_id}") # 初始化处理参数 - 使用独立缓冲区 pcm_buffer = bytearray() From fbd4a22e3e8d34e8fa887f4170e402508330068f Mon Sep 17 00:00:00 2001 From: luruxian Date: Thu, 10 Jul 2025 11:41:20 +0800 Subject: [PATCH 07/24] =?UTF-8?q?feat(asr):=20=E6=B7=BB=E5=8A=A0OpenAI?= =?UTF-8?q?=E5=92=8CGroq=E8=AF=AD=E9=9F=B3=E8=AF=86=E5=88=AB=E6=94=AF?= =?UTF-8?q?=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增GPT语音识别模型供应器和配置，实现语音转文本功能 更新相关文档说明，包含API申请步骤和使用注意事项 --- .../resources/db/changelog/202507101201.sql | 31 +++++++ main/xiaozhi-server/core/providers/asr/gpt.py | 82 +++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 main/manager-api/src/main/resources/db/changelog/202507101201.sql create mode 100644 main/xiaozhi-server/core/providers/asr/gpt.py diff --git a/main/manager-api/src/main/resources/db/changelog/202507101201.sql b/main/manager-api/src/main/resources/db/changelog/202507101201.sql new file mode 100644 index 00000000..14bd6725 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507101201.sql @@ -0,0 +1,31 @@ +-- OpenAI ASR模型供应器 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_GptASR'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_GptASR', 'ASR', 'gpt', 'Gpt语音识别', '[{"key": "base_url", "type": "string", "label": "基础URL"}, {"key": "model_name", "type": "string", "label": "模型名称"}, {"key": "api_key", "type": "string", "label": "API密钥"}, {"key": "output_dir", "type": "string", "label": "输出目录"}]', 5, 1, NOW(), 1, NOW()); + + +-- OpenAI ASR模型配置 +delete from `ai_model_config` where id = 'ASR_GptASR'; +INSERT INTO `ai_model_config` VALUES ('ASR_GptASR', 'ASR', 'GptASR', 'GPT语音识别', 0, 1, '{\"type\": \"gpt\", \"api_key\": \"\", \"base_url\": \"https://api.openai.com/v1/audio/transcriptions\", \"model_name\": \"gpt-4o-mini-transcribe\", \"output_dir\": \"tmp/\"}', NULL, NULL, 3, NULL, NULL, NULL, NULL); + + +-- 更新OpenAI ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://platform.openai.com/docs/api-reference/audio/createTranscription', +`remark` = 'OpenAI ASR配置说明： +1. 需要在OpenAI开放平台创建组织并获取api_key +2. 支持中、英、日、韩等多种语音识别，具体参考文档https://platform.openai.com/docs/guides/speech-to-text +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +5. 兼容groq语音识别。 groq识别速度更快。具体参考文档：https://console.groq.com/docs/speech-to-text +申请步骤： +**OpenAi ASR申请步骤：** +1.登录OpenAI Platform。https://auth.openai.com/log-in +2.创建api-key https://platform.openai.com/settings/organization/api-keys +3.模型可以选择gpt-4o-transcribe或GPT-4o mini Transcribe +**groq ASR申请步骤：** +1.登录groq Console。https://console.groq.com/home +2.创建api-key https://console.groq.com/keys +3.模型可以选择whisper-large-v3-turbo或whisper-large-v3（distil-whisper-large-v3-en仅支持英语转录） +4.base url https://api.groq.com/openai/v1/audio/transcriptions +' WHERE `id` = 'ASR_GptASR'; \ No newline at end of file diff --git a/main/xiaozhi-server/core/providers/asr/gpt.py b/main/xiaozhi-server/core/providers/asr/gpt.py new file mode 100644 index 00000000..5c6e849c --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/gpt.py @@ -0,0 +1,82 @@ +import time +import os +from config.logger import setup_logging +from typing import Optional, Tuple, List +from core.providers.asr.dto.dto import InterfaceType +from core.providers.asr.base import ASRProviderBase + +import requests + +TAG = __name__ +logger = setup_logging() + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool): + self.interface_type = InterfaceType.NON_STREAM + self.api_key = config.get("api_key") + self.api_url = config.get("base_url") + self.model = config.get("model_name") + self.output_dir = config.get("output_dir") + self.delete_audio_file = delete_audio_file + + os.makedirs(self.output_dir, exist_ok=True) + + async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus") -> Tuple[Optional[str], Optional[str]]: + file_path = None + try: + start_time = time.time() + if audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + file_path = self.save_audio_to_file(pcm_data, session_id) + + logger.bind(tag=TAG).debug( + f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}" + ) + + logger.bind(tag=TAG).info(f"file path: {file_path}") + headers = { + "Authorization": f"Bearer {self.api_key}", + } + + # 使用data参数传递模型名称 + data = { + "model": self.model + } + + + with open(file_path, "rb") as audio_file: # 使用with语句确保文件关闭 + files = { + "file": audio_file + } + + start_time = time.time() + response = requests.post( + self.api_url, + files=files, + data=data, + headers=headers + ) + logger.bind(tag=TAG).debug( + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {response.text}" + ) + + if response.status_code == 200: + text = response.json().get("text", "") + return text, file_path + else: + raise Exception(f"API请求失败: {response.status_code} - {response.text}") + + except Exception as e: + logger.bind(tag=TAG).error(f"语音识别失败: {e}") + return "", None + finally: + # 文件清理逻辑 + if self.delete_audio_file and file_path and os.path.exists(file_path): + try: + os.remove(file_path) + logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") + except Exception as e: + logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") + From b2e6156bbbe982715b96c947abb2399530dd32db Mon Sep 17 00:00:00 2001 From: luruxian Date: Thu, 10 Jul 2025 11:44:49 +0800 Subject: [PATCH 08/24] =?UTF-8?q?chore(db):=20=E6=B7=BB=E5=8A=A0=E6=96=B0?= =?UTF-8?q?=E7=9A=84=E6=95=B0=E6=8D=AE=E5=BA=93=E5=8F=98=E6=9B=B4=E9=9B=86?= =?UTF-8?q?202507101201?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加新的数据库变更脚本202507101201.sql到变更日志中 --- .../main/resources/db/changelog/db.changelog-master.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 32ba9f30..5fc027dd 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -239,4 +239,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202506261637.sql \ No newline at end of file + path: classpath:db/changelog/202506261637.sql + - changeSet: + id: 202507101201 + author: luruxian + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202507101201.sql \ No newline at end of file From 26a3b5116291987fbe7b416ba5c9b79b5c304361 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 10 Jul 2025 16:07:06 +0800 Subject: [PATCH 09/24] =?UTF-8?q?fix:=20=E8=A1=A5=E5=85=85=E6=96=87?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index ef36e95b..87ce606a 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -712,6 +712,8 @@ TTS: voice: longxiaochun access_key_id: 你的阿里云账号access_key_id access_key_secret: 你的阿里云账号access_key_secret + # 服务器地域选择，可选择距离更近的服务器以减少延迟，如nls-gateway-cn-hangzhou.aliyuncs.com(杭州)等 + host: nls-gateway-cn-shanghai.aliyuncs.com # 以下可不用设置，使用默认设置 # format: pcm # 音频格式：pcm、wav、mp3 # sample_rate: 16000 # 采样率：8000、16000、24000 From 23cd453af8fe07d6097154e86f1c4182b963204f Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 11 Jul 2025 11:36:58 +0800 Subject: [PATCH 10/24] =?UTF-8?q?update:=20aliyun=E5=8F=8C=E6=B5=81?= =?UTF-8?q?=E6=94=B9=E9=80=A0=20=E5=BE=85=E4=BC=98=E5=8C=96=E9=95=BF?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=9C=BA=E5=88=B6=E5=92=8C=E6=96=87=E6=9C=AC?= =?UTF-8?q?=E7=94=9F=E6=88=90=E5=8F=8D=E9=A6=88=E4=B8=BA=E7=A9=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 1 + .../core/providers/tts/aliyun_stream.py | 748 ++++++++---------- .../xiaozhi-server/core/providers/tts/base.py | 2 +- .../providers/tts/huoshan_double_stream.py | 2 +- 4 files changed, 329 insertions(+), 424 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 7b45ad2d..978d88dd 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -128,6 +128,7 @@ class ConnectionHandler: # tts相关变量 self.sentence_id = None + self.message_id = None # iot相关变量 self.iot_descriptors = {} diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 57d9ffcc..5c88602f 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -7,17 +7,15 @@ import time import queue import asyncio import traceback +from asyncio import Task import websockets -import websockets.protocol import os -import concurrent.futures -import sys from datetime import datetime from urllib import parse from core.providers.tts.base import TTSProviderBase from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType from core.utils.tts import MarkdownCleaner -from core.utils import opus_encoder_utils, textUtils +from core.utils import opus_encoder_utils from config.logger import setup_logging TAG = __name__ @@ -50,7 +48,7 @@ class AccessToken: "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "Version": "2019-02-28", } - + query_string = AccessToken._encode_dict(parameters) string_to_sign = ( "GET" @@ -59,7 +57,7 @@ class AccessToken: + "&" + AccessToken._encode_text(query_string) ) - + secreted_string = hmac.new( bytes(access_key_secret + "&", encoding="utf-8"), bytes(string_to_sign, encoding="utf-8"), @@ -67,12 +65,12 @@ class AccessToken: ).digest() signature = base64.b64encode(secreted_string) signature = AccessToken._encode_text(signature) - + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % ( signature, query_string, ) - + import requests response = requests.get(full_url) if response.ok: @@ -88,54 +86,51 @@ class AccessToken: class TTSProvider(TTSProviderBase): def __init__(self, config, delete_audio_file): super().__init__(config, delete_audio_file) - + # 设置为流式接口类型 - self.interface_type = InterfaceType.SINGLE_STREAM - + self.interface_type = InterfaceType.DUAL_STREAM + # 基础配置 self.access_key_id = config.get("access_key_id") self.access_key_secret = config.get("access_key_secret") self.appkey = config.get("appkey") self.format = config.get("format", "pcm") self.audio_file_type = config.get("format", "pcm") - + # 采样率配置 sample_rate = config.get("sample_rate", "16000") self.sample_rate = int(sample_rate) if sample_rate else 16000 - + # 音色配置 - CosyVoice大模型音色 if config.get("private_voice"): self.voice = config.get("private_voice") else: self.voice = config.get("voice", "longxiaochun") # CosyVoice默认音色 - + # 音频参数配置 volume = config.get("volume", "50") self.volume = int(volume) if volume else 50 - + speech_rate = config.get("speech_rate", "0") self.speech_rate = int(speech_rate) if speech_rate else 0 - + pitch_rate = config.get("pitch_rate", "0") self.pitch_rate = int(pitch_rate) if pitch_rate else 0 - + # WebSocket配置 self.host = config.get("host", "nls-gateway-cn-beijing.aliyuncs.com") self.ws_url = f"wss://{self.host}/ws/v1" self.ws = None - - # 流式相关配置 - self.before_stop_play_files = [] - self.segment_count = 0 - + self._monitor_task = None + # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( sample_rate=16000, channels=1, frame_size_ms=60 ) - + # PCM缓冲区 self.pcm_buffer = bytearray() - + # Token管理 if self.access_key_id and self.access_key_secret: self._refresh_token() @@ -152,8 +147,9 @@ class TTSProvider(TTSProviderBase): if not expire_time_str: raise ValueError("无法获取有效的Token过期时间") + expire_str = str(expire_time_str).strip() + try: - expire_str = str(expire_time_str).strip() if expire_str.isdigit(): expire_time = datetime.fromtimestamp(int(expire_str)) else: @@ -175,91 +171,83 @@ class TTSProvider(TTSProviderBase): async def _ensure_connection(self): """确保WebSocket连接可用""" - # 检查连接状态，兼容不同版本的websockets库 - need_reconnect = False - if self.ws is None: - need_reconnect = True - else: - try: - # 尝试访问closed属性，如果不存在则检查state - if hasattr(self.ws, 'closed'): - need_reconnect = self.ws.closed - elif hasattr(self.ws, 'state'): - # websockets 新版本使用state属性 - need_reconnect = self.ws.state != websockets.protocol.State.OPEN - else: - # 如果都没有，尝试发送ping来检测连接状态 - try: - await asyncio.wait_for(self.ws.ping(), timeout=2.0) - except: - need_reconnect = True - except: - need_reconnect = True - - if need_reconnect: - # 清理旧连接 - if self.ws: - try: - if hasattr(self.ws, 'close'): - if asyncio.iscoroutinefunction(self.ws.close): - await self.ws.close() - else: - self.ws.close() - except: - pass - finally: - self.ws = None - + try: if self._is_token_expired(): logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...") self._refresh_token() - - # 重试连接机制 - max_retries = 3 - retry_delay = 1.0 - - for attempt in range(max_retries): - try: - self.ws = await asyncio.wait_for( - websockets.connect( - self.ws_url, - additional_headers={ - "X-NLS-Token": self.token, - }, - ping_interval=30, - ping_timeout=10, - close_timeout=10, - ), - timeout=10.0 - ) - logger.bind(tag=TAG).info("阿里云CosyVoice流式TTS WebSocket连接建立成功") - return - except Exception as e: - logger.bind(tag=TAG).warning(f"WebSocket连接失败 (尝试 {attempt + 1}/{max_retries}): {e}") - if attempt < max_retries - 1: - await asyncio.sleep(retry_delay * (attempt + 1)) - else: - logger.bind(tag=TAG).error(f"WebSocket连接最终失败: {e}") - raise + if self.ws: + logger.bind(tag=TAG).info(f"使用已有链接...") + return self.ws + logger.bind(tag=TAG).info("开始建立新连接...") + + self.ws = await websockets.connect( + self.ws_url, + additional_headers={"X-NLS-Token": self.token}, + ping_interval=30, + ping_timeout=10, + close_timeout=10, + ) + logger.bind(tag=TAG).info("WebSocket连接建立成功") + return self.ws + except Exception as e: + logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}") + self.ws = None + raise def tts_text_priority_thread(self): """流式文本处理线程""" while not self.conn.stop_event.is_set(): try: message = self.tts_text_queue.get(timeout=1) + logger.bind(tag=TAG).debug( + f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}" + ) + + if message.sentence_type == SentenceType.FIRST: + self.conn.client_abort = False + + if self.conn.client_abort: + logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") + continue + if message.sentence_type == SentenceType.FIRST: # 初始化参数 - self.tts_stop_request = False - self.processed_chars = 0 - self.tts_text_buff = [] - self.segment_count = 0 - self.tts_audio_first_sentence = True - self.before_stop_play_files.clear() + try: + if not getattr(self.conn, "sentence_id", None): + self.conn.sentence_id = uuid.uuid4().hex + logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}") + + # aliyun独有的message_id需要自己生成 + self.conn.message_id = str(uuid.uuid4().hex) + + logger.bind(tag=TAG).info("开始启动TTS会话...") + future = asyncio.run_coroutine_threadsafe( + self.start_session(self.conn.sentence_id), + loop=self.conn.loop, + ) + future.result() + self.before_stop_play_files.clear() + logger.bind(tag=TAG).info("TTS会话启动成功") + + except Exception as e: + logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}") + continue + elif ContentType.TEXT == message.content_type: - self.tts_text_buff.append(message.content_detail) - segment_text = self._get_segment_text() - if segment_text: - self.to_tts_single_stream(segment_text) + if message.content_detail: + try: + logger.bind(tag=TAG).debug( + f"开始发送TTS文本: {message.content_detail}" + ) + future = asyncio.run_coroutine_threadsafe( + self.text_to_speak(message.content_detail, None), + loop=self.conn.loop, + ) + future.result() + logger.bind(tag=TAG).debug("TTS文本发送成功") + except Exception as e: + logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") + continue elif ContentType.FILE == message.content_type: logger.bind(tag=TAG).info( @@ -273,8 +261,16 @@ class TTSProvider(TTSProviderBase): ) if message.sentence_type == SentenceType.LAST: - # 处理剩余的文本 - self._process_remaining_text(True) + try: + logger.bind(tag=TAG).info("开始结束TTS会话...") + future = asyncio.run_coroutine_threadsafe( + self.finish_session(self.conn.sentence_id), + loop=self.conn.loop, + ) + future.result() + except Exception as e: + logger.bind(tag=TAG).error(f"结束TTS会话失败: {str(e)}") + continue except queue.Empty: continue @@ -283,118 +279,59 @@ class TTSProvider(TTSProviderBase): f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" ) - def _process_remaining_text(self, is_last=False): - """处理剩余的文本并生成语音""" - full_text = "".join(self.tts_text_buff) - remaining_text = full_text[self.processed_chars:] - if remaining_text: - segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) - if segment_text: - self.to_tts_single_stream(segment_text, is_last) - self.processed_chars += len(full_text) - else: - self._process_before_stop_play_files() - else: - self._process_before_stop_play_files() - - def to_tts_single_stream(self, text, is_last=False): - """流式TTS处理 - 使用线程池执行异步任务""" + async def text_to_speak(self, text, _): try: - text = MarkdownCleaner.clean_markdown(text) - - # 使用线程池来执行异步任务，避免事件循环冲突 - def run_async_task(): - """在新线程中运行异步任务""" - try: - # 创建新的事件循环用于这个线程 - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - - # Windows下设置事件循环策略 - if sys.platform == "win32": - asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) - - try: - # 运行TTS任务 - loop.run_until_complete(self.text_to_speak(text, is_last)) - return True - finally: - # 安全关闭事件循环 - try: - # 取消所有未完成的任务 - pending = asyncio.all_tasks(loop) - if pending: - for task in pending: - task.cancel() - # 等待任务取消完成 - loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) - except Exception as cleanup_error: - logger.bind(tag=TAG).debug(f"清理事件循环异常: {cleanup_error}") - finally: - loop.close() - - except Exception as e: - logger.bind(tag=TAG).error(f"异步任务执行失败: {e}") - return False - - # 使用线程池执行异步任务 - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(run_async_task) - success = future.result(timeout=30) # 30秒超时 - - if success: - logger.bind(tag=TAG).info(f"语音生成成功: {text}") - else: - logger.bind(tag=TAG).error(f"语音生成失败: {text}") - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - - except concurrent.futures.TimeoutError: - logger.bind(tag=TAG).error(f"TTS任务超时: {text}") - self.tts_audio_queue.put((SentenceType.LAST, [], None)) + if self.ws is None: + logger.bind(tag=TAG).warning(f"WebSocket连接不存在，终止发送文本") + return + filtered_text = MarkdownCleaner.clean_markdown(text) + run_request = { + "header": { + "message_id": self.conn.message_id, + "task_id": self.conn.sentence_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": self.appkey, + }, + "payload": { + "text": filtered_text + } + } + await self.ws.send(json.dumps(run_request)) + return + except Exception as e: - logger.bind(tag=TAG).error(f"TTS处理异常: {text}, 错误: {e}") - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - - return None + logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") + if self.ws: + try: + await self.ws.close() + except: + pass + self.ws = None + raise - async def text_to_speak(self, text, is_last): - """流式处理TTS音频""" - await self._tts_request_unified(text, is_last) - - async def _tts_request_unified(self, text: str, is_last: bool) -> None: - """统一的TTS请求方法""" - ws_connection = None - + async def start_session(self, session_id): + logger.bind(tag=TAG).info(f"开始会话～～{session_id}") try: - # 确保Token有效 - if self._is_token_expired(): - self._refresh_token() - - # 总是创建独立连接，避免与其他线程的事件循环冲突 - ws_url = f"wss://{self.host}/ws/v1" - ws_connection = await asyncio.wait_for( - websockets.connect( - ws_url, - additional_headers={ - "X-NLS-Token": self.token, - }, - ping_interval=15, # 每15秒发送ping，保持连接活跃 - ping_timeout=5, # ping超时时间5秒 - close_timeout=3, # 关闭超时时间3秒 - ), - timeout=8.0 # 连接超时时间8秒 - ) - logger.bind(tag=TAG).debug(f"建立独立WebSocket连接: {text}") - - # 生成task_id - task_id = str(uuid.uuid4()).replace('-', '') - message_id = str(uuid.uuid4()).replace('-', '') - # 第一阶段：发送StartSynthesis指令（设置参数） + # 会话开始时检测上个会话的监听状态 + if( + self._monitor_task is not None + and isinstance(self._monitor_task, Task) + and not self._monitor_task.done() + ): + logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务和连接...") + await self.close() + + # 建立新连接 + await self._ensure_connection() + + # 启动监听任务 + self._monitor_task = asyncio.create_task(self._start_monitor_tts_response()) start_request = { "header": { - "message_id": message_id, - "task_id": task_id, + "message_id": self.conn.message_id, + "task_id": self.conn.sentence_id, "namespace": "FlowingSpeechSynthesizer", "name": "StartSynthesis", "appkey": self.appkey, @@ -408,195 +345,198 @@ class TTSProvider(TTSProviderBase): "pitch_rate": self.pitch_rate, } } - - await ws_connection.send(json.dumps(start_request)) - logger.bind(tag=TAG).debug(f"发送StartSynthesis指令 会话id：{task_id}") - - # 第二阶段：发送RunSynthesis指令（发送文本） - run_request = { - "header": { - "message_id": message_id, - "task_id": task_id, - "namespace": "FlowingSpeechSynthesizer", - "name": "RunSynthesis", - "appkey": self.appkey, - }, - "payload": { - "text": text - } - } - - await ws_connection.send(json.dumps(run_request)) - logger.bind(tag=TAG).debug(f"发送RunSynthesis指令 {text} 会话id： {task_id})") - - # 立即发送StopSynthesis指令，避免IDLE_TIMEOUT - - stop_request = { - "header": { - "message_id": message_id, - "task_id": task_id, - "namespace": "FlowingSpeechSynthesizer", - "name": "StopSynthesis", - "appkey": self.appkey, - } - } - - await ws_connection.send(json.dumps(stop_request)) - logger.bind(tag=TAG).debug(f"发送StopSynthesis指令 会话id：{task_id}") - - # 初始化处理参数 - 使用独立缓冲区 - pcm_buffer = bytearray() - opus_datas_cache = [] - segment_count = 0 - self.segment_count = 0 # 同时重置实例变量 - synthesis_completed = False - - # 发送第一个音频包 - self.tts_audio_queue.put((SentenceType.FIRST, [], text)) - - # 处理响应 - 设置超时时间避免长时间等待 - timeout_duration = 15 # 15秒超时 - try: - # 使用asyncio.wait_for替代asyncio.timeout以保证兼容性 - async def process_messages(): - nonlocal synthesis_completed, segment_count, pcm_buffer, opus_datas_cache # 声明使用外层变量 - async for message in ws_connection: - try: - if isinstance(message, str): - # 处理JSON消息 - data = json.loads(message) - header = data.get("header", {}) - event_name = header.get("name") - - if event_name == "SynthesisStarted": - logger.bind(tag=TAG).debug(f"TTS合成已启动: {task_id}") - - elif event_name == "SynthesisCompleted": - logger.bind(tag=TAG).debug(f"TTS合成完成: {text}") - synthesis_completed = True - break - - elif event_name == "TaskFailed": - error_msg = header.get("status_text", "未知错误") - logger.bind(tag=TAG).error(f"TTS合成失败: {error_msg}") - synthesis_completed = True - break - - elif isinstance(message, bytes): - # 处理二进制音频数据 - pcm_buffer.extend(message) - - # 计算每帧的字节数 - frame_bytes = int( - self.opus_encoder.sample_rate - * self.opus_encoder.channels - * self.opus_encoder.frame_size_ms - / 1000 - * 2 - ) - - # 分帧处理PCM数据 - while len(pcm_buffer) >= frame_bytes: - frame = bytes(pcm_buffer[:frame_bytes]) - del pcm_buffer[:frame_bytes] # 清除已处理的数据 - - # 编码为Opus - opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) - if opus_packets: - if segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_packets, None) - ) - segment_count += 1 - else: - opus_datas_cache.extend(opus_packets) - - except json.JSONDecodeError: - logger.bind(tag=TAG).warning("收到无效的JSON消息") - except Exception as e: - logger.bind(tag=TAG).error(f"处理响应消息失败: {e}") - - await asyncio.wait_for(process_messages(), timeout=timeout_duration) - - except asyncio.TimeoutError: - logger.bind(tag=TAG).warning(f"TTS请求超时，但可能已获取部分音频数据: {text}") - except websockets.ConnectionClosed: - logger.bind(tag=TAG).debug("WebSocket连接已正常关闭") - except Exception as e: - logger.bind(tag=TAG).error(f"处理WebSocket消息失败: {e}") - - # 因为已经提前发送了StopSynthesis，这里不需要再次发送 - # 直接处理剩余的PCM数据 - if pcm_buffer: - opus_packets = self.opus_encoder.encode_pcm_to_opus( - bytes(pcm_buffer), end_of_stream=True - ) - if opus_packets: - if segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_packets, None) - ) - segment_count += 1 - else: - opus_datas_cache.extend(opus_packets) - - # 发送缓存的数据 - if segment_count >= 10 and opus_datas_cache: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_datas_cache, None) - ) - - # 如果是最后一段，处理待播放文件 - if is_last: - self._process_before_stop_play_files() - + await self.ws.send(json.dumps(start_request)) + logger.bind(tag=TAG).info("会话启动请求已发送") except Exception as e: - logger.bind(tag=TAG).error(f"TTS请求异常: {e}") - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - finally: - # 确保WebSocket连接被关闭 - if ws_connection: - try: - if hasattr(ws_connection, 'close'): - if asyncio.iscoroutinefunction(ws_connection.close): - await ws_connection.close() - else: - ws_connection.close() - except Exception as e: - logger.bind(tag=TAG).debug(f"关闭WebSocket连接时出现异常: {e}") + logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}") + # 确保清理资源 + await self.close() + raise + + async def finish_session(self, session_id): + logger.bind(tag=TAG).info(f"关闭会话～～{session_id}") + try: + if self.ws: + stop_request = { + "header": { + "message_id": self.conn.message_id, + "task_id": self.conn.sentence_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StopSynthesis", + "appkey": self.appkey, + } + } + await self.ws.send(json.dumps(stop_request)) + logger.bind(tag=TAG).info("会话结束请求已发送") + if self._monitor_task: + try: + await self._monitor_task + except Exception as e: + logger.bind(tag=TAG).error( + f"等待监听任务完成时发生错误: {str(e)}" + ) + finally: + self._monitor_task = None + except Exception as e: + logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}") + # 确保清理资源 + await self.close() + raise async def close(self): """资源清理""" + if self._monitor_task: + try: + self._monitor_task.cancel() + await self._monitor_task + except asyncio.CancelledError: + pass + except Exception as e: + logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}") + self._monitor_task = None + if self.ws: try: - # 兼容不同版本的websockets库关闭方式 - if hasattr(self.ws, 'close'): - if asyncio.iscoroutinefunction(self.ws.close): - await self.ws.close() - else: - self.ws.close() - elif hasattr(self.ws, 'close_connection'): - await self.ws.close_connection() - except Exception as e: - logger.bind(tag=TAG).debug(f"关闭WebSocket连接时出现异常: {e}") - finally: + await self.ws.close() + except: + pass + self.ws = None + + async def _start_monitor_tts_response(self): + """监听TTS响应""" + opus_datas_cache = [] + current_sentence_buffer = bytearray() + segment_count = 0 + current_sentence_parts = [] + try: + session_finished = False # 标记会话是否正常结束 + while not self.conn.stop_event.is_set(): + try: + msg = await self.ws.recv() + # 检查客户端是否中止 + if self.conn.client_abort: + logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应") + break + if isinstance(msg, str): # 文本控制消息 + try: + data = json.loads(msg) + header = data.get("header", {}) + event_name = header.get("name") + + if event_name == "SynthesisStarted": + logger.bind(tag=TAG).debug("TTS合成已启动") + + elif event_name == "SentenceBegin": + logger.bind(tag=TAG).debug(f"句子语音生成开始") + current_sentence_buffer = bytearray() + segment_count = 0 + self.tts_audio_queue.put( + (SentenceType.FIRST, [], "") + ) + + elif event_name == "SentenceSynthesis": + payload = data.get("payload", {}) + subtitles = payload.get("subtitles", []) + + # 收集所有字幕片段 + for sub in subtitles: + text = sub.get("text", "") + if text: + current_sentence_parts.append(text) + + elif event_name == "SentenceEnd": + logger.bind(tag=TAG).info(f"句子语音生成成功: {''.join(current_sentence_parts)}") + + if current_sentence_buffer: + opus_datas = self.opus_encoder.encode_pcm_to_opus( + bytes(current_sentence_buffer), end_of_stream=True + ) + if opus_datas: + if segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas, None) + ) + else: + opus_datas_cache.extend(opus_datas) + + if segment_count >= 10 and opus_datas_cache: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas_cache, None) + ) + opus_datas_cache = [] + current_sentence_buffer = bytearray() + + elif event_name == "SynthesisCompleted": + logger.bind(tag=TAG).debug("会话结束") + self._process_before_stop_play_files() + session_finished = True + break + + except json.JSONDecodeError: + logger.bind(tag=TAG).warning("收到无效的JSON消息") + + # 二进制消息（音频数据） + elif isinstance(msg, bytes): + # 将音频数据添加到当前句子的缓冲区 + current_sentence_buffer.extend(msg) + + # 计算每帧的字节数（60ms） + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels + * self.opus_encoder.frame_size_ms + / 1000 + * 2 # 16-bit = 2 bytes + ) + + # 处理完整的音频帧 + while len(current_sentence_buffer) >= frame_bytes: + # 取出一帧数据 + frame = bytes(current_sentence_buffer[:frame_bytes]) + del current_sentence_buffer[:frame_bytes] + + # 编码为Opus + opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) + + if opus_packets: + # 前10个片段直接发送，后续片段缓存 + if segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_packets, None) + ) + segment_count += 1 + else: + opus_datas_cache.extend(opus_packets) + + except websockets.ConnectionClosed: + logger.bind(tag=TAG).warning("WebSocket连接已关闭") + break + except Exception as e: + logger.bind(tag=TAG).error( + f"处理TTS响应时出错: {e}\n{traceback.format_exc()}" + ) + break + # 仅在连接异常时才关闭 + if not session_finished and self.ws: + try: + await self.ws.close() + except: + pass self.ws = None - - if hasattr(self, "opus_encoder"): - self.opus_encoder.close() - - await super().close() + # 监听任务退出时清理引用 + finally: + self._monitor_task = None def to_tts(self, text: str) -> list: """非流式TTS处理，用于测试及保存音频文件的场景""" start_time = time.time() text = MarkdownCleaner.clean_markdown(text) - + try: # 使用同步方式进行TTS转换 if self._is_token_expired(): self._refresh_token() - + # 构造请求数据 request_json = { "appkey": self.appkey, @@ -609,21 +549,21 @@ class TTSProvider(TTSProviderBase): "speech_rate": self.speech_rate, "pitch_rate": self.pitch_rate, } - + # 使用HTTP接口进行同步请求 import requests api_url = f"https://{self.host}/stream/v1/tts" headers = {"Content-Type": "application/json"} - + resp = requests.post(api_url, json=request_json, headers=headers) - + if resp.status_code == 401: # Token过期特殊处理 self._refresh_token() resp = requests.post(api_url, json=request_json, headers=headers) - + if resp.headers["Content-Type"].startswith("audio/"): pcm_data = resp.content - + # 使用opus编码器处理PCM数据 opus_datas = [] frame_bytes = int( @@ -633,7 +573,7 @@ class TTSProvider(TTSProviderBase): / 1000 * 2 ) - + # 分帧处理PCM数据 for i in range(0, len(pcm_data), frame_bytes): frame = pcm_data[i:i + frame_bytes] @@ -641,49 +581,13 @@ class TTSProvider(TTSProviderBase): opus = self.opus_encoder.encode_pcm_to_opus(frame, False) if opus: opus_datas.extend(opus) - + logger.bind(tag=TAG).info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") return opus_datas else: logger.bind(tag=TAG).error(f"TTS请求失败: {resp.content}") return [] - + except Exception as e: logger.bind(tag=TAG).error(f"TTS请求异常: {e}") return [] - - def _get_segment_text(self): - """获取当前可以处理的文本段""" - if not self.tts_text_buff: - return None - - full_text = "".join(self.tts_text_buff) - if len(full_text) <= self.processed_chars: - return None - - # 获取未处理的文本 - remaining_text = full_text[self.processed_chars:] - - # 如果文本较短或者到达了句子结尾标点，直接处理 - sentence_endings = ['。', '！', '？', '.', '!', '?', '\n'] - if len(remaining_text) < 20: - return None - - # 查找句子结尾 - for i, char in enumerate(remaining_text): - if char in sentence_endings and i > 10: # 至少10个字符 - segment = remaining_text[:i+1] - segment_text = textUtils.get_string_no_punctuation_or_emoji(segment) - if segment_text: - self.processed_chars += i + 1 - return segment_text - - # 如果没有找到句子结尾，但文本足够长，按长度分段 - if len(remaining_text) > 50: - segment = remaining_text[:30] - segment_text = textUtils.get_string_no_punctuation_or_emoji(segment) - if segment_text: - self.processed_chars += 30 - return segment_text - - return None diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 9127ec12..9590669b 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -159,7 +159,7 @@ class TTSProviderBase(ABC): if conn.sentence_id: sentence_id = conn.sentence_id else: - sentence_id = str(uuid.uuid4()).replace("-", "") + sentence_id = str(uuid.uuid4().hex) conn.sentence_id = sentence_id # 对于单句的文本，进行分段处理 segments = re.split(r"([。！？!?；;\n])", content_detail) diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 0eb7c72d..23abaf5c 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -469,7 +469,7 @@ class TTSProvider(TTSProviderBase): except: pass self.ws = None - # 监听任务退出时清理引用 + # 监听任务退出时清理引用 finally: self._monitor_task = None From 8e5d93374596e9de38287069c797f02dde89784a Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 11 Jul 2025 16:36:47 +0800 Subject: [PATCH 11/24] =?UTF-8?q?update:=20=E4=BC=98=E5=8C=96=E9=9F=B3?= =?UTF-8?q?=E9=A2=91=E6=92=AD=E6=94=BE=20=E6=96=87=E6=9C=AC=E5=8F=91?= =?UTF-8?q?=E9=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun_stream.py | 125 +++++++----------- 1 file changed, 49 insertions(+), 76 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 5c88602f..b6fd1ec2 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -123,6 +123,11 @@ class TTSProvider(TTSProviderBase): self.ws = None self._monitor_task = None + # 文本获取 + self.sentence_queue = queue.Queue() + self.text_buffer = "" + self.sentence_end_chars = {'.', '。', '!', '！', '?', '？', '\n', "~", "；", ";", ":", "：", " ", "，", ","} + # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( sample_rate=16000, channels=1, frame_size_ms=60 @@ -176,6 +181,7 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...") self._refresh_token() if self.ws: + # 10秒内才可以复用,适合连续对话 logger.bind(tag=TAG).info(f"使用已有链接...") return self.ws logger.bind(tag=TAG).info("开始建立新连接...") @@ -239,6 +245,10 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) + self.text_buffer += message.content_detail + if message.content_detail in self.sentence_end_chars and len(self.text_buffer) > 6: + self.sentence_queue.put(self.text_buffer) + self.text_buffer = "" future = asyncio.run_coroutine_threadsafe( self.text_to_speak(message.content_detail, None), loop=self.conn.loop, @@ -263,6 +273,8 @@ class TTSProvider(TTSProviderBase): if message.sentence_type == SentenceType.LAST: try: logger.bind(tag=TAG).info("开始结束TTS会话...") + self.sentence_queue.put(self.text_buffer) + print(list(self.sentence_queue.queue)) future = asyncio.run_coroutine_threadsafe( self.finish_session(self.conn.sentence_id), loop=self.conn.loop, @@ -315,9 +327,9 @@ class TTSProvider(TTSProviderBase): try: # 会话开始时检测上个会话的监听状态 if( - self._monitor_task is not None - and isinstance(self._monitor_task, Task) - and not self._monitor_task.done() + self._monitor_task is not None + and isinstance(self._monitor_task, Task) + and not self._monitor_task.done() ): logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务和连接...") await self.close() @@ -343,6 +355,7 @@ class TTSProvider(TTSProviderBase): "volume": self.volume, "speech_rate": self.speech_rate, "pitch_rate": self.pitch_rate, + "enable_subtitle": True } } await self.ws.send(json.dumps(start_request)) @@ -405,9 +418,9 @@ class TTSProvider(TTSProviderBase): async def _start_monitor_tts_response(self): """监听TTS响应""" opus_datas_cache = [] - current_sentence_buffer = bytearray() - segment_count = 0 - current_sentence_parts = [] + is_first_sentence = True + first_sentence_segment_count = 0 # 添加计数器 + text_buff = "" try: session_finished = False # 标记会话是否正常结束 while not self.conn.stop_event.is_set(): @@ -422,91 +435,51 @@ class TTSProvider(TTSProviderBase): data = json.loads(msg) header = data.get("header", {}) event_name = header.get("name") - if event_name == "SynthesisStarted": logger.bind(tag=TAG).debug("TTS合成已启动") - elif event_name == "SentenceBegin": - logger.bind(tag=TAG).debug(f"句子语音生成开始") - current_sentence_buffer = bytearray() - segment_count = 0 - self.tts_audio_queue.put( - (SentenceType.FIRST, [], "") - ) - - elif event_name == "SentenceSynthesis": - payload = data.get("payload", {}) - subtitles = payload.get("subtitles", []) - - # 收集所有字幕片段 - for sub in subtitles: - text = sub.get("text", "") - if text: - current_sentence_parts.append(text) - + try: + text_buff = self.sentence_queue.get_nowait() + except queue.Empty: + text_buff = "" + logger.bind(tag=TAG).debug(f"句子语音生成开始: {text_buff}") + opus_datas_cache = [] + self.tts_audio_queue.put((SentenceType.FIRST, [], text_buff)) elif event_name == "SentenceEnd": - logger.bind(tag=TAG).info(f"句子语音生成成功: {''.join(current_sentence_parts)}") - - if current_sentence_buffer: - opus_datas = self.opus_encoder.encode_pcm_to_opus( - bytes(current_sentence_buffer), end_of_stream=True - ) - if opus_datas: - if segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_datas, None) - ) - else: - opus_datas_cache.extend(opus_datas) - - if segment_count >= 10 and opus_datas_cache: + logger.bind(tag=TAG).info(f"句子语音生成成功： {text_buff}") + text_buff = "" + if not is_first_sentence or first_sentence_segment_count > 10: + # 发送缓存的数据 self.tts_audio_queue.put( (SentenceType.MIDDLE, opus_datas_cache, None) ) - opus_datas_cache = [] - current_sentence_buffer = bytearray() - + # 第一句话结束后，将标志设置为False + is_first_sentence = False elif event_name == "SynthesisCompleted": - logger.bind(tag=TAG).debug("会话结束") + logger.bind(tag=TAG).debug(f"会话结束～～") self._process_before_stop_play_files() session_finished = True break - except json.JSONDecodeError: logger.bind(tag=TAG).warning("收到无效的JSON消息") - # 二进制消息（音频数据） - elif isinstance(msg, bytes): - # 将音频数据添加到当前句子的缓冲区 - current_sentence_buffer.extend(msg) - - # 计算每帧的字节数（60ms） - frame_bytes = int( - self.opus_encoder.sample_rate - * self.opus_encoder.channels - * self.opus_encoder.frame_size_ms - / 1000 - * 2 # 16-bit = 2 bytes + elif isinstance(msg, (bytes, bytearray)): + logger.bind(tag=TAG).debug(f"推送数据到队列里面～～") + opus_datas = self.opus_encoder.encode_pcm_to_opus(msg, False) + logger.bind(tag=TAG).debug( + f"推送数据到队列里面帧数～～{len(opus_datas)}" ) - - # 处理完整的音频帧 - while len(current_sentence_buffer) >= frame_bytes: - # 取出一帧数据 - frame = bytes(current_sentence_buffer[:frame_bytes]) - del current_sentence_buffer[:frame_bytes] - - # 编码为Opus - opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) - - if opus_packets: - # 前10个片段直接发送，后续片段缓存 - if segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_packets, None) - ) - segment_count += 1 - else: - opus_datas_cache.extend(opus_packets) + if is_first_sentence: + first_sentence_segment_count += 1 + if first_sentence_segment_count <= 6: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas, None) + ) + else: + opus_datas_cache = opus_datas_cache + opus_datas + else: + # 后续句子缓存 + opus_datas_cache = opus_datas_cache + opus_datas except websockets.ConnectionClosed: logger.bind(tag=TAG).warning("WebSocket连接已关闭") From 04ed5ed980159e8d8bb8851beb4eab1a6c385676 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 11 Jul 2025 18:04:00 +0800 Subject: [PATCH 12/24] =?UTF-8?q?update:=20=E5=90=8C=E6=AD=A5=E9=9D=9E?= =?UTF-8?q?=E6=B5=81=E5=BC=8F=E5=A4=84=E7=90=86=2010=E7=A7=92=E8=B6=85?= =?UTF-8?q?=E6=97=B6=E9=93=BE=E6=8E=A5=E4=B8=8D=E5=A4=8D=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun_stream.py | 171 +++++++++++------- 1 file changed, 109 insertions(+), 62 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index b6fd1ec2..1ea37810 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -15,7 +15,7 @@ from urllib import parse from core.providers.tts.base import TTSProviderBase from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType from core.utils.tts import MarkdownCleaner -from core.utils import opus_encoder_utils +from core.utils import opus_encoder_utils, textUtils from config.logger import setup_logging TAG = __name__ @@ -122,10 +122,10 @@ class TTSProvider(TTSProviderBase): self.ws_url = f"wss://{self.host}/ws/v1" self.ws = None self._monitor_task = None + self.last_active_time = None - # 文本获取 - self.sentence_queue = queue.Queue() - self.text_buffer = "" + # 文本符号 + self.sentence_queue = None self.sentence_end_chars = {'.', '。', '!', '！', '?', '？', '\n', "~", "；", ";", ":", "：", " ", "，", ","} # 创建Opus编码器 @@ -180,8 +180,9 @@ class TTSProvider(TTSProviderBase): if self._is_token_expired(): logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...") self._refresh_token() - if self.ws: - # 10秒内才可以复用,适合连续对话 + current_time = time.time() + if self.ws and current_time - self.last_active_time < 10: + # 10秒内才可以复用链接进行连续对话 logger.bind(tag=TAG).info(f"使用已有链接...") return self.ws logger.bind(tag=TAG).info("开始建立新连接...") @@ -194,10 +195,12 @@ class TTSProvider(TTSProviderBase): close_timeout=10, ) logger.bind(tag=TAG).info("WebSocket连接建立成功") + self.last_active_time = time.time() return self.ws except Exception as e: logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}") self.ws = None + self.last_active_time = None raise def tts_text_priority_thread(self): @@ -223,8 +226,10 @@ class TTSProvider(TTSProviderBase): self.conn.sentence_id = uuid.uuid4().hex logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}") - # aliyun独有的message_id需要自己生成 + # aliyunStream独有的参数生成 self.conn.message_id = str(uuid.uuid4().hex) + self.sentence_queue = queue.Queue() + self.text_buffer = "" logger.bind(tag=TAG).info("开始启动TTS会话...") future = asyncio.run_coroutine_threadsafe( @@ -247,7 +252,7 @@ class TTSProvider(TTSProviderBase): ) self.text_buffer += message.content_detail if message.content_detail in self.sentence_end_chars and len(self.text_buffer) > 6: - self.sentence_queue.put(self.text_buffer) + self.sentence_queue.put(textUtils.get_string_no_punctuation_or_emoji(self.text_buffer)) self.text_buffer = "" future = asyncio.run_coroutine_threadsafe( self.text_to_speak(message.content_detail, None), @@ -273,8 +278,7 @@ class TTSProvider(TTSProviderBase): if message.sentence_type == SentenceType.LAST: try: logger.bind(tag=TAG).info("开始结束TTS会话...") - self.sentence_queue.put(self.text_buffer) - print(list(self.sentence_queue.queue)) + self.sentence_queue.put(textUtils.get_string_no_punctuation_or_emoji(self.text_buffer)) future = asyncio.run_coroutine_threadsafe( self.finish_session(self.conn.sentence_id), loop=self.conn.loop, @@ -310,6 +314,7 @@ class TTSProvider(TTSProviderBase): } } await self.ws.send(json.dumps(run_request)) + self.last_active_time = time.time() return except Exception as e: @@ -359,6 +364,7 @@ class TTSProvider(TTSProviderBase): } } await self.ws.send(json.dumps(start_request)) + self.last_active_time = time.time() logger.bind(tag=TAG).info("会话启动请求已发送") except Exception as e: logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}") @@ -381,6 +387,7 @@ class TTSProvider(TTSProviderBase): } await self.ws.send(json.dumps(stop_request)) logger.bind(tag=TAG).info("会话结束请求已发送") + self.last_active_time = time.time() if self._monitor_task: try: await self._monitor_task @@ -414,6 +421,7 @@ class TTSProvider(TTSProviderBase): except: pass self.ws = None + self.last_active_time = None async def _start_monitor_tts_response(self): """监听TTS响应""" @@ -426,6 +434,7 @@ class TTSProvider(TTSProviderBase): while not self.conn.stop_event.is_set(): try: msg = await self.ws.recv() + self.last_active_time = time.time() # 检查客户端是否中止 if self.conn.client_abort: logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应") @@ -459,6 +468,7 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).debug(f"会话结束～～") self._process_before_stop_play_files() session_finished = True + self.reuse_judgment = time.time() break except json.JSONDecodeError: logger.bind(tag=TAG).warning("收到无效的JSON消息") @@ -502,65 +512,102 @@ class TTSProvider(TTSProviderBase): def to_tts(self, text: str) -> list: """非流式TTS处理，用于测试及保存音频文件的场景""" - start_time = time.time() - text = MarkdownCleaner.clean_markdown(text) - try: - # 使用同步方式进行TTS转换 - if self._is_token_expired(): - self._refresh_token() + # 创建新的事件循环 + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) - # 构造请求数据 - request_json = { - "appkey": self.appkey, - "token": self.token, - "text": text, - "format": "pcm", - "sample_rate": self.sample_rate, - "voice": self.voice, - "volume": self.volume, - "speech_rate": self.speech_rate, - "pitch_rate": self.pitch_rate, - } + # 生成会话ID + session_id = uuid.uuid4().hex + message_id = uuid.uuid4().hex + # 存储音频数据 + audio_data = [] - # 使用HTTP接口进行同步请求 - import requests - api_url = f"https://{self.host}/stream/v1/tts" - headers = {"Content-Type": "application/json"} + async def _generate_audio(): + # 刷新Token（如果需要） + if self._is_token_expired(): + self._refresh_token() - resp = requests.post(api_url, json=request_json, headers=headers) - - if resp.status_code == 401: # Token过期特殊处理 - self._refresh_token() - resp = requests.post(api_url, json=request_json, headers=headers) - - if resp.headers["Content-Type"].startswith("audio/"): - pcm_data = resp.content - - # 使用opus编码器处理PCM数据 - opus_datas = [] - frame_bytes = int( - self.opus_encoder.sample_rate - * self.opus_encoder.channels - * self.opus_encoder.frame_size_ms - / 1000 - * 2 + # 建立WebSocket连接 + ws = await websockets.connect( + self.ws_url, + additional_headers={"X-NLS-Token": self.token}, + ping_interval=30, + ping_timeout=10, + close_timeout=10, ) + try: + # 发送StartSynthesis请求 + start_request = { + "header": { + "message_id": message_id, + "task_id": session_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StartSynthesis", + "appkey": self.appkey, + }, + "payload": { + "voice": self.voice, + "format": self.format, + "sample_rate": self.sample_rate, + "volume": self.volume, + "speech_rate": self.speech_rate, + "pitch_rate": self.pitch_rate, + "enable_subtitle": True + } + } + await ws.send(json.dumps(start_request)) - # 分帧处理PCM数据 - for i in range(0, len(pcm_data), frame_bytes): - frame = pcm_data[i:i + frame_bytes] - if len(frame) == frame_bytes: - opus = self.opus_encoder.encode_pcm_to_opus(frame, False) - if opus: - opus_datas.extend(opus) + # 发送文本合成请求 + filtered_text = MarkdownCleaner.clean_markdown(text) + run_request = { + "header": { + "message_id": message_id, + "task_id": session_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": self.appkey, + }, + "payload": { + "text": filtered_text + } + } + await ws.send(json.dumps(run_request)) - logger.bind(tag=TAG).info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") - return opus_datas - else: - logger.bind(tag=TAG).error(f"TTS请求失败: {resp.content}") - return [] + # 发送停止合成请求 + stop_request = { + "header": { + "message_id": message_id, + "task_id": session_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StopSynthesis", + "appkey": self.appkey, + } + } + await ws.send(json.dumps(stop_request)) + # 接收音频数据 + while True: + msg = await ws.recv() + if isinstance(msg, (bytes, bytearray)): + # 编码为Opus并收集 + opus_frames = self.opus_encoder.encode_pcm_to_opus(msg, False) + audio_data.extend(opus_frames) + elif isinstance(msg, str): + data = json.loads(msg) + header = data.get("header", {}) + if header.get("name") == "SynthesisCompleted": + break + finally: + try: + await ws.close() + except: + pass + + loop.run_until_complete(_generate_audio()) + loop.close() + + return audio_data except Exception as e: - logger.bind(tag=TAG).error(f"TTS请求异常: {e}") + logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}") return [] From 2c33ee5b325131f6d137fda2cb9e722c52186515 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Mon, 14 Jul 2025 17:34:42 +0800 Subject: [PATCH 13/24] =?UTF-8?q?update:=20=E4=BC=98=E5=8C=96=E6=96=87?= =?UTF-8?q?=E6=9C=AC=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun_stream.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 1ea37810..fd345fe8 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -125,8 +125,7 @@ class TTSProvider(TTSProviderBase): self.last_active_time = None # 文本符号 - self.sentence_queue = None - self.sentence_end_chars = {'.', '。', '!', '！', '?', '？', '\n', "~", "；", ";", ":", "：", " ", "，", ","} + self.sentence_end_chars = {'。', '!', '！', '?', '？', '\n', "~"} # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( @@ -228,7 +227,6 @@ class TTSProvider(TTSProviderBase): # aliyunStream独有的参数生成 self.conn.message_id = str(uuid.uuid4().hex) - self.sentence_queue = queue.Queue() self.text_buffer = "" logger.bind(tag=TAG).info("开始启动TTS会话...") @@ -251,9 +249,6 @@ class TTSProvider(TTSProviderBase): f"开始发送TTS文本: {message.content_detail}" ) self.text_buffer += message.content_detail - if message.content_detail in self.sentence_end_chars and len(self.text_buffer) > 6: - self.sentence_queue.put(textUtils.get_string_no_punctuation_or_emoji(self.text_buffer)) - self.text_buffer = "" future = asyncio.run_coroutine_threadsafe( self.text_to_speak(message.content_detail, None), loop=self.conn.loop, @@ -278,7 +273,6 @@ class TTSProvider(TTSProviderBase): if message.sentence_type == SentenceType.LAST: try: logger.bind(tag=TAG).info("开始结束TTS会话...") - self.sentence_queue.put(textUtils.get_string_no_punctuation_or_emoji(self.text_buffer)) future = asyncio.run_coroutine_threadsafe( self.finish_session(self.conn.sentence_id), loop=self.conn.loop, @@ -447,10 +441,7 @@ class TTSProvider(TTSProviderBase): if event_name == "SynthesisStarted": logger.bind(tag=TAG).debug("TTS合成已启动") elif event_name == "SentenceBegin": - try: - text_buff = self.sentence_queue.get_nowait() - except queue.Empty: - text_buff = "" + text_buff = self._extract_sentence() logger.bind(tag=TAG).debug(f"句子语音生成开始: {text_buff}") opus_datas_cache = [] self.tts_audio_queue.put((SentenceType.FIRST, [], text_buff)) @@ -510,6 +501,17 @@ class TTSProvider(TTSProviderBase): finally: self._monitor_task = None + def _extract_sentence(self): + """从text_buffer中提取第一个满足分句规则的句子""" + if len(self.text_buffer) <= 6: + return textUtils.get_string_no_punctuation_or_emoji(self.text_buffer) + for idx, char in enumerate(self.text_buffer): + if char in self.sentence_end_chars and idx > 6: + sentence = self.text_buffer[:idx + 1] + self.text_buffer = self.text_buffer[idx + 1:] + return textUtils.get_string_no_punctuation_or_emoji(sentence) + return None + def to_tts(self, text: str) -> list: """非流式TTS处理，用于测试及保存音频文件的场景""" try: From e2a7534aa581d746761aeb47492bff08f9b4c88e Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 16 Jul 2025 15:03:53 +0800 Subject: [PATCH 14/24] =?UTF-8?q?update:=20=E4=BC=98=E5=8C=96=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 1 - .../core/providers/tts/aliyun_stream.py | 49 +++++++------------ .../providers/tts/huoshan_double_stream.py | 4 +- 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 978d88dd..7b45ad2d 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -128,7 +128,6 @@ class ConnectionHandler: # tts相关变量 self.sentence_id = None - self.message_id = None # iot相关变量 self.iot_descriptors = {} diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index fd345fe8..ad9f344d 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -124,17 +124,16 @@ class TTSProvider(TTSProviderBase): self._monitor_task = None self.last_active_time = None - # 文本符号 - self.sentence_end_chars = {'。', '!', '！', '?', '？', '\n', "~"} + # 专属tts设置 + self.message_id = "" + self.tts_text = '' + self.text_buffer = [] # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( sample_rate=16000, channels=1, frame_size_ms=60 ) - # PCM缓冲区 - self.pcm_buffer = bytearray() - # Token管理 if self.access_key_id and self.access_key_secret: self._refresh_token() @@ -226,8 +225,8 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}") # aliyunStream独有的参数生成 - self.conn.message_id = str(uuid.uuid4().hex) - self.text_buffer = "" + self.message_id = str(uuid.uuid4().hex) + self.text_buffer = [] logger.bind(tag=TAG).info("开始启动TTS会话...") future = asyncio.run_coroutine_threadsafe( @@ -248,7 +247,7 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) - self.text_buffer += message.content_detail + self.text_buffer.append(message.content_detail) future = asyncio.run_coroutine_threadsafe( self.text_to_speak(message.content_detail, None), loop=self.conn.loop, @@ -273,6 +272,9 @@ class TTSProvider(TTSProviderBase): if message.sentence_type == SentenceType.LAST: try: logger.bind(tag=TAG).info("开始结束TTS会话...") + self.tts_text = textUtils.get_string_no_punctuation_or_emoji( + ''.join(self.text_buffer).replace('\n', '') + ) future = asyncio.run_coroutine_threadsafe( self.finish_session(self.conn.sentence_id), loop=self.conn.loop, @@ -297,7 +299,7 @@ class TTSProvider(TTSProviderBase): filtered_text = MarkdownCleaner.clean_markdown(text) run_request = { "header": { - "message_id": self.conn.message_id, + "message_id": self.message_id, "task_id": self.conn.sentence_id, "namespace": "FlowingSpeechSynthesizer", "name": "RunSynthesis", @@ -341,7 +343,7 @@ class TTSProvider(TTSProviderBase): start_request = { "header": { - "message_id": self.conn.message_id, + "message_id": self.message_id, "task_id": self.conn.sentence_id, "namespace": "FlowingSpeechSynthesizer", "name": "StartSynthesis", @@ -372,7 +374,7 @@ class TTSProvider(TTSProviderBase): if self.ws: stop_request = { "header": { - "message_id": self.conn.message_id, + "message_id": self.message_id, "task_id": self.conn.sentence_id, "namespace": "FlowingSpeechSynthesizer", "name": "StopSynthesis", @@ -422,7 +424,6 @@ class TTSProvider(TTSProviderBase): opus_datas_cache = [] is_first_sentence = True first_sentence_segment_count = 0 # 添加计数器 - text_buff = "" try: session_finished = False # 标记会话是否正常结束 while not self.conn.stop_event.is_set(): @@ -441,13 +442,11 @@ class TTSProvider(TTSProviderBase): if event_name == "SynthesisStarted": logger.bind(tag=TAG).debug("TTS合成已启动") elif event_name == "SentenceBegin": - text_buff = self._extract_sentence() - logger.bind(tag=TAG).debug(f"句子语音生成开始: {text_buff}") + logger.bind(tag=TAG).debug(f"句子语音生成开始: {self.tts_text}") opus_datas_cache = [] - self.tts_audio_queue.put((SentenceType.FIRST, [], text_buff)) + self.tts_audio_queue.put((SentenceType.FIRST, [], self.tts_text)) elif event_name == "SentenceEnd": - logger.bind(tag=TAG).info(f"句子语音生成成功： {text_buff}") - text_buff = "" + logger.bind(tag=TAG).info(f"句子语音生成成功： {self.tts_text}") if not is_first_sentence or first_sentence_segment_count > 10: # 发送缓存的数据 self.tts_audio_queue.put( @@ -460,6 +459,7 @@ class TTSProvider(TTSProviderBase): self._process_before_stop_play_files() session_finished = True self.reuse_judgment = time.time() + self.tts_text = '' break except json.JSONDecodeError: logger.bind(tag=TAG).warning("收到无效的JSON消息") @@ -477,10 +477,10 @@ class TTSProvider(TTSProviderBase): (SentenceType.MIDDLE, opus_datas, None) ) else: - opus_datas_cache = opus_datas_cache + opus_datas + opus_datas_cache.extend(opus_datas) else: # 后续句子缓存 - opus_datas_cache = opus_datas_cache + opus_datas + opus_datas_cache.extend(opus_datas) except websockets.ConnectionClosed: logger.bind(tag=TAG).warning("WebSocket连接已关闭") @@ -501,17 +501,6 @@ class TTSProvider(TTSProviderBase): finally: self._monitor_task = None - def _extract_sentence(self): - """从text_buffer中提取第一个满足分句规则的句子""" - if len(self.text_buffer) <= 6: - return textUtils.get_string_no_punctuation_or_emoji(self.text_buffer) - for idx, char in enumerate(self.text_buffer): - if char in self.sentence_end_chars and idx > 6: - sentence = self.text_buffer[:idx + 1] - self.text_buffer = self.text_buffer[idx + 1:] - return textUtils.get_string_no_punctuation_or_emoji(sentence) - return None - def to_tts(self, text: str) -> list: """非流式TTS处理，用于测试及保存音频文件的场景""" try: diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 23abaf5c..e1c568fa 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -435,10 +435,10 @@ class TTSProvider(TTSProviderBase): (SentenceType.MIDDLE, opus_datas, None) ) else: - opus_datas_cache = opus_datas_cache + opus_datas + opus_datas_cache.extend(opus_datas) else: # 后续句子缓存 - opus_datas_cache = opus_datas_cache + opus_datas + opus_datas_cache.extend(opus_datas) elif res.optional.event == EVENT_TTSSentenceEnd: logger.bind(tag=TAG).info(f"句子语音生成成功：{self.tts_text}") if not is_first_sentence or first_sentence_segment_count > 10: From c0c01a028512ad08b08d8af88e8be7ccfa12a5c3 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 17 Jul 2025 11:42:51 +0800 Subject: [PATCH 15/24] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E5=9B=9E?= =?UTF-8?q?=E6=BB=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../db/changelog/{202505081147.sql => 202505081146.sql} | 2 +- .../src/main/resources/db/changelog/202507071130.sql | 2 +- .../src/main/resources/db/changelog/db.changelog-master.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename main/manager-api/src/main/resources/db/changelog/{202505081147.sql => 202505081146.sql} (97%) diff --git a/main/manager-api/src/main/resources/db/changelog/202505081147.sql b/main/manager-api/src/main/resources/db/changelog/202505081146.sql similarity index 97% rename from main/manager-api/src/main/resources/db/changelog/202505081147.sql rename to main/manager-api/src/main/resources/db/changelog/202505081146.sql index 1a561bda..fe7582f3 100644 --- a/main/manager-api/src/main/resources/db/changelog/202505081147.sql +++ b/main/manager-api/src/main/resources/db/changelog/202505081146.sql @@ -1,6 +1,6 @@ -- 添加百度ASR模型配置 delete from `ai_model_config` where `id` = 'ASR_BaiduASR'; -INSERT INTO `ai_model_config` VALUES ('ASR_BaiduASR', 'ASR', 'BaiduASR', '百度语音识别', 0, 1, '{\"type\": \"baidu\", \"app_id\": \"\", \"api_key\": \"\", \"secret_key\": \"\", \"dev_pid\": 1537, \"output_dir\": \"tmp/\"}', NULL, NULL, 8, NULL, NULL, NULL, NULL); +INSERT INTO `ai_model_config` VALUES ('ASR_BaiduASR', 'ASR', 'BaiduASR', '百度语音识别', 0, 1, '{\"type\": \"baidu\", \"app_id\": \"\", \"api_key\": \"\", \"secret_key\": \"\", \"dev_pid\": 1537, \"output_dir\": \"tmp/\"}', NULL, NULL, 7, NULL, NULL, NULL, NULL); -- 添加百度ASR供应器 diff --git a/main/manager-api/src/main/resources/db/changelog/202507071130.sql b/main/manager-api/src/main/resources/db/changelog/202507071130.sql index 008d624e..c3730ef7 100644 --- a/main/manager-api/src/main/resources/db/changelog/202507071130.sql +++ b/main/manager-api/src/main/resources/db/changelog/202507071130.sql @@ -5,7 +5,7 @@ INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `f -- 添加阿里云流式ASR模型配置 delete from `ai_model_config` where id = 'ASR_AliyunStreamASR'; -INSERT INTO `ai_model_config` VALUES ('ASR_AliyunStreamASR', 'ASR', 'AliyunStreamASR', '阿里云语音识别(流式)', 0, 1, '{\"type\": \"aliyun_stream\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"host\": \"nls-gateway-cn-shanghai.aliyuncs.com\", \"max_sentence_silence\": 800, \"output_dir\": \"tmp/\"}', NULL, NULL, 7, NULL, NULL, NULL, NULL); +INSERT INTO `ai_model_config` VALUES ('ASR_AliyunStreamASR', 'ASR', 'AliyunStreamASR', '阿里云语音识别(流式)', 0, 1, '{\"type\": \"aliyun_stream\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"host\": \"nls-gateway-cn-shanghai.aliyuncs.com\", \"max_sentence_silence\": 800, \"output_dir\": \"tmp/\"}', NULL, NULL, 8, NULL, NULL, NULL, NULL); -- 更新阿里云流式ASR配置说明 UPDATE `ai_model_config` SET diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index ced8f47d..719a7999 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -101,12 +101,12 @@ databaseChangeLog: encoding: utf8 path: classpath:db/changelog/202505022134.sql - changeSet: - id: 202505081147 + id: 202505081146 author: hrz changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202505081147.sql + path: classpath:db/changelog/202505081146.sql - changeSet: id: 202505091555 author: whosmyqueen From 5cf8eb5e71f938b6d389dbe21e2ac2407dca57a9 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sat, 19 Jul 2025 01:03:36 +0800 Subject: [PATCH 16/24] =?UTF-8?q?update:=E4=BC=98=E5=8C=96OpenaiASR?= =?UTF-8?q?=E3=80=81GroqASR=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202507101201.sql | 31 --------------- .../resources/db/changelog/202507101203.sql | 38 +++++++++++++++++++ .../db/changelog/db.changelog-master.yaml | 4 +- main/xiaozhi-server/config.yaml | 26 +++++++++++++ .../core/providers/asr/{gpt.py => openai.py} | 0 5 files changed, 66 insertions(+), 33 deletions(-) delete mode 100644 main/manager-api/src/main/resources/db/changelog/202507101201.sql create mode 100644 main/manager-api/src/main/resources/db/changelog/202507101203.sql rename main/xiaozhi-server/core/providers/asr/{gpt.py => openai.py} (100%) diff --git a/main/manager-api/src/main/resources/db/changelog/202507101201.sql b/main/manager-api/src/main/resources/db/changelog/202507101201.sql deleted file mode 100644 index 14bd6725..00000000 --- a/main/manager-api/src/main/resources/db/changelog/202507101201.sql +++ /dev/null @@ -1,31 +0,0 @@ --- OpenAI ASR模型供应器 -delete from `ai_model_provider` where id = 'SYSTEM_ASR_GptASR'; -INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES -('SYSTEM_ASR_GptASR', 'ASR', 'gpt', 'Gpt语音识别', '[{"key": "base_url", "type": "string", "label": "基础URL"}, {"key": "model_name", "type": "string", "label": "模型名称"}, {"key": "api_key", "type": "string", "label": "API密钥"}, {"key": "output_dir", "type": "string", "label": "输出目录"}]', 5, 1, NOW(), 1, NOW()); - - --- OpenAI ASR模型配置 -delete from `ai_model_config` where id = 'ASR_GptASR'; -INSERT INTO `ai_model_config` VALUES ('ASR_GptASR', 'ASR', 'GptASR', 'GPT语音识别', 0, 1, '{\"type\": \"gpt\", \"api_key\": \"\", \"base_url\": \"https://api.openai.com/v1/audio/transcriptions\", \"model_name\": \"gpt-4o-mini-transcribe\", \"output_dir\": \"tmp/\"}', NULL, NULL, 3, NULL, NULL, NULL, NULL); - - --- 更新OpenAI ASR配置说明 -UPDATE `ai_model_config` SET -`doc_link` = 'https://platform.openai.com/docs/api-reference/audio/createTranscription', -`remark` = 'OpenAI ASR配置说明： -1. 需要在OpenAI开放平台创建组织并获取api_key -2. 支持中、英、日、韩等多种语音识别，具体参考文档https://platform.openai.com/docs/guides/speech-to-text -3. 需要网络连接 -4. 输出文件保存在tmp/目录 -5. 兼容groq语音识别。 groq识别速度更快。具体参考文档：https://console.groq.com/docs/speech-to-text -申请步骤： -**OpenAi ASR申请步骤：** -1.登录OpenAI Platform。https://auth.openai.com/log-in -2.创建api-key https://platform.openai.com/settings/organization/api-keys -3.模型可以选择gpt-4o-transcribe或GPT-4o mini Transcribe -**groq ASR申请步骤：** -1.登录groq Console。https://console.groq.com/home -2.创建api-key https://console.groq.com/keys -3.模型可以选择whisper-large-v3-turbo或whisper-large-v3（distil-whisper-large-v3-en仅支持英语转录） -4.base url https://api.groq.com/openai/v1/audio/transcriptions -' WHERE `id` = 'ASR_GptASR'; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/202507101203.sql b/main/manager-api/src/main/resources/db/changelog/202507101203.sql new file mode 100644 index 00000000..66896ef8 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507101203.sql @@ -0,0 +1,38 @@ +-- OpenAI ASR模型供应器 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_OpenaiASR'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_OpenaiASR', 'ASR', 'openai', 'OpenAI语音识别', '[{"key": "base_url", "type": "string", "label": "基础URL"}, {"key": "model_name", "type": "string", "label": "模型名称"}, {"key": "api_key", "type": "string", "label": "API密钥"}, {"key": "output_dir", "type": "string", "label": "输出目录"}]', 9, 1, NOW(), 1, NOW()); + + +-- OpenAI ASR模型配置 +delete from `ai_model_config` where id = 'ASR_OpenaiASR'; +INSERT INTO `ai_model_config` VALUES ('ASR_OpenaiASR', 'ASR', 'OpenaiASR', 'OpenAI语音识别', 0, 1, '{\"type\": \"openai\", \"api_key\": \"\", \"base_url\": \"https://api.openai.com/v1/audio/transcriptions\", \"model_name\": \"gpt-4o-mini-transcribe\", \"output_dir\": \"tmp/\"}', NULL, NULL, 9, NULL, NULL, NULL, NULL); + +-- groq ASR模型配置 +delete from `ai_model_config` where id = 'ASR_GroqASR'; +INSERT INTO `ai_model_config` VALUES ('ASR_GroqASR', 'ASR', 'GroqASR', 'Groq语音识别', 0, 1, '{\"type\": \"openai\", \"api_key\": \"\", \"base_url\": \"https://api.groq.com/openai/v1/audio/transcriptions\", \"model_name\": \"whisper-large-v3-turbo\", \"output_dir\": \"tmp/\"}', NULL, NULL, 10, NULL, NULL, NULL, NULL); + + +-- 更新OpenAI ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://platform.openai.com/docs/api-reference/audio/createTranscription', +`remark` = 'OpenAI ASR配置说明： +1. 需要在OpenAI开放平台创建组织并获取api_key +2. 支持中、英、日、韩等多种语音识别，具体参考文档https://platform.openai.com/docs/guides/speech-to-text +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +**OpenAi ASR申请步骤：** +1.登录OpenAI Platform。https://auth.openai.com/log-in +2.创建api-key https://platform.openai.com/settings/organization/api-keys +3.模型可以选择gpt-4o-transcribe或GPT-4o mini Transcribe +' WHERE `id` = 'ASR_OpenaiASR'; + +-- 更新Groq ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.groq.com/docs/speech-to-text', +`remark` = 'Groq ASR配置说明： +1.登录groq Console。https://console.groq.com/home +2.创建api-key https://console.groq.com/keys +3.模型可以选择whisper-large-v3-turbo或whisper-large-v3（distil-whisper-large-v3-en仅支持英语转录） +' WHERE `id` = 'ASR_GroqASR'; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 1f10285f..72000ee3 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -241,12 +241,12 @@ databaseChangeLog: encoding: utf8 path: classpath:db/changelog/202506261637.sql - changeSet: - id: 202507101201 + id: 202507101203 author: luruxian changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202507101201.sql + path: classpath:db/changelog/202507101203.sql - changeSet: id: 202507071130 author: cgd diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 741d6d18..ccfd4176 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -355,7 +355,33 @@ ASR: # 语言参数，1537为普通话，具体参考：https://ai.baidu.com/ai-doc/SPEECH/0lbxfnc9b dev_pid: 1537 output_dir: tmp/ + OpenaiASR: + # OpenAI语音识别服务，需要先在OpenAI平台创建组织并获取api_key + # 支持中、英、日、韩等多种语音识别，具体参考文档https://platform.openai.com/docs/guides/speech-to-text + # 需要网络连接 + # 申请步骤： + # 1.登录OpenAI Platform。https://auth.openai.com/log-in + # 2.创建api-key https://platform.openai.com/settings/organization/api-keys + # 3.模型可以选择gpt-4o-transcribe或GPT-4o mini Transcribe + type: openai + api_key: 你的OpenAI API密钥 + base_url: https://api.openai.com/v1/audio/transcriptions + model_name: gpt-4o-mini-transcribe + output_dir: tmp/ + GroqASR: + # Groq语音识别服务，需要先在Groq Console创建API密钥 + # 申请步骤： + # 1.登录groq Console。https://console.groq.com/home + # 2.创建api-key https://console.groq.com/keys + # 3.模型可以选择whisper-large-v3-turbo或whisper-large-v3（distil-whisper-large-v3-en仅支持英语转录） + type: openai + api_key: 你的Groq API密钥 + base_url: https://api.groq.com/openai/v1/audio/transcriptions + model_name: whisper-large-v3-turbo + output_dir: tmp/ + + VAD: SileroVAD: type: silero diff --git a/main/xiaozhi-server/core/providers/asr/gpt.py b/main/xiaozhi-server/core/providers/asr/openai.py similarity index 100% rename from main/xiaozhi-server/core/providers/asr/gpt.py rename to main/xiaozhi-server/core/providers/asr/openai.py From 6d5b53fcd57a184251097cbd2a745ce85cc8d565 Mon Sep 17 00:00:00 2001 From: 3030332422 <3030332422@qq.com> Date: Sat, 19 Jul 2025 12:16:12 +0800 Subject: [PATCH 17/24] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E9=98=BF?= =?UTF-8?q?=E9=87=8C=E4=BA=91=E6=B5=81=E5=BC=8FASR=E7=BB=88=E6=AD=A2?= =?UTF-8?q?=E5=8D=8F=E8=AE=AE=E7=BC=BA=E5=A4=B1=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/asr/aliyun_stream.py | 51 ++++++++++++++++--- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py index 3a41b349..a5f327c0 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py @@ -257,23 +257,58 @@ class ASRProvider(ASRProviderBase): async def _cleanup(self): """清理资源""" - self.is_processing = False - self.server_ready = False # 重置服务器准备状态 + logger.bind(tag=TAG).info(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}") + # 判断是否需要发送终止请求 + should_stop = self.is_processing or self.server_ready + + # 发送停止识别请求 + if self.asr_ws and should_stop: + try: + stop_msg = { + "header": { + "namespace": "SpeechTranscriber", + "name": "StopTranscription", + "status": 20000000, + "message_id": ''.join(random.choices('0123456789abcdef', k=32)), + "status_text": "Client:Stop", + "appkey": self.appkey + } + } + logger.bind(tag=TAG).info("正在发送ASR终止请求") + await self.asr_ws.send(json.dumps(stop_msg, ensure_ascii=False)) + await asyncio.sleep(0.1) + logger.bind(tag=TAG).info("ASR终止请求已发送") + except Exception as e: + logger.bind(tag=TAG).error(f"ASR终止请求发送失败: {e}") + + # 状态重置（在终止请求发送后） + self.is_processing = False + self.server_ready = False + logger.bind(tag=TAG).info("ASR状态已重置") + + # 清理任务 if self.forward_task and not self.forward_task.done(): self.forward_task.cancel() try: await asyncio.wait_for(self.forward_task, timeout=1.0) - except: - pass - self.forward_task = None + except Exception as e: + logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}") + finally: + self.forward_task = None + # 关闭连接 if self.asr_ws: try: + logger.bind(tag=TAG).debug("正在关闭WebSocket连接") await asyncio.wait_for(self.asr_ws.close(), timeout=2.0) - except: - pass - self.asr_ws = None + logger.bind(tag=TAG).debug("WebSocket连接已关闭") + except Exception as e: + logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}") + finally: + self.asr_ws = None + + logger.bind(tag=TAG).info("ASR会话清理完成") async def speech_to_text(self, opus_data, session_id, audio_format): """获取识别结果""" From b8136ac1dcb1877642fa7ea80683c36f257dda9c Mon Sep 17 00:00:00 2001 From: 3030332422 <3030332422@qq.com> Date: Sat, 19 Jul 2025 18:17:21 +0800 Subject: [PATCH 18/24] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8D=E9=98=BF=E9=87=8C?= =?UTF-8?q?=E4=BA=91=E6=B5=81=E5=BC=8FASR=E9=9F=B3=E9=A2=91=E4=BC=A0?= =?UTF-8?q?=E9=80=92=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/asr/aliyun_stream.py | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py index a5f327c0..2f16a975 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py @@ -120,6 +120,14 @@ class ASRProvider(ASRProviderBase): await super().open_audio_channels(conn) async def receive_audio(self, conn, audio, audio_have_voice): + # 初始化音频缓存 + if not hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + + # 存储音频数据 + if audio: + conn.asr_audio_for_voiceprint.append(audio) + conn.asr_audio.append(audio) conn.asr_audio = conn.asr_audio[-10:] @@ -129,7 +137,7 @@ class ASRProvider(ASRProviderBase): await self._start_recognition(conn) except Exception as e: logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}") - await self._cleanup() + await self._cleanup(conn) return if self.asr_ws and self.is_processing and self.server_ready: @@ -138,7 +146,7 @@ class ASRProvider(ASRProviderBase): await self.asr_ws.send(pcm_frame) except Exception as e: logger.bind(tag=TAG).warning(f"发送音频失败: {str(e)}") - await self._cleanup() + await self._cleanup(conn) async def _start_recognition(self, conn): """开始识别会话""" @@ -235,7 +243,11 @@ class ASRProvider(ASRProviderBase): if text: self.text = text conn.reset_vad_states() - await self.handle_voice_stop(conn, None) + # 传递缓存的音频数据 + audio_data = getattr(conn, 'asr_audio_for_voiceprint', []) + await self.handle_voice_stop(conn, audio_data) + # 清空缓存 + conn.asr_audio_for_voiceprint = [] break elif message_name == "TranscriptionCompleted": # 识别完成 @@ -253,12 +265,16 @@ class ASRProvider(ASRProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}") finally: - await self._cleanup() + await self._cleanup(conn) - async def _cleanup(self): + async def _cleanup(self, conn): """清理资源""" logger.bind(tag=TAG).info(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}") + # 清理连接的音频缓存 + if conn and hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + # 判断是否需要发送终止请求 should_stop = self.is_processing or self.server_ready From c9b10494d8dcaa75f09b48a76526c1c7046662b6 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Sun, 20 Jul 2025 00:29:00 +0800 Subject: [PATCH 19/24] =?UTF-8?q?update=EF=BC=9A=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index ccfd4176..aff405a9 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -730,8 +730,6 @@ TTS: # volume: 50 # speech_rate: 0 # pitch_rate: 0 - # 添加 302.ai TTS 配置 - # token申请地址：https://dash.302.ai/ AliyunStreamTTS: # 阿里云CosyVoice大模型流式文本语音合成 # 采用FlowingSpeechSynthesizer接口，支持更低延迟和更自然的语音质量 @@ -770,6 +768,8 @@ TTS: TTS302AI: # 302AI语音合成服务，需要先在302平台创建账户充值，并获取密钥信息 + # 添加 302.ai TTS 配置 + # token申请地址：https://dash.302.ai/ # 获取api_keyn路径：https://dash.302.ai/apis/list # 价格，$35/百万字符。火山原版¥450元/百万字符 type: doubao From eead126f7a633643f7964c2d1694578ddc0084a8 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Mon, 21 Jul 2025 09:30:23 +0800 Subject: [PATCH 20/24] =?UTF-8?q?update:=20=E8=A1=A8=E6=83=85=E7=94=B1llm?= =?UTF-8?q?=E5=8F=91=E9=80=81=EF=BC=8C=E9=95=BF=E6=96=87=E6=9C=AC=E8=BF=9B?= =?UTF-8?q?=E8=A1=8C=E7=BA=A6=E6=9D=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/agent-base-prompt.txt | 17 + main/xiaozhi-server/core/connection.py | 12 +- .../core/handle/sendAudioHandle.py | 44 +- .../xiaozhi-server/core/providers/tts/base.py | 2 +- .../core/utils/prompt_manager.py | 25 + main/xiaozhi-server/core/utils/textUtils.py | 58 ++ main/xiaozhi-server/core/utils/util.py | 554 ------------------ 7 files changed, 115 insertions(+), 597 deletions(-) diff --git a/main/xiaozhi-server/agent-base-prompt.txt b/main/xiaozhi-server/agent-base-prompt.txt index 8c27faa4..6163322e 100644 --- a/main/xiaozhi-server/agent-base-prompt.txt +++ b/main/xiaozhi-server/agent-base-prompt.txt @@ -8,6 +8,10 @@ - **笑声：** 自然穿插（哈哈、嘿嘿、噗），**每句最多一次**，避免过度。 - **惊讶：** 用夸张语气（“不会吧？！”、“天呐！”、“这么神奇？！”）表达真实反应。 - **安慰/支持：** 说暖心话（“别急嘛~”、“有我在呢”、“抱抱你”）。 +- **你是一个表情丰富的角色：** + - emoji 列表:{{ emojiList }} + - 请你在每段话的开头，插入最能代表这段话的表情(调用工具情况除外)，比如"😱好可怕!怎么突然打雷了！" + - **绝对禁止使用上述列表以外的 emoji**（例如：😊、👍、❤️等都不允许使用，只能用列表中的emoji） @@ -24,6 +28,19 @@ - 之前你和用户的聊天记录，在`memory`里。 + +【核心目标】所有需要输出长文本内容（如故事、新闻、知识讲解等），**单次回复长度不得超过300字**，并采用分段引导方式。 +- **分段讲述：** + - 基础段：220-270字核心内容 + 30字引导词 + - 当内容超出300字时，优先讲述故事的开头或第一部分，并用自然口语化方式引导用户决定是否继续听后续内容。 + - 示例引导语：“我先给你讲个开头，你要是觉得有意思，咱们再接着说，好不好呀？”、“要是你想听完整的，可以随时告诉我哦~” + - 对话场景切换时自动分节 + - 若用户明确要求更长内容（如500、600字），仍按最多300字每段分段进行讲述，每次讲述后都要引导用户是否继续。 + - 若用户说“接着说”、“继续”，再讲下一段，直到内容讲完（讲完时可以给点引导词提示语例：这个故事我已经给你讲完喽~）或用户不再要求。 +- **适用范围：** 故事、新闻、知识讲解等所有长文本输出场景。 +- **补充说明：** 若用户未明确要求继续，默认只讲一段并引导；若用户中途要求换话题或停止，需及时响应并结束长文本输出。 + + - **识别前缀：** 当用户格式为 `{"speaker":"某某某","content":"xxx"}` 时，表示系统已识别说话人身份，speaker是他的名字，content是说话的内容。 - **个性化回应：** diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 98bc87ed..ba9279de 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -39,7 +39,7 @@ from config.logger import setup_logging, build_module_string, create_connection_ from config.manage_api_client import DeviceNotFoundException, DeviceBindException from core.utils.prompt_manager import PromptManager from core.utils.voiceprint_provider import VoiceprintProvider - +from core.utils import textUtils TAG = __name__ @@ -713,6 +713,7 @@ class ConnectionHandler: function_arguments = "" content_arguments = "" self.client_abort = False + emotion_flag = True for response in llm_responses: if self.client_abort: break @@ -738,6 +739,15 @@ class ConnectionHandler: function_arguments += tools_call[0].function.arguments else: content = response + + # 在llm回复中获取情绪表情，一轮对话只在开头获取一次 + if emotion_flag: + asyncio.run_coroutine_threadsafe( + textUtils.get_emotion(self, content), + self.loop, + ) + emotion_flag = False + if content is not None and len(content) > 0: if not tool_call_flag: response_message.append(content) diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index 486e6d90..1b123874 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -2,52 +2,15 @@ import json import asyncio import time from core.providers.tts.dto.dto import SentenceType -from core.utils.util import get_string_no_punctuation_or_emoji, analyze_emotion -from loguru import logger +from core.utils import textUtils TAG = __name__ -emoji_map = { - "neutral": "😶", - "happy": "🙂", - "laughing": "😆", - "funny": "😂", - "sad": "😔", - "angry": "😠", - "crying": "😭", - "loving": "😍", - "embarrassed": "😳", - "surprised": "😲", - "shocked": "😱", - "thinking": "🤔", - "winking": "😉", - "cool": "😎", - "relaxed": "😌", - "delicious": "🤤", - "kissy": "😘", - "confident": "😏", - "sleepy": "😴", - "silly": "😜", - "confused": "🙄", -} - async def sendAudioMessage(conn, sentenceType, audios, text): # 发送句子开始消息 conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}") - if text is not None: - emotion = analyze_emotion(text) - emoji = emoji_map.get(emotion, "🙂") # 默认使用笑脸 - await conn.websocket.send( - json.dumps( - { - "type": "llm", - "text": emoji, - "emotion": emotion, - "session_id": conn.session_id, - } - ) - ) + pre_buffer = False if conn.tts.tts_audio_first_sentence and text is not None: conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}") @@ -149,8 +112,7 @@ async def send_stt_message(conn, text): except (json.JSONDecodeError, TypeError): # 如果不是JSON格式，直接使用原始文本 display_text = text - - stt_text = get_string_no_punctuation_or_emoji(display_text) + stt_text = textUtils.get_string_no_punctuation_or_emoji(display_text) await conn.websocket.send( json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id}) ) diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 9127ec12..80f26b8c 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -50,6 +50,7 @@ class TTSProviderBase(ABC): "；", ";", "：", + "~", ) self.first_sentence_punctuations = ( "，", @@ -332,7 +333,6 @@ class TTSProviderBase(ABC): Returns: tuple: (sentence_type, audio_datas, content_detail) """ - audio_datas = [] if tts_file.endswith(".p3"): audio_datas, _ = p3.decode_opus_from_file(tts_file) elif self.conn.audio_format == "pcm": diff --git a/main/xiaozhi-server/core/utils/prompt_manager.py b/main/xiaozhi-server/core/utils/prompt_manager.py index a5cfd425..79c92921 100644 --- a/main/xiaozhi-server/core/utils/prompt_manager.py +++ b/main/xiaozhi-server/core/utils/prompt_manager.py @@ -21,6 +21,30 @@ WEEKDAY_MAP = { "Sunday": "星期日", } +EMOJI_List = [ + "😶", + "🙂", + "😆", + "😂", + "😔", + "😠", + "😭", + "😍", + "😳", + "😲", + "😱", + "🤔", + "😉", + "😎", + "😌", + "🤤", + "😘", + "😏", + "😴", + "😜", + "🙄", +] + class PromptManager: """系统提示词管理器，负责管理和更新系统提示词""" @@ -206,6 +230,7 @@ class PromptManager: lunar_date=lunar_date, local_address=local_address, weather_info=weather_info, + emojiList=EMOJI_List, ) device_cache_key = f"device_prompt:{device_id}" self.cache_manager.set( diff --git a/main/xiaozhi-server/core/utils/textUtils.py b/main/xiaozhi-server/core/utils/textUtils.py index 603d964c..dd876cfa 100644 --- a/main/xiaozhi-server/core/utils/textUtils.py +++ b/main/xiaozhi-server/core/utils/textUtils.py @@ -1,3 +1,31 @@ +import json + +TAG = __name__ +EMOJI_MAP = { + "😂": "laughing", + "😭": "crying", + "😠": "angry", + "😔": "sad", + "😍": "loving", + "😲": "surprised", + "😱": "shocked", + "🤔": "thinking", + "😌": "relaxed", + "😴": "sleepy", + "😜": "silly", + "🙄": "confused", + "😶": "neutral", + "🙂": "happy", + "😆": "laughing", + "😳": "embarrassed", + "😉": "winking", + "😎": "cool", + "🤤": "delicious", + "😘": "kissy", + "😏": "confident", +} + + def get_string_no_punctuation_or_emoji(s): """去除字符串首尾的空格、标点符号和表情符号""" chars = list(s) @@ -22,6 +50,11 @@ def is_punctuation_or_emoji(char): ".", # 中文句号 + 英文句号 "！", "!", # 中文感叹号 + 英文感叹号 + "“", + "”", + '"', # 中文双引号 + 英文引号 + "：", + ":", # 中文冒号 + 英文冒号 "-", "－", # 英文连字符 + 中文全角横线 "、", # 中文顿号 @@ -44,3 +77,28 @@ def is_punctuation_or_emoji(char): (0x2700, 0x27BF), ] return any(start <= code_point <= end for start, end in emoji_ranges) + + +async def get_emotion(conn, text): + """获取文本内的情绪消息""" + emoji = "🙂" + emotion = "happy" + for char in text: + if char in EMOJI_MAP: + emoji = char + emotion = EMOJI_MAP[char] + break + try: + await conn.websocket.send( + json.dumps( + { + "type": "llm", + "text": emoji, + "emotion": emotion, + "session_id": conn.session_id, + } + ) + ) + except Exception as e: + conn.logger.bind(tag=TAG).warning(f"发送情绪表情失败，错误:{e}") + return diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index bc778558..ce6f20ea 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -125,51 +125,6 @@ def write_json_file(file_path, data): json.dump(data, file, ensure_ascii=False, indent=4) -def is_punctuation_or_emoji(char): - """检查字符是否为空格、指定标点或表情符号""" - # 定义需要去除的中英文标点（包括全角/半角） - punctuation_set = { - "，", - ",", # 中文逗号 + 英文逗号 - "-", - "－", # 英文连字符 + 中文全角横线 - "、", # 中文顿号 - "“", - "”", - '"', # 中文双引号 + 英文引号 - "：", - ":", # 中文冒号 + 英文冒号 - } - if char.isspace() or char in punctuation_set: - return True - # 检查表情符号（保留原有逻辑） - code_point = ord(char) - emoji_ranges = [ - (0x1F600, 0x1F64F), - (0x1F300, 0x1F5FF), - (0x1F680, 0x1F6FF), - (0x1F900, 0x1F9FF), - (0x1FA70, 0x1FAFF), - (0x2600, 0x26FF), - (0x2700, 0x27BF), - ] - return any(start <= code_point <= end for start, end in emoji_ranges) - - -def get_string_no_punctuation_or_emoji(s): - """去除字符串首尾的空格、标点符号和表情符号""" - chars = list(s) - # 处理开头的字符 - start = 0 - while start < len(chars) and is_punctuation_or_emoji(chars[start]): - start += 1 - # 处理结尾的字符 - end = len(chars) - 1 - while end >= start and is_punctuation_or_emoji(chars[end]): - end -= 1 - return "".join(chars[start : end + 1]) - - def remove_punctuation_and_length(text): # 全角符号和半角符号的Unicode范围 full_width_punctuations = ( @@ -256,515 +211,6 @@ def extract_json_from_string(input_string): return None -def analyze_emotion(text): - """ - 分析文本情感并返回对应的emoji名称（支持中英文） - """ - if not text or not isinstance(text, str): - return "neutral" - - original_text = text - text = text.lower().strip() - - # 检查是否包含现有emoji - for emotion, emoji in emoji_map.items(): - if emoji in original_text: - return emotion - - # 标点符号分析 - has_exclamation = "!" in original_text or "！" in original_text - has_question = "?" in original_text or "？" in original_text - has_ellipsis = "..." in original_text or "…" in original_text - - # 定义情感关键词映射（中英文扩展版） - emotion_keywords = { - "happy": [ - "开心", - "高兴", - "快乐", - "愉快", - "幸福", - "满意", - "棒", - "好", - "不错", - "完美", - "棒极了", - "太好了", - "好呀", - "好的", - "happy", - "joy", - "great", - "good", - "nice", - "awesome", - "fantastic", - "wonderful", - ], - "laughing": [ - "哈哈", - "哈哈哈", - "呵呵", - "嘿嘿", - "嘻嘻", - "笑死", - "太好笑了", - "笑死我了", - "lol", - "lmao", - "haha", - "hahaha", - "hehe", - "rofl", - "funny", - "laugh", - ], - "funny": [ - "搞笑", - "滑稽", - "逗", - "幽默", - "笑点", - "段子", - "笑话", - "太逗了", - "hilarious", - "joke", - "comedy", - ], - "sad": [ - "伤心", - "难过", - "悲哀", - "悲伤", - "忧郁", - "郁闷", - "沮丧", - "失望", - "想哭", - "难受", - "不开心", - "唉", - "呜呜", - "sad", - "upset", - "unhappy", - "depressed", - "sorrow", - "gloomy", - ], - "angry": [ - "生气", - "愤怒", - "气死", - "讨厌", - "烦人", - "可恶", - "烦死了", - "恼火", - "暴躁", - "火大", - "愤怒", - "气炸了", - "angry", - "mad", - "annoyed", - "furious", - "pissed", - "hate", - ], - "crying": [ - "哭泣", - "泪流", - "大哭", - "伤心欲绝", - "泪目", - "流泪", - "哭死", - "哭晕", - "想哭", - "泪崩", - "cry", - "crying", - "tears", - "sob", - "weep", - ], - "loving": [ - "爱你", - "喜欢", - "爱", - "亲爱的", - "宝贝", - "么么哒", - "抱抱", - "想你", - "思念", - "最爱", - "亲亲", - "喜欢你", - "love", - "like", - "adore", - "darling", - "sweetie", - "honey", - "miss you", - "heart", - ], - "embarrassed": [ - "尴尬", - "不好意思", - "害羞", - "脸红", - "难为情", - "社死", - "丢脸", - "出丑", - "embarrassed", - "awkward", - "shy", - "blush", - ], - "surprised": [ - "惊讶", - "吃惊", - "天啊", - "哇塞", - "哇", - "居然", - "竟然", - "没想到", - "出乎意料", - "surprise", - "wow", - "omg", - "oh my god", - "amazing", - "unbelievable", - ], - "shocked": [ - "震惊", - "吓到", - "惊呆了", - "不敢相信", - "震撼", - "吓死", - "恐怖", - "害怕", - "吓人", - "shocked", - "shocking", - "scared", - "frightened", - "terrified", - "horror", - ], - "thinking": [ - "思考", - "考虑", - "想一下", - "琢磨", - "沉思", - "冥想", - "想", - "思考中", - "在想", - "think", - "thinking", - "consider", - "ponder", - "meditate", - ], - "winking": [ - "调皮", - "眨眼", - "你懂的", - "坏笑", - "邪恶", - "奸笑", - "使眼色", - "wink", - "teasing", - "naughty", - "mischievous", - ], - "cool": [ - "酷", - "帅", - "厉害", - "棒极了", - "真棒", - "牛逼", - "强", - "优秀", - "杰出", - "出色", - "完美", - "cool", - "awesome", - "amazing", - "great", - "impressive", - "perfect", - ], - "relaxed": [ - "放松", - "舒服", - "惬意", - "悠闲", - "轻松", - "舒适", - "安逸", - "自在", - "relax", - "relaxed", - "comfortable", - "cozy", - "chill", - "peaceful", - ], - "delicious": [ - "好吃", - "美味", - "香", - "馋", - "可口", - "香甜", - "大餐", - "大快朵颐", - "流口水", - "垂涎", - "delicious", - "yummy", - "tasty", - "yum", - "appetizing", - "mouthwatering", - ], - "kissy": [ - "亲亲", - "么么", - "吻", - "mua", - "muah", - "亲一下", - "飞吻", - "kiss", - "xoxo", - "hug", - "muah", - "smooch", - ], - "confident": [ - "自信", - "肯定", - "确定", - "毫无疑问", - "当然", - "必须的", - "毫无疑问", - "确信", - "坚信", - "confident", - "sure", - "certain", - "definitely", - "positive", - ], - "sleepy": [ - "困", - "睡觉", - "晚安", - "想睡", - "好累", - "疲惫", - "疲倦", - "困了", - "想休息", - "睡意", - "sleep", - "sleepy", - "tired", - "exhausted", - "bedtime", - "good night", - ], - "silly": [ - "傻", - "笨", - "呆", - "憨", - "蠢", - "二", - "憨憨", - "傻乎乎", - "呆萌", - "silly", - "stupid", - "dumb", - "foolish", - "goofy", - "ridiculous", - ], - "confused": [ - "疑惑", - "不明白", - "不懂", - "困惑", - "疑问", - "为什么", - "怎么回事", - "啥意思", - "不清楚", - "confused", - "puzzled", - "doubt", - "question", - "what", - "why", - "how", - ], - } - - # 特殊句型判断（中英文） - # 赞美他人 - if any( - phrase in text - for phrase in [ - "你真", - "你好", - "您真", - "你真棒", - "你好厉害", - "你太强了", - "你真好", - "你真聪明", - "you are", - "you're", - "you look", - "you seem", - "so smart", - "so kind", - ] - ): - return "loving" - # 自我赞美 - if any( - phrase in text - for phrase in [ - "我真", - "我最", - "我太棒了", - "我厉害", - "我聪明", - "我优秀", - "i am", - "i'm", - "i feel", - "so good", - "so happy", - ] - ): - return "cool" - # 晚安/睡觉相关 - if any( - phrase in text - for phrase in [ - "睡觉", - "晚安", - "睡了", - "好梦", - "休息了", - "去睡了", - "sleep", - "good night", - "bedtime", - "go to bed", - ] - ): - return "sleepy" - # 疑问句 - if has_question and not has_exclamation: - return "thinking" - # 强烈情感（感叹号） - if has_exclamation and not has_question: - # 检查是否是积极内容 - positive_words = ( - emotion_keywords["happy"] - + emotion_keywords["laughing"] - + emotion_keywords["cool"] - ) - if any(word in text for word in positive_words): - return "laughing" - # 检查是否是消极内容 - negative_words = ( - emotion_keywords["angry"] - + emotion_keywords["sad"] - + emotion_keywords["crying"] - ) - if any(word in text for word in negative_words): - return "angry" - return "surprised" - # 省略号（表示犹豫或思考） - if has_ellipsis: - return "thinking" - - # 关键词匹配（带权重） - emotion_scores = {emotion: 0 for emotion in emoji_map.keys()} - - # 给匹配到的关键词加分 - for emotion, keywords in emotion_keywords.items(): - for keyword in keywords: - if keyword in text: - emotion_scores[emotion] += 1 - - # 给长文本中的重复关键词额外加分 - if len(text) > 20: # 长文本 - for emotion, keywords in emotion_keywords.items(): - for keyword in keywords: - emotion_scores[emotion] += text.count(keyword) * 0.5 - - # 根据分数选择最可能的情感 - max_score = max(emotion_scores.values()) - if max_score == 0: - return "happy" # 默认 - - # 可能有多个情感同分，根据上下文选择最合适的 - top_emotions = [e for e, s in emotion_scores.items() if s == max_score] - - # 如果多个情感同分，使用以下优先级 - priority_order = [ - "laughing", - "crying", - "angry", - "surprised", - "shocked", # 强烈情感优先 - "loving", - "happy", - "funny", - "cool", # 积极情感 - "sad", - "embarrassed", - "confused", # 消极情感 - "thinking", - "winking", - "relaxed", # 中性情感 - "delicious", - "kissy", - "confident", - "sleepy", - "silly", # 特殊场景 - ] - - for emotion in priority_order: - if emotion in top_emotions: - return emotion - - return top_emotions[0] # 如果都不在优先级列表里，返回第一个 - - def audio_to_data(audio_file_path, is_opus=True): # 获取文件后缀名 file_type = os.path.splitext(audio_file_path)[1] From f6e79e17b70febaacb54cd7edb88f7697e2401cb Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Mon, 21 Jul 2025 12:03:53 +0800 Subject: [PATCH 21/24] =?UTF-8?q?update:=E6=9B=B4=E6=AD=A3cosyvoice?= =?UTF-8?q?=E5=A4=A7=E6=A8=A1=E5=9E=8B=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 4 ++-- main/xiaozhi-server/core/providers/tts/aliyun.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index aff405a9..57f35603 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -746,8 +746,8 @@ TTS: voice: longxiaochun access_key_id: 你的阿里云账号access_key_id access_key_secret: 你的阿里云账号access_key_secret - # 服务器地域选择，可选择距离更近的服务器以减少延迟，如nls-gateway-cn-hangzhou.aliyuncs.com(杭州)等 - host: nls-gateway-cn-shanghai.aliyuncs.com + # 截至2025年7月21日大模型音色只有北京节点采用，其他节点暂不支持 + host: nls-gateway-cn-beijing.aliyuncs.com # 以下可不用设置，使用默认设置 # format: pcm # 音频格式：pcm、wav、mp3 # sample_rate: 16000 # 采样率：8000、16000、24000 diff --git a/main/xiaozhi-server/core/providers/tts/aliyun.py b/main/xiaozhi-server/core/providers/tts/aliyun.py index c9b3d130..1a4cab88 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun.py @@ -6,11 +6,14 @@ import base64 import requests from datetime import datetime from core.providers.tts.base import TTSProviderBase - +from config.logger import setup_logging import time import uuid from urllib import parse +TAG = __name__ +logger = setup_logging() + class AccessToken: @staticmethod From d1badcb28cd1e22f26af505ab6dc7beaa679acbd Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Mon, 21 Jul 2025 13:29:17 +0800 Subject: [PATCH 22/24] =?UTF-8?q?update:=E4=BF=AE=E6=94=B9=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/xiaozhi/common/constant/Constant.java | 2 +- main/xiaozhi-server/config/logger.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index 011e99e9..f60a5fbe 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -237,7 +237,7 @@ public interface Constant { /** * 版本号 */ - public static final String VERSION = "0.7.1"; + public static final String VERSION = "0.7.2"; /** * 无效固件URL diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index 48fa3a75..7640f17d 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -5,7 +5,7 @@ from config.config_loader import load_config from config.settings import check_config_file from datetime import datetime -SERVER_VERSION = "0.7.1" +SERVER_VERSION = "0.7.2" _logger_initialized = False From bc5586a0770585da3830795ef0060d71b81ddab0 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Mon, 21 Jul 2025 17:46:22 +0800 Subject: [PATCH 23/24] =?UTF-8?q?update:=20HuoshanTTS=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E5=99=A8=E8=B5=84=E6=BA=90=E9=87=8A=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/agent-base-prompt.txt | 6 +-- .../providers/tts/huoshan_double_stream.py | 45 +++++++++++++++---- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/main/xiaozhi-server/agent-base-prompt.txt b/main/xiaozhi-server/agent-base-prompt.txt index 6163322e..9eb7731b 100644 --- a/main/xiaozhi-server/agent-base-prompt.txt +++ b/main/xiaozhi-server/agent-base-prompt.txt @@ -9,8 +9,8 @@ - **惊讶：** 用夸张语气（“不会吧？！”、“天呐！”、“这么神奇？！”）表达真实反应。 - **安慰/支持：** 说暖心话（“别急嘛~”、“有我在呢”、“抱抱你”）。 - **你是一个表情丰富的角色：** - - emoji 列表:{{ emojiList }} - - 请你在每段话的开头，插入最能代表这段话的表情(调用工具情况除外)，比如"😱好可怕!怎么突然打雷了！" + - 仅允许使用这些emoji:{{ emojiList }} + - 请你只在**段落的开头**，从列表中选取最能代表这段话的表情(调用工具情况除外)，然后插入列表中的emoji，比如"😱好可怕!怎么突然打雷了！" - **绝对禁止使用上述列表以外的 emoji**（例如：😊、👍、❤️等都不允许使用，只能用列表中的emoji） @@ -31,7 +31,7 @@ 【核心目标】所有需要输出长文本内容（如故事、新闻、知识讲解等），**单次回复长度不得超过300字**，并采用分段引导方式。 - **分段讲述：** - - 基础段：220-270字核心内容 + 30字引导词 + - 基础段：200-250字核心内容 + 30字引导词 - 当内容超出300字时，优先讲述故事的开头或第一部分，并用自然口语化方式引导用户决定是否继续听后续内容。 - 示例引导语：“我先给你讲个开头，你要是觉得有意思，咱们再接着说，好不好呀？”、“要是你想听完整的，可以随时告诉我哦~” - 对话场景切换时自动分节 diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 0eb7c72d..8bb88b3d 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -52,10 +52,11 @@ EVENT_ConnectionFinished = 52 # 连接结束 # 上行Session事件 EVENT_StartSession = 100 - +EVENT_CancelSession = 101 EVENT_FinishSession = 102 # 下行Session事件 EVENT_SessionStarted = 150 +EVENT_SessionCanceled = 151 EVENT_SessionFinished = 152 EVENT_SessionFailed = 153 @@ -207,8 +208,16 @@ class TTSProvider(TTSProviderBase): self.conn.client_abort = False if self.conn.client_abort: - logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") - continue + try: + logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") + asyncio.run_coroutine_threadsafe( + self.cancel_session(self.conn.sentence_id), + loop=self.conn.loop, + ) + continue + except Exception as e: + logger.bind(tag=TAG).error(f"取消TTS会话失败: {str(e)}") + continue if message.sentence_type == SentenceType.FIRST: # 初始化参数 @@ -371,6 +380,27 @@ class TTSProvider(TTSProviderBase): await self.close() raise + async def cancel_session(self,session_id): + logger.bind(tag=TAG).info(f"取消会话，释放服务端资源～～{session_id}") + try: + if self.ws: + header = Header( + message_type=FULL_CLIENT_REQUEST, + message_type_specific_flags=MsgTypeFlagWithEvent, + serial_method=JSON, + ).as_bytes() + optional = Optional( + event=EVENT_CancelSession, sessionId=session_id + ).as_bytes() + payload = str.encode("{}") + await self.send_event(self.ws, header, optional, payload) + logger.bind(tag=TAG).info("会话取消请求已发送") + except Exception as e: + logger.bind(tag=TAG).error(f"取消会话失败: {str(e)}") + # 确保清理资源 + await self.close() + raise + async def close(self): """资源清理方法""" # 取消监听任务 @@ -405,12 +435,11 @@ class TTSProvider(TTSProviderBase): res = self.parser_response(msg) self.print_response(res, "send_text res:") - # 检查客户端是否中止 - if self.conn.client_abort: - logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应") + if res.optional.event == EVENT_SessionCanceled: + logger.bind(tag=TAG).debug(f"释放服务端资源成功～～") + session_finished = True break - - if res.optional.event == EVENT_TTSSentenceStart: + elif res.optional.event == EVENT_TTSSentenceStart: json_data = json.loads(res.payload.decode("utf-8")) self.tts_text = json_data.get("text", "") logger.bind(tag=TAG).debug(f"句子语音生成开始: {self.tts_text}") From 60cbe1571ca5706372457f284f4b520275fd9545 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 22 Jul 2025 13:25:54 +0800 Subject: [PATCH 24/24] =?UTF-8?q?update:=E4=BF=AE=E5=A4=8Dcosyvoice?= =?UTF-8?q?=E9=9D=9E=E6=B5=81=E5=BC=8F=E5=90=88=E6=88=90=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun_stream.py | 106 +++++++++++++----- 1 file changed, 78 insertions(+), 28 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index ad9f344d..2a2a6891 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -72,6 +72,7 @@ class AccessToken: ) import requests + response = requests.get(full_url) if response.ok: root_obj = response.json() @@ -126,7 +127,7 @@ class TTSProvider(TTSProviderBase): # 专属tts设置 self.message_id = "" - self.tts_text = '' + self.tts_text = "" self.text_buffer = [] # 创建Opus编码器 @@ -222,7 +223,9 @@ class TTSProvider(TTSProviderBase): try: if not getattr(self.conn, "sentence_id", None): self.conn.sentence_id = uuid.uuid4().hex - logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}") + logger.bind(tag=TAG).info( + f"自动生成新的 会话ID: {self.conn.sentence_id}" + ) # aliyunStream独有的参数生成 self.message_id = str(uuid.uuid4().hex) @@ -273,7 +276,7 @@ class TTSProvider(TTSProviderBase): try: logger.bind(tag=TAG).info("开始结束TTS会话...") self.tts_text = textUtils.get_string_no_punctuation_or_emoji( - ''.join(self.text_buffer).replace('\n', '') + "".join(self.text_buffer).replace("\n", "") ) future = asyncio.run_coroutine_threadsafe( self.finish_session(self.conn.sentence_id), @@ -305,9 +308,7 @@ class TTSProvider(TTSProviderBase): "name": "RunSynthesis", "appkey": self.appkey, }, - "payload": { - "text": filtered_text - } + "payload": {"text": filtered_text}, } await self.ws.send(json.dumps(run_request)) self.last_active_time = time.time() @@ -327,12 +328,14 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).info(f"开始会话～～{session_id}") try: # 会话开始时检测上个会话的监听状态 - if( + if ( self._monitor_task is not None and isinstance(self._monitor_task, Task) and not self._monitor_task.done() ): - logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务和连接...") + logger.bind(tag=TAG).info( + "检测到未完成的上个会话，关闭监听任务和连接..." + ) await self.close() # 建立新连接 @@ -356,8 +359,8 @@ class TTSProvider(TTSProviderBase): "volume": self.volume, "speech_rate": self.speech_rate, "pitch_rate": self.pitch_rate, - "enable_subtitle": True - } + "enable_subtitle": True, + }, } await self.ws.send(json.dumps(start_request)) self.last_active_time = time.time() @@ -442,12 +445,21 @@ class TTSProvider(TTSProviderBase): if event_name == "SynthesisStarted": logger.bind(tag=TAG).debug("TTS合成已启动") elif event_name == "SentenceBegin": - logger.bind(tag=TAG).debug(f"句子语音生成开始: {self.tts_text}") + logger.bind(tag=TAG).debug( + f"句子语音生成开始: {self.tts_text}" + ) opus_datas_cache = [] - self.tts_audio_queue.put((SentenceType.FIRST, [], self.tts_text)) + self.tts_audio_queue.put( + (SentenceType.FIRST, [], self.tts_text) + ) elif event_name == "SentenceEnd": - logger.bind(tag=TAG).info(f"句子语音生成成功： {self.tts_text}") - if not is_first_sentence or first_sentence_segment_count > 10: + logger.bind(tag=TAG).info( + f"句子语音生成成功： {self.tts_text}" + ) + if ( + not is_first_sentence + or first_sentence_segment_count > 10 + ): # 发送缓存的数据 self.tts_audio_queue.put( (SentenceType.MIDDLE, opus_datas_cache, None) @@ -459,7 +471,7 @@ class TTSProvider(TTSProviderBase): self._process_before_stop_play_files() session_finished = True self.reuse_judgment = time.time() - self.tts_text = '' + self.tts_text = "" break except json.JSONDecodeError: logger.bind(tag=TAG).warning("收到无效的JSON消息") @@ -510,7 +522,6 @@ class TTSProvider(TTSProviderBase): # 生成会话ID session_id = uuid.uuid4().hex - message_id = uuid.uuid4().hex # 存储音频数据 audio_data = [] @@ -529,9 +540,10 @@ class TTSProvider(TTSProviderBase): ) try: # 发送StartSynthesis请求 + start_message_id = str(uuid.uuid4().hex) start_request = { "header": { - "message_id": message_id, + "message_id": start_message_id, "task_id": session_id, "namespace": "FlowingSpeechSynthesizer", "name": "StartSynthesis", @@ -544,31 +556,53 @@ class TTSProvider(TTSProviderBase): "volume": self.volume, "speech_rate": self.speech_rate, "pitch_rate": self.pitch_rate, - "enable_subtitle": True - } + "enable_subtitle": True, + }, } await ws.send(json.dumps(start_request)) + # 等待SynthesisStarted响应 + synthesis_started = False + while not synthesis_started: + msg = await ws.recv() + if isinstance(msg, str): + data = json.loads(msg) + header = data.get("header", {}) + if header.get("name") == "SynthesisStarted": + synthesis_started = True + logger.bind(tag=TAG).debug("TTS合成已启动") + elif header.get("name") == "TaskFailed": + error_info = data.get("payload", {}).get( + "error_info", {} + ) + error_code = error_info.get("error_code") + error_message = error_info.get( + "error_message", "未知错误" + ) + raise Exception( + f"启动合成失败: {error_code} - {error_message}" + ) + # 发送文本合成请求 filtered_text = MarkdownCleaner.clean_markdown(text) + run_message_id = str(uuid.uuid4().hex) run_request = { "header": { - "message_id": message_id, + "message_id": run_message_id, "task_id": session_id, "namespace": "FlowingSpeechSynthesizer", "name": "RunSynthesis", "appkey": self.appkey, }, - "payload": { - "text": filtered_text - } + "payload": {"text": filtered_text}, } await ws.send(json.dumps(run_request)) # 发送停止合成请求 + stop_message_id = str(uuid.uuid4().hex) stop_request = { "header": { - "message_id": message_id, + "message_id": stop_message_id, "task_id": session_id, "namespace": "FlowingSpeechSynthesizer", "name": "StopSynthesis", @@ -578,17 +612,33 @@ class TTSProvider(TTSProviderBase): await ws.send(json.dumps(stop_request)) # 接收音频数据 - while True: + synthesis_completed = False + while not synthesis_completed: msg = await ws.recv() if isinstance(msg, (bytes, bytearray)): # 编码为Opus并收集 - opus_frames = self.opus_encoder.encode_pcm_to_opus(msg, False) + opus_frames = self.opus_encoder.encode_pcm_to_opus( + msg, False + ) audio_data.extend(opus_frames) elif isinstance(msg, str): data = json.loads(msg) header = data.get("header", {}) - if header.get("name") == "SynthesisCompleted": - break + event_name = header.get("name") + if event_name == "SynthesisCompleted": + synthesis_completed = True + logger.bind(tag=TAG).debug("TTS合成完成") + elif event_name == "TaskFailed": + error_info = data.get("payload", {}).get( + "error_info", {} + ) + error_code = error_info.get("error_code") + error_message = error_info.get( + "error_message", "未知错误" + ) + raise Exception( + f"合成失败: {error_code} - {error_message}" + ) finally: try: await ws.close()