From 90a96314501ec8aeae5d50cad17be69edc33a322 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Mon, 30 Jun 2025 14:41:53 +0800 Subject: [PATCH 01/19] =?UTF-8?q?update:=E6=B7=BB=E5=8A=A0=E9=98=BF?= =?UTF-8?q?=E9=87=8C=E4=BA=91=E6=B5=81=E5=BC=8Fasr?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 22 ++ .../core/providers/asr/aliyun_stream.py | 286 ++++++++++++++++++ 2 files changed, 308 insertions(+) create mode 100644 main/xiaozhi-server/core/providers/asr/aliyun_stream.py diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 71c13ec7..bd9f024b 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -298,9 +298,13 @@ ASR: output_dir: tmp/ AliyunASR: # 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息 + # HTTP POST请求，一次性处理完整音频 # 平台地址：https://nls-portal.console.aliyun.com/ # appkey地址：https://nls-portal.console.aliyun.com/applist # token地址：https://nls-portal.console.aliyun.com/overview + # AliyunASR和AliyunStreamASR的区别是：AliyunASR是批量处理场景，AliyunStreamASR是实时交互场景 + # 一般来说非流式ASR更便宜（0.004元/秒，¥0.24/分钟） + # 但是AliyunStreamASR实时性更好（0.005元/秒，¥0.3/分钟） # 定义ASR API类型 type: aliyun appkey: 你的阿里云智能语音交互服务项目Appkey @@ -308,6 +312,24 @@ ASR: access_key_id: 你的阿里云账号access_key_id access_key_secret: 你的阿里云账号access_key_secret output_dir: tmp/ + AliyunStreamASR: + # 阿里云智能语音交互服务 - 实时流式语音识别 + # WebSocket连接，实时处理音频流 + # 平台地址：https://nls-portal.console.aliyun.com/ + # appkey地址：https://nls-portal.console.aliyun.com/applist + # token地址：https://nls-portal.console.aliyun.com/overview + # AliyunASR和AliyunStreamASR的区别是：AliyunASR是批量处理场景，AliyunStreamASR是实时交互场景 + # 一般来说非流式ASR更便宜（0.004元/秒，¥0.24/分钟） + # 但是AliyunStreamASR实时性更好（0.005元/秒，¥0.3/分钟） + # 定义ASR API类型 + type: aliyun + appkey: 你的阿里云智能语音交互服务项目Appkey + token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret + access_key_id: 你的阿里云账号access_key_id + access_key_secret: 你的阿里云账号access_key_secret + # 服务器地域选择，如nls-gateway-cn-hangzhou.aliyuncs.com(杭州)等 + host: 根据自己的地理位置选择最近的服务器地域，不填默认为nls-gateway-cn-shanghai.aliyuncs.com(上海) + output_dir: tmp/ BaiduASR: # 获取AppID、API Key、Secret Key：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list # 查看资源额度：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/overview/resource/list diff --git a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py new file mode 100644 index 00000000..94727798 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py @@ -0,0 +1,286 @@ +import json +import time +import uuid +import hmac +import base64 +import hashlib +import asyncio +import requests +import websockets +import opuslib_next +import random +from typing import Optional, Tuple, List +from urllib import parse +from datetime import datetime +from config.logger import setup_logging +from core.providers.asr.base import ASRProviderBase +from core.providers.asr.dto.dto import InterfaceType + +TAG = __name__ +logger = setup_logging() + + +class AccessToken: + @staticmethod + def _encode_text(text): + encoded_text = parse.quote_plus(text) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def _encode_dict(dic): + keys = dic.keys() + dic_sorted = [(key, dic[key]) for key in sorted(keys)] + encoded_text = parse.urlencode(dic_sorted) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def create_token(access_key_id, access_key_secret): + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } + query_string = AccessToken._encode_dict(parameters) + string_to_sign = ( + "GET" + "&" + AccessToken._encode_text("/") + "&" + AccessToken._encode_text(query_string) + ) + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() + signature = base64.b64encode(secreted_string) + signature = AccessToken._encode_text(signature) + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (signature, query_string) + response = requests.get(full_url) + if response.ok: + root_obj = response.json() + if "Token" in root_obj: + return root_obj["Token"]["Id"], root_obj["Token"]["ExpireTime"] + return None, None + + +class ASRProvider(ASRProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__() + self.interface_type = InterfaceType.STREAM + self.config = config + self.text = "" + self.decoder = opuslib_next.Decoder(16000, 1) + self.asr_ws = None + self.forward_task = None + self.is_processing = False + self.server_ready = False # 服务器准备状态 + + # 基础配置 + self.access_key_id = config.get("access_key_id") + self.access_key_secret = config.get("access_key_secret") + self.appkey = config.get("appkey") + self.token = config.get("token") + self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com") + self.ws_url = f"wss://{self.host}/ws/v1" + self.max_sentence_silence = config.get("max_sentence_silence") + self.output_dir = config.get("output_dir", "./audio_output") + self.delete_audio_file = delete_audio_file + self.expire_time = None + + # Token管理 + if self.access_key_id and self.access_key_secret: + self._refresh_token() + elif not self.token: + raise ValueError("必须提供access_key_id+access_key_secret或者直接提供token") + + def _refresh_token(self): + """刷新Token""" + self.token, expire_time_str = AccessToken.create_token(self.access_key_id, self.access_key_secret) + if not self.token: + raise ValueError("无法获取有效的访问Token") + + try: + expire_str = str(expire_time_str).strip() + if expire_str.isdigit(): + expire_time = datetime.fromtimestamp(int(expire_str)) + else: + expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ") + self.expire_time = expire_time.timestamp() - 60 + except: + self.expire_time = None + + def _is_token_expired(self): + """检查Token是否过期""" + return self.expire_time and time.time() > self.expire_time + + async def open_audio_channels(self, conn): + await super().open_audio_channels(conn) + + async def receive_audio(self, conn, audio, audio_have_voice): + conn.asr_audio.append(audio) + conn.asr_audio = conn.asr_audio[-10:] + + # 参考豆包ASR：只在有声音且没有连接时建立连接 + if audio_have_voice and not self.is_processing: + try: + await self._start_recognition(conn) + except Exception as e: + logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}") + await self._cleanup() + return + + if self.asr_ws and self.is_processing and self.server_ready: + try: + pcm_frame = self.decoder.decode(audio, 960) + await self.asr_ws.send(pcm_frame) + except Exception as e: + logger.bind(tag=TAG).warning(f"发送音频失败: {str(e)}") + await self._cleanup() + + async def _start_recognition(self, conn): + """开始识别会话""" + if self._is_token_expired(): + self._refresh_token() + + # 建立连接 + headers = {"X-NLS-Token": self.token} + self.asr_ws = await websockets.connect( + self.ws_url, + additional_headers=headers, + max_size=1000000000, + ping_interval=None, + ping_timeout=None, + close_timeout=5, + ) + + self.is_processing = True + self.server_ready = False # 重置服务器准备状态 + self.forward_task = asyncio.create_task(self._forward_results(conn)) + + # 发送开始请求 + start_request = { + "header": { + "namespace": "SpeechTranscriber", + "name": "StartTranscription", + "status": 20000000, + "message_id": ''.join(random.choices('0123456789abcdef', k=32)), + "task_id": ''.join(random.choices('0123456789abcdef', k=32)), + "status_text": "Gateway:SUCCESS:Success.", + "appkey": self.appkey + }, + "payload": { + "format": "pcm", + "sample_rate": 16000, + "enable_intermediate_result": True, + "enable_punctuation_prediction": True, + "enable_inverse_text_normalization": True, + "max_sentence_silence": self.max_sentence_silence, + "enable_voice_detection": False, + } + } + await self.asr_ws.send(json.dumps(start_request, ensure_ascii=False)) + logger.bind(tag=TAG).info("已发送开始请求，等待服务器准备...") + + async def _forward_results(self, conn): + """转发识别结果""" + try: + while self.asr_ws and not conn.stop_event.is_set(): + try: + response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0) + result = json.loads(response) + + header = result.get("header", {}) + payload = result.get("payload", {}) + message_name = header.get("name", "") + status = header.get("status", 0) + + if status != 20000000: + if status in [40000004, 40010004]: # 连接超时或客户端断开 + logger.bind(tag=TAG).warning(f"连接问题，状态码: {status}") + break + elif status in [40270002, 40270003]: # 音频问题 + logger.bind(tag=TAG).warning(f"音频处理问题，状态码: {status}") + continue + else: + logger.bind(tag=TAG).error(f"识别错误，状态码: {status}, 消息: {header.get('status_text', '')}") + continue + + # 收到TranscriptionStarted表示服务器准备好接收音频数据 + if message_name == "TranscriptionStarted": + self.server_ready = True + logger.bind(tag=TAG).info("服务器已准备，开始发送缓存音频...") + + # 发送缓存音频 + if conn.asr_audio: + for cached_audio in conn.asr_audio[-10:]: + try: + pcm_frame = self.decoder.decode(cached_audio, 960) + await self.asr_ws.send(pcm_frame) + except Exception as e: + logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}") + break + continue + + if message_name == "TranscriptionResultChanged": + # 中间结果 + text = payload.get("result", "") + if text: + self.text = text + elif message_name == "SentenceEnd": + # 最终结果 + text = payload.get("result", "") + if text: + self.text = text + conn.reset_vad_states() + await self.handle_voice_stop(conn, None) + break + elif message_name == "TranscriptionCompleted": + # 识别完成 + self.is_processing = False + break + + except asyncio.TimeoutError: + continue + except websockets.exceptions.ConnectionClosed: + break + except Exception as e: + logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}") + break + + except Exception as e: + logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}") + finally: + await self._cleanup() + + async def _cleanup(self): + """清理资源""" + self.is_processing = False + self.server_ready = False # 重置服务器准备状态 + + if self.forward_task and not self.forward_task.done(): + self.forward_task.cancel() + try: + await asyncio.wait_for(self.forward_task, timeout=1.0) + except: + pass + self.forward_task = None + + if self.asr_ws: + try: + await asyncio.wait_for(self.asr_ws.close(), timeout=2.0) + except: + pass + self.asr_ws = None + + async def speech_to_text(self, opus_data, session_id, audio_format): + """获取识别结果""" + result = self.text + self.text = "" + return result, None + + async def close(self): + """关闭资源""" + await self._cleanup() From 039badd2650bd11a524fce631bc537d46f5e05f7 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Tue, 1 Jul 2025 18:14:36 +0800 Subject: [PATCH 02/19] =?UTF-8?q?update:=E6=B7=BB=E5=8A=A0=E4=BA=86?= =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91CosyVoice=E6=B5=81=E5=BC=8FTTS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 22 + .../core/providers/tts/aliyun_stream.py | 891 ++++++++++++++++++ 2 files changed, 913 insertions(+) create mode 100644 main/xiaozhi-server/core/providers/tts/aliyun_stream.py diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index bd9f024b..a4b4ccc0 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -690,6 +690,28 @@ TTS: # pitch_rate: 0 # 添加 302.ai TTS 配置 # token申请地址：https://dash.302.ai/ + AliyunStreamTTS: + # 阿里云CosyVoice大模型流式文本语音合成 + # 采用FlowingSpeechSynthesizer接口，支持更低延迟和更自然的语音质量 + # 流式文本语音合成仅提供商用版，不支持试用，详情请参见试用版和商用版。要使用该功能，请开通商用版。 + # 支持龙系列专用音色：longxiaochun、longyu、longchen等 + # 平台地址：https://nls-portal.console.aliyun.com/ + # appkey地址：https://nls-portal.console.aliyun.com/applist + # token地址：https://nls-portal.console.aliyun.com/overview + # 使用三阶段流式交互：StartSynthesis -> RunSynthesis -> StopSynthesis + type: aliyun + output_dir: tmp/ + appkey: 你的阿里云智能语音交互服务项目Appkey + token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret + voice: longxiaochun + access_key_id: 你的阿里云账号access_key_id + access_key_secret: 你的阿里云账号access_key_secret + # 以下可不用设置，使用默认设置 + # format: pcm # 音频格式：pcm、wav、mp3 + # sample_rate: 16000 # 采样率：8000、16000、24000 + # volume: 50 # 音量：0-100 + # speech_rate: 0 # 语速：-500到500 + # pitch_rate: 0 # 语调：-500到500 TencentTTS: # 腾讯云智能语音交互服务，需要先在腾讯云平台开通服务 # appid、secret_id、secret_key申请地址：https://console.cloud.tencent.com/cam/capi diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py new file mode 100644 index 00000000..9aad9614 --- /dev/null +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -0,0 +1,891 @@ +import uuid +import json +import hmac +import hashlib +import base64 +import time +import queue +import asyncio +import traceback +import websockets +import websockets.protocol +import os +import random +import threading +import concurrent.futures +import sys +from datetime import datetime +from urllib import parse +from typing import Optional +from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType +from core.utils.tts import MarkdownCleaner +from core.utils import opus_encoder_utils, textUtils +from config.logger import setup_logging + +TAG = __name__ +logger = setup_logging() + + +class AccessToken: + @staticmethod + def _encode_text(text): + encoded_text = parse.quote_plus(text) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def _encode_dict(dic): + keys = dic.keys() + dic_sorted = [(key, dic[key]) for key in sorted(keys)] + encoded_text = parse.urlencode(dic_sorted) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def create_token(access_key_id, access_key_secret): + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", # 使用上海地域进行Token获取 + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } + + query_string = AccessToken._encode_dict(parameters) + string_to_sign = ( + "GET" + + "&" + + AccessToken._encode_text("/") + + "&" + + AccessToken._encode_text(query_string) + ) + + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() + signature = base64.b64encode(secreted_string) + signature = AccessToken._encode_text(signature) + + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % ( + signature, + query_string, + ) + + import requests + response = requests.get(full_url) + if response.ok: + root_obj = response.json() + key = "Token" + if key in root_obj: + token = root_obj[key]["Id"] + expire_time = root_obj[key]["ExpireTime"] + return token, expire_time + return None, None + + +class TTSProvider(TTSProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__(config, delete_audio_file) + + # 设置为流式接口类型 + self.interface_type = InterfaceType.SINGLE_STREAM + + # 基础配置 + self.access_key_id = config.get("access_key_id") + self.access_key_secret = config.get("access_key_secret") + self.appkey = config.get("appkey") + self.format = config.get("format", "pcm") + self.audio_file_type = config.get("format", "pcm") + + # 采样率配置 + sample_rate = config.get("sample_rate", "16000") + self.sample_rate = int(sample_rate) if sample_rate else 16000 + + # 音色配置 - CosyVoice大模型音色 + if config.get("private_voice"): + self.voice = config.get("private_voice") + else: + self.voice = config.get("voice", "longxiaochun") # CosyVoice默认音色 + + # 音频参数配置 + volume = config.get("volume", "50") + self.volume = int(volume) if volume else 50 + + speech_rate = config.get("speech_rate", "0") + self.speech_rate = int(speech_rate) if speech_rate else 0 + + pitch_rate = config.get("pitch_rate", "0") + self.pitch_rate = int(pitch_rate) if pitch_rate else 0 + + # WebSocket配置 + self.host = config.get("host", "nls-gateway-cn-beijing.aliyuncs.com") + self.ws_url = f"wss://{self.host}/ws/v1" + self.ws = None + + # 流式相关配置 + self.before_stop_play_files = [] + self.segment_count = 0 + + # 创建Opus编码器 + self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( + sample_rate=16000, channels=1, frame_size_ms=60 + ) + + # PCM缓冲区 + self.pcm_buffer = bytearray() + + # Token管理 + if self.access_key_id and self.access_key_secret: + self._refresh_token() + else: + self.token = config.get("token") + self.expire_time = None + + def _refresh_token(self): + """刷新Token并记录过期时间""" + if self.access_key_id and self.access_key_secret: + self.token, expire_time_str = AccessToken.create_token( + self.access_key_id, self.access_key_secret + ) + if not expire_time_str: + raise ValueError("无法获取有效的Token过期时间") + + try: + expire_str = str(expire_time_str).strip() + if expire_str.isdigit(): + expire_time = datetime.fromtimestamp(int(expire_str)) + else: + expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ") + self.expire_time = expire_time.timestamp() - 60 + except Exception as e: + raise ValueError(f"无效的过期时间格式: {expire_str}") from e + else: + self.expire_time = None + + if not self.token: + raise ValueError("无法获取有效的访问Token") + + def _is_token_expired(self): + """检查Token是否过期""" + if not self.expire_time: + return False + return time.time() > self.expire_time + + async def _ensure_connection(self): + """确保WebSocket连接可用""" + # 检查连接状态，兼容不同版本的websockets库 + need_reconnect = False + if self.ws is None: + need_reconnect = True + else: + try: + # 尝试访问closed属性，如果不存在则检查state + if hasattr(self.ws, 'closed'): + need_reconnect = self.ws.closed + elif hasattr(self.ws, 'state'): + # websockets 新版本使用state属性 + need_reconnect = self.ws.state != websockets.protocol.State.OPEN + else: + # 如果都没有，尝试发送ping来检测连接状态 + try: + await asyncio.wait_for(self.ws.ping(), timeout=2.0) + except: + need_reconnect = True + except: + need_reconnect = True + + if need_reconnect: + # 清理旧连接 + if self.ws: + try: + if hasattr(self.ws, 'close'): + if asyncio.iscoroutinefunction(self.ws.close): + await self.ws.close() + else: + self.ws.close() + except: + pass + finally: + self.ws = None + + if self._is_token_expired(): + logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...") + self._refresh_token() + + # 重试连接机制 + max_retries = 3 + retry_delay = 1.0 + + for attempt in range(max_retries): + try: + self.ws = await asyncio.wait_for( + websockets.connect( + self.ws_url, + additional_headers={ + "X-NLS-Token": self.token, + }, + ping_interval=30, + ping_timeout=10, + close_timeout=10, + ), + timeout=10.0 + ) + logger.bind(tag=TAG).info("阿里云CosyVoice流式TTS WebSocket连接建立成功") + return + except Exception as e: + logger.bind(tag=TAG).warning(f"WebSocket连接失败 (尝试 {attempt + 1}/{max_retries}): {e}") + if attempt < max_retries - 1: + await asyncio.sleep(retry_delay * (attempt + 1)) + else: + logger.bind(tag=TAG).error(f"WebSocket连接最终失败: {e}") + raise + + def tts_text_priority_thread(self): + """流式文本处理线程""" + while not self.conn.stop_event.is_set(): + try: + message = self.tts_text_queue.get(timeout=1) + if message.sentence_type == SentenceType.FIRST: + # 初始化参数 + self.tts_stop_request = False + self.processed_chars = 0 + self.tts_text_buff = [] + self.segment_count = 0 + self.tts_audio_first_sentence = True + self.before_stop_play_files.clear() + elif ContentType.TEXT == message.content_type: + self.tts_text_buff.append(message.content_detail) + segment_text = self._get_segment_text() + if segment_text: + self.to_tts_single_stream(segment_text) + + elif ContentType.FILE == message.content_type: + logger.bind(tag=TAG).info( + f"添加音频文件到待播放列表: {message.content_file}" + ) + self.before_stop_play_files.append( + (message.content_file, message.content_detail) + ) + + if message.sentence_type == SentenceType.LAST: + # 处理剩余的文本 + self._process_remaining_text(True) + + except queue.Empty: + continue + except Exception as e: + logger.bind(tag=TAG).error( + f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" + ) + + def _process_remaining_text(self, is_last=False): + """处理剩余的文本并生成语音""" + full_text = "".join(self.tts_text_buff) + remaining_text = full_text[self.processed_chars:] + if remaining_text: + segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) + if segment_text: + self.to_tts_single_stream(segment_text, is_last) + self.processed_chars += len(full_text) + else: + self._process_before_stop_play_files() + else: + self._process_before_stop_play_files() + + def to_tts_single_stream(self, text, is_last=False): + """流式TTS处理 - 使用线程池执行异步任务""" + try: + text = MarkdownCleaner.clean_markdown(text) + + # 使用线程池来执行异步任务，避免事件循环冲突 + def run_async_task(): + """在新线程中运行异步任务""" + try: + # 创建新的事件循环用于这个线程 + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # Windows下设置事件循环策略 + if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + + try: + # 运行TTS任务 + loop.run_until_complete(self._tts_request_unified(text, is_last)) + return True + finally: + # 安全关闭事件循环 + try: + # 取消所有未完成的任务 + pending = asyncio.all_tasks(loop) + if pending: + for task in pending: + task.cancel() + # 等待任务取消完成 + loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + except Exception as cleanup_error: + logger.bind(tag=TAG).debug(f"清理事件循环异常: {cleanup_error}") + finally: + loop.close() + + except Exception as e: + logger.bind(tag=TAG).error(f"异步任务执行失败: {e}") + return False + + # 使用线程池执行异步任务 + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(run_async_task) + success = future.result(timeout=30) # 30秒超时 + + if success: + logger.bind(tag=TAG).info(f"语音生成成功: {text}") + else: + logger.bind(tag=TAG).error(f"语音生成失败: {text}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + + except concurrent.futures.TimeoutError: + logger.bind(tag=TAG).error(f"TTS任务超时: {text}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + except Exception as e: + logger.bind(tag=TAG).error(f"TTS处理异常: {text}, 错误: {e}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + + return None + + async def text_to_speak(self, text, is_last=False): + """流式处理TTS音频""" + try: + # 确保连接可用 + await self._ensure_connection() + ws_connection = self.ws + + # 确保Token有效 + if self._is_token_expired(): + self._refresh_token() + + # 生成task_id和message_id + task_id = str(uuid.uuid4()).replace('-', '') + + # 第一阶段：发送StartSynthesis指令（设置参数） + start_message_id = str(uuid.uuid4()).replace('-', '') + start_request = { + "header": { + "message_id": start_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StartSynthesis", + "appkey": self.appkey, + }, + "payload": { + "voice": self.voice, + "format": self.format, + "sample_rate": self.sample_rate, + "volume": self.volume, + "speech_rate": self.speech_rate, + "pitch_rate": self.pitch_rate, + } + } + + await ws_connection.send(json.dumps(start_request)) + logger.bind(tag=TAG).debug(f"发送StartSynthesis指令: {start_message_id}") + + # 第二阶段：发送RunSynthesis指令（发送文本） + run_message_id = str(uuid.uuid4()).replace('-', '') + run_request = { + "header": { + "message_id": run_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": self.appkey, + }, + "payload": { + "text": text + } + } + + await ws_connection.send(json.dumps(run_request)) + logger.bind(tag=TAG).debug(f"发送RunSynthesis指令: {text} (message_id: {run_message_id})") + + # 立即发送StopSynthesis指令，避免IDLE_TIMEOUT + stop_message_id = str(uuid.uuid4()).replace('-', '') + stop_request = { + "header": { + "message_id": stop_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StopSynthesis", + "appkey": self.appkey, + } + } + + await ws_connection.send(json.dumps(stop_request)) + logger.bind(tag=TAG).debug(f"发送StopSynthesis指令: {stop_message_id}") + + # 初始化缓冲区和计数器 + self.pcm_buffer.clear() + self.segment_count = 0 # 重置分段计数器 + opus_datas_cache = [] + + # 发送第一个音频包 + self.tts_audio_queue.put((SentenceType.FIRST, [], text)) + + # 标记是否需要发送StopSynthesis + synthesis_completed = False + + # 处理响应 - 设置超时避免长时间等待 + try: + async def process_messages(): + nonlocal synthesis_completed + async for message in ws_connection: + try: + if isinstance(message, str): + # 处理JSON消息 + data = json.loads(message) + header = data.get("header", {}) + event_name = header.get("name") + + if event_name == "SynthesisStarted": + logger.bind(tag=TAG).debug(f"TTS合成已启动: {task_id}") + + elif event_name == "SynthesisCompleted": + logger.bind(tag=TAG).debug(f"TTS合成完成: {text}") + synthesis_completed = True + break + + elif event_name == "TaskFailed": + error_msg = header.get("status_text", "未知错误") + logger.bind(tag=TAG).error(f"TTS合成失败: {error_msg}") + synthesis_completed = True + break + + elif isinstance(message, bytes): + # 处理二进制音频数据 + self.pcm_buffer.extend(message) + + # 计算每帧的字节数 + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels + * self.opus_encoder.frame_size_ms + / 1000 + * 2 + ) + + # 分帧处理PCM数据 + while len(self.pcm_buffer) >= frame_bytes: + frame = bytes(self.pcm_buffer[:frame_bytes]) + del self.pcm_buffer[:frame_bytes] + + # 编码为Opus + opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) + if opus_packets: + if self.segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_packets, None) + ) + self.segment_count += 1 + else: + opus_datas_cache.extend(opus_packets) + + except json.JSONDecodeError: + logger.bind(tag=TAG).warning("收到无效的JSON消息") + except Exception as e: + logger.bind(tag=TAG).error(f"处理响应消息失败: {e}") + + await asyncio.wait_for(process_messages(), timeout=15) # 15秒超时 + + except asyncio.TimeoutError: + logger.bind(tag=TAG).warning(f"TTS请求超时，但可能已获取部分音频数据: {text}") + except websockets.ConnectionClosed: + logger.bind(tag=TAG).debug("WebSocket连接已正常关闭") + except Exception as e: + logger.bind(tag=TAG).error(f"处理WebSocket消息失败: {e}") + + # 因为已经提前发送了StopSynthesis，这里不需要再次发送 + # 直接处理剩余的PCM数据 + if self.pcm_buffer: + opus_packets = self.opus_encoder.encode_pcm_to_opus( + bytes(self.pcm_buffer), end_of_stream=True + ) + if opus_packets: + if self.segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_packets, None) + ) + self.segment_count += 1 + else: + opus_datas_cache.extend(opus_packets) + self.pcm_buffer.clear() + + # 发送缓存的数据 + if self.segment_count >= 10 and opus_datas_cache: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas_cache, None) + ) + + # 如果是最后一段，处理待播放文件 + if is_last: + self._process_before_stop_play_files() + + except Exception as e: + logger.bind(tag=TAG).error(f"TTS异步请求异常: {e}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + + async def _tts_request_unified(self, text: str, is_last: bool) -> None: + """统一的TTS请求方法""" + ws_connection = None + + try: + # 确保Token有效 + if self._is_token_expired(): + self._refresh_token() + + # 总是创建独立连接，避免与其他线程的事件循环冲突 + ws_url = f"wss://{self.host}/ws/v1" + ws_connection = await asyncio.wait_for( + websockets.connect( + ws_url, + additional_headers={ + "X-NLS-Token": self.token, + }, + ping_interval=15, # 每15秒发送ping，保持连接活跃 + ping_timeout=5, # ping超时时间5秒 + close_timeout=3, # 关闭超时时间3秒 + ), + timeout=8.0 # 连接超时时间8秒 + ) + logger.bind(tag=TAG).debug(f"建立独立WebSocket连接: {text}") + + # 生成task_id + task_id = str(uuid.uuid4()).replace('-', '') + + # 第一阶段：发送StartSynthesis指令（设置参数） + start_message_id = str(uuid.uuid4()).replace('-', '') + start_request = { + "header": { + "message_id": start_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StartSynthesis", + "appkey": self.appkey, + }, + "payload": { + "voice": self.voice, + "format": self.format, + "sample_rate": self.sample_rate, + "volume": self.volume, + "speech_rate": self.speech_rate, + "pitch_rate": self.pitch_rate, + } + } + + await ws_connection.send(json.dumps(start_request)) + logger.bind(tag=TAG).debug(f"发送StartSynthesis指令: {start_message_id}") + + # 第二阶段：发送RunSynthesis指令（发送文本） + run_message_id = str(uuid.uuid4()).replace('-', '') + run_request = { + "header": { + "message_id": run_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": self.appkey, + }, + "payload": { + "text": text + } + } + + await ws_connection.send(json.dumps(run_request)) + logger.bind(tag=TAG).debug(f"发送RunSynthesis指令: {text} (message_id: {run_message_id})") + + # 立即发送StopSynthesis指令，避免IDLE_TIMEOUT + stop_message_id = str(uuid.uuid4()).replace('-', '') + stop_request = { + "header": { + "message_id": stop_message_id, + "task_id": task_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StopSynthesis", + "appkey": self.appkey, + } + } + + await ws_connection.send(json.dumps(stop_request)) + logger.bind(tag=TAG).debug(f"发送StopSynthesis指令: {stop_message_id}") + + # 初始化处理参数 - 使用独立缓冲区 + pcm_buffer = bytearray() + opus_datas_cache = [] + segment_count = 0 + self.segment_count = 0 # 同时重置实例变量 + synthesis_completed = False + + # 发送第一个音频包 + self.tts_audio_queue.put((SentenceType.FIRST, [], text)) + + # 处理响应 - 设置超时时间避免长时间等待 + timeout_duration = 15 # 15秒超时 + try: + # 使用asyncio.wait_for替代asyncio.timeout以保证兼容性 + async def process_messages(): + nonlocal synthesis_completed, segment_count, pcm_buffer, opus_datas_cache # 声明使用外层变量 + async for message in ws_connection: + try: + if isinstance(message, str): + # 处理JSON消息 + data = json.loads(message) + header = data.get("header", {}) + event_name = header.get("name") + + if event_name == "SynthesisStarted": + logger.bind(tag=TAG).debug(f"TTS合成已启动: {task_id}") + + elif event_name == "SynthesisCompleted": + logger.bind(tag=TAG).debug(f"TTS合成完成: {text}") + synthesis_completed = True + break + + elif event_name == "TaskFailed": + error_msg = header.get("status_text", "未知错误") + logger.bind(tag=TAG).error(f"TTS合成失败: {error_msg}") + synthesis_completed = True + break + + elif isinstance(message, bytes): + # 处理二进制音频数据 + pcm_buffer.extend(message) + + # 计算每帧的字节数 + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels + * self.opus_encoder.frame_size_ms + / 1000 + * 2 + ) + + # 分帧处理PCM数据 + while len(pcm_buffer) >= frame_bytes: + frame = bytes(pcm_buffer[:frame_bytes]) + del pcm_buffer[:frame_bytes] # 清除已处理的数据 + + # 编码为Opus + opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) + if opus_packets: + if segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_packets, None) + ) + segment_count += 1 + else: + opus_datas_cache.extend(opus_packets) + + except json.JSONDecodeError: + logger.bind(tag=TAG).warning("收到无效的JSON消息") + except Exception as e: + logger.bind(tag=TAG).error(f"处理响应消息失败: {e}") + + await asyncio.wait_for(process_messages(), timeout=timeout_duration) + + except asyncio.TimeoutError: + logger.bind(tag=TAG).warning(f"TTS请求超时，但可能已获取部分音频数据: {text}") + except websockets.ConnectionClosed: + logger.bind(tag=TAG).debug("WebSocket连接已正常关闭") + except Exception as e: + logger.bind(tag=TAG).error(f"处理WebSocket消息失败: {e}") + + # 因为已经提前发送了StopSynthesis，这里不需要再次发送 + # 直接处理剩余的PCM数据 + if pcm_buffer: + opus_packets = self.opus_encoder.encode_pcm_to_opus( + bytes(pcm_buffer), end_of_stream=True + ) + if opus_packets: + if segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_packets, None) + ) + segment_count += 1 + else: + opus_datas_cache.extend(opus_packets) + + # 发送缓存的数据 + if segment_count >= 10 and opus_datas_cache: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas_cache, None) + ) + + # 如果是最后一段，处理待播放文件 + if is_last: + self._process_before_stop_play_files() + + except Exception as e: + logger.bind(tag=TAG).error(f"TTS请求异常: {e}") + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + finally: + # 确保WebSocket连接被关闭 + if ws_connection: + try: + if hasattr(ws_connection, 'close'): + if asyncio.iscoroutinefunction(ws_connection.close): + await ws_connection.close() + else: + ws_connection.close() + except Exception as e: + logger.bind(tag=TAG).debug(f"关闭WebSocket连接时出现异常: {e}") + async def close(self): + """资源清理""" + if self.ws: + try: + # 兼容不同版本的websockets库关闭方式 + if hasattr(self.ws, 'close'): + if asyncio.iscoroutinefunction(self.ws.close): + await self.ws.close() + else: + self.ws.close() + elif hasattr(self.ws, 'close_connection'): + await self.ws.close_connection() + except Exception as e: + logger.bind(tag=TAG).debug(f"关闭WebSocket连接时出现异常: {e}") + finally: + self.ws = None + + if hasattr(self, "opus_encoder"): + self.opus_encoder.close() + + await super().close() + + def _process_before_stop_play_files(self): + """处理停止前的待播放文件""" + for tts_file, text in self.before_stop_play_files: + if tts_file and os.path.exists(tts_file): + audio_datas = self._process_audio_file(tts_file) + self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text)) + self.before_stop_play_files.clear() + self.tts_audio_queue.put((SentenceType.LAST, [], None)) + + def _process_audio_file(self, tts_file): + """处理音频文件并转换为指定格式""" + audio_datas = [] + if self.conn.audio_format == "pcm": + audio_datas, _ = self.audio_to_pcm_data(tts_file) + else: + audio_datas, _ = self.audio_to_opus_data(tts_file) + + if ( + self.delete_audio_file + and tts_file is not None + and os.path.exists(tts_file) + and tts_file.startswith(self.output_file) + ): + os.remove(tts_file) + return audio_datas + + def to_tts(self, text: str) -> list: + """非流式TTS处理，用于测试及保存音频文件的场景""" + start_time = time.time() + text = MarkdownCleaner.clean_markdown(text) + + try: + # 使用同步方式进行TTS转换 + if self._is_token_expired(): + self._refresh_token() + + # 构造请求数据 + request_json = { + "appkey": self.appkey, + "token": self.token, + "text": text, + "format": "pcm", + "sample_rate": self.sample_rate, + "voice": self.voice, + "volume": self.volume, + "speech_rate": self.speech_rate, + "pitch_rate": self.pitch_rate, + } + + # 使用HTTP接口进行同步请求 + import requests + api_url = f"https://{self.host}/stream/v1/tts" + headers = {"Content-Type": "application/json"} + + resp = requests.post(api_url, json=request_json, headers=headers) + + if resp.status_code == 401: # Token过期特殊处理 + self._refresh_token() + resp = requests.post(api_url, json=request_json, headers=headers) + + if resp.headers["Content-Type"].startswith("audio/"): + pcm_data = resp.content + + # 使用opus编码器处理PCM数据 + opus_datas = [] + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels + * self.opus_encoder.frame_size_ms + / 1000 + * 2 + ) + + # 分帧处理PCM数据 + for i in range(0, len(pcm_data), frame_bytes): + frame = pcm_data[i:i + frame_bytes] + if len(frame) == frame_bytes: + opus = self.opus_encoder.encode_pcm_to_opus(frame, False) + if opus: + opus_datas.extend(opus) + + logger.bind(tag=TAG).info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") + return opus_datas + else: + logger.bind(tag=TAG).error(f"TTS请求失败: {resp.content}") + return [] + + except Exception as e: + logger.bind(tag=TAG).error(f"TTS请求异常: {e}") + return [] + + def _get_segment_text(self): + """获取当前可以处理的文本段""" + if not self.tts_text_buff: + return None + + full_text = "".join(self.tts_text_buff) + if len(full_text) <= self.processed_chars: + return None + + # 获取未处理的文本 + remaining_text = full_text[self.processed_chars:] + + # 如果文本较短或者到达了句子结尾标点，直接处理 + sentence_endings = ['。', '！', '？', '.', '!', '?', '\n'] + if len(remaining_text) < 20: + return None + + # 查找句子结尾 + for i, char in enumerate(remaining_text): + if char in sentence_endings and i > 10: # 至少10个字符 + segment = remaining_text[:i+1] + segment_text = textUtils.get_string_no_punctuation_or_emoji(segment) + if segment_text: + self.processed_chars += i + 1 + return segment_text + + # 如果没有找到句子结尾，但文本足够长，按长度分段 + if len(remaining_text) > 50: + segment = remaining_text[:30] + segment_text = textUtils.get_string_no_punctuation_or_emoji(segment) + if segment_text: + self.processed_chars += 30 + return segment_text + + return None From 69cac9d40ab0e92533ff952c3d5e0ff7fcd32a66 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 3 Jul 2025 09:16:06 +0800 Subject: [PATCH 03/19] =?UTF-8?q?update:=20TTS=E5=A4=8D=E7=94=A8=E9=93=BE?= =?UTF-8?q?=E6=8E=A5=EF=BC=8CVAD=E5=8F=8C=E9=98=88=E5=80=BC=E5=88=A4?= =?UTF-8?q?=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 1 + main/xiaozhi-server/core/connection.py | 6 ++++ .../providers/tts/huoshan_double_stream.py | 36 ++++++++++++++----- .../core/providers/vad/silero.py | 28 +++++++++------ 4 files changed, 52 insertions(+), 19 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 5a43c995..a7a94658 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -325,6 +325,7 @@ VAD: SileroVAD: type: silero threshold: 0.5 + threshold_low: 0.3 model_dir: models/snakers4_silero-vad min_silence_duration_ms: 200 # 如果说话停顿比较长，可以把这个值设置大一些 diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 9d1949c4..375bd117 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -17,6 +17,7 @@ from core.utils.util import ( filter_sensitive_info, ) from typing import Dict, Any +from collections import deque from core.utils.modules_initialize import ( initialize_modules, initialize_tts, @@ -112,6 +113,8 @@ class ConnectionHandler: self.client_have_voice = False self.last_activity_time = 0.0 # 统一的活动时间戳（毫秒） self.client_voice_stop = False + self.client_voice_window = deque(maxlen=5) + self.last_is_voice = False # asr相关变量 # 因为实际部署时可能会用到公共的本地ASR，不能把变量暴露给公共ASR @@ -858,6 +861,9 @@ class ConnectionHandler: elif self.websocket: await self.websocket.close() + if self.tts: + await self.tts.close() + # 最后关闭线程池（避免阻塞） if self.executor: self.executor.shutdown(wait=False) diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 1ebaf15a..3fe6294e 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -1,4 +1,3 @@ -import os import uuid import json import queue @@ -10,7 +9,6 @@ from config.logger import setup_logging from core.utils import opus_encoder_utils from core.utils.util import check_model_key from core.providers.tts.base import TTSProviderBase -from core.handle.abortHandle import handleAbortMessage from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType from asyncio import Task @@ -175,6 +173,9 @@ class TTSProvider(TTSProviderBase): async def _ensure_connection(self): """建立新的WebSocket连接""" try: + if self.ws: + logger.bind(tag=TAG).info(f"使用已有链接...") + return self.ws logger.bind(tag=TAG).info("开始建立新连接...") ws_header = { "X-Api-App-Key": self.appId, @@ -368,9 +369,17 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).info("会话结束请求已发送") # 等待监听任务完成 - if hasattr(self, "_monitor_task"): + if hasattr(self, "_monitor_task") and self._monitor_task is not None: try: - await self._monitor_task + # 设置4秒超时等待监听任务结束 + await asyncio.wait_for(self._monitor_task, timeout=4) + except asyncio.TimeoutError: + logger.bind(tag=TAG).warning("等待监听任务超时，强制取消") + self._monitor_task.cancel() + try: + await self._monitor_task + except asyncio.CancelledError: + pass except Exception as e: logger.bind(tag=TAG).error( f"等待监听任务完成时发生错误: {str(e)}" @@ -378,8 +387,6 @@ class TTSProvider(TTSProviderBase): finally: self._monitor_task = None - # 关闭连接 - await self.close() except Exception as e: logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}") # 确保清理资源 @@ -400,6 +407,15 @@ class TTSProvider(TTSProviderBase): async def close(self): """资源清理方法""" + # 取消监听任务 + if self._monitor_task: + try: + self._monitor_task.cancel() + await self._monitor_task + except asyncio.CancelledError: + pass + self._monitor_task = None + if self.ws: try: await self.ws.close() @@ -413,6 +429,7 @@ class TTSProvider(TTSProviderBase): is_first_sentence = True first_sentence_segment_count = 0 # 添加计数器 try: + session_finished = False # 标记会话是否正常结束 while not self.conn.stop_event.is_set(): try: # 确保 `recv()` 运行在同一个 event loop @@ -466,6 +483,7 @@ class TTSProvider(TTSProviderBase): elif res.optional.event == EVENT_SessionFinished: logger.bind(tag=TAG).debug(f"会话结束～～") self._process_before_stop_play_files() + session_finished = True break except websockets.ConnectionClosed: logger.bind(tag=TAG).warning("WebSocket连接已关闭") @@ -476,15 +494,15 @@ class TTSProvider(TTSProviderBase): ) traceback.print_exc() break - finally: - # 确保清理资源 - if self.ws: + # 仅在连接异常时才关闭 + if not session_finished and self.ws: try: await self.ws.close() except: pass self.ws = None # 监听任务退出时清理引用 + finally: self._monitor_task = None async def send_event( diff --git a/main/xiaozhi-server/core/providers/vad/silero.py b/main/xiaozhi-server/core/providers/vad/silero.py index 2c45ba8c..b516d8fb 100644 --- a/main/xiaozhi-server/core/providers/vad/silero.py +++ b/main/xiaozhi-server/core/providers/vad/silero.py @@ -23,22 +23,24 @@ class VADProvider(VADProviderBase): # 处理空字符串的情况 threshold = config.get("threshold", "0.5") + threshold_low = config.get("threshold_low", "0.2") min_silence_duration_ms = config.get("min_silence_duration_ms", "1000") self.vad_threshold = float(threshold) if threshold else 0.5 + self.vad_threshold_low = float(threshold_low) if threshold_low else 0.2 + self.silence_threshold_ms = ( int(min_silence_duration_ms) if min_silence_duration_ms else 1000 ) + # 至少要多少帧才算有语音 + self.frame_window_threshold = 3 + def is_vad(self, conn, opus_packet): try: pcm_frame = self.decoder.decode(opus_packet, 960) conn.client_audio_buffer.extend(pcm_frame) # 将新数据加入缓冲区 - # 确保帧计数器存在 - if not hasattr(conn, "client_voice_frame_count"): - conn.client_voice_frame_count = 0 - # 处理缓冲区中的完整帧（每次处理512采样点） client_have_voice = False while len(conn.client_audio_buffer) >= 512 * 2: @@ -54,15 +56,21 @@ class VADProvider(VADProviderBase): # 检测语音活动 with torch.no_grad(): speech_prob = self.model(audio_tensor, 16000).item() - is_voice = speech_prob >= self.vad_threshold - if is_voice: - conn.client_voice_frame_count += 1 + # 双阈值判断 + if speech_prob >= self.vad_threshold: + is_voice = True + elif speech_prob <= self.vad_threshold_low: + is_voice = False else: - conn.client_voice_frame_count = 0 + is_voice = conn.last_is_voice - # 只有连续4帧检测到语音才认为有语音 - client_have_voice = conn.client_voice_frame_count >= 4 + # 声音没低于最低值则延续前一个状态，判断为有声音 + conn.last_is_voice = is_voice + + # 更新滑动窗口 + conn.client_voice_window.append(is_voice) + client_have_voice = (conn.client_voice_window.count(True) >= self.frame_window_threshold) # 如果之前有声音，但本次没有声音，且与上次有声音的时间差已经超过了静默阈值，则认为已经说完一句话 if conn.client_have_voice and not client_have_voice: From 7e5a2b4549fbd22e9727deac192fdd31179fa575 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 4 Jul 2025 11:27:49 +0800 Subject: [PATCH 04/19] =?UTF-8?q?fix:=20=E6=92=AD=E6=94=BE=E9=9F=B3?= =?UTF-8?q?=E4=B9=90=E6=97=B6=EF=BC=8C=E5=BC=95=E5=AF=BC=E8=AF=8D=E5=8D=A1?= =?UTF-8?q?=E9=A1=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/providers/tts/base.py | 6 ++---- .../core/providers/tts/huoshan_double_stream.py | 10 +++++++--- main/xiaozhi-server/core/providers/tts/linkerai.py | 10 +++++++--- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 223cb807..d196b8f9 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -362,10 +362,8 @@ class TTSProviderBase(ABC): return audio_datas def _process_before_stop_play_files(self): - for tts_file, text in self.before_stop_play_files: - if tts_file and os.path.exists(tts_file): - audio_datas = self._process_audio_file(tts_file) - self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text)) + for audio_datas, text in self.before_stop_play_files: + self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text)) self.before_stop_play_files.clear() self.tts_audio_queue.put((SentenceType.LAST, [], None)) diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 3fe6294e..6f8fcca5 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -1,3 +1,4 @@ +import os import uuid import json import queue @@ -245,9 +246,12 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).info( f"添加音频文件到待播放列表: {message.content_file}" ) - self.before_stop_play_files.append( - (message.content_file, message.content_detail) - ) + if message.content_file and os.path.exists(message.content_file): + # 先处理文件音频数据 + file_audio = self._process_audio_file(message.content_file) + self.before_stop_play_files.append( + (file_audio, message.content_detail) + ) if message.sentence_type == SentenceType.LAST: try: diff --git a/main/xiaozhi-server/core/providers/tts/linkerai.py b/main/xiaozhi-server/core/providers/tts/linkerai.py index ee108540..1046b972 100644 --- a/main/xiaozhi-server/core/providers/tts/linkerai.py +++ b/main/xiaozhi-server/core/providers/tts/linkerai.py @@ -1,3 +1,4 @@ +import os import queue import asyncio import traceback @@ -63,9 +64,12 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).info( f"添加音频文件到待播放列表: {message.content_file}" ) - self.before_stop_play_files.append( - (message.content_file, message.content_detail) - ) + if message.content_file and os.path.exists(message.content_file): + # 先处理文件音频数据 + file_audio = self._process_audio_file(message.content_file) + self.before_stop_play_files.append( + (file_audio, message.content_detail) + ) if message.sentence_type == SentenceType.LAST: # 处理剩余的文本 From 8da9dc8a3f8f761515e6b96f365cf5106ee40229 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Mon, 7 Jul 2025 11:45:37 +0800 Subject: [PATCH 05/19] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E5=BC=95?= =?UTF-8?q?=E5=AF=BC=E8=AF=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/plugins_func/functions/play_music.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main/xiaozhi-server/plugins_func/functions/play_music.py b/main/xiaozhi-server/plugins_func/functions/play_music.py index 6d184b6b..a6eb5822 100644 --- a/main/xiaozhi-server/plugins_func/functions/play_music.py +++ b/main/xiaozhi-server/plugins_func/functions/play_music.py @@ -181,10 +181,10 @@ def _get_random_play_prompt(song_name): f"正在为您播放，{clean_name}", f"请欣赏歌曲，{clean_name}", f"即将为您播放，{clean_name}", - f"为您带来，{clean_name}", - f"让我们聆听，{clean_name}", + f"现在为您带来，《{clean_name}》", + f"让我们一起聆听，《{clean_name}》", f"接下来请欣赏，{clean_name}", - f"为您献上，{clean_name}", + f"此刻为您献上，《{clean_name}》", ] # 直接使用random.choice，不设置seed return random.choice(prompts) From 155ba92ba68bcde8d2bca8b472f4d1f228689cf3 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Mon, 7 Jul 2025 15:29:13 +0800 Subject: [PATCH 06/19] =?UTF-8?q?update:=E5=A2=9E=E5=8A=A0=E4=BA=86?= =?UTF-8?q?=E6=99=BA=E6=8E=A7=E5=8F=B0=E7=9A=84sql=E8=AF=AD=E5=8F=A5?= =?UTF-8?q?=EF=BC=8C=E4=BC=98=E5=8C=96=E4=BA=86=E9=83=A8=E5=88=86=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{202505081146.sql => 202505081147.sql} | 2 +- .../resources/db/changelog/202507071130.sql | 26 +++++++++ .../resources/db/changelog/202507071530.sql | 56 +++++++++++++++++++ .../db/changelog/db.changelog-master.yaml | 20 ++++++- main/xiaozhi-server/config.yaml | 10 ++-- .../core/providers/asr/aliyun_stream.py | 2 +- 6 files changed, 107 insertions(+), 9 deletions(-) rename main/manager-api/src/main/resources/db/changelog/{202505081146.sql => 202505081147.sql} (97%) create mode 100644 main/manager-api/src/main/resources/db/changelog/202507071130.sql create mode 100644 main/manager-api/src/main/resources/db/changelog/202507071530.sql diff --git a/main/manager-api/src/main/resources/db/changelog/202505081146.sql b/main/manager-api/src/main/resources/db/changelog/202505081147.sql similarity index 97% rename from main/manager-api/src/main/resources/db/changelog/202505081146.sql rename to main/manager-api/src/main/resources/db/changelog/202505081147.sql index fe7582f3..1a561bda 100644 --- a/main/manager-api/src/main/resources/db/changelog/202505081146.sql +++ b/main/manager-api/src/main/resources/db/changelog/202505081147.sql @@ -1,6 +1,6 @@ -- 添加百度ASR模型配置 delete from `ai_model_config` where `id` = 'ASR_BaiduASR'; -INSERT INTO `ai_model_config` VALUES ('ASR_BaiduASR', 'ASR', 'BaiduASR', '百度语音识别', 0, 1, '{\"type\": \"baidu\", \"app_id\": \"\", \"api_key\": \"\", \"secret_key\": \"\", \"dev_pid\": 1537, \"output_dir\": \"tmp/\"}', NULL, NULL, 7, NULL, NULL, NULL, NULL); +INSERT INTO `ai_model_config` VALUES ('ASR_BaiduASR', 'ASR', 'BaiduASR', '百度语音识别', 0, 1, '{\"type\": \"baidu\", \"app_id\": \"\", \"api_key\": \"\", \"secret_key\": \"\", \"dev_pid\": 1537, \"output_dir\": \"tmp/\"}', NULL, NULL, 8, NULL, NULL, NULL, NULL); -- 添加百度ASR供应器 diff --git a/main/manager-api/src/main/resources/db/changelog/202507071130.sql b/main/manager-api/src/main/resources/db/changelog/202507071130.sql new file mode 100644 index 00000000..008d624e --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507071130.sql @@ -0,0 +1,26 @@ +-- 添加阿里云流式ASR供应器 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_AliyunStreamASR'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_AliyunStreamASR', 'ASR', 'aliyun_stream', '阿里云语音识别(流式)', '[{"key":"appkey","label":"应用AppKey","type":"string"},{"key":"token","label":"临时Token","type":"string"},{"key":"access_key_id","label":"AccessKey ID","type":"string"},{"key":"access_key_secret","label":"AccessKey Secret","type":"string"},{"key":"host","label":"服务地址","type":"string"},{"key":"max_sentence_silence","label":"断句检测时间","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 6, 1, NOW(), 1, NOW()); + +-- 添加阿里云流式ASR模型配置 +delete from `ai_model_config` where id = 'ASR_AliyunStreamASR'; +INSERT INTO `ai_model_config` VALUES ('ASR_AliyunStreamASR', 'ASR', 'AliyunStreamASR', '阿里云语音识别(流式)', 0, 1, '{\"type\": \"aliyun_stream\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"host\": \"nls-gateway-cn-shanghai.aliyuncs.com\", \"max_sentence_silence\": 800, \"output_dir\": \"tmp/\"}', NULL, NULL, 7, NULL, NULL, NULL, NULL); + +-- 更新阿里云流式ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://nls-portal.console.aliyun.com/', +`remark` = '阿里云流式ASR配置说明： +1. 阿里云ASR和阿里云(流式)ASR的区别是：阿里云ASR是一次性识别，阿里云(流式)ASR是实时流式识别 +2. 流式ASR具有更低的延迟和更好的实时性，适合语音交互场景 +3. 需要在阿里云智能语音交互控制台创建应用并获取认证信息 +4. 支持中文实时语音识别，支持标点符号预测和逆文本规范化 +5. 需要网络连接，输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://nls-portal.console.aliyun.com/ 开通智能语音交互服务 +2. 访问 https://nls-portal.console.aliyun.com/applist 创建项目并获取appkey +3. 访问 https://nls-portal.console.aliyun.com/overview 获取临时token（或配置access_key_id和access_key_secret自动获取） +4. 如需动态token管理，建议配置access_key_id和access_key_secret +5. max_sentence_silence参数控制断句检测时间（毫秒），默认800ms +如需了解更多参数配置，请参考：https://help.aliyun.com/zh/isi/developer-reference/real-time-speech-recognition +' WHERE `id` = 'ASR_AliyunStreamASR'; diff --git a/main/manager-api/src/main/resources/db/changelog/202507071530.sql b/main/manager-api/src/main/resources/db/changelog/202507071530.sql new file mode 100644 index 00000000..ed3dcf49 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507071530.sql @@ -0,0 +1,56 @@ +-- 添加阿里云流式TTS供应器 +delete from `ai_model_provider` where id = 'SYSTEM_TTS_AliyunStreamTTS'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_TTS_AliyunStreamTTS', 'TTS', 'aliyun_stream', '阿里云语音合成(流式)', '[{"key":"appkey","label":"应用AppKey","type":"string"},{"key":"token","label":"临时Token","type":"string"},{"key":"access_key_id","label":"AccessKey ID","type":"string"},{"key":"access_key_secret","label":"AccessKey Secret","type":"string"},{"key":"host","label":"服务地址","type":"string"},{"key":"voice","label":"默认音色","type":"string"},{"key":"format","label":"音频格式","type":"string"},{"key":"sample_rate","label":"采样率","type":"number"},{"key":"volume","label":"音量","type":"number"},{"key":"speech_rate","label":"语速","type":"number"},{"key":"pitch_rate","label":"音调","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 15, 1, NOW(), 1, NOW()); + +-- 添加阿里云流式TTS模型配置 +delete from `ai_model_config` where id = 'TTS_AliyunStreamTTS'; +INSERT INTO `ai_model_config` VALUES ('TTS_AliyunStreamTTS', 'TTS', 'AliyunStreamTTS', '阿里云语音合成(流式)', 0, 1, '{\"type\": \"aliyun_stream\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"host\": \"nls-gateway-cn-beijing.aliyuncs.com\", \"voice\": \"longxiaochun\", \"format\": \"pcm\", \"sample_rate\": 16000, \"volume\": 50, \"speech_rate\": 0, \"pitch_rate\": 0, \"output_dir\": \"tmp/\"}', NULL, NULL, 18, NULL, NULL, NULL, NULL); + +-- 更新阿里云流式TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://nls-portal.console.aliyun.com/', +`remark` = '阿里云流式TTS配置说明： +1. 阿里云TTS和阿里云(流式)TTS的区别是：阿里云TTS是一次性合成，阿里云(流式)TTS是实时流式合成 +2. 流式TTS具有更低的延迟和更好的实时性，适合语音交互场景 +3. 需要在阿里云智能语音交互控制台创建应用并获取认证信息 +4. 支持CosyVoice大模型音色，音质更加自然 +5. 支持实时调节音量、语速、音调等参数 +申请步骤： +1. 访问 https://nls-portal.console.aliyun.com/ 开通智能语音交互服务 +2. 访问 https://nls-portal.console.aliyun.com/applist 创建项目并获取appkey +3. 访问 https://nls-portal.console.aliyun.com/overview 获取临时token（或配置access_key_id和access_key_secret自动获取） +4. 如需动态token管理，建议配置access_key_id和access_key_secret +5. 可选择北京、上海等不同地域的服务器以优化延迟 +6. voice参数支持CosyVoice大模型音色，如longxiaochun、longyueyue等 +如需了解更多参数配置，请参考：https://help.aliyun.com/zh/isi/developer-reference/real-time-speech-synthesis +' WHERE `id` = 'TTS_AliyunStreamTTS'; + +-- 添加阿里云流式TTS音色 +delete from `ai_tts_voice` where tts_model_id = 'TTS_AliyunStreamTTS'; +-- 温柔女声系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0001', 'TTS_AliyunStreamTTS', '龙小淳-温柔姐姐', 'longxiaochun', '中文及中英文混合', NULL, NULL, 1, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0002', 'TTS_AliyunStreamTTS', '龙小夏-温柔女声', 'longxiaoxia', '中文及中英文混合', NULL, NULL, 2, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0003', 'TTS_AliyunStreamTTS', '龙玫-温柔女声', 'longmei', '中文及中英文混合', NULL, NULL, 3, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0004', 'TTS_AliyunStreamTTS', '龙瑰-温柔女声', 'longgui', '中文及中英文混合', NULL, NULL, 4, NULL, NULL, NULL, NULL); +-- 御姐女声系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0005', 'TTS_AliyunStreamTTS', '龙玉-御姐女声', 'longyu', '中文及中英文混合', NULL, NULL, 5, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0006', 'TTS_AliyunStreamTTS', '龙娇-御姐女声', 'longjiao', '中文及中英文混合', NULL, NULL, 6, NULL, NULL, NULL, NULL); +-- 男声系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0007', 'TTS_AliyunStreamTTS', '龙臣-译制片男声', 'longchen', '中文及中英文混合', NULL, NULL, 7, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0008', 'TTS_AliyunStreamTTS', '龙修-青年男声', 'longxiu', '中文及中英文混合', NULL, NULL, 8, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0009', 'TTS_AliyunStreamTTS', '龙橙-阳光男声', 'longcheng', '中文及中英文混合', NULL, NULL, 9, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0010', 'TTS_AliyunStreamTTS', '龙哲-成熟男声', 'longzhe', '中文及中英文混合', NULL, NULL, 10, NULL, NULL, NULL, NULL); +-- 专业播报系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0011', 'TTS_AliyunStreamTTS', 'Bella2.0-新闻女声', 'loongbella', '中文及中英文混合', NULL, NULL, 11, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0012', 'TTS_AliyunStreamTTS', 'Stella2.0-飒爽女声', 'loongstella', '中文及中英文混合', NULL, NULL, 12, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0013', 'TTS_AliyunStreamTTS', '龙书-新闻男声', 'longshu', '中文及中英文混合', NULL, NULL, 13, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0014', 'TTS_AliyunStreamTTS', '龙婧-严肃女声', 'longjing', '中文及中英文混合', NULL, NULL, 14, NULL, NULL, NULL, NULL); +-- 特色音色系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0015', 'TTS_AliyunStreamTTS', '龙奇-活泼童声', 'longqi', '中文及中英文混合', NULL, NULL, 15, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0016', 'TTS_AliyunStreamTTS', '龙华-活泼女童', 'longhua', '中文及中英文混合', NULL, NULL, 16, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0017', 'TTS_AliyunStreamTTS', '龙无-无厘头男声', 'longwu', '中文及中英文混合', NULL, NULL, 17, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0018', 'TTS_AliyunStreamTTS', '龙大锤-幽默男声', 'longdachui', '中文及中英文混合', NULL, NULL, 18, NULL, NULL, NULL, NULL); +-- 粤语系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0019', 'TTS_AliyunStreamTTS', '龙嘉怡-粤语女声', 'longjiayi', '粤语及粤英混合', NULL, NULL, 19, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0020', 'TTS_AliyunStreamTTS', '龙桃-粤语女声', 'longtao', '粤语及粤英混合', NULL, NULL, 20, NULL, NULL, NULL, NULL); diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index b60094ef..3852c545 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -101,12 +101,12 @@ databaseChangeLog: encoding: utf8 path: classpath:db/changelog/202505022134.sql - changeSet: - id: 202505081146 + id: 202505081147 author: hrz changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202505081146.sql + path: classpath:db/changelog/202505081147.sql - changeSet: id: 202505091555 author: whosmyqueen @@ -225,4 +225,18 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202506261637.sql \ No newline at end of file + path: classpath:db/changelog/202506261637.sql + - changeSet: + id: 202507071130 + author: cgd + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202507071130.sql + - changeSet: + id: 202507071530 + author: cgd + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202507071530.sql diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index a4b4ccc0..be052dc4 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -322,13 +322,15 @@ ASR: # 一般来说非流式ASR更便宜（0.004元/秒，¥0.24/分钟） # 但是AliyunStreamASR实时性更好（0.005元/秒，¥0.3/分钟） # 定义ASR API类型 - type: aliyun + type: aliyun_stream appkey: 你的阿里云智能语音交互服务项目Appkey token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret access_key_id: 你的阿里云账号access_key_id access_key_secret: 你的阿里云账号access_key_secret - # 服务器地域选择，如nls-gateway-cn-hangzhou.aliyuncs.com(杭州)等 - host: 根据自己的地理位置选择最近的服务器地域，不填默认为nls-gateway-cn-shanghai.aliyuncs.com(上海) + # 服务器地域选择，可选择距离更近的服务器以减少延迟，如nls-gateway-cn-hangzhou.aliyuncs.com(杭州)等 + host: nls-gateway-cn-shanghai.aliyuncs.com + # 断句检测时间(毫秒)，控制静音多长时间后进行断句，默认800毫秒 + max_sentence_silence: 800 output_dir: tmp/ BaiduASR: # 获取AppID、API Key、Secret Key：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list @@ -699,7 +701,7 @@ TTS: # appkey地址：https://nls-portal.console.aliyun.com/applist # token地址：https://nls-portal.console.aliyun.com/overview # 使用三阶段流式交互：StartSynthesis -> RunSynthesis -> StopSynthesis - type: aliyun + type: aliyun_stream output_dir: tmp/ appkey: 你的阿里云智能语音交互服务项目Appkey token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret diff --git a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py index 94727798..3a41b349 100644 --- a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py @@ -123,7 +123,7 @@ class ASRProvider(ASRProviderBase): conn.asr_audio.append(audio) conn.asr_audio = conn.asr_audio[-10:] - # 参考豆包ASR：只在有声音且没有连接时建立连接 + # 只在有声音且没有连接时建立连接 if audio_have_voice and not self.is_processing: try: await self._start_recognition(conn) From 44593ff3f38a807463e74b4fae2adaa39cc49975 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Tue, 8 Jul 2025 17:48:50 +0800 Subject: [PATCH 07/19] =?UTF-8?q?update:=20=E4=BC=98=E5=8C=96chat=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E6=B5=81=E7=A8=8B=20=E4=BC=98=E5=8C=96huoshan?= =?UTF-8?q?=E5=A4=84=E7=90=86=20=E4=BC=9A=E8=AF=9D=E4=BF=9D=E6=8C=81?= =?UTF-8?q?=E4=B8=80=E8=87=B4=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 38 +++++---- .../core/handle/intentHandler.py | 19 ++++- .../xiaozhi-server/core/providers/tts/base.py | 14 ---- .../providers/tts/huoshan_double_stream.py | 78 ++++++++----------- .../plugins_func/functions/play_music.py | 35 ++++----- 5 files changed, 86 insertions(+), 98 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 375bd117..e0eb90bc 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -594,13 +594,24 @@ class ConnectionHandler: # 更新系统prompt至上下文 self.dialogue.update_system_message(self.prompt) - def chat(self, query, tool_call=False): + def chat(self, query, tool_call=False, depth=0): self.logger.bind(tag=TAG).info(f"大模型收到用户消息: {query}") self.llm_finish_task = False if not tool_call: self.dialogue.put(Message(role="user", content=query)) + # 为最顶层时新建会话ID和发送FIRST请求 + if depth == 0: + self.sentence_id = str(uuid.uuid4().hex) + self.tts.tts_text_queue.put( + TTSMessageDTO( + sentence_id=self.sentence_id, + sentence_type=SentenceType.FIRST, + content_type=ContentType.ACTION, + ) + ) + # Define intent functions functions = None if self.intent_type == "function_call" and hasattr(self, "func_handler"): @@ -616,8 +627,6 @@ class ConnectionHandler: ) memory_str = future.result() - self.sentence_id = str(uuid.uuid4().hex) - if self.intent_type == "function_call" and functions is not None: # 使用支持functions的streaming接口 llm_responses = self.llm.response_with_functions( @@ -640,7 +649,6 @@ class ConnectionHandler: function_id = None function_arguments = "" content_arguments = "" - text_index = 0 self.client_abort = False for response in llm_responses: if self.client_abort: @@ -670,14 +678,6 @@ class ConnectionHandler: if content is not None and len(content) > 0: if not tool_call_flag: response_message.append(content) - if text_index == 0: - self.tts.tts_text_queue.put( - TTSMessageDTO( - sentence_id=self.sentence_id, - sentence_type=SentenceType.FIRST, - content_type=ContentType.ACTION, - ) - ) self.tts.tts_text_queue.put( TTSMessageDTO( sentence_id=self.sentence_id, @@ -686,7 +686,6 @@ class ConnectionHandler: content_detail=content, ) ) - text_index += 1 # 处理function call if tool_call_flag: bHasError = False @@ -711,6 +710,11 @@ class ConnectionHandler: f"function call error: {content_arguments}" ) if not bHasError: + # 如需要大模型先处理一轮，添加相关处理后的日志情况 + if len(response_message) > 0: + self.dialogue.put( + Message(role="assistant", content="".join(response_message)) + ) response_message.clear() self.logger.bind(tag=TAG).debug( f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}" @@ -728,14 +732,14 @@ class ConnectionHandler: ), self.loop, ).result() - self._handle_function_result(result, function_call_data) + self._handle_function_result(result, function_call_data, depth=depth) # 存储对话内容 if len(response_message) > 0: self.dialogue.put( Message(role="assistant", content="".join(response_message)) ) - if text_index > 0: + if depth == 0: self.tts.tts_text_queue.put( TTSMessageDTO( sentence_id=self.sentence_id, @@ -750,7 +754,7 @@ class ConnectionHandler: return True - def _handle_function_result(self, result, function_call_data): + def _handle_function_result(self, result, function_call_data, depth): if result.action == Action.RESPONSE: # 直接回复前端 text = result.response self.tts.tts_one_sentence(self, ContentType.TEXT, content_detail=text) @@ -787,7 +791,7 @@ class ConnectionHandler: content=text, ) ) - self.chat(text, tool_call=True) + self.chat(text, tool_call=True, depth=depth + 1) elif result.action == Action.NOTFOUND or result.action == Action.ERROR: text = result.response if result.response else result.result self.tts.tts_one_sentence(self, ContentType.TEXT, content_detail=text) diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index 2d0baca4..a398195b 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -6,9 +6,8 @@ from core.handle.helloHandle import checkWakeupWords from core.utils.util import remove_punctuation_and_length from core.providers.tts.dto.dto import ContentType from core.utils.dialogue import Message -from core.providers.tools.device_mcp import call_mcp_tool from plugins_func.register import Action, ActionResponse -from loguru import logger +from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType TAG = __name__ @@ -29,6 +28,8 @@ async def handle_user_intent(conn, text): intent_result = await analyze_intent_with_llm(conn, text) if not intent_result: return False + # 会话开始时生成sentence_id + conn.sentence_id = str(uuid.uuid4().hex) # 处理各种意图 return await process_intent_result(conn, intent_result, text) @@ -158,5 +159,19 @@ async def process_intent_result(conn, intent_result, original_text): def speak_txt(conn, text): + conn.tts.tts_text_queue.put( + TTSMessageDTO( + sentence_id=conn.sentence_id, + sentence_type=SentenceType.FIRST, + content_type=ContentType.ACTION, + ) + ) conn.tts.tts_one_sentence(conn, ContentType.TEXT, content_detail=text) + conn.tts.tts_text_queue.put( + TTSMessageDTO( + sentence_id=conn.sentence_id, + sentence_type=SentenceType.LAST, + content_type=ContentType.ACTION, + ) + ) conn.dialogue.put(Message(role="assistant", content=text)) diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index d196b8f9..c89b3247 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -161,13 +161,6 @@ class TTSProviderBase(ABC): else: sentence_id = str(uuid.uuid4()).replace("-", "") conn.sentence_id = sentence_id - self.tts_text_queue.put( - TTSMessageDTO( - sentence_id=sentence_id, - sentence_type=SentenceType.FIRST, - content_type=ContentType.ACTION, - ) - ) # 对于单句的文本，进行分段处理 segments = re.split(r"([。！？!?；;\n])", content_detail) for seg in segments: @@ -180,13 +173,6 @@ class TTSProviderBase(ABC): content_file=content_file, ) ) - self.tts_text_queue.put( - TTSMessageDTO( - sentence_id=sentence_id, - sentence_type=SentenceType.LAST, - content_type=ContentType.ACTION, - ) - ) async def open_audio_channels(self, conn): self.conn = conn diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 6f8fcca5..49643b85 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -202,6 +202,10 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).debug( f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}" ) + + if message.sentence_type == SentenceType.FIRST: + self.conn.client_abort = False + if self.conn.client_abort: logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") continue @@ -300,24 +304,23 @@ class TTSProvider(TTSProviderBase): async def start_session(self, session_id): logger.bind(tag=TAG).info(f"开始会话～～{session_id}") try: - task = self._monitor_task + # 会话开始时检测上个会话的监听状态 if ( - task is not None - and isinstance(task, Task) - and not task.done() + self._monitor_task is not None + and isinstance(self._monitor_task, Task) + and not self._monitor_task.done() ): - logger.bind(tag=TAG).info("等待上一个监听任务结束...") - if self.ws is not None: - logger.bind(tag=TAG).info("强制关闭上一个WebSocket连接以唤醒监听任务...") - try: - await self.ws.close() - except Exception as e: - logger.bind(tag=TAG).warning(f"关闭上一个ws异常: {e}") - self.ws = None + logger.bind(tag=TAG).info("取消上一个监听任务...") + self._monitor_task.cancel() try: - await asyncio.wait_for(task, timeout=8) + # 等待任务取消完成 + await asyncio.wait_for(self._monitor_task, timeout=3) + except asyncio.CancelledError: + logger.bind(tag=TAG).info("监听任务已成功取消") + except asyncio.TimeoutError: + logger.bind(tag=TAG).warning("取消监听任务超时，可能仍在运行") except Exception as e: - logger.bind(tag=TAG).warning(f"等待监听任务异常: {e}") + logger.bind(tag=TAG).warning(f"取消监听任务异常: {e}") self._monitor_task = None # 建立新连接 await self._ensure_connection() @@ -341,19 +344,7 @@ class TTSProvider(TTSProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}") # 确保清理资源 - if hasattr(self, "_monitor_task"): - try: - self._monitor_task.cancel() - await self._monitor_task - except: - pass - self._monitor_task = None - if self.ws: - try: - await self.ws.close() - except: - pass - self.ws = None + await self.close() raise async def finish_session(self, session_id): @@ -379,11 +370,14 @@ class TTSProvider(TTSProviderBase): await asyncio.wait_for(self._monitor_task, timeout=4) except asyncio.TimeoutError: logger.bind(tag=TAG).warning("等待监听任务超时，强制取消") - self._monitor_task.cancel() - try: - await self._monitor_task - except asyncio.CancelledError: - pass + if not self._monitor_task.done(): + self._monitor_task.cancel() + try: + await self._monitor_task + except asyncio.CancelledError: + pass + except Exception as e: + logger.bind(tag=TAG).warning(f"监听任务取消时发生错误: {e}") except Exception as e: logger.bind(tag=TAG).error( f"等待监听任务完成时发生错误: {str(e)}" @@ -394,31 +388,21 @@ class TTSProvider(TTSProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}") # 确保清理资源 - if hasattr(self, "_monitor_task"): - try: - self._monitor_task.cancel() - await self._monitor_task - except: - pass - self._monitor_task = None - if self.ws: - try: - await self.ws.close() - except: - pass - self.ws = None + await self.close() raise async def close(self): """资源清理方法""" # 取消监听任务 - if self._monitor_task: + if self._monitor_task and not self._monitor_task.done(): try: self._monitor_task.cancel() await self._monitor_task except asyncio.CancelledError: pass - self._monitor_task = None + except Exception as e: + logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}") + self._monitor_task = None if self.ws: try: diff --git a/main/xiaozhi-server/plugins_func/functions/play_music.py b/main/xiaozhi-server/plugins_func/functions/play_music.py index a6eb5822..6451e526 100644 --- a/main/xiaozhi-server/plugins_func/functions/play_music.py +++ b/main/xiaozhi-server/plugins_func/functions/play_music.py @@ -1,13 +1,10 @@ -from config.logger import setup_logging import os import re import time import random -import asyncio import difflib import traceback from pathlib import Path -from core.utils import p3 from core.handle.sendAudioHandle import send_stt_message from plugins_func.register import register_function, ToolType, ActionResponse, Action from core.utils.dialogue import Message @@ -51,8 +48,8 @@ def play_music(conn, song_name: str): ) # 提交异步任务 - future = asyncio.run_coroutine_threadsafe( - handle_music_command(conn, music_intent), conn.loop + task = conn.loop.create_task( + handle_music_command(conn, music_intent) # 封装异步逻辑 ) # 非阻塞回调处理 @@ -63,7 +60,7 @@ def play_music(conn, song_name: str): except Exception as e: conn.logger.bind(tag=TAG).error(f"播放失败: {e}") - future.add_done_callback(handle_done) + task.add_done_callback(handle_done) return ActionResponse( action=Action.NONE, result="指令已接收", response="正在为您播放音乐" @@ -218,13 +215,14 @@ async def play_local_music(conn, specific_file=None): await send_stt_message(conn, text) conn.dialogue.put(Message(role="assistant", content=text)) - conn.tts.tts_text_queue.put( - TTSMessageDTO( - sentence_id=conn.sentence_id, - sentence_type=SentenceType.FIRST, - content_type=ContentType.ACTION, + if conn.intent_type == "intent_llm": + conn.tts.tts_text_queue.put( + TTSMessageDTO( + sentence_id=conn.sentence_id, + sentence_type=SentenceType.FIRST, + content_type=ContentType.ACTION, + ) ) - ) conn.tts.tts_text_queue.put( TTSMessageDTO( sentence_id=conn.sentence_id, @@ -241,13 +239,14 @@ async def play_local_music(conn, specific_file=None): content_file=music_path, ) ) - conn.tts.tts_text_queue.put( - TTSMessageDTO( - sentence_id=conn.sentence_id, - sentence_type=SentenceType.LAST, - content_type=ContentType.ACTION, + if conn.intent_type == "intent_llm": + conn.tts.tts_text_queue.put( + TTSMessageDTO( + sentence_id=conn.sentence_id, + sentence_type=SentenceType.LAST, + content_type=ContentType.ACTION, + ) ) - ) except Exception as e: conn.logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}") From 62e39dd0e248be0d89689709d3c0caaaf930a626 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 9 Jul 2025 11:18:26 +0800 Subject: [PATCH 08/19] =?UTF-8?q?fix:=20=E7=AD=89=E5=BE=85=E6=97=B6?= =?UTF-8?q?=E5=8F=AF=E8=83=BD=E5=B7=B2=E7=BB=8F=E5=AE=8C=E6=88=90=EF=BC=88?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E4=B8=BANone=EF=BC=89=EF=BC=8C=E5=90=8E?= =?UTF-8?q?=E7=BB=AD=E5=AF=B9None=E9=94=99=E8=AF=AF=E8=AE=BF=E9=97=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../providers/tts/huoshan_double_stream.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 49643b85..e6e4d2fa 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -364,20 +364,9 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).info("会话结束请求已发送") # 等待监听任务完成 - if hasattr(self, "_monitor_task") and self._monitor_task is not None: + if self._monitor_task: try: - # 设置4秒超时等待监听任务结束 - await asyncio.wait_for(self._monitor_task, timeout=4) - except asyncio.TimeoutError: - logger.bind(tag=TAG).warning("等待监听任务超时，强制取消") - if not self._monitor_task.done(): - self._monitor_task.cancel() - try: - await self._monitor_task - except asyncio.CancelledError: - pass - except Exception as e: - logger.bind(tag=TAG).warning(f"监听任务取消时发生错误: {e}") + await self._monitor_task except Exception as e: logger.bind(tag=TAG).error( f"等待监听任务完成时发生错误: {str(e)}" @@ -394,7 +383,7 @@ class TTSProvider(TTSProviderBase): async def close(self): """资源清理方法""" # 取消监听任务 - if self._monitor_task and not self._monitor_task.done(): + if self._monitor_task: try: self._monitor_task.cancel() await self._monitor_task @@ -402,7 +391,7 @@ class TTSProvider(TTSProviderBase): pass except Exception as e: logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}") - self._monitor_task = None + self._monitor_task = None if self.ws: try: From 7f67459c6eb95e3f49d048e400f0b533468da2c7 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Wed, 9 Jul 2025 16:26:26 +0800 Subject: [PATCH 09/19] =?UTF-8?q?update:=E5=A2=9E=E5=8A=A0=E4=B9=A6?= =?UTF-8?q?=E5=90=8D=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/plugins_func/functions/play_music.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main/xiaozhi-server/plugins_func/functions/play_music.py b/main/xiaozhi-server/plugins_func/functions/play_music.py index 6451e526..2cbc4018 100644 --- a/main/xiaozhi-server/plugins_func/functions/play_music.py +++ b/main/xiaozhi-server/plugins_func/functions/play_music.py @@ -175,12 +175,12 @@ def _get_random_play_prompt(song_name): # 移除文件扩展名 clean_name = os.path.splitext(song_name)[0] prompts = [ - f"正在为您播放，{clean_name}", - f"请欣赏歌曲，{clean_name}", - f"即将为您播放，{clean_name}", + f"正在为您播放，《{clean_name}》", + f"请欣赏歌曲，《{clean_name}》", + f"即将为您播放，《{clean_name}》", f"现在为您带来，《{clean_name}》", f"让我们一起聆听，《{clean_name}》", - f"接下来请欣赏，{clean_name}", + f"接下来请欣赏，《{clean_name}》", f"此刻为您献上，《{clean_name}》", ] # 直接使用random.choice，不设置seed From 2625133f407125b77bfa3d2c940154655b43e519 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 9 Jul 2025 17:04:42 +0800 Subject: [PATCH 10/19] =?UTF-8?q?fix:=20=E6=89=93=E6=96=AD=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E6=9C=AA=E9=87=8D=E7=BD=AE=20=E7=9B=91=E5=90=AC?= =?UTF-8?q?=E6=9C=AA=E5=AE=8C=E6=88=90=E6=97=B6=E6=9C=8D=E5=8A=A1=E7=AB=AF?= =?UTF-8?q?=E5=8F=AF=E8=83=BD=E8=BF=98=E5=9C=A8=E5=8F=91=E9=80=81=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=20=E6=AD=A4=E6=97=B6=E5=A4=8D=E7=94=A8=E9=93=BE?= =?UTF-8?q?=E6=8E=A5=E4=BC=9A=E6=8E=A5=E6=94=B6=E4=B8=8A=E4=B8=AA=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E7=9A=84=E6=AE=8B=E4=BD=99=20=E9=9C=80=E8=A6=81?= =?UTF-8?q?=E4=B8=A4=E8=80=85=E4=B8=80=E5=90=8C=E5=85=B3=E9=97=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/providers/tts/base.py | 2 ++ .../core/providers/tts/huoshan_double_stream.py | 15 +++------------ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index c89b3247..9127ec12 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -195,6 +195,8 @@ class TTSProviderBase(ABC): while not self.conn.stop_event.is_set(): try: message = self.tts_text_queue.get(timeout=1) + if message.sentence_type == SentenceType.FIRST: + self.conn.client_abort = False if self.conn.client_abort: logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") continue diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index e6e4d2fa..0eb7c72d 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -310,18 +310,9 @@ class TTSProvider(TTSProviderBase): and isinstance(self._monitor_task, Task) and not self._monitor_task.done() ): - logger.bind(tag=TAG).info("取消上一个监听任务...") - self._monitor_task.cancel() - try: - # 等待任务取消完成 - await asyncio.wait_for(self._monitor_task, timeout=3) - except asyncio.CancelledError: - logger.bind(tag=TAG).info("监听任务已成功取消") - except asyncio.TimeoutError: - logger.bind(tag=TAG).warning("取消监听任务超时，可能仍在运行") - except Exception as e: - logger.bind(tag=TAG).warning(f"取消监听任务异常: {e}") - self._monitor_task = None + logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务和连接...") + await self.close() + # 建立新连接 await self._ensure_connection() From d533fb83eb6bb81d6afc6023ff114bf2620f5524 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 10 Jul 2025 09:52:54 +0800 Subject: [PATCH 11/19] =?UTF-8?q?fix:=20=E6=8F=92=E5=85=A5=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E4=BF=AE=E5=A4=8D=EF=BC=8C=E8=B7=9F=E9=9A=8F=E4=B8=BB?= =?UTF-8?q?=E5=88=86=E6=94=AF=E6=96=B0=E5=A2=9E=E5=88=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202507071530.sql | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/main/manager-api/src/main/resources/db/changelog/202507071530.sql b/main/manager-api/src/main/resources/db/changelog/202507071530.sql index ed3dcf49..f1cf7a9b 100644 --- a/main/manager-api/src/main/resources/db/changelog/202507071530.sql +++ b/main/manager-api/src/main/resources/db/changelog/202507071530.sql @@ -29,28 +29,28 @@ UPDATE `ai_model_config` SET -- 添加阿里云流式TTS音色 delete from `ai_tts_voice` where tts_model_id = 'TTS_AliyunStreamTTS'; -- 温柔女声系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0001', 'TTS_AliyunStreamTTS', '龙小淳-温柔姐姐', 'longxiaochun', '中文及中英文混合', NULL, NULL, 1, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0002', 'TTS_AliyunStreamTTS', '龙小夏-温柔女声', 'longxiaoxia', '中文及中英文混合', NULL, NULL, 2, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0003', 'TTS_AliyunStreamTTS', '龙玫-温柔女声', 'longmei', '中文及中英文混合', NULL, NULL, 3, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0004', 'TTS_AliyunStreamTTS', '龙瑰-温柔女声', 'longgui', '中文及中英文混合', NULL, NULL, 4, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0001', 'TTS_AliyunStreamTTS', '龙小淳-温柔姐姐', 'longxiaochun', '中文及中英文混合', NULL, NULL, NULL, NULL, 1, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0002', 'TTS_AliyunStreamTTS', '龙小夏-温柔女声', 'longxiaoxia', '中文及中英文混合', NULL, NULL, NULL, NULL, 2, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0003', 'TTS_AliyunStreamTTS', '龙玫-温柔女声', 'longmei', '中文及中英文混合', NULL, NULL, NULL, NULL, 3, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0004', 'TTS_AliyunStreamTTS', '龙瑰-温柔女声', 'longgui', '中文及中英文混合', NULL, NULL, NULL, NULL, 4, NULL, NULL, NULL, NULL); -- 御姐女声系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0005', 'TTS_AliyunStreamTTS', '龙玉-御姐女声', 'longyu', '中文及中英文混合', NULL, NULL, 5, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0006', 'TTS_AliyunStreamTTS', '龙娇-御姐女声', 'longjiao', '中文及中英文混合', NULL, NULL, 6, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0005', 'TTS_AliyunStreamTTS', '龙玉-御姐女声', 'longyu', '中文及中英文混合', NULL, NULL, NULL, NULL, 5, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0006', 'TTS_AliyunStreamTTS', '龙娇-御姐女声', 'longjiao', '中文及中英文混合', NULL, NULL, NULL, NULL, 6, NULL, NULL, NULL, NULL); -- 男声系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0007', 'TTS_AliyunStreamTTS', '龙臣-译制片男声', 'longchen', '中文及中英文混合', NULL, NULL, 7, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0008', 'TTS_AliyunStreamTTS', '龙修-青年男声', 'longxiu', '中文及中英文混合', NULL, NULL, 8, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0009', 'TTS_AliyunStreamTTS', '龙橙-阳光男声', 'longcheng', '中文及中英文混合', NULL, NULL, 9, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0010', 'TTS_AliyunStreamTTS', '龙哲-成熟男声', 'longzhe', '中文及中英文混合', NULL, NULL, 10, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0007', 'TTS_AliyunStreamTTS', '龙臣-译制片男声', 'longchen', '中文及中英文混合', NULL, NULL, NULL, NULL, 7, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0008', 'TTS_AliyunStreamTTS', '龙修-青年男声', 'longxiu', '中文及中英文混合', NULL, NULL, NULL, NULL, 8, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0009', 'TTS_AliyunStreamTTS', '龙橙-阳光男声', 'longcheng', '中文及中英文混合', NULL, NULL, NULL, NULL, 9, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0010', 'TTS_AliyunStreamTTS', '龙哲-成熟男声', 'longzhe', '中文及中英文混合', NULL, NULL, NULL, NULL, 10, NULL, NULL, NULL, NULL); -- 专业播报系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0011', 'TTS_AliyunStreamTTS', 'Bella2.0-新闻女声', 'loongbella', '中文及中英文混合', NULL, NULL, 11, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0012', 'TTS_AliyunStreamTTS', 'Stella2.0-飒爽女声', 'loongstella', '中文及中英文混合', NULL, NULL, 12, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0013', 'TTS_AliyunStreamTTS', '龙书-新闻男声', 'longshu', '中文及中英文混合', NULL, NULL, 13, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0014', 'TTS_AliyunStreamTTS', '龙婧-严肃女声', 'longjing', '中文及中英文混合', NULL, NULL, 14, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0011', 'TTS_AliyunStreamTTS', 'Bella2.0-新闻女声', 'loongbella', '中文及中英文混合', NULL, NULL, NULL, NULL, 11, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0012', 'TTS_AliyunStreamTTS', 'Stella2.0-飒爽女声', 'loongstella', '中文及中英文混合', NULL, NULL, NULL, NULL, 12, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0013', 'TTS_AliyunStreamTTS', '龙书-新闻男声', 'longshu', '中文及中英文混合', NULL, NULL, NULL, NULL, 13, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0014', 'TTS_AliyunStreamTTS', '龙婧-严肃女声', 'longjing', '中文及中英文混合', NULL, NULL, NULL, NULL, 14, NULL, NULL, NULL, NULL); -- 特色音色系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0015', 'TTS_AliyunStreamTTS', '龙奇-活泼童声', 'longqi', '中文及中英文混合', NULL, NULL, 15, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0016', 'TTS_AliyunStreamTTS', '龙华-活泼女童', 'longhua', '中文及中英文混合', NULL, NULL, 16, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0017', 'TTS_AliyunStreamTTS', '龙无-无厘头男声', 'longwu', '中文及中英文混合', NULL, NULL, 17, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0018', 'TTS_AliyunStreamTTS', '龙大锤-幽默男声', 'longdachui', '中文及中英文混合', NULL, NULL, 18, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0015', 'TTS_AliyunStreamTTS', '龙奇-活泼童声', 'longqi', '中文及中英文混合', NULL, NULL, NULL, NULL, 15, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0016', 'TTS_AliyunStreamTTS', '龙华-活泼女童', 'longhua', '中文及中英文混合', NULL, NULL, NULL, NULL, 16, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0017', 'TTS_AliyunStreamTTS', '龙无-无厘头男声', 'longwu', '中文及中英文混合', NULL, NULL, NULL, NULL, 17, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0018', 'TTS_AliyunStreamTTS', '龙大锤-幽默男声', 'longdachui', '中文及中英文混合', NULL, NULL, NULL, NULL, 18, NULL, NULL, NULL, NULL); -- 粤语系列 -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0019', 'TTS_AliyunStreamTTS', '龙嘉怡-粤语女声', 'longjiayi', '粤语及粤英混合', NULL, NULL, 19, NULL, NULL, NULL, NULL); -INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0020', 'TTS_AliyunStreamTTS', '龙桃-粤语女声', 'longtao', '粤语及粤英混合', NULL, NULL, 20, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0019', 'TTS_AliyunStreamTTS', '龙嘉怡-粤语女声', 'longjiayi', '粤语及粤英混合', NULL, NULL, NULL, NULL, 19, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0020', 'TTS_AliyunStreamTTS', '龙桃-粤语女声', 'longtao', '粤语及粤英混合', NULL, NULL, NULL, NULL, 20, NULL, NULL, NULL, NULL); From ce358dcd651debd7cebe7da71c1d3d826ed849ec Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 10 Jul 2025 10:17:07 +0800 Subject: [PATCH 12/19] =?UTF-8?q?fix:=20=E9=9F=B3=E9=A2=91=E6=92=AD?= =?UTF-8?q?=E6=94=BE=E6=96=B9=E6=B3=95=E8=B7=9F=E9=9A=8F=E4=B8=BB=E5=88=86?= =?UTF-8?q?=E6=94=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun_stream.py | 39 ++++--------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 9aad9614..daecc924 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -10,13 +10,10 @@ import traceback import websockets import websockets.protocol import os -import random -import threading import concurrent.futures import sys from datetime import datetime from urllib import parse -from typing import Optional from core.providers.tts.base import TTSProviderBase from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType from core.utils.tts import MarkdownCleaner @@ -268,9 +265,12 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).info( f"添加音频文件到待播放列表: {message.content_file}" ) - self.before_stop_play_files.append( - (message.content_file, message.content_detail) - ) + if message.content_file and os.path.exists(message.content_file): + # 先处理文件音频数据 + file_audio = self._process_before_stop_play_files(message.content_file) + self.before_stop_play_files.append( + (file_audio, message.content_detail) + ) if message.sentence_type == SentenceType.LAST: # 处理剩余的文本 @@ -741,6 +741,7 @@ class TTSProvider(TTSProviderBase): ws_connection.close() except Exception as e: logger.bind(tag=TAG).debug(f"关闭WebSocket连接时出现异常: {e}") + async def close(self): """资源清理""" if self.ws: @@ -763,32 +764,6 @@ class TTSProvider(TTSProviderBase): await super().close() - def _process_before_stop_play_files(self): - """处理停止前的待播放文件""" - for tts_file, text in self.before_stop_play_files: - if tts_file and os.path.exists(tts_file): - audio_datas = self._process_audio_file(tts_file) - self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text)) - self.before_stop_play_files.clear() - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - - def _process_audio_file(self, tts_file): - """处理音频文件并转换为指定格式""" - audio_datas = [] - if self.conn.audio_format == "pcm": - audio_datas, _ = self.audio_to_pcm_data(tts_file) - else: - audio_datas, _ = self.audio_to_opus_data(tts_file) - - if ( - self.delete_audio_file - and tts_file is not None - and os.path.exists(tts_file) - and tts_file.startswith(self.output_file) - ): - os.remove(tts_file) - return audio_datas - def to_tts(self, text: str) -> list: """非流式TTS处理，用于测试及保存音频文件的场景""" start_time = time.time() From dede73e9a50320dc343adb64be6a85a2d2d4cefc Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 10 Jul 2025 11:34:31 +0800 Subject: [PATCH 13/19] =?UTF-8?q?fix:=20id=E4=B8=80=E8=87=B4=20=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun_stream.py | 203 ++---------------- 1 file changed, 13 insertions(+), 190 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index daecc924..57d9ffcc 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -267,7 +267,7 @@ class TTSProvider(TTSProviderBase): ) if message.content_file and os.path.exists(message.content_file): # 先处理文件音频数据 - file_audio = self._process_before_stop_play_files(message.content_file) + file_audio = self._process_audio_file(message.content_file) self.before_stop_play_files.append( (file_audio, message.content_detail) ) @@ -316,7 +316,7 @@ class TTSProvider(TTSProviderBase): try: # 运行TTS任务 - loop.run_until_complete(self._tts_request_unified(text, is_last)) + loop.run_until_complete(self.text_to_speak(text, is_last)) return True finally: # 安全关闭事件循环 @@ -357,185 +357,9 @@ class TTSProvider(TTSProviderBase): return None - async def text_to_speak(self, text, is_last=False): + async def text_to_speak(self, text, is_last): """流式处理TTS音频""" - try: - # 确保连接可用 - await self._ensure_connection() - ws_connection = self.ws - - # 确保Token有效 - if self._is_token_expired(): - self._refresh_token() - - # 生成task_id和message_id - task_id = str(uuid.uuid4()).replace('-', '') - - # 第一阶段：发送StartSynthesis指令（设置参数） - start_message_id = str(uuid.uuid4()).replace('-', '') - start_request = { - "header": { - "message_id": start_message_id, - "task_id": task_id, - "namespace": "FlowingSpeechSynthesizer", - "name": "StartSynthesis", - "appkey": self.appkey, - }, - "payload": { - "voice": self.voice, - "format": self.format, - "sample_rate": self.sample_rate, - "volume": self.volume, - "speech_rate": self.speech_rate, - "pitch_rate": self.pitch_rate, - } - } - - await ws_connection.send(json.dumps(start_request)) - logger.bind(tag=TAG).debug(f"发送StartSynthesis指令: {start_message_id}") - - # 第二阶段：发送RunSynthesis指令（发送文本） - run_message_id = str(uuid.uuid4()).replace('-', '') - run_request = { - "header": { - "message_id": run_message_id, - "task_id": task_id, - "namespace": "FlowingSpeechSynthesizer", - "name": "RunSynthesis", - "appkey": self.appkey, - }, - "payload": { - "text": text - } - } - - await ws_connection.send(json.dumps(run_request)) - logger.bind(tag=TAG).debug(f"发送RunSynthesis指令: {text} (message_id: {run_message_id})") - - # 立即发送StopSynthesis指令，避免IDLE_TIMEOUT - stop_message_id = str(uuid.uuid4()).replace('-', '') - stop_request = { - "header": { - "message_id": stop_message_id, - "task_id": task_id, - "namespace": "FlowingSpeechSynthesizer", - "name": "StopSynthesis", - "appkey": self.appkey, - } - } - - await ws_connection.send(json.dumps(stop_request)) - logger.bind(tag=TAG).debug(f"发送StopSynthesis指令: {stop_message_id}") - - # 初始化缓冲区和计数器 - self.pcm_buffer.clear() - self.segment_count = 0 # 重置分段计数器 - opus_datas_cache = [] - - # 发送第一个音频包 - self.tts_audio_queue.put((SentenceType.FIRST, [], text)) - - # 标记是否需要发送StopSynthesis - synthesis_completed = False - - # 处理响应 - 设置超时避免长时间等待 - try: - async def process_messages(): - nonlocal synthesis_completed - async for message in ws_connection: - try: - if isinstance(message, str): - # 处理JSON消息 - data = json.loads(message) - header = data.get("header", {}) - event_name = header.get("name") - - if event_name == "SynthesisStarted": - logger.bind(tag=TAG).debug(f"TTS合成已启动: {task_id}") - - elif event_name == "SynthesisCompleted": - logger.bind(tag=TAG).debug(f"TTS合成完成: {text}") - synthesis_completed = True - break - - elif event_name == "TaskFailed": - error_msg = header.get("status_text", "未知错误") - logger.bind(tag=TAG).error(f"TTS合成失败: {error_msg}") - synthesis_completed = True - break - - elif isinstance(message, bytes): - # 处理二进制音频数据 - self.pcm_buffer.extend(message) - - # 计算每帧的字节数 - frame_bytes = int( - self.opus_encoder.sample_rate - * self.opus_encoder.channels - * self.opus_encoder.frame_size_ms - / 1000 - * 2 - ) - - # 分帧处理PCM数据 - while len(self.pcm_buffer) >= frame_bytes: - frame = bytes(self.pcm_buffer[:frame_bytes]) - del self.pcm_buffer[:frame_bytes] - - # 编码为Opus - opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) - if opus_packets: - if self.segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_packets, None) - ) - self.segment_count += 1 - else: - opus_datas_cache.extend(opus_packets) - - except json.JSONDecodeError: - logger.bind(tag=TAG).warning("收到无效的JSON消息") - except Exception as e: - logger.bind(tag=TAG).error(f"处理响应消息失败: {e}") - - await asyncio.wait_for(process_messages(), timeout=15) # 15秒超时 - - except asyncio.TimeoutError: - logger.bind(tag=TAG).warning(f"TTS请求超时，但可能已获取部分音频数据: {text}") - except websockets.ConnectionClosed: - logger.bind(tag=TAG).debug("WebSocket连接已正常关闭") - except Exception as e: - logger.bind(tag=TAG).error(f"处理WebSocket消息失败: {e}") - - # 因为已经提前发送了StopSynthesis，这里不需要再次发送 - # 直接处理剩余的PCM数据 - if self.pcm_buffer: - opus_packets = self.opus_encoder.encode_pcm_to_opus( - bytes(self.pcm_buffer), end_of_stream=True - ) - if opus_packets: - if self.segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_packets, None) - ) - self.segment_count += 1 - else: - opus_datas_cache.extend(opus_packets) - self.pcm_buffer.clear() - - # 发送缓存的数据 - if self.segment_count >= 10 and opus_datas_cache: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_datas_cache, None) - ) - - # 如果是最后一段，处理待播放文件 - if is_last: - self._process_before_stop_play_files() - - except Exception as e: - logger.bind(tag=TAG).error(f"TTS异步请求异常: {e}") - self.tts_audio_queue.put((SentenceType.LAST, [], None)) + await self._tts_request_unified(text, is_last) async def _tts_request_unified(self, text: str, is_last: bool) -> None: """统一的TTS请求方法""" @@ -564,12 +388,12 @@ class TTSProvider(TTSProviderBase): # 生成task_id task_id = str(uuid.uuid4()).replace('-', '') - + message_id = str(uuid.uuid4()).replace('-', '') # 第一阶段：发送StartSynthesis指令（设置参数） - start_message_id = str(uuid.uuid4()).replace('-', '') + start_request = { "header": { - "message_id": start_message_id, + "message_id": message_id, "task_id": task_id, "namespace": "FlowingSpeechSynthesizer", "name": "StartSynthesis", @@ -586,13 +410,12 @@ class TTSProvider(TTSProviderBase): } await ws_connection.send(json.dumps(start_request)) - logger.bind(tag=TAG).debug(f"发送StartSynthesis指令: {start_message_id}") + logger.bind(tag=TAG).debug(f"发送StartSynthesis指令 会话id：{task_id}") # 第二阶段：发送RunSynthesis指令（发送文本） - run_message_id = str(uuid.uuid4()).replace('-', '') run_request = { "header": { - "message_id": run_message_id, + "message_id": message_id, "task_id": task_id, "namespace": "FlowingSpeechSynthesizer", "name": "RunSynthesis", @@ -604,13 +427,13 @@ class TTSProvider(TTSProviderBase): } await ws_connection.send(json.dumps(run_request)) - logger.bind(tag=TAG).debug(f"发送RunSynthesis指令: {text} (message_id: {run_message_id})") + logger.bind(tag=TAG).debug(f"发送RunSynthesis指令 {text} 会话id： {task_id})") # 立即发送StopSynthesis指令，避免IDLE_TIMEOUT - stop_message_id = str(uuid.uuid4()).replace('-', '') + stop_request = { "header": { - "message_id": stop_message_id, + "message_id": message_id, "task_id": task_id, "namespace": "FlowingSpeechSynthesizer", "name": "StopSynthesis", @@ -619,7 +442,7 @@ class TTSProvider(TTSProviderBase): } await ws_connection.send(json.dumps(stop_request)) - logger.bind(tag=TAG).debug(f"发送StopSynthesis指令: {stop_message_id}") + logger.bind(tag=TAG).debug(f"发送StopSynthesis指令 会话id：{task_id}") # 初始化处理参数 - 使用独立缓冲区 pcm_buffer = bytearray() From 26a3b5116291987fbe7b416ba5c9b79b5c304361 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Thu, 10 Jul 2025 16:07:06 +0800 Subject: [PATCH 14/19] =?UTF-8?q?fix:=20=E8=A1=A5=E5=85=85=E6=96=87?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index ef36e95b..87ce606a 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -712,6 +712,8 @@ TTS: voice: longxiaochun access_key_id: 你的阿里云账号access_key_id access_key_secret: 你的阿里云账号access_key_secret + # 服务器地域选择，可选择距离更近的服务器以减少延迟，如nls-gateway-cn-hangzhou.aliyuncs.com(杭州)等 + host: nls-gateway-cn-shanghai.aliyuncs.com # 以下可不用设置，使用默认设置 # format: pcm # 音频格式：pcm、wav、mp3 # sample_rate: 16000 # 采样率：8000、16000、24000 From 23cd453af8fe07d6097154e86f1c4182b963204f Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 11 Jul 2025 11:36:58 +0800 Subject: [PATCH 15/19] =?UTF-8?q?update:=20aliyun=E5=8F=8C=E6=B5=81?= =?UTF-8?q?=E6=94=B9=E9=80=A0=20=E5=BE=85=E4=BC=98=E5=8C=96=E9=95=BF?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=9C=BA=E5=88=B6=E5=92=8C=E6=96=87=E6=9C=AC?= =?UTF-8?q?=E7=94=9F=E6=88=90=E5=8F=8D=E9=A6=88=E4=B8=BA=E7=A9=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 1 + .../core/providers/tts/aliyun_stream.py | 748 ++++++++---------- .../xiaozhi-server/core/providers/tts/base.py | 2 +- .../providers/tts/huoshan_double_stream.py | 2 +- 4 files changed, 329 insertions(+), 424 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 7b45ad2d..978d88dd 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -128,6 +128,7 @@ class ConnectionHandler: # tts相关变量 self.sentence_id = None + self.message_id = None # iot相关变量 self.iot_descriptors = {} diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 57d9ffcc..5c88602f 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -7,17 +7,15 @@ import time import queue import asyncio import traceback +from asyncio import Task import websockets -import websockets.protocol import os -import concurrent.futures -import sys from datetime import datetime from urllib import parse from core.providers.tts.base import TTSProviderBase from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType from core.utils.tts import MarkdownCleaner -from core.utils import opus_encoder_utils, textUtils +from core.utils import opus_encoder_utils from config.logger import setup_logging TAG = __name__ @@ -50,7 +48,7 @@ class AccessToken: "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "Version": "2019-02-28", } - + query_string = AccessToken._encode_dict(parameters) string_to_sign = ( "GET" @@ -59,7 +57,7 @@ class AccessToken: + "&" + AccessToken._encode_text(query_string) ) - + secreted_string = hmac.new( bytes(access_key_secret + "&", encoding="utf-8"), bytes(string_to_sign, encoding="utf-8"), @@ -67,12 +65,12 @@ class AccessToken: ).digest() signature = base64.b64encode(secreted_string) signature = AccessToken._encode_text(signature) - + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % ( signature, query_string, ) - + import requests response = requests.get(full_url) if response.ok: @@ -88,54 +86,51 @@ class AccessToken: class TTSProvider(TTSProviderBase): def __init__(self, config, delete_audio_file): super().__init__(config, delete_audio_file) - + # 设置为流式接口类型 - self.interface_type = InterfaceType.SINGLE_STREAM - + self.interface_type = InterfaceType.DUAL_STREAM + # 基础配置 self.access_key_id = config.get("access_key_id") self.access_key_secret = config.get("access_key_secret") self.appkey = config.get("appkey") self.format = config.get("format", "pcm") self.audio_file_type = config.get("format", "pcm") - + # 采样率配置 sample_rate = config.get("sample_rate", "16000") self.sample_rate = int(sample_rate) if sample_rate else 16000 - + # 音色配置 - CosyVoice大模型音色 if config.get("private_voice"): self.voice = config.get("private_voice") else: self.voice = config.get("voice", "longxiaochun") # CosyVoice默认音色 - + # 音频参数配置 volume = config.get("volume", "50") self.volume = int(volume) if volume else 50 - + speech_rate = config.get("speech_rate", "0") self.speech_rate = int(speech_rate) if speech_rate else 0 - + pitch_rate = config.get("pitch_rate", "0") self.pitch_rate = int(pitch_rate) if pitch_rate else 0 - + # WebSocket配置 self.host = config.get("host", "nls-gateway-cn-beijing.aliyuncs.com") self.ws_url = f"wss://{self.host}/ws/v1" self.ws = None - - # 流式相关配置 - self.before_stop_play_files = [] - self.segment_count = 0 - + self._monitor_task = None + # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( sample_rate=16000, channels=1, frame_size_ms=60 ) - + # PCM缓冲区 self.pcm_buffer = bytearray() - + # Token管理 if self.access_key_id and self.access_key_secret: self._refresh_token() @@ -152,8 +147,9 @@ class TTSProvider(TTSProviderBase): if not expire_time_str: raise ValueError("无法获取有效的Token过期时间") + expire_str = str(expire_time_str).strip() + try: - expire_str = str(expire_time_str).strip() if expire_str.isdigit(): expire_time = datetime.fromtimestamp(int(expire_str)) else: @@ -175,91 +171,83 @@ class TTSProvider(TTSProviderBase): async def _ensure_connection(self): """确保WebSocket连接可用""" - # 检查连接状态，兼容不同版本的websockets库 - need_reconnect = False - if self.ws is None: - need_reconnect = True - else: - try: - # 尝试访问closed属性，如果不存在则检查state - if hasattr(self.ws, 'closed'): - need_reconnect = self.ws.closed - elif hasattr(self.ws, 'state'): - # websockets 新版本使用state属性 - need_reconnect = self.ws.state != websockets.protocol.State.OPEN - else: - # 如果都没有，尝试发送ping来检测连接状态 - try: - await asyncio.wait_for(self.ws.ping(), timeout=2.0) - except: - need_reconnect = True - except: - need_reconnect = True - - if need_reconnect: - # 清理旧连接 - if self.ws: - try: - if hasattr(self.ws, 'close'): - if asyncio.iscoroutinefunction(self.ws.close): - await self.ws.close() - else: - self.ws.close() - except: - pass - finally: - self.ws = None - + try: if self._is_token_expired(): logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...") self._refresh_token() - - # 重试连接机制 - max_retries = 3 - retry_delay = 1.0 - - for attempt in range(max_retries): - try: - self.ws = await asyncio.wait_for( - websockets.connect( - self.ws_url, - additional_headers={ - "X-NLS-Token": self.token, - }, - ping_interval=30, - ping_timeout=10, - close_timeout=10, - ), - timeout=10.0 - ) - logger.bind(tag=TAG).info("阿里云CosyVoice流式TTS WebSocket连接建立成功") - return - except Exception as e: - logger.bind(tag=TAG).warning(f"WebSocket连接失败 (尝试 {attempt + 1}/{max_retries}): {e}") - if attempt < max_retries - 1: - await asyncio.sleep(retry_delay * (attempt + 1)) - else: - logger.bind(tag=TAG).error(f"WebSocket连接最终失败: {e}") - raise + if self.ws: + logger.bind(tag=TAG).info(f"使用已有链接...") + return self.ws + logger.bind(tag=TAG).info("开始建立新连接...") + + self.ws = await websockets.connect( + self.ws_url, + additional_headers={"X-NLS-Token": self.token}, + ping_interval=30, + ping_timeout=10, + close_timeout=10, + ) + logger.bind(tag=TAG).info("WebSocket连接建立成功") + return self.ws + except Exception as e: + logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}") + self.ws = None + raise def tts_text_priority_thread(self): """流式文本处理线程""" while not self.conn.stop_event.is_set(): try: message = self.tts_text_queue.get(timeout=1) + logger.bind(tag=TAG).debug( + f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}" + ) + + if message.sentence_type == SentenceType.FIRST: + self.conn.client_abort = False + + if self.conn.client_abort: + logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") + continue + if message.sentence_type == SentenceType.FIRST: # 初始化参数 - self.tts_stop_request = False - self.processed_chars = 0 - self.tts_text_buff = [] - self.segment_count = 0 - self.tts_audio_first_sentence = True - self.before_stop_play_files.clear() + try: + if not getattr(self.conn, "sentence_id", None): + self.conn.sentence_id = uuid.uuid4().hex + logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}") + + # aliyun独有的message_id需要自己生成 + self.conn.message_id = str(uuid.uuid4().hex) + + logger.bind(tag=TAG).info("开始启动TTS会话...") + future = asyncio.run_coroutine_threadsafe( + self.start_session(self.conn.sentence_id), + loop=self.conn.loop, + ) + future.result() + self.before_stop_play_files.clear() + logger.bind(tag=TAG).info("TTS会话启动成功") + + except Exception as e: + logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}") + continue + elif ContentType.TEXT == message.content_type: - self.tts_text_buff.append(message.content_detail) - segment_text = self._get_segment_text() - if segment_text: - self.to_tts_single_stream(segment_text) + if message.content_detail: + try: + logger.bind(tag=TAG).debug( + f"开始发送TTS文本: {message.content_detail}" + ) + future = asyncio.run_coroutine_threadsafe( + self.text_to_speak(message.content_detail, None), + loop=self.conn.loop, + ) + future.result() + logger.bind(tag=TAG).debug("TTS文本发送成功") + except Exception as e: + logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") + continue elif ContentType.FILE == message.content_type: logger.bind(tag=TAG).info( @@ -273,8 +261,16 @@ class TTSProvider(TTSProviderBase): ) if message.sentence_type == SentenceType.LAST: - # 处理剩余的文本 - self._process_remaining_text(True) + try: + logger.bind(tag=TAG).info("开始结束TTS会话...") + future = asyncio.run_coroutine_threadsafe( + self.finish_session(self.conn.sentence_id), + loop=self.conn.loop, + ) + future.result() + except Exception as e: + logger.bind(tag=TAG).error(f"结束TTS会话失败: {str(e)}") + continue except queue.Empty: continue @@ -283,118 +279,59 @@ class TTSProvider(TTSProviderBase): f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" ) - def _process_remaining_text(self, is_last=False): - """处理剩余的文本并生成语音""" - full_text = "".join(self.tts_text_buff) - remaining_text = full_text[self.processed_chars:] - if remaining_text: - segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text) - if segment_text: - self.to_tts_single_stream(segment_text, is_last) - self.processed_chars += len(full_text) - else: - self._process_before_stop_play_files() - else: - self._process_before_stop_play_files() - - def to_tts_single_stream(self, text, is_last=False): - """流式TTS处理 - 使用线程池执行异步任务""" + async def text_to_speak(self, text, _): try: - text = MarkdownCleaner.clean_markdown(text) - - # 使用线程池来执行异步任务，避免事件循环冲突 - def run_async_task(): - """在新线程中运行异步任务""" - try: - # 创建新的事件循环用于这个线程 - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - - # Windows下设置事件循环策略 - if sys.platform == "win32": - asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) - - try: - # 运行TTS任务 - loop.run_until_complete(self.text_to_speak(text, is_last)) - return True - finally: - # 安全关闭事件循环 - try: - # 取消所有未完成的任务 - pending = asyncio.all_tasks(loop) - if pending: - for task in pending: - task.cancel() - # 等待任务取消完成 - loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) - except Exception as cleanup_error: - logger.bind(tag=TAG).debug(f"清理事件循环异常: {cleanup_error}") - finally: - loop.close() - - except Exception as e: - logger.bind(tag=TAG).error(f"异步任务执行失败: {e}") - return False - - # 使用线程池执行异步任务 - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(run_async_task) - success = future.result(timeout=30) # 30秒超时 - - if success: - logger.bind(tag=TAG).info(f"语音生成成功: {text}") - else: - logger.bind(tag=TAG).error(f"语音生成失败: {text}") - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - - except concurrent.futures.TimeoutError: - logger.bind(tag=TAG).error(f"TTS任务超时: {text}") - self.tts_audio_queue.put((SentenceType.LAST, [], None)) + if self.ws is None: + logger.bind(tag=TAG).warning(f"WebSocket连接不存在，终止发送文本") + return + filtered_text = MarkdownCleaner.clean_markdown(text) + run_request = { + "header": { + "message_id": self.conn.message_id, + "task_id": self.conn.sentence_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": self.appkey, + }, + "payload": { + "text": filtered_text + } + } + await self.ws.send(json.dumps(run_request)) + return + except Exception as e: - logger.bind(tag=TAG).error(f"TTS处理异常: {text}, 错误: {e}") - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - - return None + logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") + if self.ws: + try: + await self.ws.close() + except: + pass + self.ws = None + raise - async def text_to_speak(self, text, is_last): - """流式处理TTS音频""" - await self._tts_request_unified(text, is_last) - - async def _tts_request_unified(self, text: str, is_last: bool) -> None: - """统一的TTS请求方法""" - ws_connection = None - + async def start_session(self, session_id): + logger.bind(tag=TAG).info(f"开始会话～～{session_id}") try: - # 确保Token有效 - if self._is_token_expired(): - self._refresh_token() - - # 总是创建独立连接，避免与其他线程的事件循环冲突 - ws_url = f"wss://{self.host}/ws/v1" - ws_connection = await asyncio.wait_for( - websockets.connect( - ws_url, - additional_headers={ - "X-NLS-Token": self.token, - }, - ping_interval=15, # 每15秒发送ping，保持连接活跃 - ping_timeout=5, # ping超时时间5秒 - close_timeout=3, # 关闭超时时间3秒 - ), - timeout=8.0 # 连接超时时间8秒 - ) - logger.bind(tag=TAG).debug(f"建立独立WebSocket连接: {text}") - - # 生成task_id - task_id = str(uuid.uuid4()).replace('-', '') - message_id = str(uuid.uuid4()).replace('-', '') - # 第一阶段：发送StartSynthesis指令（设置参数） + # 会话开始时检测上个会话的监听状态 + if( + self._monitor_task is not None + and isinstance(self._monitor_task, Task) + and not self._monitor_task.done() + ): + logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务和连接...") + await self.close() + + # 建立新连接 + await self._ensure_connection() + + # 启动监听任务 + self._monitor_task = asyncio.create_task(self._start_monitor_tts_response()) start_request = { "header": { - "message_id": message_id, - "task_id": task_id, + "message_id": self.conn.message_id, + "task_id": self.conn.sentence_id, "namespace": "FlowingSpeechSynthesizer", "name": "StartSynthesis", "appkey": self.appkey, @@ -408,195 +345,198 @@ class TTSProvider(TTSProviderBase): "pitch_rate": self.pitch_rate, } } - - await ws_connection.send(json.dumps(start_request)) - logger.bind(tag=TAG).debug(f"发送StartSynthesis指令 会话id：{task_id}") - - # 第二阶段：发送RunSynthesis指令（发送文本） - run_request = { - "header": { - "message_id": message_id, - "task_id": task_id, - "namespace": "FlowingSpeechSynthesizer", - "name": "RunSynthesis", - "appkey": self.appkey, - }, - "payload": { - "text": text - } - } - - await ws_connection.send(json.dumps(run_request)) - logger.bind(tag=TAG).debug(f"发送RunSynthesis指令 {text} 会话id： {task_id})") - - # 立即发送StopSynthesis指令，避免IDLE_TIMEOUT - - stop_request = { - "header": { - "message_id": message_id, - "task_id": task_id, - "namespace": "FlowingSpeechSynthesizer", - "name": "StopSynthesis", - "appkey": self.appkey, - } - } - - await ws_connection.send(json.dumps(stop_request)) - logger.bind(tag=TAG).debug(f"发送StopSynthesis指令 会话id：{task_id}") - - # 初始化处理参数 - 使用独立缓冲区 - pcm_buffer = bytearray() - opus_datas_cache = [] - segment_count = 0 - self.segment_count = 0 # 同时重置实例变量 - synthesis_completed = False - - # 发送第一个音频包 - self.tts_audio_queue.put((SentenceType.FIRST, [], text)) - - # 处理响应 - 设置超时时间避免长时间等待 - timeout_duration = 15 # 15秒超时 - try: - # 使用asyncio.wait_for替代asyncio.timeout以保证兼容性 - async def process_messages(): - nonlocal synthesis_completed, segment_count, pcm_buffer, opus_datas_cache # 声明使用外层变量 - async for message in ws_connection: - try: - if isinstance(message, str): - # 处理JSON消息 - data = json.loads(message) - header = data.get("header", {}) - event_name = header.get("name") - - if event_name == "SynthesisStarted": - logger.bind(tag=TAG).debug(f"TTS合成已启动: {task_id}") - - elif event_name == "SynthesisCompleted": - logger.bind(tag=TAG).debug(f"TTS合成完成: {text}") - synthesis_completed = True - break - - elif event_name == "TaskFailed": - error_msg = header.get("status_text", "未知错误") - logger.bind(tag=TAG).error(f"TTS合成失败: {error_msg}") - synthesis_completed = True - break - - elif isinstance(message, bytes): - # 处理二进制音频数据 - pcm_buffer.extend(message) - - # 计算每帧的字节数 - frame_bytes = int( - self.opus_encoder.sample_rate - * self.opus_encoder.channels - * self.opus_encoder.frame_size_ms - / 1000 - * 2 - ) - - # 分帧处理PCM数据 - while len(pcm_buffer) >= frame_bytes: - frame = bytes(pcm_buffer[:frame_bytes]) - del pcm_buffer[:frame_bytes] # 清除已处理的数据 - - # 编码为Opus - opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) - if opus_packets: - if segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_packets, None) - ) - segment_count += 1 - else: - opus_datas_cache.extend(opus_packets) - - except json.JSONDecodeError: - logger.bind(tag=TAG).warning("收到无效的JSON消息") - except Exception as e: - logger.bind(tag=TAG).error(f"处理响应消息失败: {e}") - - await asyncio.wait_for(process_messages(), timeout=timeout_duration) - - except asyncio.TimeoutError: - logger.bind(tag=TAG).warning(f"TTS请求超时，但可能已获取部分音频数据: {text}") - except websockets.ConnectionClosed: - logger.bind(tag=TAG).debug("WebSocket连接已正常关闭") - except Exception as e: - logger.bind(tag=TAG).error(f"处理WebSocket消息失败: {e}") - - # 因为已经提前发送了StopSynthesis，这里不需要再次发送 - # 直接处理剩余的PCM数据 - if pcm_buffer: - opus_packets = self.opus_encoder.encode_pcm_to_opus( - bytes(pcm_buffer), end_of_stream=True - ) - if opus_packets: - if segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_packets, None) - ) - segment_count += 1 - else: - opus_datas_cache.extend(opus_packets) - - # 发送缓存的数据 - if segment_count >= 10 and opus_datas_cache: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_datas_cache, None) - ) - - # 如果是最后一段，处理待播放文件 - if is_last: - self._process_before_stop_play_files() - + await self.ws.send(json.dumps(start_request)) + logger.bind(tag=TAG).info("会话启动请求已发送") except Exception as e: - logger.bind(tag=TAG).error(f"TTS请求异常: {e}") - self.tts_audio_queue.put((SentenceType.LAST, [], None)) - finally: - # 确保WebSocket连接被关闭 - if ws_connection: - try: - if hasattr(ws_connection, 'close'): - if asyncio.iscoroutinefunction(ws_connection.close): - await ws_connection.close() - else: - ws_connection.close() - except Exception as e: - logger.bind(tag=TAG).debug(f"关闭WebSocket连接时出现异常: {e}") + logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}") + # 确保清理资源 + await self.close() + raise + + async def finish_session(self, session_id): + logger.bind(tag=TAG).info(f"关闭会话～～{session_id}") + try: + if self.ws: + stop_request = { + "header": { + "message_id": self.conn.message_id, + "task_id": self.conn.sentence_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StopSynthesis", + "appkey": self.appkey, + } + } + await self.ws.send(json.dumps(stop_request)) + logger.bind(tag=TAG).info("会话结束请求已发送") + if self._monitor_task: + try: + await self._monitor_task + except Exception as e: + logger.bind(tag=TAG).error( + f"等待监听任务完成时发生错误: {str(e)}" + ) + finally: + self._monitor_task = None + except Exception as e: + logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}") + # 确保清理资源 + await self.close() + raise async def close(self): """资源清理""" + if self._monitor_task: + try: + self._monitor_task.cancel() + await self._monitor_task + except asyncio.CancelledError: + pass + except Exception as e: + logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}") + self._monitor_task = None + if self.ws: try: - # 兼容不同版本的websockets库关闭方式 - if hasattr(self.ws, 'close'): - if asyncio.iscoroutinefunction(self.ws.close): - await self.ws.close() - else: - self.ws.close() - elif hasattr(self.ws, 'close_connection'): - await self.ws.close_connection() - except Exception as e: - logger.bind(tag=TAG).debug(f"关闭WebSocket连接时出现异常: {e}") - finally: + await self.ws.close() + except: + pass + self.ws = None + + async def _start_monitor_tts_response(self): + """监听TTS响应""" + opus_datas_cache = [] + current_sentence_buffer = bytearray() + segment_count = 0 + current_sentence_parts = [] + try: + session_finished = False # 标记会话是否正常结束 + while not self.conn.stop_event.is_set(): + try: + msg = await self.ws.recv() + # 检查客户端是否中止 + if self.conn.client_abort: + logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应") + break + if isinstance(msg, str): # 文本控制消息 + try: + data = json.loads(msg) + header = data.get("header", {}) + event_name = header.get("name") + + if event_name == "SynthesisStarted": + logger.bind(tag=TAG).debug("TTS合成已启动") + + elif event_name == "SentenceBegin": + logger.bind(tag=TAG).debug(f"句子语音生成开始") + current_sentence_buffer = bytearray() + segment_count = 0 + self.tts_audio_queue.put( + (SentenceType.FIRST, [], "") + ) + + elif event_name == "SentenceSynthesis": + payload = data.get("payload", {}) + subtitles = payload.get("subtitles", []) + + # 收集所有字幕片段 + for sub in subtitles: + text = sub.get("text", "") + if text: + current_sentence_parts.append(text) + + elif event_name == "SentenceEnd": + logger.bind(tag=TAG).info(f"句子语音生成成功: {''.join(current_sentence_parts)}") + + if current_sentence_buffer: + opus_datas = self.opus_encoder.encode_pcm_to_opus( + bytes(current_sentence_buffer), end_of_stream=True + ) + if opus_datas: + if segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas, None) + ) + else: + opus_datas_cache.extend(opus_datas) + + if segment_count >= 10 and opus_datas_cache: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas_cache, None) + ) + opus_datas_cache = [] + current_sentence_buffer = bytearray() + + elif event_name == "SynthesisCompleted": + logger.bind(tag=TAG).debug("会话结束") + self._process_before_stop_play_files() + session_finished = True + break + + except json.JSONDecodeError: + logger.bind(tag=TAG).warning("收到无效的JSON消息") + + # 二进制消息（音频数据） + elif isinstance(msg, bytes): + # 将音频数据添加到当前句子的缓冲区 + current_sentence_buffer.extend(msg) + + # 计算每帧的字节数（60ms） + frame_bytes = int( + self.opus_encoder.sample_rate + * self.opus_encoder.channels + * self.opus_encoder.frame_size_ms + / 1000 + * 2 # 16-bit = 2 bytes + ) + + # 处理完整的音频帧 + while len(current_sentence_buffer) >= frame_bytes: + # 取出一帧数据 + frame = bytes(current_sentence_buffer[:frame_bytes]) + del current_sentence_buffer[:frame_bytes] + + # 编码为Opus + opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) + + if opus_packets: + # 前10个片段直接发送，后续片段缓存 + if segment_count < 10: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_packets, None) + ) + segment_count += 1 + else: + opus_datas_cache.extend(opus_packets) + + except websockets.ConnectionClosed: + logger.bind(tag=TAG).warning("WebSocket连接已关闭") + break + except Exception as e: + logger.bind(tag=TAG).error( + f"处理TTS响应时出错: {e}\n{traceback.format_exc()}" + ) + break + # 仅在连接异常时才关闭 + if not session_finished and self.ws: + try: + await self.ws.close() + except: + pass self.ws = None - - if hasattr(self, "opus_encoder"): - self.opus_encoder.close() - - await super().close() + # 监听任务退出时清理引用 + finally: + self._monitor_task = None def to_tts(self, text: str) -> list: """非流式TTS处理，用于测试及保存音频文件的场景""" start_time = time.time() text = MarkdownCleaner.clean_markdown(text) - + try: # 使用同步方式进行TTS转换 if self._is_token_expired(): self._refresh_token() - + # 构造请求数据 request_json = { "appkey": self.appkey, @@ -609,21 +549,21 @@ class TTSProvider(TTSProviderBase): "speech_rate": self.speech_rate, "pitch_rate": self.pitch_rate, } - + # 使用HTTP接口进行同步请求 import requests api_url = f"https://{self.host}/stream/v1/tts" headers = {"Content-Type": "application/json"} - + resp = requests.post(api_url, json=request_json, headers=headers) - + if resp.status_code == 401: # Token过期特殊处理 self._refresh_token() resp = requests.post(api_url, json=request_json, headers=headers) - + if resp.headers["Content-Type"].startswith("audio/"): pcm_data = resp.content - + # 使用opus编码器处理PCM数据 opus_datas = [] frame_bytes = int( @@ -633,7 +573,7 @@ class TTSProvider(TTSProviderBase): / 1000 * 2 ) - + # 分帧处理PCM数据 for i in range(0, len(pcm_data), frame_bytes): frame = pcm_data[i:i + frame_bytes] @@ -641,49 +581,13 @@ class TTSProvider(TTSProviderBase): opus = self.opus_encoder.encode_pcm_to_opus(frame, False) if opus: opus_datas.extend(opus) - + logger.bind(tag=TAG).info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") return opus_datas else: logger.bind(tag=TAG).error(f"TTS请求失败: {resp.content}") return [] - + except Exception as e: logger.bind(tag=TAG).error(f"TTS请求异常: {e}") return [] - - def _get_segment_text(self): - """获取当前可以处理的文本段""" - if not self.tts_text_buff: - return None - - full_text = "".join(self.tts_text_buff) - if len(full_text) <= self.processed_chars: - return None - - # 获取未处理的文本 - remaining_text = full_text[self.processed_chars:] - - # 如果文本较短或者到达了句子结尾标点，直接处理 - sentence_endings = ['。', '！', '？', '.', '!', '?', '\n'] - if len(remaining_text) < 20: - return None - - # 查找句子结尾 - for i, char in enumerate(remaining_text): - if char in sentence_endings and i > 10: # 至少10个字符 - segment = remaining_text[:i+1] - segment_text = textUtils.get_string_no_punctuation_or_emoji(segment) - if segment_text: - self.processed_chars += i + 1 - return segment_text - - # 如果没有找到句子结尾，但文本足够长，按长度分段 - if len(remaining_text) > 50: - segment = remaining_text[:30] - segment_text = textUtils.get_string_no_punctuation_or_emoji(segment) - if segment_text: - self.processed_chars += 30 - return segment_text - - return None diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 9127ec12..9590669b 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -159,7 +159,7 @@ class TTSProviderBase(ABC): if conn.sentence_id: sentence_id = conn.sentence_id else: - sentence_id = str(uuid.uuid4()).replace("-", "") + sentence_id = str(uuid.uuid4().hex) conn.sentence_id = sentence_id # 对于单句的文本，进行分段处理 segments = re.split(r"([。！？!?；;\n])", content_detail) diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 0eb7c72d..23abaf5c 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -469,7 +469,7 @@ class TTSProvider(TTSProviderBase): except: pass self.ws = None - # 监听任务退出时清理引用 + # 监听任务退出时清理引用 finally: self._monitor_task = None From 8e5d93374596e9de38287069c797f02dde89784a Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 11 Jul 2025 16:36:47 +0800 Subject: [PATCH 16/19] =?UTF-8?q?update:=20=E4=BC=98=E5=8C=96=E9=9F=B3?= =?UTF-8?q?=E9=A2=91=E6=92=AD=E6=94=BE=20=E6=96=87=E6=9C=AC=E5=8F=91?= =?UTF-8?q?=E9=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun_stream.py | 125 +++++++----------- 1 file changed, 49 insertions(+), 76 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 5c88602f..b6fd1ec2 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -123,6 +123,11 @@ class TTSProvider(TTSProviderBase): self.ws = None self._monitor_task = None + # 文本获取 + self.sentence_queue = queue.Queue() + self.text_buffer = "" + self.sentence_end_chars = {'.', '。', '!', '！', '?', '？', '\n', "~", "；", ";", ":", "：", " ", "，", ","} + # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( sample_rate=16000, channels=1, frame_size_ms=60 @@ -176,6 +181,7 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...") self._refresh_token() if self.ws: + # 10秒内才可以复用,适合连续对话 logger.bind(tag=TAG).info(f"使用已有链接...") return self.ws logger.bind(tag=TAG).info("开始建立新连接...") @@ -239,6 +245,10 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) + self.text_buffer += message.content_detail + if message.content_detail in self.sentence_end_chars and len(self.text_buffer) > 6: + self.sentence_queue.put(self.text_buffer) + self.text_buffer = "" future = asyncio.run_coroutine_threadsafe( self.text_to_speak(message.content_detail, None), loop=self.conn.loop, @@ -263,6 +273,8 @@ class TTSProvider(TTSProviderBase): if message.sentence_type == SentenceType.LAST: try: logger.bind(tag=TAG).info("开始结束TTS会话...") + self.sentence_queue.put(self.text_buffer) + print(list(self.sentence_queue.queue)) future = asyncio.run_coroutine_threadsafe( self.finish_session(self.conn.sentence_id), loop=self.conn.loop, @@ -315,9 +327,9 @@ class TTSProvider(TTSProviderBase): try: # 会话开始时检测上个会话的监听状态 if( - self._monitor_task is not None - and isinstance(self._monitor_task, Task) - and not self._monitor_task.done() + self._monitor_task is not None + and isinstance(self._monitor_task, Task) + and not self._monitor_task.done() ): logger.bind(tag=TAG).info("检测到未完成的上个会话，关闭监听任务和连接...") await self.close() @@ -343,6 +355,7 @@ class TTSProvider(TTSProviderBase): "volume": self.volume, "speech_rate": self.speech_rate, "pitch_rate": self.pitch_rate, + "enable_subtitle": True } } await self.ws.send(json.dumps(start_request)) @@ -405,9 +418,9 @@ class TTSProvider(TTSProviderBase): async def _start_monitor_tts_response(self): """监听TTS响应""" opus_datas_cache = [] - current_sentence_buffer = bytearray() - segment_count = 0 - current_sentence_parts = [] + is_first_sentence = True + first_sentence_segment_count = 0 # 添加计数器 + text_buff = "" try: session_finished = False # 标记会话是否正常结束 while not self.conn.stop_event.is_set(): @@ -422,91 +435,51 @@ class TTSProvider(TTSProviderBase): data = json.loads(msg) header = data.get("header", {}) event_name = header.get("name") - if event_name == "SynthesisStarted": logger.bind(tag=TAG).debug("TTS合成已启动") - elif event_name == "SentenceBegin": - logger.bind(tag=TAG).debug(f"句子语音生成开始") - current_sentence_buffer = bytearray() - segment_count = 0 - self.tts_audio_queue.put( - (SentenceType.FIRST, [], "") - ) - - elif event_name == "SentenceSynthesis": - payload = data.get("payload", {}) - subtitles = payload.get("subtitles", []) - - # 收集所有字幕片段 - for sub in subtitles: - text = sub.get("text", "") - if text: - current_sentence_parts.append(text) - + try: + text_buff = self.sentence_queue.get_nowait() + except queue.Empty: + text_buff = "" + logger.bind(tag=TAG).debug(f"句子语音生成开始: {text_buff}") + opus_datas_cache = [] + self.tts_audio_queue.put((SentenceType.FIRST, [], text_buff)) elif event_name == "SentenceEnd": - logger.bind(tag=TAG).info(f"句子语音生成成功: {''.join(current_sentence_parts)}") - - if current_sentence_buffer: - opus_datas = self.opus_encoder.encode_pcm_to_opus( - bytes(current_sentence_buffer), end_of_stream=True - ) - if opus_datas: - if segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_datas, None) - ) - else: - opus_datas_cache.extend(opus_datas) - - if segment_count >= 10 and opus_datas_cache: + logger.bind(tag=TAG).info(f"句子语音生成成功： {text_buff}") + text_buff = "" + if not is_first_sentence or first_sentence_segment_count > 10: + # 发送缓存的数据 self.tts_audio_queue.put( (SentenceType.MIDDLE, opus_datas_cache, None) ) - opus_datas_cache = [] - current_sentence_buffer = bytearray() - + # 第一句话结束后，将标志设置为False + is_first_sentence = False elif event_name == "SynthesisCompleted": - logger.bind(tag=TAG).debug("会话结束") + logger.bind(tag=TAG).debug(f"会话结束～～") self._process_before_stop_play_files() session_finished = True break - except json.JSONDecodeError: logger.bind(tag=TAG).warning("收到无效的JSON消息") - # 二进制消息（音频数据） - elif isinstance(msg, bytes): - # 将音频数据添加到当前句子的缓冲区 - current_sentence_buffer.extend(msg) - - # 计算每帧的字节数（60ms） - frame_bytes = int( - self.opus_encoder.sample_rate - * self.opus_encoder.channels - * self.opus_encoder.frame_size_ms - / 1000 - * 2 # 16-bit = 2 bytes + elif isinstance(msg, (bytes, bytearray)): + logger.bind(tag=TAG).debug(f"推送数据到队列里面～～") + opus_datas = self.opus_encoder.encode_pcm_to_opus(msg, False) + logger.bind(tag=TAG).debug( + f"推送数据到队列里面帧数～～{len(opus_datas)}" ) - - # 处理完整的音频帧 - while len(current_sentence_buffer) >= frame_bytes: - # 取出一帧数据 - frame = bytes(current_sentence_buffer[:frame_bytes]) - del current_sentence_buffer[:frame_bytes] - - # 编码为Opus - opus_packets = self.opus_encoder.encode_pcm_to_opus(frame, False) - - if opus_packets: - # 前10个片段直接发送，后续片段缓存 - if segment_count < 10: - self.tts_audio_queue.put( - (SentenceType.MIDDLE, opus_packets, None) - ) - segment_count += 1 - else: - opus_datas_cache.extend(opus_packets) + if is_first_sentence: + first_sentence_segment_count += 1 + if first_sentence_segment_count <= 6: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas, None) + ) + else: + opus_datas_cache = opus_datas_cache + opus_datas + else: + # 后续句子缓存 + opus_datas_cache = opus_datas_cache + opus_datas except websockets.ConnectionClosed: logger.bind(tag=TAG).warning("WebSocket连接已关闭") From 04ed5ed980159e8d8bb8851beb4eab1a6c385676 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 11 Jul 2025 18:04:00 +0800 Subject: [PATCH 17/19] =?UTF-8?q?update:=20=E5=90=8C=E6=AD=A5=E9=9D=9E?= =?UTF-8?q?=E6=B5=81=E5=BC=8F=E5=A4=84=E7=90=86=2010=E7=A7=92=E8=B6=85?= =?UTF-8?q?=E6=97=B6=E9=93=BE=E6=8E=A5=E4=B8=8D=E5=A4=8D=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun_stream.py | 171 +++++++++++------- 1 file changed, 109 insertions(+), 62 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index b6fd1ec2..1ea37810 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -15,7 +15,7 @@ from urllib import parse from core.providers.tts.base import TTSProviderBase from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType from core.utils.tts import MarkdownCleaner -from core.utils import opus_encoder_utils +from core.utils import opus_encoder_utils, textUtils from config.logger import setup_logging TAG = __name__ @@ -122,10 +122,10 @@ class TTSProvider(TTSProviderBase): self.ws_url = f"wss://{self.host}/ws/v1" self.ws = None self._monitor_task = None + self.last_active_time = None - # 文本获取 - self.sentence_queue = queue.Queue() - self.text_buffer = "" + # 文本符号 + self.sentence_queue = None self.sentence_end_chars = {'.', '。', '!', '！', '?', '？', '\n', "~", "；", ";", ":", "：", " ", "，", ","} # 创建Opus编码器 @@ -180,8 +180,9 @@ class TTSProvider(TTSProviderBase): if self._is_token_expired(): logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...") self._refresh_token() - if self.ws: - # 10秒内才可以复用,适合连续对话 + current_time = time.time() + if self.ws and current_time - self.last_active_time < 10: + # 10秒内才可以复用链接进行连续对话 logger.bind(tag=TAG).info(f"使用已有链接...") return self.ws logger.bind(tag=TAG).info("开始建立新连接...") @@ -194,10 +195,12 @@ class TTSProvider(TTSProviderBase): close_timeout=10, ) logger.bind(tag=TAG).info("WebSocket连接建立成功") + self.last_active_time = time.time() return self.ws except Exception as e: logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}") self.ws = None + self.last_active_time = None raise def tts_text_priority_thread(self): @@ -223,8 +226,10 @@ class TTSProvider(TTSProviderBase): self.conn.sentence_id = uuid.uuid4().hex logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}") - # aliyun独有的message_id需要自己生成 + # aliyunStream独有的参数生成 self.conn.message_id = str(uuid.uuid4().hex) + self.sentence_queue = queue.Queue() + self.text_buffer = "" logger.bind(tag=TAG).info("开始启动TTS会话...") future = asyncio.run_coroutine_threadsafe( @@ -247,7 +252,7 @@ class TTSProvider(TTSProviderBase): ) self.text_buffer += message.content_detail if message.content_detail in self.sentence_end_chars and len(self.text_buffer) > 6: - self.sentence_queue.put(self.text_buffer) + self.sentence_queue.put(textUtils.get_string_no_punctuation_or_emoji(self.text_buffer)) self.text_buffer = "" future = asyncio.run_coroutine_threadsafe( self.text_to_speak(message.content_detail, None), @@ -273,8 +278,7 @@ class TTSProvider(TTSProviderBase): if message.sentence_type == SentenceType.LAST: try: logger.bind(tag=TAG).info("开始结束TTS会话...") - self.sentence_queue.put(self.text_buffer) - print(list(self.sentence_queue.queue)) + self.sentence_queue.put(textUtils.get_string_no_punctuation_or_emoji(self.text_buffer)) future = asyncio.run_coroutine_threadsafe( self.finish_session(self.conn.sentence_id), loop=self.conn.loop, @@ -310,6 +314,7 @@ class TTSProvider(TTSProviderBase): } } await self.ws.send(json.dumps(run_request)) + self.last_active_time = time.time() return except Exception as e: @@ -359,6 +364,7 @@ class TTSProvider(TTSProviderBase): } } await self.ws.send(json.dumps(start_request)) + self.last_active_time = time.time() logger.bind(tag=TAG).info("会话启动请求已发送") except Exception as e: logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}") @@ -381,6 +387,7 @@ class TTSProvider(TTSProviderBase): } await self.ws.send(json.dumps(stop_request)) logger.bind(tag=TAG).info("会话结束请求已发送") + self.last_active_time = time.time() if self._monitor_task: try: await self._monitor_task @@ -414,6 +421,7 @@ class TTSProvider(TTSProviderBase): except: pass self.ws = None + self.last_active_time = None async def _start_monitor_tts_response(self): """监听TTS响应""" @@ -426,6 +434,7 @@ class TTSProvider(TTSProviderBase): while not self.conn.stop_event.is_set(): try: msg = await self.ws.recv() + self.last_active_time = time.time() # 检查客户端是否中止 if self.conn.client_abort: logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应") @@ -459,6 +468,7 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).debug(f"会话结束～～") self._process_before_stop_play_files() session_finished = True + self.reuse_judgment = time.time() break except json.JSONDecodeError: logger.bind(tag=TAG).warning("收到无效的JSON消息") @@ -502,65 +512,102 @@ class TTSProvider(TTSProviderBase): def to_tts(self, text: str) -> list: """非流式TTS处理，用于测试及保存音频文件的场景""" - start_time = time.time() - text = MarkdownCleaner.clean_markdown(text) - try: - # 使用同步方式进行TTS转换 - if self._is_token_expired(): - self._refresh_token() + # 创建新的事件循环 + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) - # 构造请求数据 - request_json = { - "appkey": self.appkey, - "token": self.token, - "text": text, - "format": "pcm", - "sample_rate": self.sample_rate, - "voice": self.voice, - "volume": self.volume, - "speech_rate": self.speech_rate, - "pitch_rate": self.pitch_rate, - } + # 生成会话ID + session_id = uuid.uuid4().hex + message_id = uuid.uuid4().hex + # 存储音频数据 + audio_data = [] - # 使用HTTP接口进行同步请求 - import requests - api_url = f"https://{self.host}/stream/v1/tts" - headers = {"Content-Type": "application/json"} + async def _generate_audio(): + # 刷新Token（如果需要） + if self._is_token_expired(): + self._refresh_token() - resp = requests.post(api_url, json=request_json, headers=headers) - - if resp.status_code == 401: # Token过期特殊处理 - self._refresh_token() - resp = requests.post(api_url, json=request_json, headers=headers) - - if resp.headers["Content-Type"].startswith("audio/"): - pcm_data = resp.content - - # 使用opus编码器处理PCM数据 - opus_datas = [] - frame_bytes = int( - self.opus_encoder.sample_rate - * self.opus_encoder.channels - * self.opus_encoder.frame_size_ms - / 1000 - * 2 + # 建立WebSocket连接 + ws = await websockets.connect( + self.ws_url, + additional_headers={"X-NLS-Token": self.token}, + ping_interval=30, + ping_timeout=10, + close_timeout=10, ) + try: + # 发送StartSynthesis请求 + start_request = { + "header": { + "message_id": message_id, + "task_id": session_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StartSynthesis", + "appkey": self.appkey, + }, + "payload": { + "voice": self.voice, + "format": self.format, + "sample_rate": self.sample_rate, + "volume": self.volume, + "speech_rate": self.speech_rate, + "pitch_rate": self.pitch_rate, + "enable_subtitle": True + } + } + await ws.send(json.dumps(start_request)) - # 分帧处理PCM数据 - for i in range(0, len(pcm_data), frame_bytes): - frame = pcm_data[i:i + frame_bytes] - if len(frame) == frame_bytes: - opus = self.opus_encoder.encode_pcm_to_opus(frame, False) - if opus: - opus_datas.extend(opus) + # 发送文本合成请求 + filtered_text = MarkdownCleaner.clean_markdown(text) + run_request = { + "header": { + "message_id": message_id, + "task_id": session_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": self.appkey, + }, + "payload": { + "text": filtered_text + } + } + await ws.send(json.dumps(run_request)) - logger.bind(tag=TAG).info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒") - return opus_datas - else: - logger.bind(tag=TAG).error(f"TTS请求失败: {resp.content}") - return [] + # 发送停止合成请求 + stop_request = { + "header": { + "message_id": message_id, + "task_id": session_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StopSynthesis", + "appkey": self.appkey, + } + } + await ws.send(json.dumps(stop_request)) + # 接收音频数据 + while True: + msg = await ws.recv() + if isinstance(msg, (bytes, bytearray)): + # 编码为Opus并收集 + opus_frames = self.opus_encoder.encode_pcm_to_opus(msg, False) + audio_data.extend(opus_frames) + elif isinstance(msg, str): + data = json.loads(msg) + header = data.get("header", {}) + if header.get("name") == "SynthesisCompleted": + break + finally: + try: + await ws.close() + except: + pass + + loop.run_until_complete(_generate_audio()) + loop.close() + + return audio_data except Exception as e: - logger.bind(tag=TAG).error(f"TTS请求异常: {e}") + logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}") return [] From 2c33ee5b325131f6d137fda2cb9e722c52186515 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Mon, 14 Jul 2025 17:34:42 +0800 Subject: [PATCH 18/19] =?UTF-8?q?update:=20=E4=BC=98=E5=8C=96=E6=96=87?= =?UTF-8?q?=E6=9C=AC=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/providers/tts/aliyun_stream.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index 1ea37810..fd345fe8 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -125,8 +125,7 @@ class TTSProvider(TTSProviderBase): self.last_active_time = None # 文本符号 - self.sentence_queue = None - self.sentence_end_chars = {'.', '。', '!', '！', '?', '？', '\n', "~", "；", ";", ":", "：", " ", "，", ","} + self.sentence_end_chars = {'。', '!', '！', '?', '？', '\n', "~"} # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( @@ -228,7 +227,6 @@ class TTSProvider(TTSProviderBase): # aliyunStream独有的参数生成 self.conn.message_id = str(uuid.uuid4().hex) - self.sentence_queue = queue.Queue() self.text_buffer = "" logger.bind(tag=TAG).info("开始启动TTS会话...") @@ -251,9 +249,6 @@ class TTSProvider(TTSProviderBase): f"开始发送TTS文本: {message.content_detail}" ) self.text_buffer += message.content_detail - if message.content_detail in self.sentence_end_chars and len(self.text_buffer) > 6: - self.sentence_queue.put(textUtils.get_string_no_punctuation_or_emoji(self.text_buffer)) - self.text_buffer = "" future = asyncio.run_coroutine_threadsafe( self.text_to_speak(message.content_detail, None), loop=self.conn.loop, @@ -278,7 +273,6 @@ class TTSProvider(TTSProviderBase): if message.sentence_type == SentenceType.LAST: try: logger.bind(tag=TAG).info("开始结束TTS会话...") - self.sentence_queue.put(textUtils.get_string_no_punctuation_or_emoji(self.text_buffer)) future = asyncio.run_coroutine_threadsafe( self.finish_session(self.conn.sentence_id), loop=self.conn.loop, @@ -447,10 +441,7 @@ class TTSProvider(TTSProviderBase): if event_name == "SynthesisStarted": logger.bind(tag=TAG).debug("TTS合成已启动") elif event_name == "SentenceBegin": - try: - text_buff = self.sentence_queue.get_nowait() - except queue.Empty: - text_buff = "" + text_buff = self._extract_sentence() logger.bind(tag=TAG).debug(f"句子语音生成开始: {text_buff}") opus_datas_cache = [] self.tts_audio_queue.put((SentenceType.FIRST, [], text_buff)) @@ -510,6 +501,17 @@ class TTSProvider(TTSProviderBase): finally: self._monitor_task = None + def _extract_sentence(self): + """从text_buffer中提取第一个满足分句规则的句子""" + if len(self.text_buffer) <= 6: + return textUtils.get_string_no_punctuation_or_emoji(self.text_buffer) + for idx, char in enumerate(self.text_buffer): + if char in self.sentence_end_chars and idx > 6: + sentence = self.text_buffer[:idx + 1] + self.text_buffer = self.text_buffer[idx + 1:] + return textUtils.get_string_no_punctuation_or_emoji(sentence) + return None + def to_tts(self, text: str) -> list: """非流式TTS处理，用于测试及保存音频文件的场景""" try: From e2a7534aa581d746761aeb47492bff08f9b4c88e Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Wed, 16 Jul 2025 15:03:53 +0800 Subject: [PATCH 19/19] =?UTF-8?q?update:=20=E4=BC=98=E5=8C=96=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 1 - .../core/providers/tts/aliyun_stream.py | 49 +++++++------------ .../providers/tts/huoshan_double_stream.py | 4 +- 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 978d88dd..7b45ad2d 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -128,7 +128,6 @@ class ConnectionHandler: # tts相关变量 self.sentence_id = None - self.message_id = None # iot相关变量 self.iot_descriptors = {} diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py index fd345fe8..ad9f344d 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -124,17 +124,16 @@ class TTSProvider(TTSProviderBase): self._monitor_task = None self.last_active_time = None - # 文本符号 - self.sentence_end_chars = {'。', '!', '！', '?', '？', '\n', "~"} + # 专属tts设置 + self.message_id = "" + self.tts_text = '' + self.text_buffer = [] # 创建Opus编码器 self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( sample_rate=16000, channels=1, frame_size_ms=60 ) - # PCM缓冲区 - self.pcm_buffer = bytearray() - # Token管理 if self.access_key_id and self.access_key_secret: self._refresh_token() @@ -226,8 +225,8 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).info(f"自动生成新的 会话ID: {self.conn.sentence_id}") # aliyunStream独有的参数生成 - self.conn.message_id = str(uuid.uuid4().hex) - self.text_buffer = "" + self.message_id = str(uuid.uuid4().hex) + self.text_buffer = [] logger.bind(tag=TAG).info("开始启动TTS会话...") future = asyncio.run_coroutine_threadsafe( @@ -248,7 +247,7 @@ class TTSProvider(TTSProviderBase): logger.bind(tag=TAG).debug( f"开始发送TTS文本: {message.content_detail}" ) - self.text_buffer += message.content_detail + self.text_buffer.append(message.content_detail) future = asyncio.run_coroutine_threadsafe( self.text_to_speak(message.content_detail, None), loop=self.conn.loop, @@ -273,6 +272,9 @@ class TTSProvider(TTSProviderBase): if message.sentence_type == SentenceType.LAST: try: logger.bind(tag=TAG).info("开始结束TTS会话...") + self.tts_text = textUtils.get_string_no_punctuation_or_emoji( + ''.join(self.text_buffer).replace('\n', '') + ) future = asyncio.run_coroutine_threadsafe( self.finish_session(self.conn.sentence_id), loop=self.conn.loop, @@ -297,7 +299,7 @@ class TTSProvider(TTSProviderBase): filtered_text = MarkdownCleaner.clean_markdown(text) run_request = { "header": { - "message_id": self.conn.message_id, + "message_id": self.message_id, "task_id": self.conn.sentence_id, "namespace": "FlowingSpeechSynthesizer", "name": "RunSynthesis", @@ -341,7 +343,7 @@ class TTSProvider(TTSProviderBase): start_request = { "header": { - "message_id": self.conn.message_id, + "message_id": self.message_id, "task_id": self.conn.sentence_id, "namespace": "FlowingSpeechSynthesizer", "name": "StartSynthesis", @@ -372,7 +374,7 @@ class TTSProvider(TTSProviderBase): if self.ws: stop_request = { "header": { - "message_id": self.conn.message_id, + "message_id": self.message_id, "task_id": self.conn.sentence_id, "namespace": "FlowingSpeechSynthesizer", "name": "StopSynthesis", @@ -422,7 +424,6 @@ class TTSProvider(TTSProviderBase): opus_datas_cache = [] is_first_sentence = True first_sentence_segment_count = 0 # 添加计数器 - text_buff = "" try: session_finished = False # 标记会话是否正常结束 while not self.conn.stop_event.is_set(): @@ -441,13 +442,11 @@ class TTSProvider(TTSProviderBase): if event_name == "SynthesisStarted": logger.bind(tag=TAG).debug("TTS合成已启动") elif event_name == "SentenceBegin": - text_buff = self._extract_sentence() - logger.bind(tag=TAG).debug(f"句子语音生成开始: {text_buff}") + logger.bind(tag=TAG).debug(f"句子语音生成开始: {self.tts_text}") opus_datas_cache = [] - self.tts_audio_queue.put((SentenceType.FIRST, [], text_buff)) + self.tts_audio_queue.put((SentenceType.FIRST, [], self.tts_text)) elif event_name == "SentenceEnd": - logger.bind(tag=TAG).info(f"句子语音生成成功： {text_buff}") - text_buff = "" + logger.bind(tag=TAG).info(f"句子语音生成成功： {self.tts_text}") if not is_first_sentence or first_sentence_segment_count > 10: # 发送缓存的数据 self.tts_audio_queue.put( @@ -460,6 +459,7 @@ class TTSProvider(TTSProviderBase): self._process_before_stop_play_files() session_finished = True self.reuse_judgment = time.time() + self.tts_text = '' break except json.JSONDecodeError: logger.bind(tag=TAG).warning("收到无效的JSON消息") @@ -477,10 +477,10 @@ class TTSProvider(TTSProviderBase): (SentenceType.MIDDLE, opus_datas, None) ) else: - opus_datas_cache = opus_datas_cache + opus_datas + opus_datas_cache.extend(opus_datas) else: # 后续句子缓存 - opus_datas_cache = opus_datas_cache + opus_datas + opus_datas_cache.extend(opus_datas) except websockets.ConnectionClosed: logger.bind(tag=TAG).warning("WebSocket连接已关闭") @@ -501,17 +501,6 @@ class TTSProvider(TTSProviderBase): finally: self._monitor_task = None - def _extract_sentence(self): - """从text_buffer中提取第一个满足分句规则的句子""" - if len(self.text_buffer) <= 6: - return textUtils.get_string_no_punctuation_or_emoji(self.text_buffer) - for idx, char in enumerate(self.text_buffer): - if char in self.sentence_end_chars and idx > 6: - sentence = self.text_buffer[:idx + 1] - self.text_buffer = self.text_buffer[idx + 1:] - return textUtils.get_string_no_punctuation_or_emoji(sentence) - return None - def to_tts(self, text: str) -> list: """非流式TTS处理，用于测试及保存音频文件的场景""" try: diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 23abaf5c..e1c568fa 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -435,10 +435,10 @@ class TTSProvider(TTSProviderBase): (SentenceType.MIDDLE, opus_datas, None) ) else: - opus_datas_cache = opus_datas_cache + opus_datas + opus_datas_cache.extend(opus_datas) else: # 后续句子缓存 - opus_datas_cache = opus_datas_cache + opus_datas + opus_datas_cache.extend(opus_datas) elif res.optional.event == EVENT_TTSSentenceEnd: logger.bind(tag=TAG).info(f"句子语音生成成功：{self.tts_text}") if not is_first_sentence or first_sentence_segment_count > 10: