From 50ecee99ccb00a756fd3f49d1cbf7d57edd2a5bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AC=A3=E5=8D=97=E7=A7=91=E6=8A=80?= Date: Sat, 15 Mar 2025 00:19:45 +0800 Subject: [PATCH] =?UTF-8?q?ASR=E5=8F=A5=E9=A6=96=E4=B8=A2=E5=AD=97?= =?UTF-8?q?=E9=97=AE=E9=A2=98=20(#346)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: ASR句首丢字问题 (#338) * feat: 支持dify的多轮对话, chat工作流支持持久化conversation_id #296 (#313) * TTS功能的一些修改 (#307) * fix: tts音频转码兼容 * feat: 支持自定义tts接口服务 * feat: tts超时时间作为可配置项 * feat: 自定义tts音频格式兼容 * style: 使命名更符合语义 --------- Co-authored-by: Jad Co-authored-by: yanyige Co-authored-by: Jad --- main/xiaozhi-server/config.yaml | 17 +++++++++ main/xiaozhi-server/core/connection.py | 5 +-- .../core/handle/musicHandler.py | 2 +- .../core/handle/receiveAudioHandle.py | 5 +-- .../core/providers/llm/dify/dify.py | 9 ++++- .../xiaozhi-server/core/providers/tts/base.py | 15 ++++---- .../core/providers/tts/custom.py | 35 +++++++++++++++++++ 7 files changed, 75 insertions(+), 13 deletions(-) create mode 100644 main/xiaozhi-server/core/providers/tts/custom.py diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index da98f1b4..6f93e9c1 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -58,6 +58,8 @@ delete_audio: true # 没有语音输入多久后断开连接(秒)，默认2分钟，即120秒 close_connection_no_voice_time: 120 +# TTS请求超时时间(秒) +tts_timeout: 10 CMD_exit: - "退出" @@ -412,6 +414,21 @@ TTS: # 语速范围0.25-4.0 speed: 1 output_file: tmp/ + CustomTTS: + # 自定义的TTS接口服务，请求参数可自定义 + # 要求接口使用GET方式请求，并返回音频文件 + type: custom + url: "http://127.0.0.1:9880/tts" + params: # 自定义请求参数 + # text: "{prompt_text}" # {prompt_text}会被替换为实际的提示词内容 + # speaker: jok老师 + # speed: 1 + # foo: bar + # testabc: 123456 + headers: # 自定义请求头 + # Authorization: Bearer xxxx + format: wav # 接口返回的音频格式 + output_file: tmp/ # 模块测试配置 module_test: test_sentences: # 自定义测试语句 diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 0ff1ac0e..85da8c39 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -451,7 +451,8 @@ class ConnectionHandler: opus_datas, text_index, tts_file = [], 0, None try: self.logger.bind(tag=TAG).debug("正在处理TTS任务...") - tts_file, text, text_index = future.result(timeout=10) + tts_timeout = self.config.get("tts_timeout", 10) + tts_file, text, text_index = future.result(timeout=tts_timeout) if text is None or len(text) <= 0: self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty") elif tts_file is None: @@ -459,7 +460,7 @@ class ConnectionHandler: else: self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}") if os.path.exists(tts_file): - opus_datas, duration = self.tts.wav_to_opus_data(tts_file) + opus_datas, duration = self.tts.audio_to_opus_data(tts_file) else: self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}") except TimeoutError: diff --git a/main/xiaozhi-server/core/handle/musicHandler.py b/main/xiaozhi-server/core/handle/musicHandler.py index 022d6dc4..b650b847 100644 --- a/main/xiaozhi-server/core/handle/musicHandler.py +++ b/main/xiaozhi-server/core/handle/musicHandler.py @@ -131,7 +131,7 @@ class MusicHandler: if music_path.endswith(".p3"): opus_packets, duration = p3.decode_opus_from_file(music_path) else: - opus_packets, duration = conn.tts.wav_to_opus_data(music_path) + opus_packets, duration = conn.tts.audio_to_opus_data(music_path) conn.audio_play_queue.put((opus_packets, selected_music, 0)) except Exception as e: diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index bf6d4c36..c5b4135c 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -20,7 +20,8 @@ async def handleAudioMessage(conn, audio): # 如果本次没有声音，本段也没声音，就把声音丢弃了 if have_voice == False and conn.client_have_voice == False: await no_voice_close_connect(conn) - conn.asr_audio.clear() + conn.asr_audio.append(audio) + conn.asr_audio = conn.asr_audio[-5:] # 保留最新的5帧音频内容，解决ASR句首丢字问题 return conn.client_no_voice_last_time = 0.0 conn.asr_audio.append(audio) @@ -29,7 +30,7 @@ async def handleAudioMessage(conn, audio): conn.client_abort = False conn.asr_server_receive = False # 音频太短了，无法识别 - if len(conn.asr_audio) < 3: + if len(conn.asr_audio) < 10: conn.asr_server_receive = True else: text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id) diff --git a/main/xiaozhi-server/core/providers/llm/dify/dify.py b/main/xiaozhi-server/core/providers/llm/dify/dify.py index 0e9c821a..f3c62639 100644 --- a/main/xiaozhi-server/core/providers/llm/dify/dify.py +++ b/main/xiaozhi-server/core/providers/llm/dify/dify.py @@ -10,11 +10,13 @@ class LLMProvider(LLMProviderBase): def __init__(self, config): self.api_key = config["api_key"] self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/') + self.session_conversation_map = {} # 存储session_id和conversation_id的映射 def response(self, session_id, dialogue): try: # 取最后一条用户消息 last_msg = next(m for m in reversed(dialogue) if m["role"] == "user") + conversation_id = self.session_conversation_map.get(session_id) # 发起流式请求 with requests.post( @@ -24,13 +26,18 @@ class LLMProvider(LLMProviderBase): "query": last_msg["content"], "response_mode": "streaming", "user": session_id, - "inputs": {} + "inputs": {}, + "conversation_id": conversation_id }, stream=True ) as r: for line in r.iter_lines(): if line.startswith(b'data: '): event = json.loads(line[6:]) + # 如果没有找到conversation_id，则获取此次conversation_id + if not conversation_id: + conversation_id = event.get('conversation_id') + self.session_conversation_map[session_id] = conversation_id # 更新映射 if event.get('answer'): yield event['answer'] diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 84737608..e6f013ca 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -41,19 +41,20 @@ class TTSProviderBase(ABC): async def text_to_speak(self, text, output_file): pass - def wav_to_opus_data(self, wav_file_path): - # 使用pydub加载PCM文件 + def audio_to_opus_data(self, audio_file_path): + """音频文件转换为Opus编码""" # 获取文件后缀名 - file_type = os.path.splitext(wav_file_path)[1] + file_type = os.path.splitext(audio_file_path)[1] if file_type: file_type = file_type.lstrip('.') - audio = AudioSegment.from_file(wav_file_path, format=file_type) + audio = AudioSegment.from_file(audio_file_path, format=file_type) + # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配） + audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2) + + # 音频时长(秒) duration = len(audio) / 1000.0 - # 转换为单声道和16kHz采样率（确保与编码器匹配） - audio = audio.set_channels(1).set_frame_rate(16000) - # 获取原始PCM数据（16位小端） raw_data = audio.raw_data diff --git a/main/xiaozhi-server/core/providers/tts/custom.py b/main/xiaozhi-server/core/providers/tts/custom.py new file mode 100644 index 00000000..d5447878 --- /dev/null +++ b/main/xiaozhi-server/core/providers/tts/custom.py @@ -0,0 +1,35 @@ +import os +import uuid +import requests +from config.logger import setup_logging +from datetime import datetime +from core.providers.tts.base import TTSProviderBase + +TAG = __name__ +logger = setup_logging() + +class TTSProvider(TTSProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__(config, delete_audio_file) + self.url = config.get("url") + self.headers = config.get("headers", {}) + self.params = config.get("params") + self.format = config.get("format", "wav") + self.output_file = config.get("output_file", "tmp/") + + def generate_filename(self): + return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}.{self.format}") + + async def text_to_speak(self, text, output_file): + request_params = {} + for k, v in self.params.items(): + if isinstance(v, str) and "{prompt_text}" in v: + v = v.replace("{prompt_text}", text) + request_params[k] = v + + resp = requests.get(self.url, params=request_params, headers=self.headers) + if resp.status_code == 200: + with open(output_file, "wb") as file: + file.write(resp.content) + else: + logger.bind(tag=TAG).error(f"Custom TTS请求失败: {resp.status_code} - {resp.text}")