ASR句首丢字问题 (#346)

* fix: ASR句首丢字问题 (#338)

* feat: 支持dify的多轮对话, chat工作流支持持久化conversation_id #296 (#313)

* TTS功能的一些修改 (#307)

* fix: tts音频转码兼容

* feat: 支持自定义tts接口服务

* feat: tts超时时间作为可配置项

* feat: 自定义tts音频格式兼容

* style: 使命名更符合语义

---------

Co-authored-by: Jad <i@nocilol.me>
Co-authored-by: yanyige <yige.yan@qq.com>
Co-authored-by: Jad <journey.adc@gmail.com>

main/xiaozhi-server/config.yaml

@@ -58,6 +58,8 @@ delete_audio: true
 
 # 没有语音输入多久后断开连接(秒)，默认2分钟，即120秒
 close_connection_no_voice_time: 120
+# TTS请求超时时间(秒)
+tts_timeout: 10
 
 CMD_exit:
   - "退出"
@@ -412,6 +414,21 @@ TTS:
     # 语速范围0.25-4.0
     speed: 1
     output_file: tmp/
+  CustomTTS:
+    # 自定义的TTS接口服务，请求参数可自定义
+    # 要求接口使用GET方式请求，并返回音频文件
+    type: custom
+    url: "http://127.0.0.1:9880/tts"
+    params: # 自定义请求参数
+      # text: "{prompt_text}" # {prompt_text}会被替换为实际的提示词内容
+      # speaker: jok老师
+      # speed: 1
+      # foo: bar
+      # testabc: 123456
+    headers: # 自定义请求头
+      # Authorization: Bearer xxxx
+    format: wav # 接口返回的音频格式
+    output_file: tmp/
 # 模块测试配置
 module_test:
   test_sentences:  # 自定义测试语句

main/xiaozhi-server/core/connection.py

@@ -451,7 +451,8 @@ class ConnectionHandler:
                 opus_datas, text_index, tts_file = [], 0, None
                 try:
                     self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
-                    tts_file, text, text_index = future.result(timeout=10)
+                    tts_timeout = self.config.get("tts_timeout", 10)
+                    tts_file, text, text_index = future.result(timeout=tts_timeout)
                     if text is None or len(text) <= 0:
                         self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
                     elif tts_file is None:
@@ -459,7 +460,7 @@ class ConnectionHandler:
                     else:
                         self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
                         if os.path.exists(tts_file):
-                            opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
+                            opus_datas, duration = self.tts.audio_to_opus_data(tts_file)
                         else:
                             self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
                 except TimeoutError:

main/xiaozhi-server/core/handle/musicHandler.py

@@ -131,7 +131,7 @@ class MusicHandler:
             if music_path.endswith(".p3"):
                 opus_packets, duration = p3.decode_opus_from_file(music_path)
             else:
-                opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
+                opus_packets, duration = conn.tts.audio_to_opus_data(music_path)
             conn.audio_play_queue.put((opus_packets, selected_music, 0))
 
         except Exception as e:

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -20,7 +20,8 @@ async def handleAudioMessage(conn, audio):
     # 如果本次没有声音，本段也没声音，就把声音丢弃了
     if have_voice == False and conn.client_have_voice == False:
         await no_voice_close_connect(conn)
-        conn.asr_audio.clear()
+        conn.asr_audio.append(audio)
+        conn.asr_audio = conn.asr_audio[-5:]  # 保留最新的5帧音频内容，解决ASR句首丢字问题
         return
     conn.client_no_voice_last_time = 0.0
     conn.asr_audio.append(audio)
@@ -29,7 +30,7 @@ async def handleAudioMessage(conn, audio):
         conn.client_abort = False
         conn.asr_server_receive = False
         # 音频太短了，无法识别
-        if len(conn.asr_audio) < 3:
+        if len(conn.asr_audio) < 10:
             conn.asr_server_receive = True
         else:
             text, file_path = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -10,11 +10,13 @@ class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.api_key = config["api_key"]
         self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
+        self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
 
     def response(self, session_id, dialogue):
         try:
             # 取最后一条用户消息
             last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+            conversation_id = self.session_conversation_map.get(session_id)
 
             # 发起流式请求
             with requests.post(
@@ -24,13 +26,18 @@ class LLMProvider(LLMProviderBase):
                         "query": last_msg["content"],
                         "response_mode": "streaming",
                         "user": session_id,
-                        "inputs": {}
+                        "inputs": {},
+                        "conversation_id": conversation_id
                     },
                     stream=True
             ) as r:
                 for line in r.iter_lines():
                     if line.startswith(b'data: '):
                         event = json.loads(line[6:])
+                        # 如果没有找到conversation_id，则获取此次conversation_id
+                        if not conversation_id:
+                            conversation_id = event.get('conversation_id')
+                            self.session_conversation_map[session_id] = conversation_id  # 更新映射
                         if event.get('answer'):
                             yield event['answer']
 

main/xiaozhi-server/core/providers/tts/base.py

@@ -41,19 +41,20 @@ class TTSProviderBase(ABC):
     async def text_to_speak(self, text, output_file):
         pass
 
-    def wav_to_opus_data(self, wav_file_path):
+    def audio_to_opus_data(self, audio_file_path):
-        # 使用pydub加载PCM文件
+        """音频文件转换为Opus编码"""
         # 获取文件后缀名
-        file_type = os.path.splitext(wav_file_path)[1]
+        file_type = os.path.splitext(audio_file_path)[1]
         if file_type:
             file_type = file_type.lstrip('.')
-        audio = AudioSegment.from_file(wav_file_path, format=file_type)
+        audio = AudioSegment.from_file(audio_file_path, format=file_type)
 
+        # 转换为单声道/16kHz采样率/16位小端编码（确保与编码器匹配）
+        audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
+
+        # 音频时长(秒)
         duration = len(audio) / 1000.0
 
-        # 转换为单声道和16kHz采样率（确保与编码器匹配）
-        audio = audio.set_channels(1).set_frame_rate(16000)
-
         # 获取原始PCM数据（16位小端）
         raw_data = audio.raw_data
 

main/xiaozhi-server/core/providers/tts/custom.py

@@ -0,0 +1,35 @@
+import os
+import uuid
+import requests
+from config.logger import setup_logging
+from datetime import datetime
+from core.providers.tts.base import TTSProviderBase
+
+TAG = __name__
+logger = setup_logging()
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.url = config.get("url")
+        self.headers = config.get("headers", {})
+        self.params = config.get("params")
+        self.format = config.get("format", "wav")
+        self.output_file = config.get("output_file", "tmp/")
+
+    def generate_filename(self):
+        return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}.{self.format}")
+
+    async def text_to_speak(self, text, output_file):
+        request_params = {}
+        for k, v in self.params.items():
+            if isinstance(v, str) and "{prompt_text}" in v:
+                v = v.replace("{prompt_text}", text)
+            request_params[k] = v
+
+        resp = requests.get(self.url, params=request_params, headers=self.headers)
+        if resp.status_code == 200:
+            with open(output_file, "wb") as file:
+                file.write(resp.content)
+        else:
+            logger.bind(tag=TAG).error(f"Custom TTS请求失败: {resp.status_code} - {resp.text}")