Merge branch 'main' into agent-plugin

# Conflicts:
#	main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

main/xiaozhi-server/core/connection.py

@@ -656,7 +656,7 @@ class ConnectionHandler:
                     # print("content_arguments", content_arguments)
                     tool_call_flag = True
 
-                if tools_call is not None:
+                if tools_call is not None and len(tools_call) > 0:
                     tool_call_flag = True
                     if tools_call[0].id is not None:
                         function_id = tools_call[0].id

main/xiaozhi-server/core/handle/helloHandle.py

@@ -1,30 +1,31 @@
-import os
 import time
 import json
 import random
-import shutil
 import asyncio
-from core.handle.sendAudioHandle import send_stt_message
-from core.utils.util import remove_punctuation_and_length
-from core.providers.tts.dto.dto import ContentType, InterfaceType
+from core.utils.util import audio_to_data
+from core.handle.sendAudioHandle import sendAudioMessage, send_stt_message
+from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes
+from core.providers.tts.dto.dto import ContentType, SentenceType
 from core.handle.mcpHandle import (
     MCPClient,
     send_mcp_initialize_message,
     send_mcp_tools_list_request,
 )
-
+from core.utils.wakeup_word import WakeupWordsConfig
 
 TAG = __name__
 
 WAKEUP_CONFIG = {
-    "dir": "config/assets/",
-    "file_name": "wakeup_words",
-    "create_time": time.time(),
-    "refresh_time": 10,
-    "words": ["你好小智", "你好啊小智", "小智你好", "小智"],
-    "text": "",
+    "refresh_time": 5,
+    "words": ["你好", "你好啊", "嘿，你好", "嗨"],
 }
 
+# 创建全局的唤醒词配置管理器
+wakeup_words_config = WakeupWordsConfig()
+
+# 用于防止并发调用wakeupWordsResponse的锁
+_wakeup_response_lock = asyncio.Lock()
+
 
 async def handleHelloMessage(conn, msg_json):
     """处理hello消息"""
@@ -53,85 +54,75 @@ async def checkWakeupWords(conn, text):
     enable_wakeup_words_response_cache = conn.config[
         "enable_wakeup_words_response_cache"
     ]
-    """是否用的是非流式tts"""
-    if conn.tts and conn.tts.interface_type != InterfaceType.NON_STREAM:
+
+    if not enable_wakeup_words_response_cache or not conn.tts:
         return False
 
-    """是否开启唤醒词加速"""
-    if not enable_wakeup_words_response_cache:
-        return False
-    """检查是否是唤醒词"""
     _, filtered_text = remove_punctuation_and_length(text)
-    if filtered_text in conn.config.get("wakeup_words"):
-        # 设置刚刚被唤醒的标志
-        conn.just_woken_up = True
-        await send_stt_message(conn, text)
+    if filtered_text not in conn.config.get("wakeup_words"):
+        return False
 
-        file = getWakeupWordFile(WAKEUP_CONFIG["file_name"])
-        if file is None:
+    conn.just_woken_up = True
+    await send_stt_message(conn, text)
+
+    # 获取当前音色
+    voice = getattr(conn.tts, "voice", "default")
+
+    # 获取唤醒词回复配置
+    response = wakeup_words_config.get_wakeup_response(voice)
+
+    # 播放唤醒词回复
+    conn.client_abort = False
+    opus_packets, _ = audio_to_data(response["file_path"])
+
+    conn.logger.bind(tag=TAG).info(f"播放唤醒词回复: {response['text']}")
+    await sendAudioMessage(conn, SentenceType.FIRST, opus_packets, response["text"])
+    await sendAudioMessage(conn, SentenceType.LAST, [], None)
+
+    # 检查是否需要更新唤醒词回复
+    if time.time() - response["time"] > WAKEUP_CONFIG["refresh_time"]:
+        if not _wakeup_response_lock.locked():
             asyncio.create_task(wakeupWordsResponse(conn))
-            return False
-        text_hello = WAKEUP_CONFIG["text"]
-        if not text_hello:
-            text_hello = text
-        if conn.tts is None:
-            return False
-        conn.tts.tts_one_sentence(
-            conn, ContentType.FILE, content_file=file, content_detail=text_hello
-        )
-        if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
-            asyncio.create_task(wakeupWordsResponse(conn))
-        return True
-    return False
-
-
-def getWakeupWordFile(file_name):
-    for file in os.listdir(WAKEUP_CONFIG["dir"]):
-        if file.startswith("my_" + file_name):
-            """避免缓存文件是一个空文件"""
-            if os.stat(f"config/assets/{file}").st_size > (15 * 1024):
-                return f"config/assets/{file}"
-
-    """查找config/assets/目录下名称为wakeup_words的文件"""
-    for file in os.listdir(WAKEUP_CONFIG["dir"]):
-        if file.startswith(file_name):
-            return f"config/assets/{file}"
-    return None
+    return True
 
 
 async def wakeupWordsResponse(conn):
-    wait_max_time = 5
-    while conn.llm is None or not conn.llm.response_no_stream:
-        await asyncio.sleep(1)
-        wait_max_time -= 1
-        if wait_max_time <= 0:
-            conn.logger.bind(tag=TAG).error("连接对象没有llm")
+    if not conn.tts or not conn.llm or not conn.llm.response_no_stream:
+        return
+
+    try:
+        # 尝试获取锁，如果获取不到就返回
+        if not await _wakeup_response_lock.acquire():
             return
 
-    """唤醒词响应"""
-    wakeup_word = random.choice(WAKEUP_CONFIG["words"])
-    question = (
-        "此刻用户正在和你说```"
-        + wakeup_word
-        + "```。\n请你根据以上用户的内容，进行简短回复，文字内容控制在15个字以内。\n"
-        + "请勿对这条内容本身进行任何解释和回应，仅返回对用户的内容的回复。"
-    )
-    result = conn.llm.response_no_stream(conn.config["prompt"], question)
-    if result is None or result == "":
-        return
-    tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
-
-    if tts_file is not None and os.path.exists(tts_file):
-        file_type = os.path.splitext(tts_file)[1]
-        if file_type:
-            file_type = file_type.lstrip(".")
-        old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"])
-        if old_file is not None:
-            os.remove(old_file)
-        """将文件挪到"wakeup_words.mp3"""
-        shutil.move(
-            tts_file,
-            WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type,
+        # 生成唤醒词回复
+        wakeup_word = random.choice(WAKEUP_CONFIG["words"])
+        question = (
+            "此刻用户正在和你说```"
+            + wakeup_word
+            + "```。\n请你根据以上用户的内容进行简短回复。要像一个人正常人一样说话，不要像机器人一样说话。\n"
+            + "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。"
         )
-        WAKEUP_CONFIG["create_time"] = time.time()
-        WAKEUP_CONFIG["text"] = result
+
+        result = conn.llm.response_no_stream(conn.config["prompt"], question)
+        if not result or len(result) == 0:
+            return
+
+        # 生成TTS音频
+        tts_result = await asyncio.to_thread(conn.tts.to_tts, result)
+        if not tts_result:
+            return
+
+        # 获取当前音色
+        voice = getattr(conn.tts, "voice", "default")
+
+        wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
+        file_path = wakeup_words_config.generate_file_path(voice)
+        with open(file_path, "wb") as f:
+            f.write(wav_bytes)
+        # 更新配置
+        wakeup_words_config.update_wakeup_response(voice, file_path, result)
+    finally:
+        # 确保在任何情况下都释放锁
+        if _wakeup_response_lock.locked():
+            _wakeup_response_lock.release()

main/xiaozhi-server/core/handle/mcpHandle.py

@@ -1,7 +1,7 @@
 import json
 import asyncio
 from concurrent.futures import Future
-from core.utils.util import get_vision_url
+from core.utils.util import get_vision_url, sanitize_tool_name
 from core.utils.auth import AuthToken
 
 TAG = __name__
@@ -11,7 +11,8 @@ class MCPClient:
     """MCPClient，用于管理MCP状态和工具"""
 
     def __init__(self):
-        self.tools = {}  # Dictionary for O(1) lookup
+        self.tools = {}  # sanitized_name -> tool_data
+        self.name_mapping = {}
         self.ready = False
         self.call_results = {}  # To store Futures for tool call responses
         self.next_id = 1
@@ -30,7 +31,7 @@ class MCPClient:
         result = []
         for tool_name, tool_data in self.tools.items():
             function_def = {
-                "name": tool_data["name"],
+                "name": tool_name,
                 "description": tool_data["description"],
                 "parameters": {
                     "type": tool_data["inputSchema"].get("type", "object"),
@@ -53,7 +54,9 @@ class MCPClient:
 
     async def add_tool(self, tool_data: dict):
         async with self.lock:
-            self.tools[tool_data["name"]] = tool_data
+            sanitized_name = sanitize_tool_name(tool_data["name"])
+            self.tools[sanitized_name] = tool_data
+            self.name_mapping[sanitized_name] = tool_data["name"]
             self._cached_available_tools = (
                 None  # Invalidate the cache when a tool is added
             )
@@ -133,9 +136,6 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict):
                 conn.logger.bind(tag=TAG).info(
                     f"客户端MCP服务器信息: name={name}, version={version}"
                 )
-            await send_mcp_tools_list_request(
-                conn
-            )  # After initialization, request tool list
             return
 
         elif msg_id == 2:  # mcpToolsListID
@@ -174,6 +174,20 @@ async def handle_mcp_message(conn, mcp_client: MCPClient, payload: dict):
                     await mcp_client.add_tool(new_tool)
                     conn.logger.bind(tag=TAG).debug(f"客户端工具 #{i+1}: {name}")
 
+                # 替换所有工具描述中的工具名称
+                for tool_data in mcp_client.tools.values():
+                    if "description" in tool_data:
+                        description = tool_data["description"]
+                        # 遍历所有工具名称进行替换
+                        for (
+                            sanitized_name,
+                            original_name,
+                        ) in mcp_client.name_mapping.items():
+                            description = description.replace(
+                                original_name, sanitized_name
+                            )
+                        tool_data["description"] = description
+
                 next_cursor = result.get("nextCursor", "")
                 if next_cursor:
                     conn.logger.bind(tag=TAG).info(
@@ -219,8 +233,6 @@ async def send_mcp_initialize_message(conn):
         "token": token,
     }
 
-    conn.logger.bind(tag=TAG).info(f"视觉服务信息: {vision}")
-
     payload = {
         "jsonrpc": "2.0",
         "id": 1,  # mcpInitializeID
@@ -333,15 +345,16 @@ async def call_mcp_tool(
             raise ValueError(f"参数处理失败: {str(e)}")
         raise e
 
+    actual_name = mcp_client.name_mapping.get(tool_name, tool_name)
     payload = {
         "jsonrpc": "2.0",
         "id": tool_call_id,
         "method": "tools/call",
-        "params": {"name": tool_name, "arguments": arguments},
+        "params": {"name": actual_name, "arguments": arguments},
     }
 
     conn.logger.bind(tag=TAG).info(
-        f"发送客户端mcp工具调用请求: {tool_name}，参数: {args}"
+        f"发送客户端mcp工具调用请求: {actual_name}，参数: {args}"
     )
     await send_mcp_message(conn, payload)
 
@@ -349,7 +362,7 @@ async def call_mcp_tool(
         # Wait for response or timeout
         raw_result = await asyncio.wait_for(result_future, timeout=timeout)
         conn.logger.bind(tag=TAG).info(
-            f"客户端mcp工具调用 {tool_name} 成功，原始结果: {raw_result}"
+            f"客户端mcp工具调用 {actual_name} 成功，原始结果: {raw_result}"
         )
 
         if isinstance(raw_result, dict):

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -13,12 +13,14 @@ TAG = __name__
 async def handleAudioMessage(conn, audio):
     # 当前片段是否有人说话
     have_voice = conn.vad.is_vad(conn, audio)
-
     # 如果设备刚刚被唤醒，短暂忽略VAD检测
-    if hasattr(conn, "just_woken_up") and conn.just_woken_up:
+    if have_voice and hasattr(conn, "just_woken_up") and conn.just_woken_up:
         have_voice = False
         # 设置一个短暂延迟后恢复VAD检测
-        asyncio.create_task(resume_vad_detection(conn))
+        conn.asr_audio.clear()
+        if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
+            conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
+        return
 
     if have_voice:
         if conn.client_is_speaking:
@@ -31,7 +33,7 @@ async def handleAudioMessage(conn, audio):
 
 async def resume_vad_detection(conn):
     # 等待2秒后恢复VAD检测
-    await asyncio.sleep(2)
+    await asyncio.sleep(1)
     conn.just_woken_up = False
 
 

main/xiaozhi-server/core/handle/sendAudioHandle.py

@@ -34,6 +34,7 @@ emoji_map = {
 
 async def sendAudioMessage(conn, sentenceType, audios, text):
     # 发送句子开始消息
+    conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}")
     if text is not None:
         emotion = analyze_emotion(text)
         emoji = emoji_map.get(emotion, "🙂")  # 默认使用笑脸

main/xiaozhi-server/core/mcp/MCPClient.py

@@ -9,6 +9,7 @@ from mcp import ClientSession, StdioServerParameters
 from mcp.client.stdio import stdio_client
 from mcp.client.sse import sse_client
 from config.logger import setup_logging
+from core.utils.util import sanitize_tool_name
 
 TAG = __name__
 
@@ -23,7 +24,9 @@ class MCPClient:
         self._shutdown_evt = asyncio.Event()
 
         self.session: Optional[ClientSession] = None
-        self.tools: List = []
+        self.tools: List = []  # original tool objects
+        self.tools_dict: Dict[str, Any] = {}
+        self.name_mapping: Dict[str, str] = {}
 
     async def initialize(self):
         if self._worker_task:
@@ -32,7 +35,7 @@ class MCPClient:
         await self._ready_evt.wait()
 
         self.logger.bind(tag=TAG).info(
-            f"Connected, tools = {[t.name for t in self.tools]}"
+            f"Connected, tools = {[name for name in self.name_mapping.values()]}"
         )
 
     async def cleanup(self):
@@ -48,27 +51,28 @@ class MCPClient:
             self._worker_task = None
 
     def has_tool(self, name: str) -> bool:
-        return any(t.name == name for t in self.tools)
+        return name in self.tools_dict
 
     def get_available_tools(self):
         return [
             {
                 "type": "function",
                 "function": {
-                    "name": t.name,
-                    "description": t.description,
-                    "parameters": t.inputSchema,
+                    "name": name,
+                    "description": tool.description,
+                    "parameters": tool.inputSchema,
                 },
             }
-            for t in self.tools
+            for name, tool in self.tools_dict.items()
         ]
 
     async def call_tool(self, name: str, args: dict):
         if not self.session:
             raise RuntimeError("MCPClient not initialized")
 
+        real_name = self.name_mapping.get(name, name)
         loop = self._worker_task.get_loop()
-        coro = self.session.call_tool(name, args)
+        coro = self.session.call_tool(real_name, args)
 
         if loop is asyncio.get_running_loop():
             return await coro
@@ -123,6 +127,10 @@ class MCPClient:
 
                 # 获取工具
                 self.tools = (await self.session.list_tools()).tools
+                for t in self.tools:
+                    sanitized = sanitize_tool_name(t.name)
+                    self.tools_dict[sanitized] = t
+                    self.name_mapping[sanitized] = t.name
 
                 self._ready_evt.set()
 

main/xiaozhi-server/core/providers/asr/doubao.py

@@ -168,6 +168,7 @@ class ASRProvider(ASRProviderBase):
                 if (
                     "payload_msg" in result
                     and result["payload_msg"]["code"] != self.success_code
+                    and result["payload_msg"]["code"] != 1013  # 忽略无有效语音的错误
                 ):
                     logger.bind(tag=TAG).error(f"ASR error: {result}")
                     return None
@@ -203,6 +204,9 @@ class ASRProvider(ASRProviderBase):
                     if len(result["payload_msg"]["result"]) > 0:
                         return result["payload_msg"]["result"][0]["text"]
                     return None
+                elif "payload_msg" in result and result["payload_msg"]["code"] == 1013:
+                    # 无有效语音，返回空字符串
+                    return ""
                 else:
                     logger.bind(tag=TAG).error(f"ASR error: {result}")
                     return None

main/xiaozhi-server/core/providers/asr/doubao_stream.py

@@ -157,6 +157,11 @@ class ASRProvider(ASRProviderBase):
 
                     if "payload_msg" in result:
                         payload = result["payload_msg"]
+                        # 检查是否是错误码1013（无有效语音）
+                        if "code" in payload and payload["code"] == 1013:
+                            # 静默处理，不记录错误日志
+                            continue
+
                         if "result" in payload:
                             utterances = payload["result"].get("utterances", [])
                             # 检查duration和空文本的情况

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -2,6 +2,7 @@ import time
 import os
 import sys
 import io
+import psutil
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
 from core.providers.asr.base import ASRProviderBase
@@ -37,6 +38,13 @@ class CaptureOutput:
 class ASRProvider(ASRProviderBase):
     def __init__(self, config: dict, delete_audio_file: bool):
         super().__init__()
+        
+        # 内存检测，要求大于2G
+        min_mem_bytes = 2 * 1024 * 1024 * 1024
+        total_mem = psutil.virtual_memory().total
+        if total_mem < min_mem_bytes:
+            logger.bind(tag=TAG).error(f"可用内存不足2G，当前仅有 {total_mem / (1024*1024):.2f} MB，可能无法启动FunASR")
+        
         self.interface_type = InterfaceType.LOCAL
         self.model_dir = config.get("model_dir")
         self.output_dir = config.get("output_dir")  # 修正配置键名

main/xiaozhi-server/core/providers/tts/aliyun.py

@@ -91,7 +91,7 @@ class TTSProvider(TTSProviderBase):
 
         self.appkey = config.get("appkey")
         self.format = config.get("format", "wav")
-
+        self.audio_file_type = config.get("format", "wav")
         sample_rate = config.get("sample_rate", "16000")
         self.sample_rate = int(sample_rate) if sample_rate else 16000
 
@@ -188,9 +188,12 @@ class TTSProvider(TTSProviderBase):
                 )
             # 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的
             if resp.headers["Content-Type"].startswith("audio/"):
-                with open(output_file, "wb") as f:
-                    f.write(resp.content)
-                return output_file
+                if output_file:
+                    with open(output_file, "wb") as f:
+                        f.write(resp.content)
+                    return output_file
+                else:
+                    return resp.content
             else:
                 raise Exception(
                     f"{__name__} status_code: {resp.status_code} response: {resp.content}"

main/xiaozhi-server/core/providers/tts/base.py

@@ -8,7 +8,7 @@ from datetime import datetime
 from core.utils import textUtils
 from abc import ABC, abstractmethod
 from config.logger import setup_logging
-from core.utils.util import audio_to_data
+from core.utils.util import audio_to_data, audio_bytes_to_data
 from core.utils.tts import MarkdownCleaner
 from core.utils.output_counter import add_device_output
 from core.handle.reportHandle import enqueue_tts_report
@@ -20,7 +20,6 @@ from core.providers.tts.dto.dto import (
     InterfaceType,
 )
 
-
 import traceback
 
 TAG = __name__
@@ -33,10 +32,12 @@ class TTSProviderBase(ABC):
         self.conn = None
         self.tts_timeout = 10
         self.delete_audio_file = delete_audio_file
+        self.audio_file_type = "wav"
         self.output_file = config.get("output_dir", "tmp/")
         self.tts_text_queue = queue.Queue()
         self.tts_audio_queue = queue.Queue()
         self.tts_audio_first_sentence = True
+        self.before_stop_play_files = []
 
         self.tts_text_buff = []
         self.punctuations = (
@@ -77,35 +78,62 @@ class TTSProviderBase(ABC):
         )
 
     def to_tts(self, text):
-        tmp_file = self.generate_filename()
-        try:
-            max_repeat_time = 5
-            text = MarkdownCleaner.clean_markdown(text)
-            while not os.path.exists(tmp_file) and max_repeat_time > 0:
+        text = MarkdownCleaner.clean_markdown(text)
+        max_repeat_time = 5
+        if self.delete_audio_file:
+            # 需要删除文件的直接转为音频数据
+            while max_repeat_time > 0:
                 try:
-                    asyncio.run(self.text_to_speak(text, tmp_file))
+                    audio_bytes = asyncio.run(self.text_to_speak(text, None))
+                    if audio_bytes:
+                        audio_datas, _ = audio_bytes_to_data(
+                            audio_bytes, file_type=self.audio_file_type, is_opus=True
+                        )
+                        return audio_datas
+                    else:
+                        max_repeat_time -= 1
                 except Exception as e:
                     logger.bind(tag=TAG).warning(
                         f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}"
                     )
-                    # 未执行成功，删除文件
-                    if os.path.exists(tmp_file):
-                        os.remove(tmp_file)
                     max_repeat_time -= 1
-
             if max_repeat_time > 0:
                 logger.bind(tag=TAG).info(
-                    f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次"
+                    f"语音生成成功: {text}，重试{5 - max_repeat_time}次"
                 )
             else:
                 logger.bind(tag=TAG).error(
                     f"语音生成失败: {text}，请检查网络或服务是否正常"
                 )
-
-            return tmp_file
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
             return None
+        else:
+            tmp_file = self.generate_filename()
+            try:
+                while not os.path.exists(tmp_file) and max_repeat_time > 0:
+                    try:
+                        asyncio.run(self.text_to_speak(text, tmp_file))
+                    except Exception as e:
+                        logger.bind(tag=TAG).warning(
+                            f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}"
+                        )
+                        # 未执行成功，删除文件
+                        if os.path.exists(tmp_file):
+                            os.remove(tmp_file)
+                        max_repeat_time -= 1
+
+                if max_repeat_time > 0:
+                    logger.bind(tag=TAG).info(
+                        f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次"
+                    )
+                else:
+                    logger.bind(tag=TAG).error(
+                        f"语音生成失败: {text}，请检查网络或服务是否正常"
+                    )
+
+                return tmp_file
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
+                return None
 
     @abstractmethod
     async def text_to_speak(self, text, output_file):
@@ -193,12 +221,19 @@ class TTSProviderBase(ABC):
                     self.tts_text_buff.append(message.content_detail)
                     segment_text = self._get_segment_text()
                     if segment_text:
-                        tts_file = self.to_tts(segment_text)
-                        if tts_file:
-                            audio_datas = self._process_audio_file(tts_file)
-                            self.tts_audio_queue.put(
-                                (message.sentence_type, audio_datas, segment_text)
-                            )
+                        if self.delete_audio_file:
+                            audio_datas = self.to_tts(segment_text)
+                            if audio_datas:
+                                self.tts_audio_queue.put(
+                                    (message.sentence_type, audio_datas, segment_text)
+                                )
+                        else:
+                            tts_file = self.to_tts(segment_text)
+                            if tts_file:
+                                audio_datas = self._process_audio_file(tts_file)
+                                self.tts_audio_queue.put(
+                                    (message.sentence_type, audio_datas, segment_text)
+                                )
                 elif ContentType.FILE == message.content_type:
                     self._process_remaining_text()
                     tts_file = message.content_file
@@ -324,6 +359,14 @@ class TTSProviderBase(ABC):
             os.remove(tts_file)
         return audio_datas
 
+    def _process_before_stop_play_files(self):
+        for tts_file, text in self.before_stop_play_files:
+            if tts_file and os.path.exists(tts_file):
+                audio_datas = self._process_audio_file(tts_file)
+                self.tts_audio_queue.put((SentenceType.MIDDLE, audio_datas, text))
+        self.before_stop_play_files.clear()
+        self.tts_audio_queue.put((SentenceType.LAST, [], None))
+
     def _process_remaining_text(self):
         """处理剩余的文本并生成语音
 
@@ -335,11 +378,18 @@ class TTSProviderBase(ABC):
         if remaining_text:
             segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text)
             if segment_text:
-                tts_file = self.to_tts(segment_text)
-                audio_datas = self._process_audio_file(tts_file)
-                self.tts_audio_queue.put(
-                    (SentenceType.MIDDLE, audio_datas, segment_text)
-                )
+                if self.delete_audio_file:
+                    audio_datas = self.to_tts(segment_text)
+                    if audio_datas:
+                        self.tts_audio_queue.put(
+                            (SentenceType.MIDDLE, audio_datas, segment_text)
+                        )
+                else:
+                    tts_file = self.to_tts(segment_text)
+                    audio_datas = self._process_audio_file(tts_file)
+                    self.tts_audio_queue.put(
+                        (SentenceType.MIDDLE, audio_datas, segment_text)
+                    )
                 self.processed_chars += len(full_text)
                 return True
         return False

main/xiaozhi-server/core/providers/tts/cozecn.py

@@ -11,8 +11,8 @@ class TTSProvider(TTSProviderBase):
             self.voice = config.get("private_voice")
         else:
             self.voice = config.get("voice")
-        self.response_format = config.get("response_format")
-
+        self.response_format = config.get("response_format", "mp3")
+        self.audio_file_type = config.get("response_format", "mp3")
         self.host = "api.coze.cn"
         self.api_url = f"https://{self.host}/v1/audio/speech"
 
@@ -33,7 +33,10 @@ class TTSProvider(TTSProviderBase):
                 "POST", self.api_url, json=request_json, headers=headers
             )
             data = response.content
-            file_to_save = open(output_file, "wb")
-            file_to_save.write(data)
+            if output_file:
+                with open(output_file, "wb") as file_to_save:
+                    file_to_save.write(data)
+            else:
+                return data
         except Exception as e:
             raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/custom.py

@@ -16,8 +16,8 @@ class TTSProvider(TTSProviderBase):
         self.method = config.get("method", "GET")
         self.headers = config.get("headers", {})
         self.format = config.get("format", "wav")
+        self.audio_file_type = config.get("format", "wav")
         self.output_file = config.get("output_dir", "tmp/")
-
         self.params = config.get("params")
 
         if isinstance(self.params, str):
@@ -43,8 +43,11 @@ class TTSProvider(TTSProviderBase):
         else:
             resp = requests.get(self.url, params=request_params, headers=self.headers)
         if resp.status_code == 200:
-            with open(output_file, "wb") as file:
-                file.write(resp.content)
+            if output_file:
+                with open(output_file, "wb") as file:
+                    file.write(resp.content)
+            else:
+                return resp.content
         else:
             error_msg = f"Custom TTS请求失败: {resp.status_code} - {resp.text}"
             logger.bind(tag=TAG).error(error_msg)

main/xiaozhi-server/core/providers/tts/doubao.py

@@ -29,7 +29,7 @@ class TTSProvider(TTSProviderBase):
         speed_ratio = config.get("speed_ratio", "1.0")
         volume_ratio = config.get("volume_ratio", "1.0")
         pitch_ratio = config.get("pitch_ratio", "1.0")
-
+        self.audio_file_type = config.get("format", "wav")
         self.speed_ratio = float(speed_ratio) if speed_ratio else 1.0
         self.volume_ratio = float(volume_ratio) if volume_ratio else 1.0
         self.pitch_ratio = float(pitch_ratio) if pitch_ratio else 1.0
@@ -49,7 +49,7 @@ class TTSProvider(TTSProviderBase):
             "user": {"uid": "1"},
             "audio": {
                 "voice_type": self.voice,
-                "encoding": "wav",
+                "encoding": self.audio_file_type,
                 "speed_ratio": self.speed_ratio,
                 "volume_ratio": self.volume_ratio,
                 "pitch_ratio": self.pitch_ratio,
@@ -70,8 +70,12 @@ class TTSProvider(TTSProviderBase):
             )
             if "data" in resp.json():
                 data = resp.json()["data"]
-                file_to_save = open(output_file, "wb")
-                file_to_save.write(base64.b64decode(data))
+                audio_bytes = base64.b64decode(data)
+                if output_file:
+                    with open(output_file, "wb") as file_to_save:
+                        file_to_save.write(audio_bytes)
+                else:
+                    return audio_bytes
             else:
                 raise Exception(
                     f"{__name__} status_code: {resp.status_code} response: {resp.content}"

main/xiaozhi-server/core/providers/tts/edge.py

@@ -12,6 +12,7 @@ class TTSProvider(TTSProviderBase):
             self.voice = config.get("private_voice")
         else:
             self.voice = config.get("voice")
+        self.audio_file_type = config.get("format", "mp3")
 
     def generate_filename(self, extension=".mp3"):
         return os.path.join(
@@ -22,16 +23,24 @@ class TTSProvider(TTSProviderBase):
     async def text_to_speak(self, text, output_file):
         try:
             communicate = edge_tts.Communicate(text, voice=self.voice)
-            # 确保目录存在并创建空文件
-            os.makedirs(os.path.dirname(output_file), exist_ok=True)
-            with open(output_file, "wb") as f:
-                pass
+            if output_file:
+                # 确保目录存在并创建空文件
+                os.makedirs(os.path.dirname(output_file), exist_ok=True)
+                with open(output_file, "wb") as f:
+                    pass
 
-            # 流式写入音频数据
-            with open(output_file, "ab") as f:  # 改为追加模式避免覆盖
+                # 流式写入音频数据
+                with open(output_file, "ab") as f:  # 改为追加模式避免覆盖
+                    async for chunk in communicate.stream():
+                        if chunk["type"] == "audio":  # 只处理音频数据块
+                            f.write(chunk["data"])
+            else:
+                # 返回音频二进制数据
+                audio_bytes = b""
                 async for chunk in communicate.stream():
-                    if chunk["type"] == "audio":  # 只处理音频数据块
-                        f.write(chunk["data"])
+                    if chunk["type"] == "audio":
+                        audio_bytes += chunk["data"]
+                return audio_bytes
         except Exception as e:
             error_msg = f"Edge TTS请求失败: {e}"
             raise Exception(error_msg)  # 抛出异常，让调用方捕获

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -88,7 +88,7 @@ class TTSProvider(TTSProviderBase):
         self.reference_audio = parse_string_to_list(config.get("reference_audio"))
         self.reference_text = parse_string_to_list(config.get("reference_text"))
         self.format = config.get("response_format", "wav")
-
+        self.audio_file_type = config.get("response_format", "wav")
         self.api_key = config.get("api_key", "YOUR_API_KEY")
         have_key = check_model_key("FishSpeech TTS", self.api_key)
         if not have_key:
@@ -170,8 +170,11 @@ class TTSProvider(TTSProviderBase):
         if response.status_code == 200:
             audio_content = response.content
 
-            with open(output_file, "wb") as audio_file:
-                audio_file.write(audio_content)
+            if output_file:
+                with open(output_file, "wb") as audio_file:
+                    audio_file.write(audio_content)
+            else:
+                return audio_content
 
         else:
             error_msg = f"Request failed with status code {response.status_code}"

main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py

@@ -65,6 +65,7 @@ class TTSProvider(TTSProviderBase):
         self.aux_ref_audio_paths = parse_string_to_list(
             config.get("aux_ref_audio_paths")
         )
+        self.audio_file_type = config.get("format", "wav")
 
     async def text_to_speak(self, text, output_file):
         request_json = {
@@ -91,8 +92,11 @@ class TTSProvider(TTSProviderBase):
 
         resp = requests.post(self.url, json=request_json)
         if resp.status_code == 200:
-            with open(output_file, "wb") as file:
-                file.write(resp.content)
+            if output_file:
+                with open(output_file, "wb") as file:
+                    file.write(resp.content)
+            else:
+                return resp.content
         else:
             error_msg = f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}"
             logger.bind(tag=TAG).error(error_msg)

main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py

@@ -32,6 +32,7 @@ class TTSProvider(TTSProviderBase):
         self.cut_punc = config.get("cut_punc", "")
         self.inp_refs = parse_string_to_list(config.get("inp_refs"))
         self.if_sr = str(config.get("if_sr", False)).lower() in ("true", "1", "yes")
+        self.audio_file_type = config.get("format", "wav")
 
     async def text_to_speak(self, text, output_file):
         request_params = {
@@ -52,8 +53,11 @@ class TTSProvider(TTSProviderBase):
 
         resp = requests.get(self.url, params=request_params)
         if resp.status_code == 200:
-            with open(output_file, "wb") as file:
-                file.write(resp.content)
+            if output_file:
+                with open(output_file, "wb") as file:
+                    file.write(resp.content)
+            else:
+                return resp.content
         else:
             error_msg = f"GPT_SoVITS_V3 TTS请求失败: {resp.status_code} - {resp.text}"
             logger.bind(tag=TAG).error(error_msg)

main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py

@@ -5,6 +5,7 @@ import queue
 import asyncio
 import traceback
 import websockets
+from core.utils.tts import MarkdownCleaner
 from config.logger import setup_logging
 from core.utils import opus_encoder_utils
 from core.utils.util import check_model_key
@@ -145,16 +146,14 @@ class TTSProvider(TTSProviderBase):
         self.cluster = config.get("cluster")
         self.resource_id = config.get("resource_id")
         if config.get("private_voice"):
-            self.speaker = config.get("private_voice")
+            self.voice = config.get("private_voice")
         else:
-            self.speaker = config.get("speaker")
-        self.voice = config.get("voice")
+            self.voice = config.get("speaker")
         self.ws_url = config.get("ws_url")
         self.authorization = config.get("authorization")
         self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
         self.enable_two_way = True
         self.tts_text = ""
-        self.before_stop_play_files = []
         self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
             sample_rate=16000, channels=1, frame_size_ms=60
         )
@@ -190,10 +189,8 @@ class TTSProvider(TTSProviderBase):
 
     def tts_text_priority_thread(self):
         """火山引擎双流式TTS的文本处理线程"""
-        logger.bind(tag=TAG).info("TTS文本处理线程启动")
         while not self.conn.stop_event.is_set():
             try:
-                logger.bind(tag=TAG).debug("等待TTS文本队列消息...")
                 message = self.tts_text_queue.get(timeout=1)
                 logger.bind(tag=TAG).debug(
                     f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}"
@@ -270,8 +267,12 @@ class TTSProvider(TTSProviderBase):
                 await handleAbortMessage(self.conn)
                 logger.bind(tag=TAG).error(f"WebSocket连接不存在，终止发送文本")
                 return
+            
+            #  过滤Markdown
+            filtered_text = MarkdownCleaner.clean_markdown(text)
+
             # 发送文本
-            await self.send_text(self.speaker, text, self.conn.sentence_id)
+            await self.send_text(self.voice, filtered_text, self.conn.sentence_id)
             return
         except Exception as e:
             logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
@@ -301,9 +302,9 @@ class TTSProvider(TTSProviderBase):
                 event=EVENT_StartSession, sessionId=session_id
             ).as_bytes()
             payload = self.get_payload_bytes(
-                event=EVENT_StartSession, speaker=self.speaker
+                event=EVENT_StartSession, speaker=self.voice
             )
-            await self.send_event(header, optional, payload)
+            await self.send_event(self.ws, header, optional, payload)
             logger.bind(tag=TAG).info("会话启动请求已发送")
         except Exception as e:
             logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
@@ -336,7 +337,7 @@ class TTSProvider(TTSProviderBase):
                     event=EVENT_FinishSession, sessionId=session_id
                 ).as_bytes()
                 payload = str.encode("{}")
-                await self.send_event(header, optional, payload)
+                await self.send_event(self.ws, header, optional, payload)
                 logger.bind(tag=TAG).info("会话结束请求已发送")
 
                 # 等待监听任务完成
@@ -383,6 +384,7 @@ class TTSProvider(TTSProviderBase):
         """监听TTS响应"""
         opus_datas_cache = []
         is_first_sentence = True
+        first_sentence_segment_count = 0  # 添加计数器
         try:
             while not self.conn.stop_event.is_set():
                 try:
@@ -404,6 +406,7 @@ class TTSProvider(TTSProviderBase):
                             (SentenceType.FIRST, [], self.tts_text)
                         )
                         opus_datas_cache = []
+                        first_sentence_segment_count = 0  # 重置计数器
                     elif (
                         res.optional.event == EVENT_TTSResponse
                         and res.header.message_type == AUDIO_ONLY_RESPONSE
@@ -414,32 +417,28 @@ class TTSProvider(TTSProviderBase):
                             f"推送数据到队列里面帧数～～{len(opus_datas)}"
                         )
                         if is_first_sentence:
-                            # 第一句话直接发送
-                            self.tts_audio_queue.put(
-                                (SentenceType.MIDDLE, opus_datas, self.tts_text)
-                            )
+                            first_sentence_segment_count += 1
+                            if first_sentence_segment_count <= 6:
+                                self.tts_audio_queue.put(
+                                    (SentenceType.MIDDLE, opus_datas, None)
+                                )
+                            else:
+                                opus_datas_cache = opus_datas_cache + opus_datas
                         else:
                             # 后续句子缓存
                             opus_datas_cache = opus_datas_cache + opus_datas
                     elif res.optional.event == EVENT_TTSSentenceEnd:
                         logger.bind(tag=TAG).info(f"句子语音生成成功：{self.tts_text}")
-                        if not is_first_sentence:
-                            # 只有非第一句话才发送缓存的数据
+                        if not is_first_sentence or first_sentence_segment_count > 10:
+                            # 发送缓存的数据
                             self.tts_audio_queue.put(
-                                (SentenceType.MIDDLE, opus_datas_cache, self.tts_text)
+                                (SentenceType.MIDDLE, opus_datas_cache, None)
                             )
                         # 第一句话结束后，将标志设置为False
                         is_first_sentence = False
                     elif res.optional.event == EVENT_SessionFinished:
                         logger.bind(tag=TAG).debug(f"会话结束～～")
-                        for tts_file, text in self.before_stop_play_files:
-                            if tts_file and os.path.exists(tts_file):
-                                audio_datas = self._process_audio_file(tts_file)
-                                self.tts_audio_queue.put(
-                                    (SentenceType.MIDDLE, audio_datas, text)
-                                )
-                        self.before_stop_play_files.clear()
-                        self.tts_audio_queue.put((SentenceType.LAST, [], None))
+                        self._process_before_stop_play_files()
                         break
                 except websockets.ConnectionClosed:
                     logger.bind(tag=TAG).warning("WebSocket连接已关闭")
@@ -460,7 +459,11 @@ class TTSProvider(TTSProviderBase):
                 self.ws = None
 
     async def send_event(
-        self, header: bytes, optional: bytes | None = None, payload: bytes = None
+        self,
+        ws: websockets.WebSocketClientProtocol,
+        header: bytes,
+        optional: bytes | None = None,
+        payload: bytes = None,
     ):
         try:
             full_client_request = bytearray(header)
@@ -470,7 +473,7 @@ class TTSProvider(TTSProviderBase):
                 payload_size = len(payload).to_bytes(4, "big", signed=True)
                 full_client_request.extend(payload_size)
                 full_client_request.extend(payload)
-            await self.ws.send(full_client_request)
+            await ws.send(full_client_request)
         except websockets.ConnectionClosed:
             logger.bind(tag=TAG).error(f"ConnectionClosed")
             raise
@@ -485,7 +488,7 @@ class TTSProvider(TTSProviderBase):
         payload = self.get_payload_bytes(
             event=EVENT_TaskRequest, text=text, speaker=speaker
         )
-        return await self.send_event(header, optional, payload)
+        return await self.send_event(self.ws, header, optional, payload)
 
     # 读取 res 数组某段 字符串内容
     def read_res_content(self, res: bytes, offset: int):
@@ -563,7 +566,7 @@ class TTSProvider(TTSProviderBase):
         ).as_bytes()
         optional = Optional(event=EVENT_Start_Connection).as_bytes()
         payload = str.encode("{}")
-        return await self.send_event(header, optional, payload)
+        return await self.send_event(self.ws, header, optional, payload)
 
     def print_response(self, res, tag_msg: str):
         logger.bind(tag=TAG).debug(f"===>{tag_msg} header:{res.header.__dict__}")
@@ -599,3 +602,107 @@ class TTSProvider(TTSProviderBase):
     def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False):
         opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end)
         return opus_datas
+
+    def to_tts(self, text: str) -> list:
+        """非流式生成音频数据，用于生成音频及测试场景
+
+        Args:
+            text: 要转换的文本
+
+        Returns:
+            list: 音频数据列表
+        """
+        try:
+            # 创建事件循环
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+            # 生成会话ID
+            session_id = uuid.uuid4().__str__().replace("-", "")
+
+            # 存储音频数据
+            audio_data = []
+
+            async def _generate_audio():
+                # 创建新的WebSocket连接
+                ws_header = {
+                    "X-Api-App-Key": self.appId,
+                    "X-Api-Access-Key": self.access_token,
+                    "X-Api-Resource-Id": self.resource_id,
+                    "X-Api-Connect-Id": uuid.uuid4(),
+                }
+                ws = await websockets.connect(
+                    self.ws_url, additional_headers=ws_header, max_size=1000000000
+                )
+
+                try:
+                    # 启动会话
+                    header = Header(
+                        message_type=FULL_CLIENT_REQUEST,
+                        message_type_specific_flags=MsgTypeFlagWithEvent,
+                        serial_method=JSON,
+                    ).as_bytes()
+                    optional = Optional(
+                        event=EVENT_StartSession, sessionId=session_id
+                    ).as_bytes()
+                    payload = self.get_payload_bytes(
+                        event=EVENT_StartSession, speaker=self.voice
+                    )
+                    await self.send_event(ws, header, optional, payload)
+
+                    # 发送文本
+                    header = Header(
+                        message_type=FULL_CLIENT_REQUEST,
+                        message_type_specific_flags=MsgTypeFlagWithEvent,
+                        serial_method=JSON,
+                    ).as_bytes()
+                    optional = Optional(
+                        event=EVENT_TaskRequest, sessionId=session_id
+                    ).as_bytes()
+                    payload = self.get_payload_bytes(
+                        event=EVENT_TaskRequest, text=text, speaker=self.voice
+                    )
+                    await self.send_event(ws, header, optional, payload)
+
+                    # 发送结束会话请求
+                    header = Header(
+                        message_type=FULL_CLIENT_REQUEST,
+                        message_type_specific_flags=MsgTypeFlagWithEvent,
+                        serial_method=JSON,
+                    ).as_bytes()
+                    optional = Optional(
+                        event=EVENT_FinishSession, sessionId=session_id
+                    ).as_bytes()
+                    payload = str.encode("{}")
+                    await self.send_event(ws, header, optional, payload)
+
+                    # 接收音频数据
+                    while True:
+                        msg = await ws.recv()
+                        res = self.parser_response(msg)
+
+                        if (
+                            res.optional.event == EVENT_TTSResponse
+                            and res.header.message_type == AUDIO_ONLY_RESPONSE
+                        ):
+                            opus_datas = self.wav_to_opus_data_audio_raw(res.payload)
+                            audio_data.extend(opus_datas)
+                        elif res.optional.event == EVENT_SessionFinished:
+                            break
+
+                finally:
+                    # 清理资源
+                    try:
+                        await ws.close()
+                    except:
+                        pass
+
+            # 运行异步任务
+            loop.run_until_complete(_generate_audio())
+            loop.close()
+
+            return audio_data
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}")
+            return []

main/xiaozhi-server/core/providers/tts/linkerai.py

@@ -0,0 +1,303 @@
+import queue
+import asyncio
+import traceback
+import aiohttp
+import requests
+import time
+from config.logger import setup_logging
+from core.utils.tts import MarkdownCleaner
+from core.providers.tts.base import TTSProviderBase
+from core.utils import opus_encoder_utils, textUtils
+from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+
+class TTSProvider(TTSProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__(config, delete_audio_file)
+        self.interface_type = InterfaceType.SINGLE_STREAM
+        self.access_token = config.get("access_token")
+        self.voice = config.get("voice")
+        self.api_url = config.get("api_url")
+        self.audio_format = "pcm"
+        self.before_stop_play_files = []
+        self.segment_count = 0  # 添加片段计数器
+
+        # 创建Opus编码器
+        self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
+            sample_rate=16000, channels=1, frame_size_ms=60
+        )
+
+        # 添加文本缓冲区
+        self.text_buffer = ""
+
+        # PCM缓冲区
+        self.pcm_buffer = bytearray()
+
+    ###################################################################################
+    # linkerai单流式TTS重写父类的方法--开始
+    ###################################################################################
+
+    def tts_text_priority_thread(self):
+        """流式文本处理线程"""
+        while not self.conn.stop_event.is_set():
+            try:
+                message = self.tts_text_queue.get(timeout=1)
+                if message.sentence_type == SentenceType.FIRST:
+                    # 初始化参数
+                    self.tts_stop_request = False
+                    self.processed_chars = 0
+                    self.tts_text_buff = []
+                    self.segment_count = 0
+                    self.tts_audio_first_sentence = True
+                    self.before_stop_play_files.clear()
+                elif ContentType.TEXT == message.content_type:
+                    self.tts_text_buff.append(message.content_detail)
+                    segment_text = self._get_segment_text()
+                    if segment_text:
+                        self.to_tts_single_stream(segment_text)
+
+                elif ContentType.FILE == message.content_type:
+                    logger.bind(tag=TAG).info(
+                        f"添加音频文件到待播放列表: {message.content_file}"
+                    )
+                    self.before_stop_play_files.append(
+                        (message.content_file, message.content_detail)
+                    )
+
+                if message.sentence_type == SentenceType.LAST:
+                    # 处理剩余的文本
+                    self._process_remaining_text(True)
+
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.bind(tag=TAG).error(
+                    f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
+                )
+
+    def _process_remaining_text(self, is_last=False):
+        """处理剩余的文本并生成语音
+
+        Returns:
+            bool: 是否成功处理了文本
+        """
+        full_text = "".join(self.tts_text_buff)
+        remaining_text = full_text[self.processed_chars :]
+        if remaining_text:
+            segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text)
+            if segment_text:
+                self.to_tts_single_stream(segment_text, is_last)
+                self.processed_chars += len(full_text)
+            else:
+                self._process_before_stop_play_files()
+        else:
+            self._process_before_stop_play_files()
+
+    def to_tts_single_stream(self, text, is_last=False):
+        try:
+            max_repeat_time = 5
+            text = MarkdownCleaner.clean_markdown(text)
+            try:
+                asyncio.run(self.text_to_speak(text, is_last))
+            except Exception as e:
+                logger.bind(tag=TAG).warning(
+                    f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}"
+                )
+                max_repeat_time -= 1
+
+            if max_repeat_time > 0:
+                logger.bind(tag=TAG).info(
+                    f"语音生成成功: {text}，重试{5 - max_repeat_time}次"
+                )
+            else:
+                logger.bind(tag=TAG).error(
+                    f"语音生成失败: {text}，请检查网络或服务是否正常"
+                )
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
+        finally:
+            return None
+
+    ###################################################################################
+    # linkerai单流式TTS重写父类的方法--结束
+    ###################################################################################
+
+    async def text_to_speak(self, text, is_last):
+        """流式处理TTS音频，每句只推送一次音频列表"""
+        await self._tts_request(text, is_last)
+
+    async def close(self):
+        """资源清理"""
+        await super().close()
+        if hasattr(self, "opus_encoder"):
+            self.opus_encoder.close()
+
+    async def _tts_request(self, text: str, is_last: bool) -> None:
+        params = {
+            "tts_text": text,
+            "spk_id": self.voice,
+            "frame_durition": 60,
+            "stream": "true",
+            "target_sr": 16000,
+            "audio_format": "pcm",
+            "instruct_text": "请生成一段自然流畅的语音",
+        }
+        headers = {
+            "Authorization": f"Bearer {self.access_token}",
+            "Content-Type": "application/json",
+        }
+
+        # 一帧 PCM 所需字节数：60 ms × 16 kHz × 1 ch × 2 B = 1 920
+        frame_bytes = int(
+            self.opus_encoder.sample_rate
+            * self.opus_encoder.channels  # 1
+            * self.opus_encoder.frame_size_ms
+            / 1000
+            * 2
+        )  # 16-bit = 2 bytes
+
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(
+                    self.api_url, params=params, headers=headers, timeout=10
+                ) as resp:
+
+                    if resp.status != 200:
+                        logger.error(f"TTS请求失败: {resp.status}, {await resp.text()}")
+                        self.tts_audio_queue.put((SentenceType.LAST, [], None))
+                        return
+
+                    self.pcm_buffer.clear()
+                    opus_datas_cache = []
+
+                    self.tts_audio_queue.put((SentenceType.FIRST, [], text))
+
+                    # 兼容 iter_chunked / iter_chunks / iter_any
+                    async for chunk in resp.content.iter_any():
+                        data = chunk[0] if isinstance(chunk, (list, tuple)) else chunk
+                        if not data:
+                            continue
+
+                        # 拼到 buffer
+                        self.pcm_buffer.extend(data)
+
+                        # 够一帧就编码
+                        while len(self.pcm_buffer) >= frame_bytes:
+                            frame = bytes(self.pcm_buffer[:frame_bytes])
+                            del self.pcm_buffer[:frame_bytes]
+
+                            opus = self.opus_encoder.encode_pcm_to_opus(
+                                frame, end_of_stream=False
+                            )
+                            if opus:
+                                if self.segment_count < 10:  # 前10个片段直接发送
+                                    self.tts_audio_queue.put(
+                                        (SentenceType.MIDDLE, opus, None)
+                                    )
+                                    self.segment_count += 1
+                                else:
+                                    opus_datas_cache.extend(opus)
+
+                    # flush 剩余不足一帧的数据
+                    if self.pcm_buffer:
+                        opus = self.opus_encoder.encode_pcm_to_opus(
+                            bytes(self.pcm_buffer), end_of_stream=True
+                        )
+                        if opus:
+                            if self.segment_count < 10:  # 前10个片段直接发送
+                                # 直接发送
+                                self.tts_audio_queue.put(
+                                    (SentenceType.MIDDLE, opus, None)
+                                )
+                                self.segment_count += 1
+                            else:
+                                # 后续片段缓存
+                                opus_datas_cache.extend(opus)
+                        self.pcm_buffer.clear()
+
+                    # 如果不是前10个片段，发送缓存的数据
+                    if self.segment_count >= 10 and opus_datas_cache:
+                        self.tts_audio_queue.put(
+                            (SentenceType.MIDDLE, opus_datas_cache, None)
+                        )
+
+                    # 如果是最后一段，输出音频获取完毕
+                    if is_last:
+                        self._process_before_stop_play_files()
+
+        except Exception as e:
+            logger.error(f"TTS请求异常: {e}")
+            self.tts_audio_queue.put((SentenceType.LAST, [], None))
+
+    def to_tts(self, text: str) -> list:
+        """非流式TTS处理，用于测试及保存音频文件的场景
+
+        Args:
+            text: 要转换的文本
+
+        Returns:
+            list: 返回opus编码后的音频数据列表
+        """
+        start_time = time.time()
+        text = MarkdownCleaner.clean_markdown(text)
+
+        params = {
+            "tts_text": text,
+            "spk_id": self.voice,
+            "frame_duration": 60,
+            "stream": False,
+            "target_sr": 16000,
+            "audio_format": self.audio_format,
+            "instruct_text": "请生成一段自然流畅的语音",
+        }
+        headers = {
+            "Authorization": f"Bearer {self.access_token}",
+            "Content-Type": "application/json",
+        }
+
+        try:
+            with requests.get(
+                self.api_url, params=params, headers=headers, timeout=5
+            ) as response:
+                if response.status_code != 200:
+                    logger.error(
+                        f"TTS请求失败: {response.status_code}, {response.text}"
+                    )
+                    return []
+
+                logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒")
+
+                # 使用opus编码器处理PCM数据
+                opus_datas = []
+                pcm_data = response.content
+
+                # 计算每帧的字节数
+                frame_bytes = int(
+                    self.opus_encoder.sample_rate
+                    * self.opus_encoder.channels
+                    * self.opus_encoder.frame_size_ms
+                    / 1000
+                    * 2
+                )
+
+                # 分帧处理PCM数据
+                for i in range(0, len(pcm_data), frame_bytes):
+                    frame = pcm_data[i : i + frame_bytes]
+                    if len(frame) < frame_bytes:
+                        # 最后一帧可能不足，用0填充
+                        frame = frame + b"\x00" * (frame_bytes - len(frame))
+
+                    opus = self.opus_encoder.encode_pcm_to_opus(
+                        frame, end_of_stream=(i + frame_bytes >= len(pcm_data))
+                    )
+                    if opus:
+                        opus_datas.extend(opus)
+
+                return opus_datas
+
+        except Exception as e:
+            logger.error(f"TTS请求异常: {e}")
+            return []

main/xiaozhi-server/core/providers/tts/minimax.py

@@ -14,9 +14,9 @@ class TTSProvider(TTSProviderBase):
         self.api_key = config.get("api_key")
         self.model = config.get("model")
         if config.get("private_voice"):
-            self.voice_id = config.get("private_voice")
+            self.voice = config.get("private_voice")
         else:
-            self.voice_id = config.get("voice_id")
+            self.voice = config.get("voice_id")
 
         default_voice_setting = {
             "voice_id": "female-shaonv",
@@ -43,8 +43,8 @@ class TTSProvider(TTSProviderBase):
         self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
         self.timber_weights = parse_string_to_list(config.get("timber_weights"))
 
-        if self.voice_id:
-            self.voice_setting["voice_id"] = self.voice_id
+        if self.voice:
+            self.voice_setting["voice_id"] = self.voice
 
         self.host = "api.minimax.chat"
         self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
@@ -52,6 +52,7 @@ class TTSProvider(TTSProviderBase):
             "Content-Type": "application/json",
             "Authorization": f"Bearer {self.api_key}",
         }
+        self.audio_file_type = defult_audio_setting.get("format", "mp3")
 
     def generate_filename(self, extension=".mp3"):
         return os.path.join(
@@ -80,8 +81,12 @@ class TTSProvider(TTSProviderBase):
             # 检查返回请求数据的status_code是否为0
             if resp.json()["base_resp"]["status_code"] == 0:
                 data = resp.json()["data"]["audio"]
-                file_to_save = open(output_file, "wb")
-                file_to_save.write(bytes.fromhex(data))
+                audio_bytes = bytes.fromhex(data)
+                if output_file:
+                    with open(output_file, "wb") as file_to_save:
+                        file_to_save.write(audio_bytes)
+                else:
+                    return audio_bytes
             else:
                 raise Exception(
                     f"{__name__} status_code: {resp.status_code} response: {resp.content}"

main/xiaozhi-server/core/providers/tts/openai.py

@@ -17,7 +17,8 @@ class TTSProvider(TTSProviderBase):
             self.voice = config.get("private_voice")
         else:
             self.voice = config.get("voice", "alloy")
-        self.response_format = "wav"
+        self.response_format = config.get("format", "wav")
+        self.audio_file_type = config.get("format", "wav")
 
         # 处理空字符串的情况
         speed = config.get("speed", "1.0")
@@ -40,8 +41,11 @@ class TTSProvider(TTSProviderBase):
         }
         response = requests.post(self.api_url, json=data, headers=headers)
         if response.status_code == 200:
-            with open(output_file, "wb") as audio_file:
-                audio_file.write(response.content)
+            if output_file:
+                with open(output_file, "wb") as audio_file:
+                    audio_file.write(response.content)
+            else:
+                return response.content
         else:
             raise Exception(
                 f"OpenAI TTS请求失败: {response.status_code} - {response.text}"

main/xiaozhi-server/core/providers/tts/siliconflow.py

@@ -11,7 +11,8 @@ class TTSProvider(TTSProviderBase):
             self.voice = config.get("private_voice")
         else:
             self.voice = config.get("voice")
-        self.response_format = config.get("response_format")
+        self.response_format = config.get("response_format", "mp3")
+        self.audio_file_type = config.get("response_format", "mp3")
         self.sample_rate = config.get("sample_rate")
         self.speed = float(config.get("speed", 1.0))
         self.gain = config.get("gain")
@@ -35,7 +36,10 @@ class TTSProvider(TTSProviderBase):
                 "POST", self.api_url, json=request_json, headers=headers
             )
             data = response.content
-            file_to_save = open(output_file, "wb")
-            file_to_save.write(data)
+            if output_file:
+                with open(output_file, "wb") as file_to_save:
+                    file_to_save.write(data)
+            else:
+                return data
         except Exception as e:
             raise Exception(f"{__name__} error: {e}")

main/xiaozhi-server/core/providers/tts/tencent.py

@@ -22,6 +22,7 @@ class TTSProvider(TTSProviderBase):
         self.api_url = "https://tts.tencentcloudapi.com"  # 正确的API端点
         self.region = config.get("region")
         self.output_file = config.get("output_dir")
+        self.audio_file_type = config.get("format", "wav")
 
     def _get_auth_headers(self, request_body):
         """生成鉴权请求头"""
@@ -148,12 +149,14 @@ class TTSProvider(TTSProviderBase):
                         f"API返回错误: {error_info['Code']}: {error_info['Message']}"
                     )
 
-                # 提取音频数据
-                audio_data = response_data["Response"].get("Audio")
-                if audio_data:
-                    # 解码Base64音频数据并保存
-                    with open(output_file, "wb") as f:
-                        f.write(base64.b64decode(audio_data))
+                # 解码Base64音频数据
+                audio_bytes = base64.b64decode(response_data["Response"].get("Audio"))
+                if audio_bytes:
+                    if output_file:
+                        with open(output_file, "wb") as f:
+                            f.write(audio_bytes)
+                    else:
+                        return audio_bytes
                 else:
                     raise Exception(f"{__name__}: 没有返回音频数据: {response_data}")
             else:

main/xiaozhi-server/core/providers/tts/ttson.py

@@ -19,9 +19,9 @@ class TTSProvider(TTSProviderBase):
             "https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=",
         )
         if config.get("private_voice"):
-            self.voice_id = int(config.get("private_voice"))
+            self.voice = int(config.get("private_voice"))
         else:
-            self.voice_id = int(config.get("voice_id", 1695))
+            self.voice = int(config.get("voice_id", 1695))
         self.token = config.get("token")
         self.to_lang = config.get("to_lang")
         self.volume_change_dB = int(config.get("volume_change_dB", 0))
@@ -30,6 +30,7 @@ class TTSProvider(TTSProviderBase):
         self.output_file = config.get("output_dir")
         self.pitch_factor = int(config.get("pitch_factor", 0))
         self.format = config.get("format", "mp3")
+        self.audio_file_type = config.get("format", "mp3")
         self.emotion = int(config.get("emotion", 1))
         self.header = {"Content-Type": "application/json"}
 
@@ -49,7 +50,7 @@ class TTSProvider(TTSProviderBase):
                 "emotion": self.emotion,
                 "format": self.format,
                 "volume_change_dB": self.volume_change_dB,
-                "voice_id": self.voice_id,
+                "voice_id": self.voice,
                 "pitch_factor": self.pitch_factor,
                 "speed_factor": self.speed_factor,
                 "token": self.token,
@@ -73,9 +74,11 @@ class TTSProvider(TTSProviderBase):
             )
 
             audio_content = requests.get(result)
-            with open(output_file, "wb") as f:
-                f.write(audio_content.content)
-                return True
+            if output_file:
+                with open(output_file, "wb") as f:
+                    f.write(audio_content.content)
+            else:
+                return audio_content.content
             voice_path = resp_json.get("voice_path")
             des_path = output_file
             shutil.move(voice_path, des_path)

main/xiaozhi-server/core/utils/p3.py

@@ -29,5 +29,31 @@ def decode_opus_from_file(input_file):
             total_frames += 1
 
     # 计算总时长
+    total_duration = (total_frames * frame_duration_ms) / 1000.0
+    return opus_datas, total_duration
+
+def decode_opus_from_bytes(input_bytes):
+    """
+    从p3二进制数据中解码 Opus 数据，并返回一个 Opus 数据包的列表以及总时长。
+    """
+    import io
+    opus_datas = []
+    total_frames = 0
+    sample_rate = 16000  # 文件采样率
+    frame_duration_ms = 60  # 帧时长
+    frame_size = int(sample_rate * frame_duration_ms / 1000)
+
+    f = io.BytesIO(input_bytes)
+    while True:
+        header = f.read(4)
+        if not header:
+            break
+        _, _, data_len = struct.unpack('>BBH', header)
+        opus_data = f.read(data_len)
+        if len(opus_data) != data_len:
+            raise ValueError(f"Data length({len(opus_data)}) mismatch({data_len}) in the bytes.")
+        opus_datas.append(opus_data)
+        total_frames += 1
+
     total_duration = (total_frames * frame_duration_ms) / 1000.0
     return opus_datas, total_duration

main/xiaozhi-server/core/utils/util.py

@@ -3,6 +3,9 @@ import socket
 import subprocess
 import re
 import os
+import wave
+from io import BytesIO
+from core.utils import p3
 import numpy as np
 import requests
 import opuslib_next
@@ -773,6 +776,22 @@ def audio_to_data(audio_file_path, is_opus=True):
     return pcm_to_data(raw_data, is_opus), duration
 
 
+def audio_bytes_to_data(audio_bytes, file_type, is_opus=True):
+    """
+    直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3
+    """
+    if file_type == "p3":
+        # 直接用p3解码
+        return p3.decode_opus_from_bytes(audio_bytes)
+    else:
+        # 其他格式用pydub
+        audio = AudioSegment.from_file(BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"])
+        audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
+        duration = len(audio) / 1000.0
+        raw_data = audio.raw_data
+        return pcm_to_data(raw_data, is_opus), duration
+
+
 def pcm_to_data(raw_data, is_opus=True):
     # 初始化Opus编码器
     encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
@@ -804,6 +823,33 @@ def pcm_to_data(raw_data, is_opus=True):
     return datas
 
 
+def opus_datas_to_wav_bytes(opus_datas, sample_rate=16000, channels=1):
+    """
+    将opus帧列表解码为wav字节流
+    """
+    decoder = opuslib_next.Decoder(sample_rate, channels)
+    pcm_datas = []
+
+    frame_duration = 60  # ms
+    frame_size = int(sample_rate * frame_duration / 1000)  # 960
+
+    for opus_frame in opus_datas:
+        # 解码为PCM（返回bytes，2字节/采样点）
+        pcm = decoder.decode(opus_frame, frame_size)
+        pcm_datas.append(pcm)
+
+    pcm_bytes = b''.join(pcm_datas)
+
+    # 写入wav字节流
+    wav_buffer = BytesIO()
+    with wave.open(wav_buffer, 'wb') as wf:
+        wf.setnchannels(channels)
+        wf.setsampwidth(2)  # 16bit
+        wf.setframerate(sample_rate)
+        wf.writeframes(pcm_bytes)
+    return wav_buffer.getvalue()
+
+
 def check_vad_update(before_config, new_config):
     if (
         new_config.get("selected_module") is None
@@ -930,3 +976,8 @@ def is_valid_image_file(file_data: bytes) -> bool:
             return True
 
     return False
+
+
+def sanitize_tool_name(name: str) -> str:
+    """Sanitize tool names for OpenAI compatibility."""
+    return re.sub(r"[^a-zA-Z0-9_-]", "_", name)

main/xiaozhi-server/core/utils/wakeup_word.py

@@ -0,0 +1,142 @@
+import os
+import yaml
+import time
+import hashlib
+import portalocker
+from typing import Dict
+
+
+class FileLock:
+    def __init__(self, file, timeout=5):
+        self.file = file
+        self.timeout = timeout
+        self.start_time = None
+
+    def __enter__(self):
+        self.start_time = time.time()
+        while True:
+            try:
+                portalocker.lock(self.file, portalocker.LOCK_EX | portalocker.LOCK_NB)
+                return self.file
+            except portalocker.LockException:
+                if time.time() - self.start_time > self.timeout:
+                    raise TimeoutError("获取文件锁超时")
+                time.sleep(0.1)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        portalocker.unlock(self.file)
+
+
+class WakeupWordsConfig:
+    def __init__(self):
+        self.config_file = "data/.wakeup_words.yaml"
+        self.assets_dir = "config/assets/wakeup_words"
+        self._ensure_directories()
+        self._config_cache = None
+        self._last_load_time = 0
+        self._cache_ttl = 1  # 缓存有效期（秒）
+        self._lock_timeout = 5  # 文件锁超时时间（秒）
+
+    def _ensure_directories(self):
+        """确保必要的目录存在"""
+        os.makedirs(os.path.dirname(self.config_file), exist_ok=True)
+        os.makedirs(self.assets_dir, exist_ok=True)
+
+    def _load_config(self) -> Dict:
+        """加载配置文件，使用缓存机制"""
+        current_time = time.time()
+
+        # 如果缓存有效，直接返回缓存
+        if (
+            self._config_cache is not None
+            and current_time - self._last_load_time < self._cache_ttl
+        ):
+            return self._config_cache
+
+        try:
+            with open(self.config_file, "a+") as f:
+                with FileLock(f, timeout=self._lock_timeout):
+                    f.seek(0)
+                    content = f.read()
+                    config = yaml.safe_load(content) if content else {}
+                    self._config_cache = config
+                    self._last_load_time = current_time
+                    return config
+        except (TimeoutError, IOError) as e:
+            print(f"加载配置文件失败: {e}")
+            return {}
+        except Exception as e:
+            print(f"加载配置文件时发生未知错误: {e}")
+            return {}
+
+    def _save_config(self, config: Dict):
+        """保存配置到文件，使用文件锁保护"""
+        try:
+            with open(self.config_file, "w") as f:
+                with FileLock(f, timeout=self._lock_timeout):
+                    yaml.dump(config, f, allow_unicode=True)
+                    self._config_cache = config
+                    self._last_load_time = time.time()
+        except (TimeoutError, IOError) as e:
+            print(f"保存配置文件失败: {e}")
+            raise
+        except Exception as e:
+            print(f"保存配置文件时发生未知错误: {e}")
+            raise
+
+    def get_wakeup_response(self, voice: str) -> Dict:
+        voice = hashlib.md5(voice.encode()).hexdigest()
+        """获取唤醒词回复配置"""
+        config = self._load_config()
+        default_response = {
+            "voice": "default",
+            "file_path": "config/assets/wakeup_words.wav",
+            "time": 0,
+            "text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",
+        }
+
+        if not config or voice not in config:
+            return default_response
+
+        # 检查文件大小
+        file_path = config[voice]["file_path"]
+        if not os.path.exists(file_path) or os.stat(file_path).st_size < (15 * 1024):
+            return default_response
+
+        return config[voice]
+
+    def update_wakeup_response(self, voice: str, file_path: str, text: str):
+        """更新唤醒词回复配置"""
+        try:
+            config = self._load_config()
+            voice_hash = hashlib.md5(voice.encode()).hexdigest()
+            config[voice_hash] = {
+                "voice": voice,
+                "file_path": file_path,
+                "time": time.time(),
+                "text": text,
+            }
+            self._save_config(config)
+        except Exception as e:
+            print(f"更新唤醒词回复配置失败: {e}")
+            raise
+
+    def generate_file_path(self, voice: str) -> str:
+        """生成音频文件路径，使用voice的哈希值作为文件名"""
+        try:
+            # 生成voice的哈希值
+            voice_hash = hashlib.md5(voice.encode()).hexdigest()
+            file_path = os.path.join(self.assets_dir, f"{voice_hash}.wav")
+
+            # 如果文件已存在，先删除
+            if os.path.exists(file_path):
+                try:
+                    os.remove(file_path)
+                except Exception as e:
+                    print(f"删除已存在的音频文件失败: {e}")
+                    raise
+
+            return file_path
+        except Exception as e:
+            print(f"生成音频文件路径失败: {e}")
+            raise