Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 07:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1496 from xinnan-tech/py_Text

Browse Source 
优化唤醒词答复速度
This commit is contained in:
欣南科技
2025-06-07 00:12:21 +08:00
committed by GitHub
parent 6023af2f21 98174bcc16
commit 837bb74576
22 changed files with 690 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/config/logger.py
+32 -8
View File
@@ -3,6 +3,7 @@ import sys
from loguru import logger
from config.config_loader import load_config
from config.settings import check_config_file
from datetime import datetime
SERVER_VERSION = "0.5.4"
_logger_initialized = False
@@ -59,7 +60,7 @@ def setup_logging():
)
log_format_file = log_config.get(
"log_format_file",
"{time:YYYY-MM-DD HH:mm:ss} - {version_{extra[selected_module]}} - {name} - {level} - {extra[tag]} - {message}",
"{time:YYYY-MM-DD HH:mm:ss} - {version}_{extra[selected_module]} - {name} - {level} - {extra[tag]} - {message}",
)
selected_module_str = logger._core.extra["selected_module"]
@@ -84,12 +85,23 @@ def setup_logging():
# 输出到控制台
logger.add(sys.stdout, format=log_format, level=log_level, filter=formatter)
# 输出到文件
# 输出到文件 - 统一目录，按大小轮转
# 日志文件完整路径
log_file_path = os.path.join(log_dir, log_file)
# 添加日志处理器
logger.add(
os.path.join(log_dir, log_file),
log_file_path,
format=log_format_file,
level=log_level,
filter=formatter,
rotation="10 MB", # 每个文件最大10MB
retention="30 days", # 保留30天
compression=None,
encoding="utf-8",
enqueue=True, # 异步安全
backtrace=True,
diagnose=True,
)
_logger_initialized = True # 标记为已初始化
@@ -116,7 +128,7 @@ def update_module_string(selected_module_str):
)
log_format_file = log_config.get(
"log_format_file",
"{time:YYYY-MM-DD HH:mm:ss} - {version_{extra[selected_module]}} - {name} - {level} - {extra[tag]} - {message}",
"{time:YYYY-MM-DD HH:mm:ss} - {version}_{extra[selected_module]} - {name} - {level} - {extra[tag]} - {message}",
)
log_format = log_format.replace("{version}", SERVER_VERSION)
@@ -133,14 +145,26 @@ def update_module_string(selected_module_str):
level=log_config.get("log_level", "INFO"),
filter=formatter,
)
# 更新文件日志配置 - 统一目录，按大小轮转
log_dir = log_config.get("log_dir", "tmp")
log_file = log_config.get("log_file", "server.log")
# 日志文件完整路径
log_file_path = os.path.join(log_dir, log_file)
logger.add(
os.path.join(
log_config.get("log_dir", "tmp"),
log_config.get("log_file", "server.log"),
),
log_file_path,
format=log_format_file,
level=log_config.get("log_level", "INFO"),
filter=formatter,
rotation="10 MB", # 每个文件最大10MB
retention="30 days", # 保留30天
compression=None,
encoding="utf-8",
enqueue=True, # 异步安全
backtrace=True,
diagnose=True,
)
except Exception as e:
main/xiaozhi-server/core/handle/helloHandle.py
+70 -81
View File
@@ -1,30 +1,31 @@
import os
import time
import json
import random
import shutil
import asyncio
from core.utils.util import audio_to_data
from core.handle.sendAudioHandle import send_stt_message
from core.utils.util import remove_punctuation_and_length
from core.providers.tts.dto.dto import ContentType, InterfaceType
from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes
from core.providers.tts.dto.dto import ContentType, SentenceType
from core.handle.mcpHandle import (
MCPClient,
send_mcp_initialize_message,
send_mcp_tools_list_request,
)
from core.utils.wakeup_word import WakeupWordsConfig
TAG = __name__
WAKEUP_CONFIG = {
"dir": "config/assets/",
"file_name": "wakeup_words",
"create_time": time.time(),
"refresh_time": 10,
"refresh_time": 5,
"words": ["你好小智", "你好啊小智", "小智你好", "小智"],
"text": "",
}
# 创建全局的唤醒词配置管理器
wakeup_words_config = WakeupWordsConfig()
# 用于防止并发调用wakeupWordsResponse的锁
_wakeup_response_lock = asyncio.Lock()
async def handleHelloMessage(conn, msg_json):
"""处理hello消息"""
@@ -53,85 +54,73 @@ async def checkWakeupWords(conn, text):
enable_wakeup_words_response_cache = conn.config[
"enable_wakeup_words_response_cache"
]
"""是否用的是非流式tts"""
if conn.tts and conn.tts.interface_type != InterfaceType.NON_STREAM:
if not enable_wakeup_words_response_cache or not conn.tts:
return False
"""是否开启唤醒词加速"""
if not enable_wakeup_words_response_cache:
return False
"""检查是否是唤醒词"""
_, filtered_text = remove_punctuation_and_length(text)
if filtered_text in conn.config.get("wakeup_words"):
# 设置刚刚被唤醒的标志
conn.just_woken_up = True
await send_stt_message(conn, text)
if filtered_text not in conn.config.get("wakeup_words"):
return False
file = getWakeupWordFile(WAKEUP_CONFIG["file_name"])
if file is None:
conn.just_woken_up = True
await send_stt_message(conn, text)
# 获取当前音色
voice = getattr(conn.tts, "voice", "default")
# 获取唤醒词回复配置
response = wakeup_words_config.get_wakeup_response(voice)
# 播放唤醒词回复
conn.client_abort = False
opus_packets, _ = audio_to_data(response["file_path"])
conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, response["text"]))
conn.tts.tts_audio_queue.put((SentenceType.LAST, [], None))
# 检查是否需要更新唤醒词回复
if time.time() - response["time"] > WAKEUP_CONFIG["refresh_time"]:
if not _wakeup_response_lock.locked():
asyncio.create_task(wakeupWordsResponse(conn))
return False
text_hello = WAKEUP_CONFIG["text"]
if not text_hello:
text_hello = text
if conn.tts is None:
return False
conn.tts.tts_one_sentence(
conn, ContentType.FILE, content_file=file, content_detail=text_hello
)
if time.time() - WAKEUP_CONFIG["create_time"] > WAKEUP_CONFIG["refresh_time"]:
asyncio.create_task(wakeupWordsResponse(conn))
return True
return False
def getWakeupWordFile(file_name):
for file in os.listdir(WAKEUP_CONFIG["dir"]):
if file.startswith("my_" + file_name):
"""避免缓存文件是一个空文件"""
if os.stat(f"config/assets/{file}").st_size > (15 * 1024):
return f"config/assets/{file}"
"""查找config/assets/目录下名称为wakeup_words的文件"""
for file in os.listdir(WAKEUP_CONFIG["dir"]):
if file.startswith(file_name):
return f"config/assets/{file}"
return None
return True
async def wakeupWordsResponse(conn):
wait_max_time = 5
while conn.llm is None or not conn.llm.response_no_stream:
await asyncio.sleep(1)
wait_max_time -= 1
if wait_max_time <= 0:
conn.logger.bind(tag=TAG).error("连接对象没有llm")
if not conn.tts or not conn.llm or not conn.llm.response_no_stream:
return
try:
# 尝试获取锁，如果获取不到就返回
if not await _wakeup_response_lock.acquire():
return
"""唤醒词响应"""
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
question = (
"此刻用户正在和你说```"
+ wakeup_word
+ "```。\n请你根据以上用户的内容，进行简短回复，文字内容控制在15个字以内。\n"
+ "请勿对这条内容本身进行任何解释和回应，仅返回对用户的内容的回复。"
)
result = conn.llm.response_no_stream(conn.config["prompt"], question)
if result is None or result == "":
return
tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
if tts_file is not None and os.path.exists(tts_file):
file_type = os.path.splitext(tts_file)[1]
if file_type:
file_type = file_type.lstrip(".")
old_file = getWakeupWordFile("my_" + WAKEUP_CONFIG["file_name"])
if old_file is not None:
os.remove(old_file)
"""将文件挪到"wakeup_words.mp3"""
shutil.move(
tts_file,
WAKEUP_CONFIG["dir"] + "my_" + WAKEUP_CONFIG["file_name"] + "." + file_type,
# 生成唤醒词回复
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
question = (
"此刻用户正在和你说```"
+ wakeup_word
+ "```。\n请你根据以上用户的内容进行简短回复。要像一个人正常人一样说话，不要像机器人一样说话。\n"
+ "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。"
)
WAKEUP_CONFIG["create_time"] = time.time()
WAKEUP_CONFIG["text"] = result
result = conn.llm.response_no_stream(conn.config["prompt"], question)
if not result or len(result) == 0:
return
# 生成TTS音频
tts_result = await asyncio.to_thread(conn.tts.to_tts, result)
if not tts_result:
return
# 获取当前音色
voice = getattr(conn.tts, "voice", "default")
wav_bytes = opus_datas_to_wav_bytes(tts_result, sample_rate=16000)
file_path = wakeup_words_config.generate_file_path(voice)
with open(file_path, "wb") as f:
f.write(wav_bytes)
# 更新配置
wakeup_words_config.update_wakeup_response(voice, file_path, result)
finally:
# 确保在任何情况下都释放锁
if _wakeup_response_lock.locked():
_wakeup_response_lock.release()
main/xiaozhi-server/core/providers/asr/fun_local.py
+8
View File
@@ -2,6 +2,7 @@ import time
import os
import sys
import io
import psutil
from config.logger import setup_logging
from typing import Optional, Tuple, List
from core.providers.asr.base import ASRProviderBase
@@ -37,6 +38,13 @@ class CaptureOutput:
class ASRProvider(ASRProviderBase):
def __init__(self, config: dict, delete_audio_file: bool):
super().__init__()
# 内存检测，要求大于2G
min_mem_bytes = 2 * 1024 * 1024 * 1024
total_mem = psutil.virtual_memory().total
if total_mem < min_mem_bytes:
logger.bind(tag=TAG).error(f"可用内存不足2G，当前仅有 {total_mem / (1024*1024):.2f} MB，可能无法启动FunASR")
self.interface_type = InterfaceType.LOCAL
self.model_dir = config.get("model_dir")
self.output_dir = config.get("output_dir") # 修正配置键名
main/xiaozhi-server/core/providers/tts/aliyun.py
+7 -4
View File
@@ -91,7 +91,7 @@ class TTSProvider(TTSProviderBase):
self.appkey = config.get("appkey")
self.format = config.get("format", "wav")
self.audio_file_type = config.get("format", "wav")
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
@@ -188,9 +188,12 @@ class TTSProvider(TTSProviderBase):
)
# 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的
if resp.headers["Content-Type"].startswith("audio/"):
with open(output_file, "wb") as f:
f.write(resp.content)
return output_file
if output_file:
with open(output_file, "wb") as f:
f.write(resp.content)
return output_file
else:
return resp.content
else:
raise Exception(
f"{__name__} status_code: {resp.status_code} response: {resp.content}"
main/xiaozhi-server/core/providers/tts/base.py
+66 -26
View File
@@ -8,7 +8,7 @@ from datetime import datetime
from core.utils import textUtils
from abc import ABC, abstractmethod
from config.logger import setup_logging
from core.utils.util import audio_to_data
from core.utils.util import audio_to_data, audio_bytes_to_data
from core.utils.tts import MarkdownCleaner
from core.utils.output_counter import add_device_output
from core.handle.reportHandle import enqueue_tts_report
@@ -20,7 +20,6 @@ from core.providers.tts.dto.dto import (
InterfaceType,
)
import traceback
TAG = __name__
@@ -33,6 +32,7 @@ class TTSProviderBase(ABC):
self.conn = None
self.tts_timeout = 10
self.delete_audio_file = delete_audio_file
self.audio_file_type = "wav"
self.output_file = config.get("output_dir", "tmp/")
self.tts_text_queue = queue.Queue()
self.tts_audio_queue = queue.Queue()
@@ -78,35 +78,62 @@ class TTSProviderBase(ABC):
)
def to_tts(self, text):
tmp_file = self.generate_filename()
try:
max_repeat_time = 5
text = MarkdownCleaner.clean_markdown(text)
while not os.path.exists(tmp_file) and max_repeat_time > 0:
text = MarkdownCleaner.clean_markdown(text)
max_repeat_time = 5
if self.delete_audio_file:
# 需要删除文件的直接转为音频数据
while max_repeat_time > 0:
try:
asyncio.run(self.text_to_speak(text, tmp_file))
audio_bytes = asyncio.run(self.text_to_speak(text, None))
if audio_bytes:
audio_datas, _ = audio_bytes_to_data(
audio_bytes, file_type=self.audio_file_type, is_opus=True
)
return audio_datas
else:
max_repeat_time -= 1
except Exception as e:
logger.bind(tag=TAG).warning(
f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}"
)
# 未执行成功，删除文件
if os.path.exists(tmp_file):
os.remove(tmp_file)
max_repeat_time -= 1
if max_repeat_time > 0:
logger.bind(tag=TAG).info(
f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次"
f"语音生成成功: {text}，重试{5 - max_repeat_time}次"
)
else:
logger.bind(tag=TAG).error(
f"语音生成失败: {text}，请检查网络或服务是否正常"
)
return tmp_file
except Exception as e:
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
return None
else:
tmp_file = self.generate_filename()
try:
while not os.path.exists(tmp_file) and max_repeat_time > 0:
try:
asyncio.run(self.text_to_speak(text, tmp_file))
except Exception as e:
logger.bind(tag=TAG).warning(
f"语音生成失败{5 - max_repeat_time + 1}次: {text}，错误: {e}"
)
# 未执行成功，删除文件
if os.path.exists(tmp_file):
os.remove(tmp_file)
max_repeat_time -= 1
if max_repeat_time > 0:
logger.bind(tag=TAG).info(
f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次"
)
else:
logger.bind(tag=TAG).error(
f"语音生成失败: {text}，请检查网络或服务是否正常"
)
return tmp_file
except Exception as e:
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
return None
@abstractmethod
async def text_to_speak(self, text, output_file):
@@ -194,12 +221,19 @@ class TTSProviderBase(ABC):
self.tts_text_buff.append(message.content_detail)
segment_text = self._get_segment_text()
if segment_text:
tts_file = self.to_tts(segment_text)
if tts_file:
audio_datas = self._process_audio_file(tts_file)
self.tts_audio_queue.put(
(message.sentence_type, audio_datas, segment_text)
)
if self.delete_audio_file:
audio_datas = self.to_tts(segment_text)
if audio_datas:
self.tts_audio_queue.put(
(message.sentence_type, audio_datas, segment_text)
)
else:
tts_file = self.to_tts(segment_text)
if tts_file:
audio_datas = self._process_audio_file(tts_file)
self.tts_audio_queue.put(
(message.sentence_type, audio_datas, segment_text)
)
elif ContentType.FILE == message.content_type:
self._process_remaining_text()
tts_file = message.content_file
@@ -344,12 +378,18 @@ class TTSProviderBase(ABC):
if remaining_text:
segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text)
if segment_text:
tts_file = self.to_tts(segment_text)
if tts_file:
if self.delete_audio_file:
audio_datas = self.to_tts(segment_text)
if audio_datas:
self.tts_audio_queue.put(
(SentenceType.MIDDLE, audio_datas, segment_text)
)
else:
tts_file = self.to_tts(segment_text)
audio_datas = self._process_audio_file(tts_file)
self.tts_audio_queue.put(
(SentenceType.MIDDLE, audio_datas, segment_text)
)
self.processed_chars += len(full_text)
self.processed_chars += len(full_text)
return True
return False
main/xiaozhi-server/core/providers/tts/cozecn.py
+7 -4
View File
@@ -11,8 +11,8 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("private_voice")
else:
self.voice = config.get("voice")
self.response_format = config.get("response_format")
self.response_format = config.get("response_format", "mp3")
self.audio_file_type = config.get("response_format", "mp3")
self.host = "api.coze.cn"
self.api_url = f"https://{self.host}/v1/audio/speech"
@@ -33,7 +33,10 @@ class TTSProvider(TTSProviderBase):
"POST", self.api_url, json=request_json, headers=headers
)
data = response.content
file_to_save = open(output_file, "wb")
file_to_save.write(data)
if output_file:
with open(output_file, "wb") as file_to_save:
file_to_save.write(data)
else:
return data
except Exception as e:
raise Exception(f"{__name__} error: {e}")
main/xiaozhi-server/core/providers/tts/custom.py
+6 -3
View File
@@ -16,8 +16,8 @@ class TTSProvider(TTSProviderBase):
self.method = config.get("method", "GET")
self.headers = config.get("headers", {})
self.format = config.get("format", "wav")
self.audio_file_type = config.get("format", "wav")
self.output_file = config.get("output_dir", "tmp/")
self.params = config.get("params")
if isinstance(self.params, str):
@@ -43,8 +43,11 @@ class TTSProvider(TTSProviderBase):
else:
resp = requests.get(self.url, params=request_params, headers=self.headers)
if resp.status_code == 200:
with open(output_file, "wb") as file:
file.write(resp.content)
if output_file:
with open(output_file, "wb") as file:
file.write(resp.content)
else:
return resp.content
else:
error_msg = f"Custom TTS请求失败: {resp.status_code} - {resp.text}"
logger.bind(tag=TAG).error(error_msg)
main/xiaozhi-server/core/providers/tts/doubao.py
+8 -4
View File
@@ -29,7 +29,7 @@ class TTSProvider(TTSProviderBase):
speed_ratio = config.get("speed_ratio", "1.0")
volume_ratio = config.get("volume_ratio", "1.0")
pitch_ratio = config.get("pitch_ratio", "1.0")
self.audio_file_type = config.get("format", "wav")
self.speed_ratio = float(speed_ratio) if speed_ratio else 1.0
self.volume_ratio = float(volume_ratio) if volume_ratio else 1.0
self.pitch_ratio = float(pitch_ratio) if pitch_ratio else 1.0
@@ -49,7 +49,7 @@ class TTSProvider(TTSProviderBase):
"user": {"uid": "1"},
"audio": {
"voice_type": self.voice,
"encoding": "wav",
"encoding": self.audio_file_type,
"speed_ratio": self.speed_ratio,
"volume_ratio": self.volume_ratio,
"pitch_ratio": self.pitch_ratio,
@@ -70,8 +70,12 @@ class TTSProvider(TTSProviderBase):
)
if "data" in resp.json():
data = resp.json()["data"]
file_to_save = open(output_file, "wb")
file_to_save.write(base64.b64decode(data))
audio_bytes = base64.b64decode(data)
if output_file:
with open(output_file, "wb") as file_to_save:
file_to_save.write(audio_bytes)
else:
return audio_bytes
else:
raise Exception(
f"{__name__} status_code: {resp.status_code} response: {resp.content}"
main/xiaozhi-server/core/providers/tts/edge.py
+17 -8
View File
@@ -12,6 +12,7 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("private_voice")
else:
self.voice = config.get("voice")
self.audio_file_type = config.get("format", "mp3")
def generate_filename(self, extension=".mp3"):
return os.path.join(
@@ -22,16 +23,24 @@ class TTSProvider(TTSProviderBase):
async def text_to_speak(self, text, output_file):
try:
communicate = edge_tts.Communicate(text, voice=self.voice)
# 确保目录存在并创建空文件
os.makedirs(os.path.dirname(output_file), exist_ok=True)
with open(output_file, "wb") as f:
pass
if output_file:
# 确保目录存在并创建空文件
os.makedirs(os.path.dirname(output_file), exist_ok=True)
with open(output_file, "wb") as f:
pass
# 流式写入音频数据
with open(output_file, "ab") as f: # 改为追加模式避免覆盖
# 流式写入音频数据
with open(output_file, "ab") as f: # 改为追加模式避免覆盖
async for chunk in communicate.stream():
if chunk["type"] == "audio": # 只处理音频数据块
f.write(chunk["data"])
else:
# 返回音频二进制数据
audio_bytes = b""
async for chunk in communicate.stream():
if chunk["type"] == "audio": # 只处理音频数据块
f.write(chunk["data"])
if chunk["type"] == "audio":
audio_bytes += chunk["data"]
return audio_bytes
except Exception as e:
error_msg = f"Edge TTS请求失败: {e}"
raise Exception(error_msg) # 抛出异常，让调用方捕获
main/xiaozhi-server/core/providers/tts/fishspeech.py
+6 -3
View File
@@ -88,7 +88,7 @@ class TTSProvider(TTSProviderBase):
self.reference_audio = parse_string_to_list(config.get("reference_audio"))
self.reference_text = parse_string_to_list(config.get("reference_text"))
self.format = config.get("response_format", "wav")
self.audio_file_type = config.get("response_format", "wav")
self.api_key = config.get("api_key", "YOUR_API_KEY")
have_key = check_model_key("FishSpeech TTS", self.api_key)
if not have_key:
@@ -170,8 +170,11 @@ class TTSProvider(TTSProviderBase):
if response.status_code == 200:
audio_content = response.content
with open(output_file, "wb") as audio_file:
audio_file.write(audio_content)
if output_file:
with open(output_file, "wb") as audio_file:
audio_file.write(audio_content)
else:
return audio_content
else:
error_msg = f"Request failed with status code {response.status_code}"
main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py
+6 -2
View File
@@ -65,6 +65,7 @@ class TTSProvider(TTSProviderBase):
self.aux_ref_audio_paths = parse_string_to_list(
config.get("aux_ref_audio_paths")
)
self.audio_file_type = config.get("format", "wav")
async def text_to_speak(self, text, output_file):
request_json = {
@@ -91,8 +92,11 @@ class TTSProvider(TTSProviderBase):
resp = requests.post(self.url, json=request_json)
if resp.status_code == 200:
with open(output_file, "wb") as file:
file.write(resp.content)
if output_file:
with open(output_file, "wb") as file:
file.write(resp.content)
else:
return resp.content
else:
error_msg = f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}"
logger.bind(tag=TAG).error(error_msg)
main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py
+6 -2
View File
@@ -32,6 +32,7 @@ class TTSProvider(TTSProviderBase):
self.cut_punc = config.get("cut_punc", "")
self.inp_refs = parse_string_to_list(config.get("inp_refs"))
self.if_sr = str(config.get("if_sr", False)).lower() in ("true", "1", "yes")
self.audio_file_type = config.get("format", "wav")
async def text_to_speak(self, text, output_file):
request_params = {
@@ -52,8 +53,11 @@ class TTSProvider(TTSProviderBase):
resp = requests.get(self.url, params=request_params)
if resp.status_code == 200:
with open(output_file, "wb") as file:
file.write(resp.content)
if output_file:
with open(output_file, "wb") as file:
file.write(resp.content)
else:
return resp.content
else:
error_msg = f"GPT_SoVITS_V3 TTS请求失败: {resp.status_code} - {resp.text}"
logger.bind(tag=TAG).error(error_msg)
main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py
+118 -11
View File
@@ -145,10 +145,9 @@ class TTSProvider(TTSProviderBase):
self.cluster = config.get("cluster")
self.resource_id = config.get("resource_id")
if config.get("private_voice"):
self.speaker = config.get("private_voice")
self.voice = config.get("private_voice")
else:
self.speaker = config.get("speaker")
self.voice = config.get("voice")
self.voice = config.get("speaker")
self.ws_url = config.get("ws_url")
self.authorization = config.get("authorization")
self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
@@ -270,7 +269,7 @@ class TTSProvider(TTSProviderBase):
logger.bind(tag=TAG).error(f"WebSocket连接不存在，终止发送文本")
return
# 发送文本
await self.send_text(self.speaker, text, self.conn.sentence_id)
await self.send_text(self.voice, text, self.conn.sentence_id)
return
except Exception as e:
logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
@@ -300,9 +299,9 @@ class TTSProvider(TTSProviderBase):
event=EVENT_StartSession, sessionId=session_id
).as_bytes()
payload = self.get_payload_bytes(
event=EVENT_StartSession, speaker=self.speaker
event=EVENT_StartSession, speaker=self.voice
)
await self.send_event(header, optional, payload)
await self.send_event(self.ws, header, optional, payload)
logger.bind(tag=TAG).info("会话启动请求已发送")
except Exception as e:
logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
@@ -335,7 +334,7 @@ class TTSProvider(TTSProviderBase):
event=EVENT_FinishSession, sessionId=session_id
).as_bytes()
payload = str.encode("{}")
await self.send_event(header, optional, payload)
await self.send_event(self.ws, header, optional, payload)
logger.bind(tag=TAG).info("会话结束请求已发送")
# 等待监听任务完成
@@ -452,7 +451,11 @@ class TTSProvider(TTSProviderBase):
self.ws = None
async def send_event(
self, header: bytes, optional: bytes | None = None, payload: bytes = None
self,
ws: websockets.WebSocketClientProtocol,
header: bytes,
optional: bytes | None = None,
payload: bytes = None,
):
try:
full_client_request = bytearray(header)
@@ -462,7 +465,7 @@ class TTSProvider(TTSProviderBase):
payload_size = len(payload).to_bytes(4, "big", signed=True)
full_client_request.extend(payload_size)
full_client_request.extend(payload)
await self.ws.send(full_client_request)
await ws.send(full_client_request)
except websockets.ConnectionClosed:
logger.bind(tag=TAG).error(f"ConnectionClosed")
raise
@@ -477,7 +480,7 @@ class TTSProvider(TTSProviderBase):
payload = self.get_payload_bytes(
event=EVENT_TaskRequest, text=text, speaker=speaker
)
return await self.send_event(header, optional, payload)
return await self.send_event(self.ws, header, optional, payload)
# 读取 res 数组某段 字符串内容
def read_res_content(self, res: bytes, offset: int):
@@ -555,7 +558,7 @@ class TTSProvider(TTSProviderBase):
).as_bytes()
optional = Optional(event=EVENT_Start_Connection).as_bytes()
payload = str.encode("{}")
return await self.send_event(header, optional, payload)
return await self.send_event(self.ws, header, optional, payload)
def print_response(self, res, tag_msg: str):
logger.bind(tag=TAG).debug(f"===>{tag_msg} header:{res.header.__dict__}")
@@ -591,3 +594,107 @@ class TTSProvider(TTSProviderBase):
def wav_to_opus_data_audio_raw(self, raw_data_var, is_end=False):
opus_datas = self.opus_encoder.encode_pcm_to_opus(raw_data_var, is_end)
return opus_datas
def to_tts(self, text: str) -> list:
"""非流式生成音频数据，用于生成音频及测试场景
Args:
text: 要转换的文本
Returns:
list: 音频数据列表
"""
try:
# 创建事件循环
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
# 生成会话ID
session_id = uuid.uuid4().__str__().replace("-", "")
# 存储音频数据
audio_data = []
async def _generate_audio():
# 创建新的WebSocket连接
ws_header = {
"X-Api-App-Key": self.appId,
"X-Api-Access-Key": self.access_token,
"X-Api-Resource-Id": self.resource_id,
"X-Api-Connect-Id": uuid.uuid4(),
}
ws = await websockets.connect(
self.ws_url, additional_headers=ws_header, max_size=1000000000
)
try:
# 启动会话
header = Header(
message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON,
).as_bytes()
optional = Optional(
event=EVENT_StartSession, sessionId=session_id
).as_bytes()
payload = self.get_payload_bytes(
event=EVENT_StartSession, speaker=self.voice
)
await self.send_event(ws, header, optional, payload)
# 发送文本
header = Header(
message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON,
).as_bytes()
optional = Optional(
event=EVENT_TaskRequest, sessionId=session_id
).as_bytes()
payload = self.get_payload_bytes(
event=EVENT_TaskRequest, text=text, speaker=self.voice
)
await self.send_event(ws, header, optional, payload)
# 发送结束会话请求
header = Header(
message_type=FULL_CLIENT_REQUEST,
message_type_specific_flags=MsgTypeFlagWithEvent,
serial_method=JSON,
).as_bytes()
optional = Optional(
event=EVENT_FinishSession, sessionId=session_id
).as_bytes()
payload = str.encode("{}")
await self.send_event(ws, header, optional, payload)
# 接收音频数据
while True:
msg = await ws.recv()
res = self.parser_response(msg)
if (
res.optional.event == EVENT_TTSResponse
and res.header.message_type == AUDIO_ONLY_RESPONSE
):
opus_datas = self.wav_to_opus_data_audio_raw(res.payload)
audio_data.extend(opus_datas)
elif res.optional.event == EVENT_SessionFinished:
break
finally:
# 清理资源
try:
await ws.close()
except:
pass
# 运行异步任务
loop.run_until_complete(_generate_audio())
loop.close()
return audio_data
except Exception as e:
logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}")
return []
main/xiaozhi-server/core/providers/tts/linkerai.py
+75 -3
View File
@@ -2,6 +2,8 @@ import queue
import asyncio
import traceback
import aiohttp
import requests
import time
from config.logger import setup_logging
from core.utils.tts import MarkdownCleaner
from core.providers.tts.base import TTSProviderBase
@@ -55,7 +57,7 @@ class TTSProvider(TTSProviderBase):
self.tts_text_buff.append(message.content_detail)
segment_text = self._get_segment_text()
if segment_text:
self.to_tts(segment_text)
self.to_tts_single_stream(segment_text)
elif ContentType.FILE == message.content_type:
logger.bind(tag=TAG).info(
@@ -87,12 +89,12 @@ class TTSProvider(TTSProviderBase):
if remaining_text:
segment_text = textUtils.get_string_no_punctuation_or_emoji(remaining_text)
if segment_text:
self.to_tts(segment_text, is_last)
self.to_tts_single_stream(segment_text, is_last)
self.processed_chars += len(full_text)
else:
self._process_before_stop_play_files()
def to_tts(self, text, is_last=False):
def to_tts_single_stream(self, text, is_last=False):
try:
max_repeat_time = 5
text = MarkdownCleaner.clean_markdown(text)
@@ -225,3 +227,73 @@ class TTSProvider(TTSProviderBase):
except Exception as e:
logger.error(f"TTS请求异常: {e}")
self.tts_audio_queue.put((SentenceType.LAST, [], None))
def to_tts(self, text: str) -> list:
"""非流式TTS处理，用于测试及保存音频文件的场景
Args:
text: 要转换的文本
Returns:
list: 返回opus编码后的音频数据列表
"""
start_time = time.time()
text = MarkdownCleaner.clean_markdown(text)
params = {
"tts_text": text,
"spk_id": self.voice,
"frame_duration": 60,
"stream": False,
"target_sr": 16000,
"audio_format": self.audio_format,
"instruct_text": "请生成一段自然流畅的语音",
}
headers = {
"Authorization": f"Bearer {self.access_token}",
"Content-Type": "application/json",
}
try:
with requests.get(
self.api_url, params=params, headers=headers, timeout=5
) as response:
if response.status_code != 200:
logger.error(
f"TTS请求失败: {response.status_code}, {response.text}"
)
return []
logger.info(f"TTS请求成功: {text}, 耗时: {time.time() - start_time}秒")
# 使用opus编码器处理PCM数据
opus_datas = []
pcm_data = response.content
# 计算每帧的字节数
frame_bytes = int(
self.opus_encoder.sample_rate
* self.opus_encoder.channels
* self.opus_encoder.frame_size_ms
/ 1000
* 2
)
# 分帧处理PCM数据
for i in range(0, len(pcm_data), frame_bytes):
frame = pcm_data[i : i + frame_bytes]
if len(frame) < frame_bytes:
# 最后一帧可能不足，用0填充
frame = frame + b"\x00" * (frame_bytes - len(frame))
opus = self.opus_encoder.encode_pcm_to_opus(
frame, end_of_stream=(i + frame_bytes >= len(pcm_data))
)
if opus:
opus_datas.extend(opus)
return opus_datas
except Exception as e:
logger.error(f"TTS请求异常: {e}")
return []
main/xiaozhi-server/core/providers/tts/minimax.py
+11 -6
View File
@@ -14,9 +14,9 @@ class TTSProvider(TTSProviderBase):
self.api_key = config.get("api_key")
self.model = config.get("model")
if config.get("private_voice"):
self.voice_id = config.get("private_voice")
self.voice = config.get("private_voice")
else:
self.voice_id = config.get("voice_id")
self.voice = config.get("voice_id")
default_voice_setting = {
"voice_id": "female-shaonv",
@@ -43,8 +43,8 @@ class TTSProvider(TTSProviderBase):
self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
self.timber_weights = parse_string_to_list(config.get("timber_weights"))
if self.voice_id:
self.voice_setting["voice_id"] = self.voice_id
if self.voice:
self.voice_setting["voice_id"] = self.voice
self.host = "api.minimax.chat"
self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
@@ -52,6 +52,7 @@ class TTSProvider(TTSProviderBase):
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
self.audio_file_type = defult_audio_setting.get("format", "mp3")
def generate_filename(self, extension=".mp3"):
return os.path.join(
@@ -80,8 +81,12 @@ class TTSProvider(TTSProviderBase):
# 检查返回请求数据的status_code是否为0
if resp.json()["base_resp"]["status_code"] == 0:
data = resp.json()["data"]["audio"]
file_to_save = open(output_file, "wb")
file_to_save.write(bytes.fromhex(data))
audio_bytes = bytes.fromhex(data)
if output_file:
with open(output_file, "wb") as file_to_save:
file_to_save.write(audio_bytes)
else:
return audio_bytes
else:
raise Exception(
f"{__name__} status_code: {resp.status_code} response: {resp.content}"
main/xiaozhi-server/core/providers/tts/openai.py
+7 -3
View File
@@ -17,7 +17,8 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("private_voice")
else:
self.voice = config.get("voice", "alloy")
self.response_format = "wav"
self.response_format = config.get("format", "wav")
self.audio_file_type = config.get("format", "wav")
# 处理空字符串的情况
speed = config.get("speed", "1.0")
@@ -40,8 +41,11 @@ class TTSProvider(TTSProviderBase):
}
response = requests.post(self.api_url, json=data, headers=headers)
if response.status_code == 200:
with open(output_file, "wb") as audio_file:
audio_file.write(response.content)
if output_file:
with open(output_file, "wb") as audio_file:
audio_file.write(response.content)
else:
return response.content
else:
raise Exception(
f"OpenAI TTS请求失败: {response.status_code} - {response.text}"
main/xiaozhi-server/core/providers/tts/siliconflow.py
+7 -3
View File
@@ -11,7 +11,8 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("private_voice")
else:
self.voice = config.get("voice")
self.response_format = config.get("response_format")
self.response_format = config.get("response_format", "mp3")
self.audio_file_type = config.get("response_format", "mp3")
self.sample_rate = config.get("sample_rate")
self.speed = float(config.get("speed", 1.0))
self.gain = config.get("gain")
@@ -35,7 +36,10 @@ class TTSProvider(TTSProviderBase):
"POST", self.api_url, json=request_json, headers=headers
)
data = response.content
file_to_save = open(output_file, "wb")
file_to_save.write(data)
if output_file:
with open(output_file, "wb") as file_to_save:
file_to_save.write(data)
else:
return data
except Exception as e:
raise Exception(f"{__name__} error: {e}")
main/xiaozhi-server/core/providers/tts/tencent.py
+9 -6
View File
@@ -22,6 +22,7 @@ class TTSProvider(TTSProviderBase):
self.api_url = "https://tts.tencentcloudapi.com" # 正确的API端点
self.region = config.get("region")
self.output_file = config.get("output_dir")
self.audio_file_type = config.get("format", "wav")
def _get_auth_headers(self, request_body):
"""生成鉴权请求头"""
@@ -148,12 +149,14 @@ class TTSProvider(TTSProviderBase):
f"API返回错误: {error_info['Code']}: {error_info['Message']}"
)
# 提取音频数据
audio_data = response_data["Response"].get("Audio")
if audio_data:
# 解码Base64音频数据并保存
with open(output_file, "wb") as f:
f.write(base64.b64decode(audio_data))
# 解码Base64音频数据
audio_bytes = base64.b64decode(response_data["Response"].get("Audio"))
if audio_bytes:
if output_file:
with open(output_file, "wb") as f:
f.write(audio_bytes)
else:
return audio_bytes
else:
raise Exception(f"{__name__}: 没有返回音频数据: {response_data}")
else:
main/xiaozhi-server/core/providers/tts/ttson.py
+9 -6
View File
@@ -19,9 +19,9 @@ class TTSProvider(TTSProviderBase):
"https://u95167-bd74-2aef8085.westx.seetacloud.com:8443/flashsummary/tts?token=",
)
if config.get("private_voice"):
self.voice_id = int(config.get("private_voice"))
self.voice = int(config.get("private_voice"))
else:
self.voice_id = int(config.get("voice_id", 1695))
self.voice = int(config.get("voice_id", 1695))
self.token = config.get("token")
self.to_lang = config.get("to_lang")
self.volume_change_dB = int(config.get("volume_change_dB", 0))
@@ -30,6 +30,7 @@ class TTSProvider(TTSProviderBase):
self.output_file = config.get("output_dir")
self.pitch_factor = int(config.get("pitch_factor", 0))
self.format = config.get("format", "mp3")
self.audio_file_type = config.get("format", "mp3")
self.emotion = int(config.get("emotion", 1))
self.header = {"Content-Type": "application/json"}
@@ -49,7 +50,7 @@ class TTSProvider(TTSProviderBase):
"emotion": self.emotion,
"format": self.format,
"volume_change_dB": self.volume_change_dB,
"voice_id": self.voice_id,
"voice_id": self.voice,
"pitch_factor": self.pitch_factor,
"speed_factor": self.speed_factor,
"token": self.token,
@@ -73,9 +74,11 @@ class TTSProvider(TTSProviderBase):
)
audio_content = requests.get(result)
with open(output_file, "wb") as f:
f.write(audio_content.content)
return True
if output_file:
with open(output_file, "wb") as f:
f.write(audio_content.content)
else:
return audio_content.content
voice_path = resp_json.get("voice_path")
des_path = output_file
shutil.move(voice_path, des_path)
main/xiaozhi-server/core/utils/p3.py
+26
View File
@@ -29,5 +29,31 @@ def decode_opus_from_file(input_file):
total_frames += 1
# 计算总时长
total_duration = (total_frames * frame_duration_ms) / 1000.0
return opus_datas, total_duration
def decode_opus_from_bytes(input_bytes):
"""
从p3二进制数据中解码 Opus 数据，并返回一个 Opus 数据包的列表以及总时长。
"""
import io
opus_datas = []
total_frames = 0
sample_rate = 16000 # 文件采样率
frame_duration_ms = 60 # 帧时长
frame_size = int(sample_rate * frame_duration_ms / 1000)
f = io.BytesIO(input_bytes)
while True:
header = f.read(4)
if not header:
break
_, _, data_len = struct.unpack('>BBH', header)
opus_data = f.read(data_len)
if len(opus_data) != data_len:
raise ValueError(f"Data length({len(opus_data)}) mismatch({data_len}) in the bytes.")
opus_datas.append(opus_data)
total_frames += 1
total_duration = (total_frames * frame_duration_ms) / 1000.0
return opus_datas, total_duration
main/xiaozhi-server/core/utils/util.py
+46
View File
@@ -3,6 +3,9 @@ import socket
import subprocess
import re
import os
import wave
from io import BytesIO
from core.utils import p3
import numpy as np
import requests
import opuslib_next
@@ -773,6 +776,22 @@ def audio_to_data(audio_file_path, is_opus=True):
return pcm_to_data(raw_data, is_opus), duration
def audio_bytes_to_data(audio_bytes, file_type, is_opus=True):
"""
直接用音频二进制数据转为opus/pcm数据，支持wav、mp3、p3
"""
if file_type == "p3":
# 直接用p3解码
return p3.decode_opus_from_bytes(audio_bytes)
else:
# 其他格式用pydub
audio = AudioSegment.from_file(BytesIO(audio_bytes), format=file_type, parameters=["-nostdin"])
audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)
duration = len(audio) / 1000.0
raw_data = audio.raw_data
return pcm_to_data(raw_data, is_opus), duration
def pcm_to_data(raw_data, is_opus=True):
# 初始化Opus编码器
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
@@ -804,6 +823,33 @@ def pcm_to_data(raw_data, is_opus=True):
return datas
def opus_datas_to_wav_bytes(opus_datas, sample_rate=16000, channels=1):
"""
将opus帧列表解码为wav字节流
"""
decoder = opuslib_next.Decoder(sample_rate, channels)
pcm_datas = []
frame_duration = 60 # ms
frame_size = int(sample_rate * frame_duration / 1000) # 960
for opus_frame in opus_datas:
# 解码为PCM（返回bytes，2字节/采样点）
pcm = decoder.decode(opus_frame, frame_size)
pcm_datas.append(pcm)
pcm_bytes = b''.join(pcm_datas)
# 写入wav字节流
wav_buffer = BytesIO()
with wave.open(wav_buffer, 'wb') as wf:
wf.setnchannels(channels)
wf.setsampwidth(2) # 16bit
wf.setframerate(sample_rate)
wf.writeframes(pcm_bytes)
return wav_buffer.getvalue()
def check_vad_update(before_config, new_config):
if (
new_config.get("selected_module") is None
main/xiaozhi-server/core/utils/wakeup_word.py
+143
View File
@@ -0,0 +1,143 @@
import os
import yaml
import time
import hashlib
import fcntl
from typing import Dict
from contextlib import contextmanager
class FileLock:
def __init__(self, file, timeout=5):
self.file = file
self.timeout = timeout
self.start_time = None
def __enter__(self):
self.start_time = time.time()
while True:
try:
fcntl.flock(self.file, fcntl.LOCK_EX | fcntl.LOCK_NB)
return self.file
except IOError:
if time.time() - self.start_time > self.timeout:
raise TimeoutError("获取文件锁超时")
time.sleep(0.1)
def __exit__(self, exc_type, exc_val, exc_tb):
fcntl.flock(self.file, fcntl.LOCK_UN)
class WakeupWordsConfig:
def __init__(self):
self.config_file = "data/.wakeup_words.yaml"
self.assets_dir = "config/assets/wakeup_words"
self._ensure_directories()
self._config_cache = None
self._last_load_time = 0
self._cache_ttl = 1 # 缓存有效期（秒）
self._lock_timeout = 5 # 文件锁超时时间（秒）
def _ensure_directories(self):
"""确保必要的目录存在"""
os.makedirs(os.path.dirname(self.config_file), exist_ok=True)
os.makedirs(self.assets_dir, exist_ok=True)
def _load_config(self) -> Dict:
"""加载配置文件，使用缓存机制"""
current_time = time.time()
# 如果缓存有效，直接返回缓存
if (
self._config_cache is not None
and current_time - self._last_load_time < self._cache_ttl
):
return self._config_cache
try:
with open(self.config_file, "a+") as f:
with FileLock(f, timeout=self._lock_timeout):
f.seek(0)
content = f.read()
config = yaml.safe_load(content) if content else {}
self._config_cache = config
self._last_load_time = current_time
return config
except (TimeoutError, IOError) as e:
print(f"加载配置文件失败: {e}")
return {}
except Exception as e:
print(f"加载配置文件时发生未知错误: {e}")
return {}
def _save_config(self, config: Dict):
"""保存配置到文件，使用文件锁保护"""
try:
with open(self.config_file, "w") as f:
with FileLock(f, timeout=self._lock_timeout):
yaml.dump(config, f, allow_unicode=True)
self._config_cache = config
self._last_load_time = time.time()
except (TimeoutError, IOError) as e:
print(f"保存配置文件失败: {e}")
raise
except Exception as e:
print(f"保存配置文件时发生未知错误: {e}")
raise
def get_wakeup_response(self, voice: str) -> Dict:
voice = hashlib.md5(voice.encode()).hexdigest()
"""获取唤醒词回复配置"""
config = self._load_config()
default_response = {
"voice": "default",
"file_path": "config/assets/wakeup_words.wav",
"time": 0,
"text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",
}
if not config or voice not in config:
return default_response
# 检查文件大小
file_path = config[voice]["file_path"]
if not os.path.exists(file_path) or os.stat(file_path).st_size < (15 * 1024):
return default_response
return config[voice]
def update_wakeup_response(self, voice: str, file_path: str, text: str):
"""更新唤醒词回复配置"""
try:
config = self._load_config()
voice_hash = hashlib.md5(voice.encode()).hexdigest()
config[voice_hash] = {
"voice": voice,
"file_path": file_path,
"time": time.time(),
"text": text,
}
self._save_config(config)
except Exception as e:
print(f"更新唤醒词回复配置失败: {e}")
raise
def generate_file_path(self, voice: str) -> str:
"""生成音频文件路径，使用voice的哈希值作为文件名"""
try:
# 生成voice的哈希值
voice_hash = hashlib.md5(voice.encode()).hexdigest()
file_path = os.path.join(self.assets_dir, f"{voice_hash}.wav")
# 如果文件已存在，先删除
if os.path.exists(file_path):
try:
os.remove(file_path)
except Exception as e:
print(f"删除已存在的音频文件失败: {e}")
raise
return file_path
except Exception as e:
print(f"生成音频文件路径失败: {e}")
raise