mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-27 09:33:55 +08:00
重新划分目录 (#214)
* 🦄 refactor(web): 修改zhikongtaiweb到web * 🦄 refactor: 重写前端 路由守护尚未写完 * 🦄 refactor: 标准化路由 * update:前端重写，保留后端 * update:添加前端代码 * update:pip转成poetry启动 * update:增加mem0ai包依赖 * update:文档增加mem0ai的描述 * feat: play online mp3 (#181) Co-authored-by: 欣南科技 <huangrongzhuang@xin-nan.com> * 修改前端代码 * update:调整项目目录 * update:优化 * update:配置文件去除8002端口 * update:增加开发说明 * update:更新开发协议 --------- Co-authored-by: kalicyh <34980061+kaliCYH@users.noreply.github.com> Co-authored-by: hrz <1710360675@qq.com> Co-authored-by: freshlife001 <talent@mises.site> Co-authored-by: CGD <3030332422@qq.com>
This commit is contained in:
欣南科技
co-authored by
kalicyh
hrz
freshlife001
CGD
kalicyh
hrz
freshlife001
CGD
parent
8b151d07c2
commit
0e43748fdc
@@ -1,19 +0,0 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional, Tuple, List
|
||||
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ASRProviderBase(ABC):
|
||||
@abstractmethod
|
||||
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
|
||||
"""解码Opus数据并保存为WAV文件"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
pass
|
||||
@@ -1,286 +0,0 @@
|
||||
import time
|
||||
import io
|
||||
import wave
|
||||
import os
|
||||
from typing import Optional, Tuple, List
|
||||
import uuid
|
||||
import websockets
|
||||
import json
|
||||
import gzip
|
||||
|
||||
import opuslib_next
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
CLIENT_FULL_REQUEST = 0b0001
|
||||
CLIENT_AUDIO_ONLY_REQUEST = 0b0010
|
||||
|
||||
NO_SEQUENCE = 0b0000
|
||||
NEG_SEQUENCE = 0b0010
|
||||
|
||||
SERVER_FULL_RESPONSE = 0b1001
|
||||
SERVER_ACK = 0b1011
|
||||
SERVER_ERROR_RESPONSE = 0b1111
|
||||
|
||||
NO_SERIALIZATION = 0b0000
|
||||
JSON = 0b0001
|
||||
THRIFT = 0b0011
|
||||
CUSTOM_TYPE = 0b1111
|
||||
NO_COMPRESSION = 0b0000
|
||||
GZIP = 0b0001
|
||||
CUSTOM_COMPRESSION = 0b1111
|
||||
|
||||
|
||||
def parse_response(res):
|
||||
"""
|
||||
protocol_version(4 bits), header_size(4 bits),
|
||||
message_type(4 bits), message_type_specific_flags(4 bits)
|
||||
serialization_method(4 bits) message_compression(4 bits)
|
||||
reserved （8bits) 保留字段
|
||||
header_extensions 扩展头(大小等于 8 * 4 * (header_size - 1) )
|
||||
payload 类似与http 请求体
|
||||
"""
|
||||
protocol_version = res[0] >> 4
|
||||
header_size = res[0] & 0x0f
|
||||
message_type = res[1] >> 4
|
||||
message_type_specific_flags = res[1] & 0x0f
|
||||
serialization_method = res[2] >> 4
|
||||
message_compression = res[2] & 0x0f
|
||||
reserved = res[3]
|
||||
header_extensions = res[4:header_size * 4]
|
||||
payload = res[header_size * 4:]
|
||||
result = {}
|
||||
payload_msg = None
|
||||
payload_size = 0
|
||||
if message_type == SERVER_FULL_RESPONSE:
|
||||
payload_size = int.from_bytes(payload[:4], "big", signed=True)
|
||||
payload_msg = payload[4:]
|
||||
elif message_type == SERVER_ACK:
|
||||
seq = int.from_bytes(payload[:4], "big", signed=True)
|
||||
result['seq'] = seq
|
||||
if len(payload) >= 8:
|
||||
payload_size = int.from_bytes(payload[4:8], "big", signed=False)
|
||||
payload_msg = payload[8:]
|
||||
elif message_type == SERVER_ERROR_RESPONSE:
|
||||
code = int.from_bytes(payload[:4], "big", signed=False)
|
||||
result['code'] = code
|
||||
payload_size = int.from_bytes(payload[4:8], "big", signed=False)
|
||||
payload_msg = payload[8:]
|
||||
if payload_msg is None:
|
||||
return result
|
||||
if message_compression == GZIP:
|
||||
payload_msg = gzip.decompress(payload_msg)
|
||||
if serialization_method == JSON:
|
||||
payload_msg = json.loads(str(payload_msg, "utf-8"))
|
||||
elif serialization_method != NO_SERIALIZATION:
|
||||
payload_msg = str(payload_msg, "utf-8")
|
||||
result['payload_msg'] = payload_msg
|
||||
result['payload_size'] = payload_size
|
||||
return result
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool):
|
||||
self.appid = config.get("appid")
|
||||
self.cluster = config.get("cluster")
|
||||
self.access_token = config.get("access_token")
|
||||
self.output_dir = config.get("output_dir")
|
||||
|
||||
self.host = "openspeech.bytedance.com"
|
||||
self.ws_url = f"wss://{self.host}/api/v2/asr"
|
||||
self.success_code = 1000
|
||||
self.seg_duration = 15000
|
||||
|
||||
# 确保输出目录存在
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
|
||||
"""将Opus音频数据解码并保存为WAV文件"""
|
||||
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
|
||||
file_path = os.path.join(self.output_dir, file_name)
|
||||
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
with wave.open(file_path, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2) # 2 bytes = 16-bit
|
||||
wf.setframerate(16000)
|
||||
wf.writeframes(b"".join(pcm_data))
|
||||
|
||||
return file_path
|
||||
|
||||
@staticmethod
|
||||
def _generate_header(message_type=CLIENT_FULL_REQUEST, message_type_specific_flags=NO_SEQUENCE) -> bytearray:
|
||||
"""Generate protocol header."""
|
||||
header = bytearray()
|
||||
header_size = 1
|
||||
header.append((0b0001 << 4) | header_size) # Protocol version
|
||||
header.append((message_type << 4) | message_type_specific_flags)
|
||||
header.append((0b0001 << 4) | 0b0001) # JSON serialization & GZIP compression
|
||||
header.append(0x00) # reserved
|
||||
return header
|
||||
|
||||
def _construct_request(self, reqid) -> dict:
|
||||
"""Construct the request payload."""
|
||||
return {
|
||||
"app": {
|
||||
"appid": f"{self.appid}",
|
||||
"cluster": self.cluster,
|
||||
"token": self.access_token,
|
||||
},
|
||||
"user": {
|
||||
"uid": str(uuid.uuid4()),
|
||||
},
|
||||
"request": {
|
||||
"reqid": reqid,
|
||||
"show_utterances": False,
|
||||
"sequence": 1
|
||||
},
|
||||
"audio": {
|
||||
"format": "wav",
|
||||
"rate": 16000,
|
||||
"language": "zh-CN",
|
||||
"bits": 16,
|
||||
"channel": 1,
|
||||
"codec": "raw",
|
||||
},
|
||||
}
|
||||
|
||||
async def _send_request(self, audio_data: List[bytes], segment_size: int) -> Optional[str]:
|
||||
"""Send request to Volcano ASR service."""
|
||||
try:
|
||||
auth_header = {'Authorization': 'Bearer; {}'.format(self.access_token)}
|
||||
async with websockets.connect(self.ws_url, additional_headers=auth_header) as websocket:
|
||||
# Prepare request data
|
||||
request_params = self._construct_request(str(uuid.uuid4()))
|
||||
print(request_params)
|
||||
payload_bytes = str.encode(json.dumps(request_params))
|
||||
payload_bytes = gzip.compress(payload_bytes)
|
||||
full_client_request = self._generate_header()
|
||||
full_client_request.extend((len(payload_bytes)).to_bytes(4, 'big')) # payload size(4 bytes)
|
||||
full_client_request.extend(payload_bytes) # payload
|
||||
|
||||
# Send header and metadata
|
||||
# full_client_request
|
||||
await websocket.send(full_client_request)
|
||||
res = await websocket.recv()
|
||||
result = parse_response(res)
|
||||
if 'payload_msg' in result and result['payload_msg']['code'] != self.success_code:
|
||||
logger.bind(tag=TAG).error(f"ASR error: {result}")
|
||||
return None
|
||||
|
||||
for seq, (chunk, last) in enumerate(self.slice_data(audio_data, segment_size), 1):
|
||||
if last:
|
||||
audio_only_request = self._generate_header(
|
||||
message_type=CLIENT_AUDIO_ONLY_REQUEST,
|
||||
message_type_specific_flags=NEG_SEQUENCE
|
||||
)
|
||||
else:
|
||||
audio_only_request = self._generate_header(
|
||||
message_type=CLIENT_AUDIO_ONLY_REQUEST
|
||||
)
|
||||
payload_bytes = gzip.compress(chunk)
|
||||
audio_only_request.extend((len(payload_bytes)).to_bytes(4, 'big')) # payload size(4 bytes)
|
||||
audio_only_request.extend(payload_bytes) # payload
|
||||
# Send audio data
|
||||
await websocket.send(audio_only_request)
|
||||
|
||||
# Receive response
|
||||
response = await websocket.recv()
|
||||
result = parse_response(response)
|
||||
|
||||
if 'payload_msg' in result and result['payload_msg']['code'] == self.success_code:
|
||||
if len(result['payload_msg']['result']) > 0:
|
||||
return result['payload_msg']['result'][0]["text"]
|
||||
return None
|
||||
else:
|
||||
logger.bind(tag=TAG).error(f"ASR error: {result}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
|
||||
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
return pcm_data
|
||||
|
||||
@staticmethod
|
||||
def read_wav_info(data: io.BytesIO = None) -> (int, int, int, int, int):
|
||||
with io.BytesIO(data) as _f:
|
||||
wave_fp = wave.open(_f, 'rb')
|
||||
nchannels, sampwidth, framerate, nframes = wave_fp.getparams()[:4]
|
||||
wave_bytes = wave_fp.readframes(nframes)
|
||||
return nchannels, sampwidth, framerate, nframes, len(wave_bytes)
|
||||
|
||||
@staticmethod
|
||||
def slice_data(data: bytes, chunk_size: int) -> (list, bool):
|
||||
"""
|
||||
slice data
|
||||
:param data: wav data
|
||||
:param chunk_size: the segment size in one request
|
||||
:return: segment data, last flag
|
||||
"""
|
||||
data_len = len(data)
|
||||
offset = 0
|
||||
while offset + chunk_size < data_len:
|
||||
yield data[offset: offset + chunk_size], False
|
||||
offset += chunk_size
|
||||
else:
|
||||
yield data[offset: data_len], True
|
||||
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
try:
|
||||
# 合并所有opus数据包
|
||||
pcm_data = self.decode_opus(opus_data, session_id)
|
||||
combined_pcm_data = b''.join(pcm_data)
|
||||
|
||||
wav_buffer = io.BytesIO()
|
||||
|
||||
with wave.open(wav_buffer, "wb") as wav_file:
|
||||
wav_file.setnchannels(1) # 设置声道数
|
||||
wav_file.setsampwidth(2) # 设置采样宽度
|
||||
wav_file.setframerate(16000) # 设置采样率
|
||||
wav_file.writeframes(combined_pcm_data) # 写入 PCM 数据
|
||||
|
||||
# 获取封装后的 WAV 数据
|
||||
wav_data = wav_buffer.getvalue()
|
||||
nchannels, sampwidth, framerate, nframes, wav_len = self.read_wav_info(wav_data)
|
||||
size_per_sec = nchannels * sampwidth * framerate
|
||||
segment_size = int(size_per_sec * self.seg_duration / 1000)
|
||||
|
||||
# 语音识别
|
||||
start_time = time.time()
|
||||
text = await self._send_request(wav_data, segment_size)
|
||||
if text:
|
||||
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
|
||||
return text, None
|
||||
return "", None
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", None
|
||||
@@ -1,110 +0,0 @@
|
||||
import time
|
||||
import wave
|
||||
import os
|
||||
import sys
|
||||
import io
|
||||
from config.logger import setup_logging
|
||||
from typing import Optional, Tuple, List
|
||||
import uuid
|
||||
import opuslib_next
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
|
||||
from funasr import AutoModel
|
||||
from funasr.utils.postprocess_utils import rich_transcription_postprocess
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
# 捕获标准输出
|
||||
class CaptureOutput:
|
||||
def __enter__(self):
|
||||
self._output = io.StringIO()
|
||||
self._original_stdout = sys.stdout
|
||||
sys.stdout = self._output
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
sys.stdout = self._original_stdout
|
||||
self.output = self._output.getvalue()
|
||||
self._output.close()
|
||||
|
||||
# 将捕获到的内容通过 logger 输出
|
||||
if self.output:
|
||||
logger.bind(tag=TAG).info(self.output.strip())
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool):
|
||||
self.model_dir = config.get("model_dir")
|
||||
self.output_dir = config.get("output_dir") # 修正配置键名
|
||||
self.delete_audio_file = delete_audio_file
|
||||
|
||||
# 确保输出目录存在
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
with CaptureOutput():
|
||||
self.model = AutoModel(
|
||||
model=self.model_dir,
|
||||
vad_kwargs={"max_single_segment_time": 30000},
|
||||
disable_update=True,
|
||||
hub="hf"
|
||||
# device="cuda:0", # 启用GPU加速
|
||||
)
|
||||
|
||||
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
|
||||
"""将Opus音频数据解码并保存为WAV文件"""
|
||||
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
|
||||
file_path = os.path.join(self.output_dir, file_name)
|
||||
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
with wave.open(file_path, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2) # 2 bytes = 16-bit
|
||||
wf.setframerate(16000)
|
||||
wf.writeframes(b"".join(pcm_data))
|
||||
|
||||
return file_path
|
||||
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""语音转文本主处理逻辑"""
|
||||
file_path = None
|
||||
try:
|
||||
# 保存音频文件
|
||||
start_time = time.time()
|
||||
file_path = self.save_audio_to_file(opus_data, session_id)
|
||||
logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
|
||||
|
||||
# 语音识别
|
||||
start_time = time.time()
|
||||
result = self.model.generate(
|
||||
input=file_path,
|
||||
cache={},
|
||||
language="auto",
|
||||
use_itn=True,
|
||||
batch_size_s=60,
|
||||
)
|
||||
text = rich_transcription_postprocess(result[0]["text"])
|
||||
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
|
||||
|
||||
return text, file_path
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", None
|
||||
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
|
||||
@@ -1,8 +0,0 @@
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class LLMProviderBase(ABC):
|
||||
@abstractmethod
|
||||
def response(self, session_id, dialogue):
|
||||
"""LLM response generator"""
|
||||
pass
|
||||
@@ -1,37 +0,0 @@
|
||||
from config.logger import setup_logging
|
||||
import requests
|
||||
import json
|
||||
import re
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
import os
|
||||
# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
|
||||
from cozepy import COZE_CN_BASE_URL
|
||||
from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType # noqa
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.personal_access_token = config.get("personal_access_token")
|
||||
self.bot_id = config.get("bot_id")
|
||||
self.user_id = config.get("user_id")
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
coze_api_token = self.personal_access_token
|
||||
coze_api_base = COZE_CN_BASE_URL
|
||||
|
||||
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
|
||||
|
||||
coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
|
||||
|
||||
for event in coze.chat.stream(
|
||||
bot_id=self.bot_id,
|
||||
user_id=self.user_id,
|
||||
additional_messages=[
|
||||
Message.build_user_question_text(last_msg["content"]),
|
||||
],
|
||||
):
|
||||
if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
|
||||
print(event.message.content, end="", flush=True)
|
||||
yield event.message.content
|
||||
@@ -1,39 +0,0 @@
|
||||
import json
|
||||
from config.logger import setup_logging
|
||||
import requests
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.api_key = config["api_key"]
|
||||
self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
try:
|
||||
# 取最后一条用户消息
|
||||
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
|
||||
|
||||
# 发起流式请求
|
||||
with requests.post(
|
||||
f"{self.base_url}/chat-messages",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json={
|
||||
"query": last_msg["content"],
|
||||
"response_mode": "streaming",
|
||||
"user": session_id,
|
||||
"inputs": {}
|
||||
},
|
||||
stream=True
|
||||
) as r:
|
||||
for line in r.iter_lines():
|
||||
if line.startswith(b'data: '):
|
||||
event = json.loads(line[6:])
|
||||
if event.get('answer'):
|
||||
yield event['answer']
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
|
||||
yield "【服务响应异常】"
|
||||
@@ -1,81 +0,0 @@
|
||||
from config.logger import setup_logging
|
||||
import google.generativeai as genai
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
"""初始化Gemini LLM Provider"""
|
||||
self.model_name = config.get("model_name", "gemini-1.5-pro")
|
||||
self.api_key = config.get("api_key")
|
||||
|
||||
if not self.api_key or "你" in self.api_key:
|
||||
logger.bind(tag=TAG).error("你还没配置Gemini LLM的密钥，请在配置文件中配置密钥，否则无法正常工作")
|
||||
return
|
||||
|
||||
try:
|
||||
# 初始化Gemini客户端
|
||||
genai.configure(api_key=self.api_key)
|
||||
self.model = genai.GenerativeModel(self.model_name)
|
||||
|
||||
# 设置生成参数
|
||||
self.generation_config = {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.9,
|
||||
"top_k": 40,
|
||||
"max_output_tokens": 2048,
|
||||
}
|
||||
self.chat = None
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Gemini初始化失败: {e}")
|
||||
self.model = None
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
"""生成Gemini对话响应"""
|
||||
if not self.model:
|
||||
yield "【Gemini服务未正确初始化】"
|
||||
return
|
||||
|
||||
try:
|
||||
# 处理对话历史
|
||||
chat_history = []
|
||||
for msg in dialogue[:-1]: # 历史对话
|
||||
role = "model" if msg["role"] == "assistant" else "user"
|
||||
content = msg["content"].strip()
|
||||
if content:
|
||||
chat_history.append({
|
||||
"role": role,
|
||||
"parts": [content]
|
||||
})
|
||||
|
||||
# 获取当前消息
|
||||
current_msg = dialogue[-1]["content"]
|
||||
|
||||
# 创建新的聊天会话
|
||||
chat = self.model.start_chat(history=chat_history)
|
||||
|
||||
# 发送消息并获取流式响应
|
||||
response = chat.send_message(
|
||||
current_msg,
|
||||
stream=True,
|
||||
generation_config=self.generation_config
|
||||
)
|
||||
|
||||
# 处理流式响应
|
||||
for chunk in response:
|
||||
if hasattr(chunk, 'text') and chunk.text:
|
||||
yield chunk.text
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}")
|
||||
|
||||
# 针对不同错误返回友好提示
|
||||
if "Rate limit" in error_msg:
|
||||
yield "【Gemini服务请求太频繁,请稍后再试】"
|
||||
elif "Invalid API key" in error_msg:
|
||||
yield "【Gemini API key无效】"
|
||||
else:
|
||||
yield f"【Gemini服务响应异常: {error_msg}】"
|
||||
@@ -1,62 +0,0 @@
|
||||
import requests
|
||||
from requests.exceptions import RequestException
|
||||
from config.logger import setup_logging
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.agent_id = config.get("agent_id") # 对应 agent_id
|
||||
self.api_key = config.get("api_key")
|
||||
self.base_url = config.get("base_url", config.get("url")) # 默认使用 base_url
|
||||
self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
print(dialogue)
|
||||
try:
|
||||
# home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
|
||||
|
||||
# 提取最后一个 role 为 'user' 的 content
|
||||
input_text = None
|
||||
if isinstance(dialogue, list): # 确保 dialogue 是一个列表
|
||||
# 逆序遍历，找到最后一个 role 为 'user' 的消息
|
||||
for message in reversed(dialogue):
|
||||
if message.get("role") == "user": # 找到 role 为 'user' 的消息
|
||||
input_text = message.get("content", "")
|
||||
break # 找到后立即退出循环
|
||||
|
||||
# 构造请求数据
|
||||
payload = {
|
||||
"text": input_text,
|
||||
"agent_id": self.agent_id,
|
||||
"conversation_id": session_id # 使用 session_id 作为 conversation_id
|
||||
}
|
||||
# 设置请求头
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
# 发起 POST 请求
|
||||
response = requests.post(self.api_url, json=payload, headers=headers)
|
||||
|
||||
# 检查请求是否成功
|
||||
response.raise_for_status()
|
||||
|
||||
# 解析返回数据
|
||||
data = response.json()
|
||||
speech = data.get("response", {}).get("speech", {}).get("plain", {}).get("speech", "")
|
||||
|
||||
# 返回生成的内容
|
||||
if speech:
|
||||
yield speech
|
||||
else:
|
||||
logger.bind(tag=TAG).warning("API 返回数据中没有 speech 内容")
|
||||
|
||||
except RequestException as e:
|
||||
logger.bind(tag=TAG).error(f"HTTP 请求错误: {e}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"生成响应时出错: {e}")
|
||||
@@ -1,46 +0,0 @@
|
||||
from config.logger import setup_logging
|
||||
import requests, json
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
|
||||
self.model_name = config.get("model_name")
|
||||
self.base_url = config.get("base_url", "http://localhost:11434")
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
try:
|
||||
# Convert dialogue format to Ollama format
|
||||
prompt = ""
|
||||
for msg in dialogue:
|
||||
if msg["role"] == "system":
|
||||
prompt += f"System: {msg['content']}\n"
|
||||
elif msg["role"] == "user":
|
||||
prompt += f"User: {msg['content']}\n"
|
||||
elif msg["role"] == "assistant":
|
||||
prompt += f"Assistant: {msg['content']}\n"
|
||||
|
||||
# Make request to Ollama API
|
||||
response = requests.post(
|
||||
f"{self.base_url}/api/generate",
|
||||
json={
|
||||
"model": self.model_name,
|
||||
"prompt": prompt,
|
||||
"stream": True
|
||||
},
|
||||
stream=True
|
||||
)
|
||||
|
||||
for line in response.iter_lines():
|
||||
if line:
|
||||
json_response = json.loads(line)
|
||||
if "response" in json_response:
|
||||
yield json_response["response"]
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in Ollama response generation: {e}")
|
||||
yield "【Ollama服务响应异常】"
|
||||
@@ -1,49 +0,0 @@
|
||||
from config.logger import setup_logging
|
||||
import openai
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.model_name = config.get("model_name")
|
||||
self.api_key = config.get("api_key")
|
||||
if 'base_url' in config:
|
||||
self.base_url = config.get("base_url")
|
||||
else:
|
||||
self.base_url = config.get("url")
|
||||
if "你" in self.api_key:
|
||||
logger.bind(tag=TAG).error("你还没配置LLM的密钥，请在配置文件中配置密钥，否则无法正常工作")
|
||||
self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
try:
|
||||
responses = self.client.chat.completions.create(
|
||||
model=self.model_name,
|
||||
messages=dialogue,
|
||||
stream=True
|
||||
)
|
||||
|
||||
is_active = True
|
||||
for chunk in responses:
|
||||
try:
|
||||
# 检查是否存在有效的choice且content不为空
|
||||
delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
|
||||
content = delta.content if hasattr(delta, 'content') else ''
|
||||
except IndexError:
|
||||
content = ''
|
||||
if content:
|
||||
# 处理标签跨多个chunk的情况
|
||||
if '<think>' in content:
|
||||
is_active = False
|
||||
content = content.split('<think>')[0]
|
||||
if '</think>' in content:
|
||||
is_active = True
|
||||
content = content.split('</think>')[-1]
|
||||
if is_active:
|
||||
yield content
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in response generation: {e}")
|
||||
@@ -1,23 +0,0 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class MemoryProviderBase(ABC):
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.role_id = None
|
||||
|
||||
@abstractmethod
|
||||
async def save_memory(self, msgs):
|
||||
"""Save a new memory for specific role and return memory ID"""
|
||||
print("this is base func", msgs)
|
||||
|
||||
@abstractmethod
|
||||
async def query_memory(self, query: str) -> str:
|
||||
"""Query memories for specific role based on similarity"""
|
||||
return "please implement query method"
|
||||
|
||||
def set_role_id(self, role_id: str):
|
||||
self.role_id = role_id
|
||||
@@ -1,74 +0,0 @@
|
||||
from ..base import MemoryProviderBase, logger
|
||||
from mem0 import MemoryClient
|
||||
|
||||
TAG = __name__
|
||||
|
||||
class MemoryProvider(MemoryProviderBase):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.api_key = config.get("api_key", "")
|
||||
self.api_version = config.get("api_version", "v1.1")
|
||||
if len(self.api_key) == 0 or "你" in self.api_key:
|
||||
logger.bind(tag=TAG).error("你还没配置Mem0ai的密钥，请在配置文件中配置密钥，否则无法提供记忆服务")
|
||||
self.use_mem0 = False
|
||||
return
|
||||
else:
|
||||
self.use_mem0 = True
|
||||
self.client = MemoryClient(api_key=self.api_key)
|
||||
|
||||
async def save_memory(self, msgs):
|
||||
if not self.use_mem0:
|
||||
return None
|
||||
if len(msgs) < 2:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Format the content as a message list for mem0
|
||||
messages = [
|
||||
{"role": message.role, "content": message.content}
|
||||
for message in msgs if message.role != "system"
|
||||
]
|
||||
result = self.client.add(messages, user_id=self.role_id, output_format=self.api_version)
|
||||
logger.bind(tag=TAG).debug(f"Save memory result: {result}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"保存记忆失败: {str(e)}")
|
||||
return None
|
||||
|
||||
async def query_memory(self, query: str)-> str:
|
||||
if not self.use_mem0:
|
||||
return ""
|
||||
try:
|
||||
results = self.client.search(
|
||||
query,
|
||||
user_id=self.role_id,
|
||||
output_format=self.api_version
|
||||
)
|
||||
if not results or 'results' not in results:
|
||||
return ""
|
||||
|
||||
# Format each memory entry with its update time up to minutes
|
||||
memories = []
|
||||
for entry in results['results']:
|
||||
timestamp = entry.get('updated_at', '')
|
||||
if timestamp:
|
||||
try:
|
||||
# Parse and reformat the timestamp
|
||||
dt = timestamp.split('.')[0] # Remove milliseconds
|
||||
formatted_time = dt.replace('T', ' ')
|
||||
except:
|
||||
formatted_time = timestamp
|
||||
memory = entry.get('memory', '')
|
||||
if timestamp and memory:
|
||||
# Store tuple of (timestamp, formatted_string) for sorting
|
||||
memories.append((timestamp, f"[{formatted_time}] {memory}"))
|
||||
|
||||
# Sort by timestamp in descending order (newest first)
|
||||
memories.sort(key=lambda x: x[0], reverse=True)
|
||||
|
||||
# Extract only the formatted strings
|
||||
memories_str = "\n".join(f"- {memory[1]}" for memory in memories)
|
||||
logger.bind(tag=TAG).debug(f"Query results: {memories_str}")
|
||||
return memories_str
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"查询记忆失败: {str(e)}")
|
||||
return ""
|
||||
@@ -1,57 +0,0 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
import http.client
|
||||
import urllib.parse
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.appkey = config.get("appkey")
|
||||
self.token = config.get("token")
|
||||
self.format = config.get("format", "wav")
|
||||
self.sample_rate = config.get("sample_rate", 16000)
|
||||
self.voice = config.get("voice", "xiaoyun")
|
||||
self.volume = config.get("volume", 50)
|
||||
self.speech_rate = config.get("speech_rate", 0)
|
||||
self.pitch_rate = config.get("pitch_rate", 0)
|
||||
|
||||
self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
|
||||
self.api_url = f"https://{self.host}/stream/v1/tts"
|
||||
self.header = {
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"appkey": self.appkey,
|
||||
"token": self.token,
|
||||
"text": text,
|
||||
"format": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"voice": self.voice,
|
||||
"volume": self.volume,
|
||||
"speech_rate": self.speech_rate,
|
||||
"pitch_rate": self.pitch_rate
|
||||
}
|
||||
|
||||
print(self.api_url, json.dumps(request_json, ensure_ascii=False))
|
||||
try:
|
||||
resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
|
||||
# 检查返回请求数据的mime类型是否是audio/***，是则保存到指定路径下；返回的是binary格式的
|
||||
if resp.headers['Content-Type'].startswith('audio/'):
|
||||
with open(output_file, 'wb') as f:
|
||||
f.write(resp.content)
|
||||
return output_file
|
||||
else:
|
||||
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
@@ -1,84 +0,0 @@
|
||||
import asyncio
|
||||
from config.logger import setup_logging
|
||||
import os
|
||||
import numpy as np
|
||||
import opuslib_next
|
||||
from pydub import AudioSegment
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class TTSProviderBase(ABC):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
self.delete_audio_file = delete_audio_file
|
||||
self.output_file = config.get("output_file")
|
||||
|
||||
@abstractmethod
|
||||
def generate_filename(self):
|
||||
pass
|
||||
|
||||
def to_tts(self, text):
|
||||
tmp_file = self.generate_filename()
|
||||
try:
|
||||
max_repeat_time = 5
|
||||
while not os.path.exists(tmp_file) and max_repeat_time > 0:
|
||||
asyncio.run(self.text_to_speak(text, tmp_file))
|
||||
if not os.path.exists(tmp_file):
|
||||
max_repeat_time = max_repeat_time - 1
|
||||
logger.bind(tag=TAG).error(f"语音生成失败: {text}:{tmp_file}，再试{max_repeat_time}次")
|
||||
|
||||
if max_repeat_time > 0:
|
||||
logger.bind(tag=TAG).info(f"语音生成成功: {text}:{tmp_file}，重试{5 - max_repeat_time}次")
|
||||
|
||||
return tmp_file
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).info(f"Failed to generate TTS file: {e}")
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
async def text_to_speak(self, text, output_file):
|
||||
pass
|
||||
|
||||
def wav_to_opus_data(self, wav_file_path):
|
||||
# 使用pydub加载PCM文件
|
||||
# 获取文件后缀名
|
||||
file_type = os.path.splitext(wav_file_path)[1]
|
||||
if file_type:
|
||||
file_type = file_type.lstrip('.')
|
||||
audio = AudioSegment.from_file(wav_file_path, format=file_type)
|
||||
|
||||
duration = len(audio) / 1000.0
|
||||
|
||||
# 转换为单声道和16kHz采样率（确保与编码器匹配）
|
||||
audio = audio.set_channels(1).set_frame_rate(16000)
|
||||
|
||||
# 获取原始PCM数据（16位小端）
|
||||
raw_data = audio.raw_data
|
||||
|
||||
# 初始化Opus编码器
|
||||
encoder = opuslib_next.Encoder(16000, 1, opuslib_next.APPLICATION_AUDIO)
|
||||
|
||||
# 编码参数
|
||||
frame_duration = 60 # 60ms per frame
|
||||
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
|
||||
|
||||
opus_datas = []
|
||||
# 按帧处理所有音频数据（包括最后一帧可能补零）
|
||||
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
|
||||
# 获取当前帧的二进制数据
|
||||
chunk = raw_data[i:i + frame_size * 2]
|
||||
|
||||
# 如果最后一帧不足，补零
|
||||
if len(chunk) < frame_size * 2:
|
||||
chunk += b'\x00' * (frame_size * 2 - len(chunk))
|
||||
|
||||
# 转换为numpy数组处理
|
||||
np_frame = np.frombuffer(chunk, dtype=np.int16)
|
||||
|
||||
# 编码Opus数据
|
||||
opus_data = encoder.encode(np_frame.tobytes(), frame_size)
|
||||
opus_datas.append(opus_data)
|
||||
|
||||
return opus_datas, duration
|
||||
@@ -1,38 +0,0 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import base64
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.model = config.get("model")
|
||||
self.access_token = config.get("access_token")
|
||||
self.voice = config.get("voice")
|
||||
self.response_format = config.get("response_format")
|
||||
|
||||
self.host = "api.coze.cn"
|
||||
self.api_url = f"https://{self.host}/v1/audio/speech"
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"model": self.model,
|
||||
"input": text,
|
||||
"voice_id": self.voice,
|
||||
"response_format": self.response_format,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.access_token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
response = requests.request("POST", self.api_url, json=request_json, headers=headers)
|
||||
data = response.content
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(data)
|
||||
@@ -1,60 +0,0 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import base64
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.appid = config.get("appid")
|
||||
self.access_token = config.get("access_token")
|
||||
self.cluster = config.get("cluster")
|
||||
self.voice = config.get("voice")
|
||||
self.api_url = config.get("api_url")
|
||||
self.authorization = config.get("authorization")
|
||||
self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"app": {
|
||||
"appid": f"{self.appid}",
|
||||
"token": "access_token",
|
||||
"cluster": self.cluster
|
||||
},
|
||||
"user": {
|
||||
"uid": "1"
|
||||
},
|
||||
"audio": {
|
||||
"voice_type": self.voice,
|
||||
"encoding": "wav",
|
||||
"speed_ratio": 1.0,
|
||||
"volume_ratio": 1.0,
|
||||
"pitch_ratio": 1.0,
|
||||
},
|
||||
"request": {
|
||||
"reqid": str(uuid.uuid4()),
|
||||
"text": text,
|
||||
"text_type": "plain",
|
||||
"operation": "query",
|
||||
"with_frontend": 1,
|
||||
"frontend_type": "unitTson"
|
||||
}
|
||||
}
|
||||
|
||||
try:
|
||||
resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
|
||||
if "data" in resp.json():
|
||||
data = resp.json()["data"]
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(base64.b64decode(data))
|
||||
else:
|
||||
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
@@ -1,18 +0,0 @@
|
||||
import os
|
||||
import uuid
|
||||
import edge_tts
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.voice = config.get("voice")
|
||||
|
||||
def generate_filename(self, extension=".mp3"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
communicate = edge_tts.Communicate(text, voice=self.voice) # Use your preferred voice
|
||||
await communicate.save(output_file)
|
||||
@@ -1,158 +0,0 @@
|
||||
|
||||
import base64
|
||||
import os
|
||||
import uuid
|
||||
import requests
|
||||
import ormsgpack
|
||||
from pathlib import Path
|
||||
from pydantic import BaseModel, Field, conint, model_validator
|
||||
from typing_extensions import Annotated
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class ServeReferenceAudio(BaseModel):
|
||||
audio: bytes
|
||||
text: str
|
||||
|
||||
@model_validator(mode="before")
|
||||
def decode_audio(cls, values):
|
||||
audio = values.get("audio")
|
||||
if (
|
||||
isinstance(audio, str) and len(audio) > 255
|
||||
): # Check if audio is a string (Base64)
|
||||
try:
|
||||
values["audio"] = base64.b64decode(audio)
|
||||
except Exception as e:
|
||||
# If the audio is not a valid base64 string, we will just ignore it and let the server handle it
|
||||
pass
|
||||
return values
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"ServeReferenceAudio(text={self.text!r}, audio_size={len(self.audio)})"
|
||||
|
||||
class ServeTTSRequest(BaseModel):
|
||||
text: str
|
||||
chunk_length: Annotated[int, conint(ge=100, le=300, strict=True)] = 200
|
||||
# Audio format
|
||||
format: Literal["wav", "pcm", "mp3"] = "wav"
|
||||
# References audios for in-context learning
|
||||
references: list[ServeReferenceAudio] = []
|
||||
# Reference id
|
||||
# For example, if you want use https://fish.audio/m/7f92f8afb8ec43bf81429cc1c9199cb1/
|
||||
# Just pass 7f92f8afb8ec43bf81429cc1c9199cb1
|
||||
reference_id: str | None = None
|
||||
seed: int | None = None
|
||||
use_memory_cache: Literal["on", "off"] = "off"
|
||||
# Normalize text for en & zh, this increase stability for numbers
|
||||
normalize: bool = True
|
||||
# not usually used below
|
||||
streaming: bool = False
|
||||
max_new_tokens: int = 1024
|
||||
top_p: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
|
||||
repetition_penalty: Annotated[float, Field(ge=0.9, le=2.0, strict=True)] = 1.2
|
||||
temperature: Annotated[float, Field(ge=0.1, le=1.0, strict=True)] = 0.7
|
||||
|
||||
class Config:
|
||||
# Allow arbitrary types for pytorch related types
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
|
||||
def audio_to_bytes(file_path):
|
||||
if not file_path or not Path(file_path).exists():
|
||||
return None
|
||||
with open(file_path, "rb") as wav_file:
|
||||
wav = wav_file.read()
|
||||
return wav
|
||||
|
||||
def read_ref_text(ref_text):
|
||||
path = Path(ref_text)
|
||||
if path.exists() and path.is_file():
|
||||
with path.open("r", encoding="utf-8") as file:
|
||||
return file.read()
|
||||
return ref_text
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
|
||||
self.reference_id = config.get("reference_id")
|
||||
self.reference_audio = config.get("reference_audio",[])
|
||||
self.reference_text = config.get("reference_text",[])
|
||||
self.format = config.get("format","wav")
|
||||
self.channels = config.get("channels",1)
|
||||
self.rate = config.get("rate",44100)
|
||||
self.api_key = config.get("api_key","YOUR_API_KEY")
|
||||
if "你" in self.api_key:
|
||||
logger.bind(tag=TAG).error("你还没配置FishSpeech TTS的密钥，请在配置文件中配置密钥，否则无法正常工作")
|
||||
return
|
||||
self.normalize = config.get("normalize",True)
|
||||
self.max_new_tokens = config.get("max_new_tokens",1024)
|
||||
self.chunk_length = config.get("chunk_length",200)
|
||||
self.top_p = config.get("top_p",0.7)
|
||||
self.repetition_penalty = config.get("repetition_penalty",1.2)
|
||||
self.temperature = config.get("temperature",0.7)
|
||||
self.streaming = config.get("streaming",False)
|
||||
self.use_memory_cache = config.get("use_memory_cache","on")
|
||||
self.seed = config.get("seed")
|
||||
self.api_url = config.get("api_url","http://127.0.0.1:8080/v1/tts")
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
# Prepare reference data
|
||||
byte_audios = [audio_to_bytes(ref_audio) for ref_audio in self.reference_audio]
|
||||
ref_texts = [read_ref_text(ref_text) for ref_text in self.reference_text]
|
||||
|
||||
data = {
|
||||
"text": text,
|
||||
"references": [
|
||||
ServeReferenceAudio(
|
||||
audio=audio if audio else b"", text=text
|
||||
)
|
||||
for text, audio in zip(ref_texts, byte_audios)
|
||||
],
|
||||
"reference_id": self.reference_id,
|
||||
"normalize": self.normalize,
|
||||
"format": self.format,
|
||||
"max_new_tokens": self.max_new_tokens,
|
||||
"chunk_length": self.chunk_length,
|
||||
"top_p": self.top_p,
|
||||
"repetition_penalty": self.repetition_penalty,
|
||||
"temperature": self.temperature,
|
||||
"streaming": self.streaming,
|
||||
"use_memory_cache": self.use_memory_cache,
|
||||
"seed": self.seed,
|
||||
}
|
||||
|
||||
pydantic_data = ServeTTSRequest(**data)
|
||||
|
||||
response = requests.post(
|
||||
self.api_url,
|
||||
data=ormsgpack.packb(pydantic_data, option=ormsgpack.OPT_SERIALIZE_PYDANTIC),
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/msgpack",
|
||||
},
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
audio_content = response.content
|
||||
|
||||
with open(output_file, "wb") as audio_file:
|
||||
audio_file.write(audio_content)
|
||||
|
||||
|
||||
|
||||
else:
|
||||
print(f"Request failed with status code {response.status_code}")
|
||||
print(response.json())
|
||||
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import base64
|
||||
import requests
|
||||
from config.logger import setup_logging
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.url = config.get("url")
|
||||
self.text_lang = config.get("text_lang", "zh")
|
||||
self.ref_audio_path = config.get("ref_audio_path")
|
||||
self.prompt_text = config.get("prompt_text")
|
||||
self.prompt_lang = config.get("prompt_lang", "zh")
|
||||
self.top_k = config.get("top_k", 5)
|
||||
self.top_p = config.get("top_p", 1)
|
||||
self.temperature = config.get("temperature", 1)
|
||||
self.text_split_method = config.get("text_split_method", "cut0")
|
||||
self.batch_size = config.get("batch_size", 1)
|
||||
self.batch_threshold = config.get("batch_threshold", 0.75)
|
||||
self.split_bucket = config.get("split_bucket", True)
|
||||
self.return_fragment = config.get("return_fragment", False)
|
||||
self.speed_factor = config.get("speed_factor", 1.0)
|
||||
self.streaming_mode = config.get("streaming_mode", False)
|
||||
self.seed = config.get("seed", -1)
|
||||
self.parallel_infer = config.get("parallel_infer", True)
|
||||
self.repetition_penalty = config.get("repetition_penalty", 1.35)
|
||||
self.aux_ref_audio_paths = config.get("aux_ref_audio_paths", [])
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"text": text,
|
||||
"text_lang": self.text_lang,
|
||||
"ref_audio_path": self.ref_audio_path,
|
||||
"aux_ref_audio_paths": self.aux_ref_audio_paths,
|
||||
"prompt_text": self.prompt_text,
|
||||
"prompt_lang": self.prompt_lang,
|
||||
"top_k": self.top_k,
|
||||
"top_p": self.top_p,
|
||||
"temperature": self.temperature,
|
||||
"text_split_method": self.text_split_method,
|
||||
"batch_size": self.batch_size,
|
||||
"batch_threshold": self.batch_threshold,
|
||||
"split_bucket": self.split_bucket,
|
||||
"return_fragment": self.return_fragment,
|
||||
"speed_factor": self.speed_factor,
|
||||
"streaming_mode": self.streaming_mode,
|
||||
"seed": self.seed,
|
||||
"parallel_infer": self.parallel_infer,
|
||||
"repetition_penalty": self.repetition_penalty
|
||||
}
|
||||
|
||||
resp = requests.post(self.url, json=request_json)
|
||||
if resp.status_code == 200:
|
||||
with open(output_file, "wb") as file:
|
||||
file.write(resp.content)
|
||||
else:
|
||||
logger.bind(tag=TAG).error(f"GPT_SoVITS_V2 TTS请求失败: {resp.status_code} - {resp.text}")
|
||||
@@ -1,77 +0,0 @@
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.group_id = config.get("group_id")
|
||||
self.api_key = config.get("api_key")
|
||||
self.model = config.get("model")
|
||||
self.voice_id = config.get("voice_id")
|
||||
|
||||
default_voice_setting = {
|
||||
"voice_id": "female-shaonv",
|
||||
"speed": 1,
|
||||
"vol": 1,
|
||||
"pitch": 0,
|
||||
"emotion": "happy"
|
||||
}
|
||||
default_pronunciation_dict = {
|
||||
"tone": [
|
||||
"处理/(chu3)(li3)", "危险/dangerous"
|
||||
]
|
||||
}
|
||||
defult_audio_setting = {
|
||||
"sample_rate": 32000,
|
||||
"bitrate": 128000,
|
||||
"format": "mp3",
|
||||
"channel": 1
|
||||
}
|
||||
self.voice_setting = {**default_voice_setting, **config.get("voice_setting", {})}
|
||||
self.pronunciation_dict = {**default_pronunciation_dict, **config.get("pronunciation_dict", {})}
|
||||
self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
|
||||
self.timber_weights = config.get("timber_weights", [])
|
||||
|
||||
if self.voice_id:
|
||||
self.voice_setting["voice_id"] = self.voice_id
|
||||
|
||||
self.host = "api.minimax.chat"
|
||||
self.api_url = f"https://{self.host}/v1/t2a_v2?GroupId={self.group_id}"
|
||||
self.header = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}"
|
||||
}
|
||||
|
||||
def generate_filename(self, extension=".mp3"):
|
||||
return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"model": self.model,
|
||||
"text": text,
|
||||
"stream": False,
|
||||
"voice_setting": self.voice_setting,
|
||||
"pronunciation_dict": self.pronunciation_dict,
|
||||
"audio_setting": self.audio_setting,
|
||||
}
|
||||
|
||||
if type(self.timber_weights) is list and len(self.timber_weights) > 0:
|
||||
request_json["timber_weights"] = self.timber_weights
|
||||
request_json["voice_setting"]["voice_id"] = ""
|
||||
|
||||
try:
|
||||
resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
|
||||
# 检查返回请求数据的status_code是否为0
|
||||
if resp.json()["base_resp"]["status_code"] == 0:
|
||||
data = resp.json()['data']['audio']
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(bytes.fromhex(data))
|
||||
else:
|
||||
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
|
||||
except Exception as e:
|
||||
raise Exception(f"{__name__} error: {e}")
|
||||
@@ -1,39 +0,0 @@
|
||||
import os
|
||||
import uuid
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
self.model = config.get("model")
|
||||
self.access_token = config.get("access_token")
|
||||
self.voice = config.get("voice")
|
||||
self.response_format = config.get("response_format")
|
||||
self.sample_rate = config.get("sample_rate")
|
||||
self.speed = config.get("speed")
|
||||
self.gain = config.get("gain")
|
||||
|
||||
self.host = "api.siliconflow.cn"
|
||||
self.api_url = f"https://{self.host}/v1/audio/speech"
|
||||
|
||||
def generate_filename(self, extension=".wav"):
|
||||
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
|
||||
|
||||
async def text_to_speak(self, text, output_file):
|
||||
request_json = {
|
||||
"model": self.model,
|
||||
"input": text,
|
||||
"voice": self.voice,
|
||||
"response_format": self.response_format,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.access_token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
response = requests.request("POST", self.api_url, json=request_json, headers=headers)
|
||||
data = response.content
|
||||
file_to_save = open(output_file, "wb")
|
||||
file_to_save.write(data)
|
||||
Reference in New Issue
Block a user