mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-22 07:03:53 +08:00
update：湾湾小何音色新渠道
This commit is contained in:
hrz
@@ -131,13 +131,6 @@ ASR:
|
||||
access_token: 你的火山引擎语音合成服务access_token
|
||||
cluster: volcengine_input_common
|
||||
output_dir: tmp/
|
||||
GizwitsASR:
|
||||
type: volcengine
|
||||
# 火山引擎作为基座，可以完全使用企业级火山引擎语音识别大模型服务，识别率更准确
|
||||
# 获取API Key地址：https://agentrouter.gizwitsapi.com/panel/token
|
||||
base_host: bytedance.gizwitsapi.com
|
||||
access_token: "你的机智云API key"
|
||||
output_dir: tmp/
|
||||
VAD:
|
||||
SileroVAD:
|
||||
threshold: 0.5
|
||||
@@ -407,17 +400,20 @@ TTS:
|
||||
type: doubao
|
||||
api_url: https://api.302ai.cn/doubao/tts_hd
|
||||
authorization: "Bearer "
|
||||
# 湾湾小何音色
|
||||
voice: "zh_female_wanwanxiaohe_moon_bigtts"
|
||||
output_dir: tmp/
|
||||
access_token: "你的302API密钥"
|
||||
GizwitsTTS:
|
||||
type: doubao
|
||||
# 火山引擎作为基座，可以完全使用企业级火山引擎语音合成服务
|
||||
# 前一万名注册的用户，将送5元体验金额
|
||||
# 获取API Key地址：https://agentrouter.gizwitsapi.com/panel/token
|
||||
api_url: https://bytedance.gizwitsapi.com/api/v1/tts
|
||||
authorization: "Bearer "
|
||||
# 湾湾小何音色
|
||||
voice: "zh_female_wanwanxiaohe_moon_bigtts"
|
||||
output_file: tmp/
|
||||
output_dir: tmp/
|
||||
access_token: "你的机智云API key"
|
||||
ACGNTTS:
|
||||
#在线网址：https://acgn.ttson.cn/
|
||||
|
||||
@@ -1,300 +0,0 @@
|
||||
import time
|
||||
import io
|
||||
import wave
|
||||
import os
|
||||
from typing import Optional, Tuple, List
|
||||
import uuid
|
||||
import websockets
|
||||
import json
|
||||
import gzip
|
||||
|
||||
import opuslib_next
|
||||
from core.utils.util import check_model_key
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
PROTOCOL_VERSION = 0b0001
|
||||
DEFAULT_HEADER_SIZE = 0b0001
|
||||
|
||||
# Message Type:
|
||||
FULL_CLIENT_REQUEST = 0b0001
|
||||
AUDIO_ONLY_REQUEST = 0b0010
|
||||
FULL_SERVER_RESPONSE = 0b1001
|
||||
SERVER_ERROR_RESPONSE = 0b1111
|
||||
|
||||
# Message Type Specific Flags
|
||||
FLAG_NO_SEQUENCE = 0b0000
|
||||
FLAG_WITH_SEQUENCE = 0b0001
|
||||
FLAG_LAST_PACKET = 0b0010
|
||||
|
||||
# Message Serialization
|
||||
NO_SERIALIZATION = 0b0000
|
||||
JSON = 0b0001
|
||||
|
||||
# Message Compression
|
||||
NO_COMPRESSION = 0b0000
|
||||
GZIP = 0b0001
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool):
|
||||
self.host = config.get("base_host", "openspeech.bytedance.com")
|
||||
self.ws_url = f"wss://{self.host}/api/v3/sauc/bigmodel"
|
||||
|
||||
self.appid = config.get("appid")
|
||||
self.access_token = config.get("access_token")
|
||||
self.resource_id = config.get("resource_id", "volc.bigasr.sauc.duration")
|
||||
check_model_key("ASR", self.access_token)
|
||||
|
||||
self.seg_duration = 15000
|
||||
# 确保输出目录存在
|
||||
self.output_dir = config.get("output_dir")
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
|
||||
"""将Opus音频数据解码并保存为WAV文件"""
|
||||
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
|
||||
file_path = os.path.join(self.output_dir, file_name)
|
||||
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
with wave.open(file_path, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2) # 2 bytes = 16-bit
|
||||
wf.setframerate(16000)
|
||||
wf.writeframes(b"".join(pcm_data))
|
||||
|
||||
return file_path
|
||||
|
||||
@staticmethod
|
||||
def _generate_header(message_type=FULL_CLIENT_REQUEST, message_type_specific_flags=FLAG_NO_SEQUENCE,
|
||||
serial_method=JSON, compression_type=GZIP, reserved_data=0x00) -> bytearray:
|
||||
"""
|
||||
protocol_version(4 bits), header_size(4 bits),
|
||||
message_type(4 bits), message_type_specific_flags(4 bits)
|
||||
serialization_method(4 bits) message_compression(4 bits)
|
||||
reserved(8bits) 保留字段
|
||||
"""
|
||||
header = bytearray()
|
||||
header.append((PROTOCOL_VERSION << 4) | DEFAULT_HEADER_SIZE)
|
||||
header.append((message_type << 4) | message_type_specific_flags)
|
||||
header.append((serial_method << 4) | compression_type)
|
||||
header.append(reserved_data)
|
||||
return header
|
||||
|
||||
@staticmethod
|
||||
def _construct_request() -> dict:
|
||||
"""Construct the request payload."""
|
||||
return {
|
||||
"user": {
|
||||
"uid": str(uuid.uuid4()),
|
||||
},
|
||||
"audio": {
|
||||
"format": "wav",
|
||||
"codec": "raw",
|
||||
"rate": 16000,
|
||||
"bits": 16,
|
||||
"channel": 1,
|
||||
"language": "zh-CN",
|
||||
},
|
||||
"request": {
|
||||
"model_name": "bigmodel",
|
||||
"enable_itn": False,
|
||||
"enable_ddc": False,
|
||||
"enable_punc": False,
|
||||
"show_utterances": False,
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _parse_response(res):
|
||||
"""
|
||||
protocol_version(4 bits), header_size(4 bits),
|
||||
message_type(4 bits), message_type_specific_flags(4 bits)
|
||||
serialization_method(4 bits) message_compression(4 bits)
|
||||
reserved(8 bits)保留字段
|
||||
header_extensions 扩展头(大小等于 8 * 4 * (header_size - 1))
|
||||
payload 类似与http 请求体
|
||||
"""
|
||||
protocol_version = res[0] >> 4
|
||||
if protocol_version != PROTOCOL_VERSION:
|
||||
return None
|
||||
header_size = res[0] & 0x0f
|
||||
message_type = res[1] >> 4
|
||||
message_type_specific_flags = res[1] & 0x0f
|
||||
serialization_method = res[2] >> 4
|
||||
message_compression = res[2] & 0x0f
|
||||
payload = res[header_size * 4:]
|
||||
|
||||
result = {}
|
||||
payload_msg = None
|
||||
if message_type == FULL_SERVER_RESPONSE:
|
||||
if message_type_specific_flags & FLAG_WITH_SEQUENCE:
|
||||
# receive frame with sequence
|
||||
result['payload_sequence'] = int.from_bytes(payload[:4], "big", signed=True)
|
||||
payload = payload[4:]
|
||||
if message_type_specific_flags & FLAG_LAST_PACKET:
|
||||
# receive last package
|
||||
result['is_last_package'] = True
|
||||
payload_size = int.from_bytes(payload[:4], "big", signed=True)
|
||||
payload_msg = payload[4:4 + payload_size]
|
||||
elif message_type == SERVER_ERROR_RESPONSE:
|
||||
result['code'] = int.from_bytes(payload[:4], "big", signed=False)
|
||||
payload_size = int.from_bytes(payload[4:8], "big", signed=False)
|
||||
payload_msg = payload[8:8 + payload_size]
|
||||
if payload_msg is not None:
|
||||
if message_compression == GZIP:
|
||||
payload_msg = gzip.decompress(payload_msg)
|
||||
if serialization_method == JSON:
|
||||
payload_msg = json.loads(str(payload_msg, "utf-8"))
|
||||
elif serialization_method != NO_SERIALIZATION:
|
||||
payload_msg = str(payload_msg, "utf-8")
|
||||
result['payload_msg'] = payload_msg
|
||||
return result
|
||||
|
||||
async def _send_request(self, audio_data: List[bytes], segment_size: int) -> Optional[str]:
|
||||
"""Send request to VolcEngine ASR service."""
|
||||
try:
|
||||
auth_header = {
|
||||
"X-Api-App-Key": self.appid,
|
||||
"X-Api-Access-Key": self.access_token,
|
||||
"X-Api-Resource-Id": self.resource_id,
|
||||
"X-Api-Request-Id": str(uuid.uuid4())
|
||||
}
|
||||
async with websockets.connect(self.ws_url, additional_headers=auth_header,
|
||||
max_size=128 * 1024 * 1024) as websocket:
|
||||
sequence = 1
|
||||
|
||||
# Send header and metadata
|
||||
request_params = self._construct_request()
|
||||
payload_bytes = str.encode(json.dumps(request_params))
|
||||
payload_bytes = gzip.compress(payload_bytes)
|
||||
full_client_request = self._generate_header(message_type_specific_flags=FLAG_WITH_SEQUENCE)
|
||||
full_client_request.extend(sequence.to_bytes(4, 'big', signed=True))
|
||||
full_client_request.extend((len(payload_bytes)).to_bytes(4, 'big')) # payload size(4 bytes)
|
||||
full_client_request.extend(payload_bytes) # payload
|
||||
await websocket.send(full_client_request)
|
||||
res = await websocket.recv()
|
||||
result = self._parse_response(res)
|
||||
if 'code' in result:
|
||||
logger.bind(tag=TAG).error(f"ASR error: {result['payload_msg']}")
|
||||
return None
|
||||
|
||||
for _, (chunk, last) in enumerate(self.slice_data(audio_data, segment_size), 1):
|
||||
sequence += 1
|
||||
if last:
|
||||
sequence = -sequence # last package
|
||||
audio_only_request = self._generate_header(
|
||||
message_type=AUDIO_ONLY_REQUEST,
|
||||
message_type_specific_flags=FLAG_WITH_SEQUENCE | FLAG_LAST_PACKET
|
||||
)
|
||||
else:
|
||||
audio_only_request = self._generate_header(
|
||||
message_type=AUDIO_ONLY_REQUEST,
|
||||
message_type_specific_flags=FLAG_WITH_SEQUENCE
|
||||
)
|
||||
audio_only_request.extend(sequence.to_bytes(4, 'big', signed=True)) # sequence
|
||||
payload_bytes = gzip.compress(chunk)
|
||||
audio_only_request.extend((len(payload_bytes)).to_bytes(4, 'big')) # payload size(4 bytes)
|
||||
audio_only_request.extend(payload_bytes) # payload
|
||||
# Send audio data
|
||||
await websocket.send(audio_only_request)
|
||||
|
||||
# Receive response
|
||||
for _ in range(1, -sequence):
|
||||
response = await websocket.recv()
|
||||
result = self._parse_response(response)
|
||||
if 'code' in result:
|
||||
logger.bind(tag=TAG).error(f"ASR error: {result['payload_msg']}")
|
||||
return None
|
||||
if 'is_last_package' in result and result['is_last_package'] is True:
|
||||
return result.get('payload_msg', {}).get('result', {}).get('text')
|
||||
|
||||
raise Exception("not received last package")
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
|
||||
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
|
||||
pcm_data = []
|
||||
|
||||
for opus_packet in opus_data:
|
||||
try:
|
||||
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
|
||||
pcm_data.append(pcm_frame)
|
||||
except opuslib_next.OpusError as e:
|
||||
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
|
||||
|
||||
return pcm_data
|
||||
|
||||
@staticmethod
|
||||
def read_wav_info(data: io.BytesIO = None) -> (int, int, int, int, int):
|
||||
with io.BytesIO(data) as _f:
|
||||
wave_fp = wave.open(_f, 'rb')
|
||||
nchannels, sampwidth, framerate, nframes = wave_fp.getparams()[:4]
|
||||
wave_bytes = wave_fp.readframes(nframes)
|
||||
return nchannels, sampwidth, framerate, nframes, len(wave_bytes)
|
||||
|
||||
@staticmethod
|
||||
def slice_data(data: bytes, chunk_size: int) -> (list, bool):
|
||||
"""
|
||||
slice data
|
||||
:param data: wav data
|
||||
:param chunk_size: the segment size in one request
|
||||
:return: segment data, last flag
|
||||
"""
|
||||
data_len = len(data)
|
||||
offset = 0
|
||||
while offset + chunk_size < data_len:
|
||||
yield data[offset: offset + chunk_size], False
|
||||
offset += chunk_size
|
||||
else:
|
||||
yield data[offset: data_len], True
|
||||
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""将语音数据转换为文本"""
|
||||
try:
|
||||
# 合并所有opus数据包
|
||||
pcm_data = self.decode_opus(opus_data, session_id)
|
||||
combined_pcm_data = b''.join(pcm_data)
|
||||
|
||||
wav_buffer = io.BytesIO()
|
||||
|
||||
with wave.open(wav_buffer, "wb") as wav_file:
|
||||
wav_file.setnchannels(1) # 设置声道数
|
||||
wav_file.setsampwidth(2) # 设置采样宽度
|
||||
wav_file.setframerate(16000) # 设置采样率
|
||||
wav_file.writeframes(combined_pcm_data) # 写入 PCM 数据
|
||||
|
||||
# 获取封装后的 WAV 数据
|
||||
wav_data = wav_buffer.getvalue()
|
||||
nchannels, sampwidth, framerate, nframes, wav_len = self.read_wav_info(wav_data)
|
||||
size_per_sec = nchannels * sampwidth * framerate
|
||||
segment_size = int(size_per_sec * self.seg_duration / 1000)
|
||||
|
||||
# 语音识别
|
||||
start_time = time.time()
|
||||
text = await self._send_request(wav_data, segment_size)
|
||||
if text:
|
||||
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
|
||||
return text, None
|
||||
return "", None
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
|
||||
return "", None
|
||||
Reference in New Issue
Block a user