Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-22 23:23:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1125 from xinnan-tech/pcm

Browse Source 
客户端上传编码为PCM时，服务端下发PCM格式的音频
This commit is contained in:
hrz
2025-05-08 11:31:49 +08:00
committed by GitHub
parent b246d9e567 64f10b28e7
commit ba8cb8c6f8
15 changed files with 160 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/connection.py
+10 -5
View File
@@ -136,6 +136,8 @@ class ConnectionHandler:
int(self.config.get("close_connection_no_voice_time", 120)) + 60
) # 在原来第一道关闭的基础上加60秒，进行二道关闭
self.audio_format = "opus"
async def handle_connection(self, ws):
try:
# 获取并验证headers
@@ -803,9 +805,12 @@ class ConnectionHandler:
f"TTS生成：文件路径: {tts_file}"
)
if os.path.exists(tts_file):
opus_datas, _ = self.tts.audio_to_opus_data(tts_file)
if self.audio_format == "pcm":
audio_datas, _ = self.tts.audio_to_pcm_data(tts_file)
else:
audio_datas, _ = self.tts.audio_to_opus_data(tts_file)
# 在这里上报TTS数据（使用文件路径）
enqueue_tts_report(self, 2, text, opus_datas)
enqueue_tts_report(self, 2, text, audio_datas)
else:
self.logger.bind(tag=TAG).error(
f"TTS出错：文件不存在{tts_file}"
@@ -816,7 +821,7 @@ class ConnectionHandler:
self.logger.bind(tag=TAG).error(f"TTS出错: {e}")
if not self.client_abort:
# 如果没有中途打断就发送语音
self.audio_play_queue.put((opus_datas, text, text_index))
self.audio_play_queue.put((audio_datas, text, text_index))
if (
self.tts.delete_audio_file
and tts_file is not None
@@ -847,13 +852,13 @@ class ConnectionHandler:
text = None
try:
try:
opus_datas, text, text_index = self.audio_play_queue.get(timeout=1)
audio_datas, text, text_index = self.audio_play_queue.get(timeout=1)
except queue.Empty:
if self.stop_event.is_set():
break
continue
future = asyncio.run_coroutine_threadsafe(
sendAudioMessage(self, opus_datas, text, text_index), self.loop
sendAudioMessage(self, audio_datas, text, text_index), self.loop
)
future.result()
except Exception as e:
main/xiaozhi-server/core/handle/helloHandle.py
+10 -2
View File
@@ -1,5 +1,4 @@
import json
from config.logger import setup_logging
from core.handle.sendAudioHandle import send_stt_message
from core.utils.util import remove_punctuation_and_length
import shutil
@@ -20,7 +19,16 @@ WAKEUP_CONFIG = {
}
async def handleHelloMessage(conn):
async def handleHelloMessage(conn, msg_json):
"""处理hello消息"""
audio_params = msg_json.get("audio_params")
if audio_params:
format = audio_params.get("format")
conn.logger.bind(tag=TAG).info(f"客户端音频格式: {format}")
conn.audio_format = format
conn.asr.set_audio_format(format)
conn.welcome_msg["audio_params"] = audio_params
await conn.websocket.send(json.dumps(conn.welcome_msg))
main/xiaozhi-server/core/handle/receiveAudioHandle.py
+5 -5
View File
@@ -5,7 +5,7 @@ from core.handle.sendAudioHandle import send_stt_message
from core.handle.intentHandler import handle_user_intent
from core.utils.output_counter import check_device_output_limit
from core.handle.ttsReportHandle import enqueue_tts_report
from core.providers.tts.base import audio_to_opus_data
from core.utils.util import audio_to_data
TAG = __name__
@@ -111,7 +111,7 @@ async def max_out_size(conn):
conn.tts_last_text_index = 0
conn.llm_finish_task = True
file_path = "config/assets/max_output_size.wav"
opus_packets, _ = audio_to_opus_data(file_path)
opus_packets, _ = audio_to_data(file_path)
conn.audio_play_queue.put((opus_packets, text, 0))
conn.close_after_chat = True
@@ -133,7 +133,7 @@ async def check_bind_device(conn):
# 播放提示音
music_path = "config/assets/bind_code.wav"
opus_packets, _ = audio_to_opus_data(music_path)
opus_packets, _ = audio_to_data(music_path)
conn.audio_play_queue.put((opus_packets, text, 0))
# 逐个播放数字
@@ -141,7 +141,7 @@ async def check_bind_device(conn):
try:
digit = conn.bind_code[i]
num_path = f"config/assets/bind_code/{digit}.wav"
num_packets, _ = audio_to_opus_data(num_path)
num_packets, _ = audio_to_data(num_path)
conn.audio_play_queue.put((num_packets, None, i + 1))
except Exception as e:
conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
@@ -153,5 +153,5 @@ async def check_bind_device(conn):
conn.tts_last_text_index = 0
conn.llm_finish_task = True
music_path = "config/assets/bind_not_found.wav"
opus_packets, _ = audio_to_opus_data(music_path)
opus_packets, _ = audio_to_data(music_path)
conn.audio_play_queue.put((opus_packets, text, 0))
main/xiaozhi-server/core/handle/sendAudioHandle.py
+1 -1
View File
@@ -115,7 +115,7 @@ async def send_tts_message(conn, state, text=None):
stop_tts_notify_voice = conn.config.get(
"stop_tts_notify_voice", "config/assets/tts_notify.mp3"
)
audios, duration = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
audios, _ = conn.tts.audio_to_opus_data(stop_tts_notify_voice)
await sendAudio(conn, audios)
# 清除服务端讲话状态
conn.clearSpeakStatus()
main/xiaozhi-server/core/handle/textHandle.py
+1 -1
View File
@@ -20,7 +20,7 @@ async def handleTextMessage(conn, message):
await conn.websocket.send(message)
return
if msg_json["type"] == "hello":
await handleHelloMessage(conn)
await handleHelloMessage(conn, msg_json)
elif msg_json["type"] == "abort":
await handleAbortMessage(conn)
elif msg_json["type"] == "listen":
main/xiaozhi-server/core/providers/asr/aliyun.py
+62 -60
View File
@@ -20,62 +20,75 @@ from core.providers.asr.base import ASRProviderBase
TAG = __name__
logger = setup_logging()
class AccessToken:
@staticmethod
def _encode_text(text):
encoded_text = parse.quote_plus(text)
return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
@staticmethod
def _encode_dict(dic):
keys = dic.keys()
dic_sorted = [(key, dic[key]) for key in sorted(keys)]
encoded_text = parse.urlencode(dic_sorted)
return encoded_text.replace('+', '%20').replace('*', '%2A').replace('%7E', '~')
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
@staticmethod
def create_token(access_key_id, access_key_secret):
parameters = {'AccessKeyId': access_key_id,
'Action': 'CreateToken',
'Format': 'JSON',
'RegionId': 'cn-shanghai',
'SignatureMethod': 'HMAC-SHA1',
'SignatureNonce': str(uuid.uuid1()),
'SignatureVersion': '1.0',
'Timestamp': time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
'Version': '2019-02-28'}
parameters = {
"AccessKeyId": access_key_id,
"Action": "CreateToken",
"Format": "JSON",
"RegionId": "cn-shanghai",
"SignatureMethod": "HMAC-SHA1",
"SignatureNonce": str(uuid.uuid1()),
"SignatureVersion": "1.0",
"Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"Version": "2019-02-28",
}
# 构造规范化的请求字符串
query_string = AccessToken._encode_dict(parameters)
# print('规范化的请求字符串: %s' % query_string)
# 构造待签名字符串
string_to_sign = 'GET' + '&' + AccessToken._encode_text('/') + '&' + AccessToken._encode_text(query_string)
string_to_sign = (
"GET"
+ "&"
+ AccessToken._encode_text("/")
+ "&"
+ AccessToken._encode_text(query_string)
)
# print('待签名的字符串: %s' % string_to_sign)
# 计算签名
secreted_string = hmac.new(bytes(access_key_secret + '&', encoding='utf-8'),
bytes(string_to_sign, encoding='utf-8'),
hashlib.sha1).digest()
secreted_string = hmac.new(
bytes(access_key_secret + "&", encoding="utf-8"),
bytes(string_to_sign, encoding="utf-8"),
hashlib.sha1,
).digest()
signature = base64.b64encode(secreted_string)
# print('签名: %s' % signature)
# 进行URL编码
signature = AccessToken._encode_text(signature)
# print('URL编码后的签名: %s' % signature)
# 调用服务
full_url = 'http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s' % (signature, query_string)
full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (
signature,
query_string,
)
# print('url: %s' % full_url)
# 提交HTTP GET请求
response = requests.get(full_url)
if response.ok:
root_obj = response.json()
key = 'Token'
key = "Token"
if key in root_obj:
token = root_obj[key]['Id']
expire_time = root_obj[key]['ExpireTime']
token = root_obj[key]["Id"]
expire_time = root_obj[key]["ExpireTime"]
return token, expire_time
# print(response.text)
return None, None
class ASRProvider(ASRProviderBase):
def __init__(self, config: dict, delete_audio_file: bool):
"""阿里云ASR初始化"""
@@ -102,28 +115,23 @@ class ASRProvider(ASRProviderBase):
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
def _refresh_token(self):
"""刷新Token并记录过期时间"""
if self.access_key_id and self.access_key_secret:
self.token, expire_time_str = AccessToken.create_token(
self.access_key_id,
self.access_key_secret
self.access_key_id, self.access_key_secret
)
if not expire_time_str:
raise ValueError("无法获取有效的Token过期时间")
try:
#统一转换为字符串处理
# 统一转换为字符串处理
expire_str = str(expire_time_str).strip()
if expire_str.isdigit():
expire_time = datetime.fromtimestamp(int(expire_str))
else:
expire_time = datetime.strptime(
expire_str,
"%Y-%m-%dT%H:%M:%SZ"
)
expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ")
self.expire_time = expire_time.timestamp() - 60
except Exception as e:
raise ValueError(f"无效的过期时间格式: {expire_str}") from e
@@ -145,9 +153,12 @@ class ASRProvider(ASRProviderBase):
# f"过期时间 {datetime.fromtimestamp(self.expire_time)} | "
# f"剩余 {remaining:.2f}秒")
return time.time() > self.expire_time
def generate_filename(self, extension=".wav"):
return os.path.join(self.output_file, f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
def generate_filename(self, extension=".wav"):
return os.path.join(
self.output_file,
f"tts-{__name__}{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
)
def _construct_request_url(self) -> str:
"""构造请求URL，包含参数"""
@@ -159,20 +170,6 @@ class ASRProvider(ASRProviderBase):
request += "&enable_voice_detection=false"
return request
def decode_opus(self, opus_data: List[bytes], session_id: str) -> List[bytes]:
"""将Opus数据解码为PCM"""
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return pcm_data
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
module_name = __name__.split(".")[-1]
@@ -183,7 +180,7 @@ class ASRProvider(ASRProviderBase):
wf.setnchannels(1) # 单声道
wf.setsampwidth(2) # 16-bit
wf.setframerate(self.sample_rate)
wf.writeframes(b''.join(pcm_data))
wf.writeframes(b"".join(pcm_data))
logger.bind(tag=TAG).debug(f"音频文件已保存至: {file_path}")
return file_path
@@ -193,9 +190,9 @@ class ASRProvider(ASRProviderBase):
try:
# 设置HTTP头
headers = {
'X-NLS-Token': self.token,
'Content-type': 'application/octet-stream',
'Content-Length': str(len(pcm_data))
"X-NLS-Token": self.token,
"Content-type": "application/octet-stream",
"Content-Length": str(len(pcm_data)),
}
# 创建连接并发送请求
@@ -203,12 +200,12 @@ class ASRProvider(ASRProviderBase):
request_url = self._construct_request_url()
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, lambda: conn.request(
method='POST',
url=request_url,
body=pcm_data,
headers=headers
))
await loop.run_in_executor(
None,
lambda: conn.request(
method="POST", url=request_url, body=pcm_data, headers=headers
),
)
# 获取响应
response = await loop.run_in_executor(None, conn.getresponse)
@@ -218,10 +215,10 @@ class ASRProvider(ASRProviderBase):
# 解析响应
try:
body_json = json.loads(body)
status = body_json.get('status')
status = body_json.get("status")
if status == 20000000:
result = body_json.get('result', '')
result = body_json.get("result", "")
logger.bind(tag=TAG).debug(f"ASR结果: {result}")
return result
else:
@@ -236,7 +233,9 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).error(f"ASR请求失败: {e}", exc_info=True)
return None
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
if self._is_token_expired():
logger.warning("Token已过期，正在自动刷新...")
@@ -245,8 +244,11 @@ class ASRProvider(ASRProviderBase):
file_path = None
try:
# 解码Opus为PCM
pcm_data = self.decode_opus(opus_data, session_id)
combined_pcm_data = b''.join(pcm_data)
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
@@ -264,4 +266,4 @@ class ASRProvider(ASRProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", file_path
return "", file_path
main/xiaozhi-server/core/providers/asr/baidu.py
+4 -17
View File
@@ -49,22 +49,6 @@ class ASRProvider(ASRProviderBase):
return file_path
@staticmethod
def decode_opus(opus_data: List[bytes]) -> bytes:
"""将Opus音频数据解码为PCM数据"""
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return pcm_data
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
@@ -81,7 +65,10 @@ class ASRProvider(ASRProviderBase):
return None, file_path
# 将Opus音频数据解码为PCM
pcm_data = self.decode_opus(opus_data)
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
main/xiaozhi-server/core/providers/asr/base.py
+21 -1
View File
@@ -1,6 +1,6 @@
from abc import ABC, abstractmethod
from typing import Optional, Tuple, List
import opuslib_next
from config.logger import setup_logging
TAG = __name__
@@ -17,3 +17,23 @@ class ASRProviderBase(ABC):
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
pass
def set_audio_format(self, format: str) -> None:
"""设置音频格式"""
self.audio_format = format
@staticmethod
def decode_opus(opus_data: List[bytes]) -> bytes:
"""将Opus音频数据解码为PCM数据"""
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return pcm_data
main/xiaozhi-server/core/providers/asr/doubao.py
+4 -16
View File
@@ -226,21 +226,6 @@ class ASRProvider(ASRProviderBase):
logger.bind(tag=TAG).error(f"ASR request failed: {e}", exc_info=True)
return None
@staticmethod
def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return pcm_data
@staticmethod
def slice_data(data: bytes, chunk_size: int) -> (list, bool):
"""
@@ -265,7 +250,10 @@ class ASRProvider(ASRProviderBase):
file_path = None
try:
# 合并所有opus数据包
pcm_data = self.decode_opus(opus_data, session_id)
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
main/xiaozhi-server/core/providers/asr/fun_local.py
+12 -21
View File
@@ -6,9 +6,7 @@ import io
from config.logger import setup_logging
from typing import Optional, Tuple, List
import uuid
import opuslib_next
from core.providers.asr.base import ASRProviderBase
from funasr import AutoModel
from funasr.utils.postprocess_utils import rich_transcription_postprocess
@@ -46,7 +44,7 @@ class ASRProvider(ASRProviderBase):
model=self.model_dir,
vad_kwargs={"max_single_segment_time": 30000},
disable_update=True,
hub="hf"
hub="hf",
# device="cuda:0", # 启用GPU加速
)
@@ -64,27 +62,18 @@ class ASRProvider(ASRProviderBase):
return file_path
@staticmethod
def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return pcm_data
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""语音转文本主处理逻辑"""
file_path = None
try:
# 合并所有opus数据包
pcm_data = self.decode_opus(opus_data, session_id)
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
@@ -103,7 +92,9 @@ class ASRProvider(ASRProviderBase):
batch_size_s=60,
)
text = rich_transcription_postprocess(result[0]["text"])
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
logger.bind(tag=TAG).debug(
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
)
return text, file_path
main/xiaozhi-server/core/providers/asr/fun_server.py
+4 -18
View File
@@ -44,23 +44,6 @@ class ASRProvider(ASRProviderBase):
return file_path
@staticmethod
def decode_opus(opus_data: List[bytes]) -> bytes:
"""将Opus音频数据解码为PCM数据"""
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return pcm_data
async def _receive_responses(self, ws) -> None:
'''
Asynchronous generator to receive messages from the WebSocket.
@@ -123,7 +106,10 @@ class ASRProvider(ASRProviderBase):
:return: Tuple containing recognized text and optional timestamp.
'''
file_path = None
pcm_data = self.decode_opus(opus_data)
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py
+4 -16
View File
@@ -97,21 +97,6 @@ class ASRProvider(ASRProviderBase):
return file_path
@staticmethod
def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return pcm_data
def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]:
"""
Args:
@@ -144,7 +129,10 @@ class ASRProvider(ASRProviderBase):
try:
# 保存音频文件
start_time = time.time()
pcm_data = self.decode_opus(opus_data, session_id)
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
file_path = self.save_audio_to_file(pcm_data, session_id)
logger.bind(tag=TAG).debug(
f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
main/xiaozhi-server/core/providers/asr/tencent.py
+4 -17
View File
@@ -45,22 +45,6 @@ class ASRProvider(ASRProviderBase):
return file_path
@staticmethod
def decode_opus(opus_data: List[bytes]) -> bytes:
"""将Opus音频数据解码为PCM数据"""
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return pcm_data
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
"""将语音数据转换为文本"""
if not opus_data:
@@ -75,7 +59,10 @@ class ASRProvider(ASRProviderBase):
return None, file_path
# 将Opus音频数据解码为PCM
pcm_data = self.decode_opus(opus_data)
if self.audio_format == "pcm":
pcm_data = opus_data
else:
pcm_data = self.decode_opus(opus_data)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
main/xiaozhi-server/core/providers/tts/base.py
+7 -2
View File
@@ -3,7 +3,7 @@ from config.logger import setup_logging
import os
from abc import ABC, abstractmethod
from core.utils.tts import MarkdownCleaner
from core.utils.util import audio_to_opus_data
from core.utils.util import audio_to_data
TAG = __name__
logger = setup_logging()
@@ -53,5 +53,10 @@ class TTSProviderBase(ABC):
async def text_to_speak(self, text, output_file):
pass
def audio_to_pcm_data(self, audio_file_path):
"""音频文件转换为PCM编码"""
return audio_to_data(audio_file_path, is_opus=False)
def audio_to_opus_data(self, audio_file_path):
return audio_to_opus_data(audio_file_path)
"""音频文件转换为Opus编码"""
return audio_to_data(audio_file_path, is_opus=True)
main/xiaozhi-server/core/utils/util.py
+11 -9
View File
@@ -862,8 +862,7 @@ def analyze_emotion(text):
return top_emotions[0] # 如果都不在优先级列表里，返回第一个
def audio_to_opus_data(audio_file_path):
"""音频文件转换为Opus编码"""
def audio_to_data(audio_file_path, is_opus=True):
# 获取文件后缀名
file_type = os.path.splitext(audio_file_path)[1]
if file_type:
@@ -889,7 +888,7 @@ def audio_to_opus_data(audio_file_path):
frame_duration = 60 # 60ms per frame
frame_size = int(16000 * frame_duration / 1000) # 960 samples/frame
opus_datas = []
datas = []
# 按帧处理所有音频数据（包括最后一帧可能补零）
for i in range(0, len(raw_data), frame_size * 2): # 16bit=2bytes/sample
# 获取当前帧的二进制数据
@@ -899,14 +898,17 @@ def audio_to_opus_data(audio_file_path):
if len(chunk) < frame_size * 2:
chunk += b"\x00" * (frame_size * 2 - len(chunk))
# 转换为numpy数组处理
np_frame = np.frombuffer(chunk, dtype=np.int16)
if is_opus:
# 转换为numpy数组处理
np_frame = np.frombuffer(chunk, dtype=np.int16)
# 编码Opus数据
frame_data = encoder.encode(np_frame.tobytes(), frame_size)
else:
frame_data = chunk if isinstance(chunk, bytes) else bytes(chunk)
# 编码Opus数据
opus_data = encoder.encode(np_frame.tobytes(), frame_size)
opus_datas.append(opus_data)
datas.append(frame_data)
return opus_datas, duration
return datas, duration
def check_vad_update(before_config, new_config):