Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-27 09:33:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1062 from xinnan-tech/test-pr

Browse Source 
合并多个提交
This commit is contained in:
欣南科技
2025-04-29 18:06:24 +08:00
committed by GitHub
parent 259886c1a8 ef662e0ccf
commit bb2143d9a0
10 changed files with 759 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/config.yaml
+18 -6
View File
@@ -209,6 +209,18 @@ ASR:
type: fun_local
model_dir: models/SenseVoiceSmall
output_dir: tmp/
FunASRServer:
# 独立部署FunASR，使用FunASR的API服务，只需要五句话
# 第一句：mkdir -p ./funasr-runtime-resources/models
# 第二句：sudo docker run -d -p 10096:10095 --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12
# 上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime
# 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 &
# 上一句话执行后会进入到容器，继续第五句：tail -f log.txt
# 第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了
# 以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md
type: fun_server
host: 127.0.0.1
port: 10096
SherpaASR:
type: sherpa_onnx_local
model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17
@@ -266,12 +278,12 @@ LLM:
DoubaoLLM:
# 定义LLM API类型
type: openai
# 先开通服务，打开以下网址，开通的服务搜索Doubao-pro-32k，开通它
# 先开通服务，打开以下网址，开通的服务搜索Doubao-1.5-pro，开通它
# 开通改地址：https://console.volcengine.com/ark/region:ark+cn-beijing/openManagement?LLM=%7B%7D&OpenTokenDrawer=false
# 免费额度500000token
# 开通后，进入这里获取密钥：https://console.volcengine.com/ark/region:ark+cn-beijing/apiKey?apikey=%7B%7D
base_url: https://ark.cn-beijing.volces.com/api/v3
model_name: doubao-pro-32k-functioncall-241028
model_name: doubao-1-5-pro-32k-250115
api_key: 你的doubao web key
DeepSeekLLM:
# 定义LLM API类型
@@ -447,12 +459,12 @@ TTS:
GPT_SOVITS_V2:
# 定义TTS API类型
#启动tts方法：
#python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/caixukun.yaml
#python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/demo.yaml
type: gpt_sovits_v2
url: "http://127.0.0.1:9880/tts"
output_dir: tmp/
text_lang: "auto"
ref_audio_path: "caixukun.wav"
ref_audio_path: "demo.wav"
prompt_text: ""
prompt_lang: "zh"
top_k: 5
@@ -585,8 +597,8 @@ TTS:
ACGNTTS:
#在线网址：https://acgn.ttson.cn/
#token购买：www.ttson.cn
#开发相关疑问请提交至3497689533@qq.com
#角色id获取地址：ctrl+f快速检索角色——网站管理者不允许发布,可询问网站管理者：1069379506
#开发相关疑问请提交至网站上的qq
#角色id获取地址：ctrl+f快速检索角色——网站管理者不允许发布,可询问网站管理者
#各参数意义见开发文档：https://www.yuque.com/alexuh/skmti9/wm6taqislegb02gd?singleDoc#
type: ttson
token: your_token
main/xiaozhi-server/core/providers/asr/base.py
+2 -2
View File
@@ -9,8 +9,8 @@ logger = setup_logging()
class ASRProviderBase(ABC):
@abstractmethod
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""解码Opus数据并保存为WAV文件"""
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
pass
@abstractmethod
main/xiaozhi-server/core/providers/asr/doubao.py
+9 -12
View File
@@ -89,6 +89,7 @@ class ASRProvider(ASRProviderBase):
self.cluster = config.get("cluster")
self.access_token = config.get("access_token")
self.output_dir = config.get("output_dir")
self.delete_audio_file = delete_audio_file
self.host = "openspeech.bytedance.com"
self.ws_url = f"wss://{self.host}/api/v2/asr"
@@ -98,21 +99,11 @@ class ASRProvider(ASRProviderBase):
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""将Opus音频数据解码并保存为WAV文件"""
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
@@ -274,6 +265,12 @@ class ASRProvider(ASRProviderBase):
pcm_data = self.decode_opus(opus_data, session_id)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
# 直接使用PCM数据
# 计算分段大小 (单声道, 16bit, 16kHz采样率)
size_per_sec = 1 * 2 * 16000 # nchannels * sampwidth * framerate
main/xiaozhi-server/core/providers/asr/fun_local.py
+26 -6
View File
@@ -51,7 +51,7 @@ class ASRProvider(ASRProviderBase):
)
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""将Opus音频数据解码并保存为WAV文件"""
"""PCM数据保存为WAV文件"""
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
@@ -72,20 +72,40 @@ class ASRProvider(ASRProviderBase):
wf.writeframes(b"".join(pcm_data))
return file_path
@staticmethod
def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return pcm_data
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
"""语音转文本主处理逻辑"""
file_path = None
try:
# 保存音频文件
start_time = time.time()
file_path = self.save_audio_to_file(opus_data, session_id)
logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
# 合并所有opus数据包
pcm_data = self.decode_opus(opus_data, session_id)
combined_pcm_data = b"".join(pcm_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
# 语音识别
start_time = time.time()
result = self.model.generate(
input=file_path,
input=combined_pcm_data,
cache={},
language="auto",
use_itn=True,
main/xiaozhi-server/core/providers/asr/fun_server.py
+144
View File
@@ -0,0 +1,144 @@
from typing import Optional, Tuple, List
import opuslib_next
from core.providers.asr.base import ASRProviderBase
import ssl
import json
import websockets
from config.logger import setup_logging
import asyncio
TAG = __name__
logger = setup_logging()
class ASRProvider(ASRProviderBase):
def __init__(self, config: dict, delete_audio_file: bool):
'''
Initialize the ASRProvider with server configuration.
:param config: Dictionary containing 'host', 'port', and 'is_ssl'.
:param delete_audio_file: Boolean to indicate whether to delete audio files after processing.
'''
self.host = config.get('host', 'localhost')
self.port = config.get('port', 10095)
self.is_ssl = config.get('is_ssl', True)
self.delete_audio_file = delete_audio_file
self.uri = f"wss://{self.host}:{self.port}" if self.is_ssl else f"ws://{self.host}:{self.port}"
self.ssl_context = ssl.SSLContext() if self.is_ssl else None
if self.ssl_context:
self.ssl_context.check_hostname = False
self.ssl_context.verify_mode = ssl.CERT_NONE
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""解码Opus数据并保存为WAV文件"""
pass
@staticmethod
def decode_opus(opus_data: List[bytes]) -> bytes:
"""将Opus音频数据解码为PCM数据"""
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
return b"".join(pcm_data)
async def _receive_responses(self, ws) -> None:
'''
Asynchronous generator to receive messages from the WebSocket.
Yields each message as it is received.
'''
text = ""
while True:
try:
response = await asyncio.wait_for(ws.recv(), timeout=5)
response_data = json.loads(response)
logger.bind(tag=TAG).debug(f"Received response: {response_data}")
if response_data.get("is_final", True):
text += response_data.get("text", "")
break
else:
text += response_data.get("text", "")
except asyncio.TimeoutError:
logger.bind(tag=TAG).error("Timeout while waiting for response from WebSocket.")
break
except websockets.exceptions.ConnectionClosed as e:
logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
break
return text
async def _send_data(self, ws, pcm_data: bytes, session_id: str) -> tuple:
'''
Internal method to handle WebSocket communication.
Reuses the persistent WebSocket connection if available.
:param pcm_data: PCM audio data to send.
:param session_id: Unique session identifier.
:return: Tuple containing recognized text and optional timestamp.
'''
# Send initial configuration message
config_message = json.dumps({
"mode": "offline",
"chunk_size": [5, 10, 5],
"chunk_interval": 10,
"wav_name": session_id,
"is_speaking": True,
"itn": False
})
await ws.send(config_message)
logger.bind(tag=TAG).debug(f"Sent configuration message: {config_message}")
# Send PCM data
await ws.send(pcm_data)
logger.bind(tag=TAG).debug(f"Sent PCM data of length: {len(pcm_data)} bytes")
# Indicate end of speech
end_message = json.dumps({"is_speaking": False})
await ws.send(end_message)
logger.bind(tag=TAG).debug(f"Sent end message: {end_message}")
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
'''
Convert speech data to text using FunASR.
:param opus_data: List of Opus-encoded audio data chunks.
:param session_id: Unique session identifier.
:return: Tuple containing recognized text and optional timestamp.
'''
pcm_data = self.decode_opus(opus_data)
async with websockets.connect(self.uri, subprotocols=["binary"], ping_interval=None, ssl=self.ssl_context) as ws:
try:
# Use asyncio to handle WebSocket communication
send_task = asyncio.create_task(self._send_data(ws, pcm_data, session_id))
receive_task = asyncio.create_task(self._receive_responses(ws))
# Gather tasks with error handling
done, pending = await asyncio.wait(
[send_task, receive_task],
return_when=asyncio.FIRST_EXCEPTION
)
# Cancel any pending tasks
for task in pending:
task.cancel()
# Check for exceptions in completed tasks
for task in done:
if task.exception():
raise task.exception()
# Get the result from the receive task
result = receive_task.result()
return result, None # Return the recognized text and timestamp (if any)
except websockets.exceptions.ConnectionClosed as e:
logger.bind(tag=TAG).error(f"WebSocket connection closed: {e}")
return "", None
except Exception as e:
logger.bind(tag=TAG).error(f"Error during speech-to-text conversion: {e}", exc_info=True)
return "", None
main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py
+31 -20
View File
@@ -43,11 +43,11 @@ class ASRProvider(ASRProviderBase):
# 确保输出目录存在
os.makedirs(self.output_dir, exist_ok=True)
# 初始化模型文件路径
model_files = {
"model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"),
"tokens.txt": os.path.join(self.model_dir, "tokens.txt")
"tokens.txt": os.path.join(self.model_dir, "tokens.txt"),
}
# 下载并检查模型文件
@@ -58,15 +58,15 @@ class ASRProvider(ASRProviderBase):
model_file_download(
model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue",
file_path=file_name,
local_dir=self.model_dir
local_dir=self.model_dir,
)
if not os.path.isfile(file_path):
raise FileNotFoundError(f"模型文件下载失败: {file_path}")
self.model_path = model_files["model.int8.onnx"]
self.tokens_path = model_files["tokens.txt"]
except Exception as e:
logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}")
raise
@@ -83,11 +83,22 @@ class ASRProvider(ASRProviderBase):
use_itn=True,
)
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""将Opus音频数据解码并保存为WAV文件"""
def save_audio_to_file(self, pcm_data: List[bytes], session_id: str) -> str:
"""PCM数据保存为WAV文件"""
file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
wf.setframerate(16000)
wf.writeframes(b"".join(pcm_data))
return file_path
@staticmethod
def decode_opus(opus_data: List[bytes], session_id: str) -> List[bytes]:
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
@@ -98,13 +109,7 @@ class ASRProvider(ASRProviderBase):
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
wf.setframerate(16000)
wf.writeframes(b"".join(pcm_data))
return file_path
return pcm_data
def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]:
"""
@@ -130,14 +135,19 @@ class ASRProvider(ASRProviderBase):
samples_float32 = samples_float32 / 32768
return samples_float32, f.getframerate()
async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
async def speech_to_text(
self, opus_data: List[bytes], session_id: str
) -> Tuple[Optional[str], Optional[str]]:
"""语音转文本主处理逻辑"""
file_path = None
try:
# 保存音频文件
start_time = time.time()
file_path = self.save_audio_to_file(opus_data, session_id)
logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
pcm_data = self.decode_opus(opus_data, session_id)
file_path = self.save_audio_to_file(pcm_data, session_id)
logger.bind(tag=TAG).debug(
f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
)
# 语音识别
start_time = time.time()
@@ -146,14 +156,15 @@ class ASRProvider(ASRProviderBase):
s.accept_waveform(sample_rate, samples)
self.model.decode_stream(s)
text = s.result.text
logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
logger.bind(tag=TAG).debug(
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
)
return text, file_path
except Exception as e:
logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
return "", None
finally:
# 文件清理逻辑
if self.delete_audio_file and file_path and os.path.exists(file_path):
main/xiaozhi-server/core/providers/asr/tencent.py
+7 -11
View File
@@ -31,21 +31,11 @@ class ASRProvider(ASRProviderBase):
os.makedirs(self.output_dir, exist_ok=True)
def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
"""将Opus音频数据解码并保存为WAV文件"""
"""PCM数据保存为WAV文件"""
file_name = f"tencent_asr_{session_id}_{uuid.uuid4()}.wav"
file_path = os.path.join(self.output_dir, file_name)
decoder = opuslib_next.Decoder(16000, 1) # 16kHz, 单声道
pcm_data = []
for opus_packet in opus_data:
try:
pcm_frame = decoder.decode(opus_packet, 960) # 960 samples = 60ms
pcm_data.append(pcm_frame)
except opuslib_next.OpusError as e:
logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
with wave.open(file_path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2) # 2 bytes = 16-bit
@@ -85,6 +75,12 @@ class ASRProvider(ASRProviderBase):
# 将Opus音频数据解码为PCM
pcm_data = self.decode_opus(opus_data)
# 判断是否保存为WAV文件
if self.delete_audio_file:
pass
else:
self.save_audio_to_file(pcm_data, session_id)
# 将音频数据转换为Base64编码
base64_audio = base64.b64encode(pcm_data).decode('utf-8')