Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-26 17:13:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2552 from Packeting1/multi_lang_tts

Browse Source 
feat: 为阿里百炼流式TTS支持多语言音色
This commit is contained in:
Sakura-RanChen
2026-01-04 15:02:24 +08:00
committed by GitHub
parent a2023a65c0 6cdbe45313
commit 85919e800e
8 changed files with 160 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/providers/asr/fun_local.py
+5 -2
View File
@@ -8,8 +8,9 @@ import asyncio
from config.logger import setup_logging
from typing import Optional, Tuple, List
from core.providers.asr.base import ASRProviderBase
from core.providers.asr.utils import lang_tag_filter
from funasr import AutoModel
from funasr.utils.postprocess_utils import rich_transcription_postprocess
from core.providers.asr.base import ASRProviderBase
from core.providers.asr.dto.dto import InterfaceType
@@ -102,7 +103,9 @@ class ASRProvider(ASRProviderBase):
use_itn=True,
batch_size_s=60,
)
text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"])
# text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"])
# 使用lang_tag_filter处理识别结果
text = await asyncio.to_thread(lang_tag_filter, result[0]["text"])
logger.bind(tag=TAG).debug(
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
)
main/xiaozhi-server/core/providers/asr/fun_server.py
+8 -4
View File
@@ -1,12 +1,12 @@
from typing import Optional, Tuple, List
from core.providers.asr.base import ASRProviderBase
from core.providers.asr.utils import lang_tag_filter
from core.providers.asr.dto.dto import InterfaceType
import ssl
import json
import websockets
from config.logger import setup_logging
import asyncio
import re
TAG = __name__
logger = setup_logging()
@@ -151,9 +151,13 @@ class ASRProvider(ASRProviderBase):
# Get the result from the receive task
result = receive_task.result()
match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result)
if match:
result = match.group(4).strip()
# match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result)
# if match:
# result = match.group(4).strip()
# Handle language tags
result = lang_tag_filter(result)
return (
result,
file_path,
main/xiaozhi-server/core/providers/asr/utils.py
+43
View File
@@ -0,0 +1,43 @@
import re
from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
def lang_tag_filter(text):
"""
过滤函数：只保留语言标签，移除其他所有标签
用于FunASR识别结果的处理，保留语言标签（如<|zh|>、<|en|>等），
但移除其他所有格式的标签（如时间戳、情感标签等）
Args:
text: ASR识别的原始文本，可能包含多种标签
Returns:
str: 处理后的文本，只保留语言标签（如果存在）
Examples:
>>> lang_tag_filter("<|zh|><|emotion:happy|>你好")
'<|zh|>你好'
>>> lang_tag_filter("<|en|>hello world")
'<|en|>hello world'
"""
# 定义语言标签模式
lang_pattern = r"<\|(zh|en|yue|ja|ko|nospeech)\|>"
lang_tags = re.findall(lang_pattern, text)
# 移除所有 < | ... | > 格式的标签
clean_text = re.sub(r"<\|.*?\|>", "", text)
# 在开头添加语言标签（如果存在）
if lang_tags:
if len(lang_tags) > 1:
logger.bind(tag=TAG).warning(
f"检测到多个语言标签: {lang_tags}，仅使用第一个: {lang_tags[0]}"
)
clean_text = f"<|{lang_tags[0]}|>{clean_text}"
return clean_text.strip()
main/xiaozhi-server/core/providers/tts/alibl_stream.py
+48 -2
View File
@@ -39,6 +39,13 @@ class TTSProvider(TTSProviderBase):
if config.get("private_voice"):
self.voice = config.get("private_voice")
# 多语言音色配置
self.voice_zh = config.get("voice_zh", self.voice) # 中文音色
self.voice_yue = config.get("voice_yue", self.voice) # 粤语音色
self.voice_en = config.get("voice_en", self.voice) # 英语音色
self.voice_ja = config.get("voice_ja", self.voice) # 日语音色
self.voice_ko = config.get("voice_ko", self.voice) # 韩语音色
# 音频参数配置
self.format = config.get("format", "pcm")
sample_rate = config.get("sample_rate", "24000")
@@ -65,6 +72,36 @@ class TTSProvider(TTSProviderBase):
sample_rate=self.sample_rate, channels=1, frame_size_ms=60
)
def get_voice_by_language(self, language_tag):
"""根据语言标签返回对应的音色（仅在FunASR语音识别时生效）"""
if not language_tag:
return self.voice
# 检查当前ASR是否为FunASR
is_funasr = False
if hasattr(self, 'conn') and self.conn and hasattr(self.conn, 'asr') and self.conn.asr:
asr_module = self.conn.asr.__class__.__module__
if 'fun_local' in asr_module or 'fun_server' in asr_module:
is_funasr = True
# 只有在使用FunASR时才应用多语言音色选择
if is_funasr:
language_tag = language_tag.lower()
voice_map = {
'zh': self.voice_zh,
'yue': self.voice_yue,
'en': self.voice_en,
'ja': self.voice_ja,
'ko': self.voice_ko
}
selected_voice = voice_map.get(language_tag, self.voice)
logger.bind(tag=TAG).info(f"FunASR语言标签 '{language_tag}' 选择音色: {selected_voice}")
return selected_voice
else:
# 非FunASR时使用默认音色
return self.voice
async def _ensure_connection(self):
"""确保WebSocket连接可用，支持60秒内连接复用"""
try:
@@ -228,6 +265,9 @@ class TTSProvider(TTSProviderBase):
# 启动监听任务
self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
# 根据当前语言标签选择音色
current_voice = self.get_voice_by_language(getattr(self.conn, 'current_language_tag', None))
# 发送run-task消息启动会话
run_task_message = {
"header": {
@@ -242,7 +282,7 @@ class TTSProvider(TTSProviderBase):
"model": self.model,
"parameters": {
"text_type": "PlainText",
"voice": self.voice,
"voice": current_voice,
"format": self.format,
"sample_rate": self.sample_rate,
"volume": self.volume,
@@ -412,6 +452,12 @@ class TTSProvider(TTSProviderBase):
)
try:
# 选择音色：优先使用当前连接的语言标签，否则使用默认音色
if hasattr(self, 'conn') and self.conn and hasattr(self.conn, 'current_language_tag'):
current_voice = self.get_voice_by_language(self.conn.current_language_tag)
else:
current_voice = self.voice
# 发送run-task消息启动会话
run_task_message = {
"header": {
@@ -426,7 +472,7 @@ class TTSProvider(TTSProviderBase):
"model": self.model,
"parameters": {
"text_type": "PlainText",
"voice": self.voice,
"voice": current_voice,
"format": self.format,
"sample_rate": self.sample_rate,
"volume": self.volume,