diff --git a/main/manager-api/src/main/resources/db/changelog/202510141200.sql b/main/manager-api/src/main/resources/db/changelog/202510141200.sql deleted file mode 100644 index 9c97a413..00000000 --- a/main/manager-api/src/main/resources/db/changelog/202510141200.sql +++ /dev/null @@ -1,14 +0,0 @@ --- 为阿里百炼流式语音合成添加多语言音色配置字段 -UPDATE `ai_model_provider` SET fields = '[{"key":"api_key","type":"string","label":"API密钥"},{"key":"output_dir","type":"string","label":"输出目录"},{"key":"model","type":"string","label":"模型名称"},{"key":"format","label":"音频格式","type":"string"},{"key":"sample_rate","label":"采样率","type":"number"},{"key": "volume", "type": "number", "label": "音量"},{"key": "rate", "type": "number", "label": "语速"},{"key": "pitch", "type": "number", "label": "音调"},{"key":"voice","type":"string","label":"默认音色"},{"key": "voice_zh", "type": "string", "label": "中文音色"},{"key": "voice_yue", "type": "string", "label": "粤语音色"},{"key": "voice_en", "type": "string", "label": "英语音色"},{"key": "voice_ja", "type": "string", "label": "日语音色"},{"key": "voice_ko", "type": "string", "label": "韩语音色"}]' WHERE id = 'SYSTEM_TTS_AliBLStreamTTS'; - --- 更新配置说明 -UPDATE `ai_model_config` SET -`doc_link` = 'https://bailian.console.aliyun.com/?apiKey=1#/api-key', -`remark` = '阿里百炼流式TTS说明： -1. 访问 https://bailian.console.aliyun.com/?apiKey=1#/api-key 创建项目并获取appkey -2. 支持实时流式合成，具有较低的延迟 -3. 支持多种音色设置和音频参数调整 -4. 使用FunASR进行语音识别时，可以自动选择对应语言音色 -5. 支持CosyVoice-V3-Flash等大模型音色，价格实惠(1元/万字符) -6. 支持实时调节音量、语速、音调等参数 -' WHERE `id` = 'TTS_AliBLStreamTTS'; \ No newline at end of file diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 3e1359bc..9009eef8 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -887,16 +887,6 @@ TTS: access_key_secret: 你的阿里云账号access_key_secret # 截至2025年7月21日大模型音色只有北京节点采用，其他节点暂不支持 host: nls-gateway-cn-beijing.aliyuncs.com - - # 多语言音色配置 - 根据ASR识别的语言标签自动切换音色 - # 多语言仅限搭配FunASR SenseVoiceSmall模型使用 - voice_zh: longxiaochun_v2 # 中文音色 - voice_en: loongeva_v2 # 英文音色 - voice_yue: longjiayi_v2 # 粤语音色 - voice_ja: oongtomoka_v2 # 日语音色 - voice_ko: loongkyong_v2 # 韩语音色 - default_voice: longxiaochun_v2 # 默认音色（当语言标签不匹配或无语言标签时使用） - # 以下可不用设置，使用默认设置 # format: pcm # 音频格式：pcm、wav、mp3 # sample_rate: 16000 # 采样率：8000、16000、24000 diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 23e2d2b1..449fbd2d 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -130,6 +130,7 @@ class ConnectionHandler: # 所以涉及到ASR的变量，需要在这里定义，属于connection的私有变量 self.asr_audio = [] self.asr_audio_queue = queue.Queue() + self.current_speaker = None # 存储当前说话人 self.current_language_tag = None # 存储当前ASR识别的语言标签 # llm相关变量 diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 42861bf2..ec1d3bc7 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -1,4 +1,3 @@ -import re import time import json import asyncio @@ -41,33 +40,8 @@ async def resume_vad_detection(conn): async def startToChat(conn, text): # 检查输入是否是JSON格式（包含说话人信息） speaker_name = None - actual_text = text language_tag = None - - # 检查当前使用的ASR是否为FunASR（本地或服务版本） - is_funasr = False - if hasattr(conn, 'asr') and conn.asr: - asr_module = conn.asr.__class__.__module__ - if 'fun_local' in asr_module or 'fun_server' in asr_module: - is_funasr = True - conn.logger.bind(tag=TAG).debug(f"检测到FunASR语音识别: {asr_module}") - - # 只有在使用FunASR时才处理语言标签 - if is_funasr: - # 检查是否包含语言标签（如<|zh|>、<|en|>等） - lang_pattern = r'<\|([a-z]{2,3})\|>' - lang_match = re.search(lang_pattern, text) - if lang_match: - language_tag = lang_match.group(1) - conn.current_language_tag = language_tag - conn.logger.bind(tag=TAG).info(f"检测到FunASR语言标签: {language_tag}") - - # 移除语言标签，保留纯文本内容 - actual_text = re.sub(lang_pattern, '', text).strip() - conn.logger.bind(tag=TAG).debug(f"移除语言标签后的文本: {actual_text}") - else: - # 没有检测到语言标签时，清空之前的标签 - conn.current_language_tag = None + actual_text = text try: # 尝试解析JSON格式的输入 @@ -75,6 +49,7 @@ async def startToChat(conn, text): data = json.loads(text) if "speaker" in data and "content" in data: speaker_name = data["speaker"] + language_tag = data["language"] actual_text = data["content"] conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}") @@ -89,10 +64,11 @@ async def startToChat(conn, text): conn.current_speaker = speaker_name else: conn.current_speaker = None - - # 如果不是FunASR，清空语言标签，不影响其他ASR - if not is_funasr: - conn.current_language_tag = None + # 保存语种信息到连接对象 + if language_tag: + conn.current_language_tag = language_tag + else: + conn.current_language_tag = "zh" if conn.need_bind: await check_bind_device(conn) diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 6c6969fd..0eb23d76 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -118,23 +118,45 @@ class ASRProviderBase(ABC): else: speaker_name = voiceprint_result - if raw_text: - logger.bind(tag=TAG).info(f"识别文本: {raw_text}") - if speaker_name: - logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}") + # 判断 ASR 结果类型 + if isinstance(raw_text, dict): + # FunASR 返回的 dict 格式 + if speaker_name: + raw_text["speaker"] = speaker_name + + # 记录识别结果 + if raw_text.get("language"): + logger.bind(tag=TAG).info(f"识别语言: {raw_text['language']}") + if raw_text.get("emotion"): + logger.bind(tag=TAG).info(f"识别情绪: {raw_text['emotion']}") + if raw_text.get("content"): + logger.bind(tag=TAG).info(f"识别文本: {raw_text['content']}") + if speaker_name: + logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}") + + # 转换为 JSON 字符串用于下游 + enhanced_text = json.dumps(raw_text, ensure_ascii=False) + content_for_length_check = raw_text.get("content", "") + else: + # 其他 ASR 返回的纯文本 + if raw_text: + logger.bind(tag=TAG).info(f"识别文本: {raw_text}") + if speaker_name: + logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}") + + # 构建包含说话人信息的JSON字符串 + enhanced_text = self._build_enhanced_text(raw_text, speaker_name) + content_for_length_check = raw_text # 性能监控 total_time = time.monotonic() - total_start_time logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s") # 检查文本长度 - text_len, _ = remove_punctuation_and_length(raw_text) + text_len, _ = remove_punctuation_and_length(content_for_length_check) self.stop_ws_connection() if text_len > 0: - # 构建包含说话人信息的JSON字符串 - enhanced_text = self._build_enhanced_text(raw_text, speaker_name) - # 使用自定义模块进行上报 await startToChat(conn, enhanced_text) enqueue_asr_report(conn, enhanced_text, asr_audio_task) @@ -145,7 +167,7 @@ class ASRProviderBase(ABC): logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}") def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str: - """构建包含说话人信息的文本""" + """构建包含说话人信息的文本（仅用于纯文本ASR）""" if speaker_name and speaker_name.strip(): return json.dumps({ "speaker": speaker_name, diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 6bbb79d8..bba53f8a 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -6,11 +6,10 @@ import shutil import psutil import asyncio +from funasr import AutoModel from config.logger import setup_logging from typing import Optional, Tuple, List -from core.providers.asr.base import ASRProviderBase from core.providers.asr.utils import lang_tag_filter -from funasr import AutoModel from core.providers.asr.base import ASRProviderBase from core.providers.asr.dto.dto import InterfaceType @@ -103,11 +102,9 @@ class ASRProvider(ASRProviderBase): use_itn=True, batch_size_s=60, ) - # text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"]) - # 使用lang_tag_filter处理识别结果 - text = await asyncio.to_thread(lang_tag_filter, result[0]["text"]) + text = lang_tag_filter(result[0]["text"]) logger.bind(tag=TAG).debug( - f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}" ) return text, file_path diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py index 4c62a3f3..c6674df9 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_server.py +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -1,12 +1,13 @@ +import ssl +import json +import asyncio +import websockets + +from config.logger import setup_logging from typing import Optional, Tuple, List from core.providers.asr.base import ASRProviderBase from core.providers.asr.utils import lang_tag_filter from core.providers.asr.dto.dto import InterfaceType -import ssl -import json -import websockets -from config.logger import setup_logging -import asyncio TAG = __name__ logger = setup_logging() @@ -156,7 +157,7 @@ class ASRProvider(ASRProviderBase): # if match: # result = match.group(4).strip() - # Handle language tags + # Handle tags result = lang_tag_filter(result) return ( result, diff --git a/main/xiaozhi-server/core/providers/asr/utils.py b/main/xiaozhi-server/core/providers/asr/utils.py index 2310584c..d1f538ae 100644 --- a/main/xiaozhi-server/core/providers/asr/utils.py +++ b/main/xiaozhi-server/core/providers/asr/utils.py @@ -4,40 +4,76 @@ from config.logger import setup_logging TAG = __name__ logger = setup_logging() +EMOTION_EMOJI_MAP = { + "HAPPY": "🙂", + "SAD": "😔", + "ANGRY": "😡", + "NEUTRAL": "😶", + "FEARFUL": "😰", + "DISGUSTED": "🤢", + "SURPRISED": "😲", + "EMO_UNKNOWN": "😶", # 未知情绪默认用中性表情 +} +# EVENT_EMOJI_MAP = { +# "<|BGM|>": "🎼", +# "<|Speech|>": "", +# "<|Applause|>": "👏", +# "<|Laughter|>": "😀", +# "<|Cry|>": "😭", +# "<|Sneeze|>": "🤧", +# "<|Breath|>": "", +# "<|Cough|>": "🤧", +# } -def lang_tag_filter(text): +def lang_tag_filter(text: str) -> dict | str: """ - 过滤函数：只保留语言标签，移除其他所有标签 - - 用于FunASR识别结果的处理，保留语言标签（如<|zh|>、<|en|>等）， - 但移除其他所有格式的标签（如时间戳、情感标签等） - + 解析 FunASR 识别结果，按顺序提取标签和纯文本内容 + Args: - text: ASR识别的原始文本，可能包含多种标签 - + text: ASR 识别的原始文本，可能包含多种标签 + Returns: - str: 处理后的文本，只保留语言标签（如果存在） - + dict: {"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好"} 如果有标签 + str: 纯文本，如果没有标签 + Examples: - >>> lang_tag_filter("<|zh|><|emotion:happy|>你好") - '<|zh|>你好' - >>> lang_tag_filter("<|en|>hello world") - '<|en|>hello world' + FunASR 输出格式：<|语种|><|情绪|><|事件|><|其他选项|>原文 + >>> lang_tag_filter("<|zh|><|SAD|><|Speech|><|withitn|>你好啊，测试测试。") + {"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好啊，测试测试。"} + >>> lang_tag_filter("<|en|><|HAPPY|><|Speech|><|withitn|>Hello hello.") + {"language": "en", "emotion": "HAPPY", "emoji": "🙂", "content": "Hello hello."} + >>> lang_tag_filter("plain text") + "plain text" """ - # 定义语言标签模式 - lang_pattern = r"<\|(zh|en|yue|ja|ko|nospeech)\|>" - lang_tags = re.findall(lang_pattern, text) + # 提取所有标签（按顺序） + tag_pattern = r"<\|([^|]+)\|>" + all_tags = re.findall(tag_pattern, text) - # 移除所有 < | ... | > 格式的标签 - clean_text = re.sub(r"<\|.*?\|>", "", text) + # 移除所有 <|...|> 格式的标签，获取纯文本 + clean_text = re.sub(tag_pattern, "", text).strip() - # 在开头添加语言标签（如果存在） - if lang_tags: - if len(lang_tags) > 1: - logger.bind(tag=TAG).warning( - f"检测到多个语言标签: {lang_tags}，仅使用第一个: {lang_tags[0]}" - ) - clean_text = f"<|{lang_tags[0]}|>{clean_text}" + # 如果没有标签，直接返回纯文本 + if not all_tags: + return clean_text - return clean_text.strip() + # 按照 FunASR 的固定顺序提取标签，返回 dict + language = all_tags[0] if len(all_tags) > 0 else "zh" + emotion = all_tags[1] if len(all_tags) > 1 else "NEUTRAL" + # event = all_tags[2] if len(all_tags) > 2 else "Speech" # 事件标签暂不使用 + + result = { + "content": clean_text, + "language": language, + "emotion": emotion, + # "event": event, + } + + # 添加 emoji 映射 + if emotion in EMOTION_EMOJI_MAP: + result["emotion"] = EMOTION_EMOJI_MAP[emotion] + # 事件标签暂不使用 + # if event in EVENT_EMOJI_MAP: + # result["event"] = EVENT_EMOJI_MAP[event] + + return result diff --git a/main/xiaozhi-server/core/providers/tts/alibl_stream.py b/main/xiaozhi-server/core/providers/tts/alibl_stream.py index 26bca1e2..3130cb57 100644 --- a/main/xiaozhi-server/core/providers/tts/alibl_stream.py +++ b/main/xiaozhi-server/core/providers/tts/alibl_stream.py @@ -39,13 +39,6 @@ class TTSProvider(TTSProviderBase): if config.get("private_voice"): self.voice = config.get("private_voice") - # 多语言音色配置 - self.voice_zh = config.get("voice_zh", self.voice) # 中文音色 - self.voice_yue = config.get("voice_yue", self.voice) # 粤语音色 - self.voice_en = config.get("voice_en", self.voice) # 英语音色 - self.voice_ja = config.get("voice_ja", self.voice) # 日语音色 - self.voice_ko = config.get("voice_ko", self.voice) # 韩语音色 - # 音频参数配置 self.format = config.get("format", "pcm") sample_rate = config.get("sample_rate", "24000") @@ -72,36 +65,6 @@ class TTSProvider(TTSProviderBase): sample_rate=self.sample_rate, channels=1, frame_size_ms=60 ) - def get_voice_by_language(self, language_tag): - """根据语言标签返回对应的音色（仅在FunASR语音识别时生效）""" - if not language_tag: - return self.voice - - # 检查当前ASR是否为FunASR - is_funasr = False - if hasattr(self, 'conn') and self.conn and hasattr(self.conn, 'asr') and self.conn.asr: - asr_module = self.conn.asr.__class__.__module__ - if 'fun_local' in asr_module or 'fun_server' in asr_module: - is_funasr = True - - # 只有在使用FunASR时才应用多语言音色选择 - if is_funasr: - language_tag = language_tag.lower() - voice_map = { - 'zh': self.voice_zh, - 'yue': self.voice_yue, - 'en': self.voice_en, - 'ja': self.voice_ja, - 'ko': self.voice_ko - } - - selected_voice = voice_map.get(language_tag, self.voice) - logger.bind(tag=TAG).info(f"FunASR语言标签 '{language_tag}' 选择音色: {selected_voice}") - return selected_voice - else: - # 非FunASR时使用默认音色 - return self.voice - async def _ensure_connection(self): """确保WebSocket连接可用，支持60秒内连接复用""" try: @@ -265,9 +228,6 @@ class TTSProvider(TTSProviderBase): # 启动监听任务 self._monitor_task = asyncio.create_task(self._start_monitor_tts_response()) - # 根据当前语言标签选择音色 - current_voice = self.get_voice_by_language(getattr(self.conn, 'current_language_tag', None)) - # 发送run-task消息启动会话 run_task_message = { "header": { @@ -282,7 +242,7 @@ class TTSProvider(TTSProviderBase): "model": self.model, "parameters": { "text_type": "PlainText", - "voice": current_voice, + "voice": self.voice, "format": self.format, "sample_rate": self.sample_rate, "volume": self.volume, @@ -452,12 +412,6 @@ class TTSProvider(TTSProviderBase): ) try: - # 选择音色：优先使用当前连接的语言标签，否则使用默认音色 - if hasattr(self, 'conn') and self.conn and hasattr(self.conn, 'current_language_tag'): - current_voice = self.get_voice_by_language(self.conn.current_language_tag) - else: - current_voice = self.voice - # 发送run-task消息启动会话 run_task_message = { "header": { @@ -472,7 +426,7 @@ class TTSProvider(TTSProviderBase): "model": self.model, "parameters": { "text_type": "PlainText", - "voice": current_voice, + "voice": self.voice, "format": self.format, "sample_rate": self.sample_rate, "volume": self.volume, @@ -565,4 +519,4 @@ class TTSProvider(TTSProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}") - return [] + return [] \ No newline at end of file diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 4547ad8f..9339da7e 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -15,29 +15,6 @@ from pydub import AudioSegment from typing import Callable, Any TAG = __name__ -emoji_map = { - "neutral": "😶", - "happy": "🙂", - "laughing": "😆", - "funny": "😂", - "sad": "😔", - "angry": "😠", - "crying": "😭", - "loving": "😍", - "embarrassed": "😳", - "surprised": "😲", - "shocked": "😱", - "thinking": "🤔", - "winking": "😉", - "cool": "😎", - "relaxed": "😌", - "delicious": "🤤", - "kissy": "😘", - "confident": "😏", - "sleepy": "😴", - "silly": "😜", - "confused": "🙄", -} def get_local_ip():