增加ASR情绪和语种返回

main/manager-api/src/main/resources/db/changelog/202510141200.sql

@@ -1,14 +0,0 @@
--- 为阿里百炼流式语音合成添加多语言音色配置字段
-UPDATE `ai_model_provider` SET fields = '[{"key":"api_key","type":"string","label":"API密钥"},{"key":"output_dir","type":"string","label":"输出目录"},{"key":"model","type":"string","label":"模型名称"},{"key":"format","label":"音频格式","type":"string"},{"key":"sample_rate","label":"采样率","type":"number"},{"key": "volume", "type": "number", "label": "音量"},{"key": "rate", "type": "number", "label": "语速"},{"key": "pitch", "type": "number", "label": "音调"},{"key":"voice","type":"string","label":"默认音色"},{"key": "voice_zh", "type": "string", "label": "中文音色"},{"key": "voice_yue", "type": "string", "label": "粤语音色"},{"key": "voice_en", "type": "string", "label": "英语音色"},{"key": "voice_ja", "type": "string", "label": "日语音色"},{"key": "voice_ko", "type": "string", "label": "韩语音色"}]' WHERE id = 'SYSTEM_TTS_AliBLStreamTTS';
-
--- 更新配置说明
-UPDATE `ai_model_config` SET
-`doc_link` = 'https://bailian.console.aliyun.com/?apiKey=1#/api-key',
-`remark` = '阿里百炼流式TTS说明：
-1. 访问 https://bailian.console.aliyun.com/?apiKey=1#/api-key 创建项目并获取appkey
-2. 支持实时流式合成，具有较低的延迟
-3. 支持多种音色设置和音频参数调整
-4. 使用FunASR进行语音识别时，可以自动选择对应语言音色
-5. 支持CosyVoice-V3-Flash等大模型音色，价格实惠(1元/万字符)
-6. 支持实时调节音量、语速、音调等参数
-' WHERE `id` = 'TTS_AliBLStreamTTS';

main/xiaozhi-server/config.yaml

@@ -887,16 +887,6 @@ TTS:
     access_key_secret: 你的阿里云账号access_key_secret
     # 截至2025年7月21日大模型音色只有北京节点采用，其他节点暂不支持
     host: nls-gateway-cn-beijing.aliyuncs.com
-    
-    # 多语言音色配置 - 根据ASR识别的语言标签自动切换音色
-    # 多语言仅限搭配FunASR SenseVoiceSmall模型使用
-    voice_zh: longxiaochun_v2   # 中文音色
-    voice_en: loongeva_v2       # 英文音色  
-    voice_yue: longjiayi_v2        # 粤语音色
-    voice_ja: oongtomoka_v2       # 日语音色
-    voice_ko: loongkyong_v2       # 韩语音色
-    default_voice: longxiaochun_v2  # 默认音色（当语言标签不匹配或无语言标签时使用）
-    
     # 以下可不用设置，使用默认设置
     # format: pcm  # 音频格式：pcm、wav、mp3
     # sample_rate: 16000  # 采样率：8000、16000、24000

main/xiaozhi-server/core/connection.py

@@ -130,6 +130,7 @@ class ConnectionHandler:
         # 所以涉及到ASR的变量，需要在这里定义，属于connection的私有变量
         self.asr_audio = []
         self.asr_audio_queue = queue.Queue()
+        self.current_speaker = None  # 存储当前说话人
         self.current_language_tag = None  # 存储当前ASR识别的语言标签
 
         # llm相关变量

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -1,4 +1,3 @@
-import re
 import time
 import json
 import asyncio
@@ -41,33 +40,8 @@ async def resume_vad_detection(conn):
 async def startToChat(conn, text):
     # 检查输入是否是JSON格式（包含说话人信息）
     speaker_name = None
-    actual_text = text
     language_tag = None
-
-    # 检查当前使用的ASR是否为FunASR（本地或服务版本）
-    is_funasr = False
-    if hasattr(conn, 'asr') and conn.asr:
-        asr_module = conn.asr.__class__.__module__
-        if 'fun_local' in asr_module or 'fun_server' in asr_module:
-            is_funasr = True
-            conn.logger.bind(tag=TAG).debug(f"检测到FunASR语音识别: {asr_module}")
-
-    # 只有在使用FunASR时才处理语言标签
-    if is_funasr:
-        # 检查是否包含语言标签（如<|zh|>、<|en|>等）
-        lang_pattern = r'<\|([a-z]{2,3})\|>'
-        lang_match = re.search(lang_pattern, text)
-        if lang_match:
-            language_tag = lang_match.group(1)
-            conn.current_language_tag = language_tag
-            conn.logger.bind(tag=TAG).info(f"检测到FunASR语言标签: {language_tag}")
-
-            # 移除语言标签，保留纯文本内容
-            actual_text = re.sub(lang_pattern, '', text).strip()
-            conn.logger.bind(tag=TAG).debug(f"移除语言标签后的文本: {actual_text}")
-        else:
-            # 没有检测到语言标签时，清空之前的标签
-            conn.current_language_tag = None
+    actual_text = text
 
     try:
         # 尝试解析JSON格式的输入
@@ -75,6 +49,7 @@ async def startToChat(conn, text):
             data = json.loads(text)
             if "speaker" in data and "content" in data:
                 speaker_name = data["speaker"]
+                language_tag = data["language"]
                 actual_text = data["content"]
                 conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}")
 
@@ -89,10 +64,11 @@ async def startToChat(conn, text):
         conn.current_speaker = speaker_name
     else:
         conn.current_speaker = None
-
-    # 如果不是FunASR，清空语言标签，不影响其他ASR
-    if not is_funasr:
-        conn.current_language_tag = None
+    # 保存语种信息到连接对象
+    if language_tag:
+        conn.current_language_tag = language_tag
+    else:
+        conn.current_language_tag = "zh"
 
     if conn.need_bind:
         await check_bind_device(conn)

main/xiaozhi-server/core/providers/asr/base.py

@@ -118,23 +118,45 @@ class ASRProviderBase(ABC):
             else:
                 speaker_name = voiceprint_result
 
-            if raw_text:
-                logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
-            if speaker_name:
-                logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
+            # 判断 ASR 结果类型
+            if isinstance(raw_text, dict):
+                # FunASR 返回的 dict 格式
+                if speaker_name:
+                    raw_text["speaker"] = speaker_name
+
+                # 记录识别结果
+                if raw_text.get("language"):
+                    logger.bind(tag=TAG).info(f"识别语言: {raw_text['language']}")
+                if raw_text.get("emotion"):
+                    logger.bind(tag=TAG).info(f"识别情绪: {raw_text['emotion']}")
+                if raw_text.get("content"):
+                    logger.bind(tag=TAG).info(f"识别文本: {raw_text['content']}")
+                if speaker_name:
+                    logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
+
+                # 转换为 JSON 字符串用于下游
+                enhanced_text = json.dumps(raw_text, ensure_ascii=False)
+                content_for_length_check = raw_text.get("content", "")
+            else:
+                # 其他 ASR 返回的纯文本
+                if raw_text:
+                    logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
+                if speaker_name:
+                    logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
+
+                # 构建包含说话人信息的JSON字符串
+                enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
+                content_for_length_check = raw_text
 
             # 性能监控
             total_time = time.monotonic() - total_start_time
             logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s")
 
             # 检查文本长度
-            text_len, _ = remove_punctuation_and_length(raw_text)
+            text_len, _ = remove_punctuation_and_length(content_for_length_check)
             self.stop_ws_connection()
 
             if text_len > 0:
-                # 构建包含说话人信息的JSON字符串
-                enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
-
                 # 使用自定义模块进行上报
                 await startToChat(conn, enhanced_text)
                 enqueue_asr_report(conn, enhanced_text, asr_audio_task)
@@ -145,7 +167,7 @@ class ASRProviderBase(ABC):
             logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
 
     def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
-        """构建包含说话人信息的文本"""
+        """构建包含说话人信息的文本（仅用于纯文本ASR）"""
         if speaker_name and speaker_name.strip():
             return json.dumps({
                 "speaker": speaker_name,

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -6,11 +6,10 @@ import shutil
 import psutil
 import asyncio
 
+from funasr import AutoModel
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
-from core.providers.asr.base import ASRProviderBase
 from core.providers.asr.utils import lang_tag_filter
-from funasr import AutoModel
 from core.providers.asr.base import ASRProviderBase
 from core.providers.asr.dto.dto import InterfaceType
 
@@ -103,11 +102,9 @@ class ASRProvider(ASRProviderBase):
                     use_itn=True,
                     batch_size_s=60,
                 )
-                # text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"])
-                # 使用lang_tag_filter处理识别结果
-                text = await asyncio.to_thread(lang_tag_filter, result[0]["text"])
+                text = lang_tag_filter(result[0]["text"])
                 logger.bind(tag=TAG).debug(
-                    f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+                    f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}"
                 )
 
                 return text, file_path

main/xiaozhi-server/core/providers/asr/fun_server.py

@@ -1,12 +1,13 @@
+import ssl
+import json
+import asyncio
+import websockets
+
+from config.logger import setup_logging
 from typing import Optional, Tuple, List
 from core.providers.asr.base import ASRProviderBase
 from core.providers.asr.utils import lang_tag_filter
 from core.providers.asr.dto.dto import InterfaceType
-import ssl
-import json
-import websockets
-from config.logger import setup_logging
-import asyncio
 
 TAG = __name__
 logger = setup_logging()
@@ -156,7 +157,7 @@ class ASRProvider(ASRProviderBase):
                 # if match:
                 #     result = match.group(4).strip()
 
-                # Handle language tags
+                # Handle tags
                 result = lang_tag_filter(result)
                 return (
                     result,

main/xiaozhi-server/core/providers/asr/utils.py

@@ -4,40 +4,76 @@ from config.logger import setup_logging
 TAG = __name__
 logger = setup_logging()
 
+EMOTION_EMOJI_MAP = {
+    "HAPPY": "🙂",
+    "SAD": "😔",
+    "ANGRY": "😡",
+    "NEUTRAL": "😶",
+    "FEARFUL": "😰",
+    "DISGUSTED": "🤢",
+    "SURPRISED": "😲",
+    "EMO_UNKNOWN": "😶",  # 未知情绪默认用中性表情
+}
+# EVENT_EMOJI_MAP = {
+#     "<|BGM|>": "🎼",
+#     "<|Speech|>": "",
+#     "<|Applause|>": "👏",
+#     "<|Laughter|>": "😀",
+#     "<|Cry|>": "😭",
+#     "<|Sneeze|>": "🤧",
+#     "<|Breath|>": "",
+#     "<|Cough|>": "🤧",
+# }
 
-def lang_tag_filter(text):
+def lang_tag_filter(text: str) -> dict | str:
     """
-    过滤函数：只保留语言标签，移除其他所有标签
-    
-    用于FunASR识别结果的处理，保留语言标签（如<|zh|>、<|en|>等），
-    但移除其他所有格式的标签（如时间戳、情感标签等）
-    
+    解析 FunASR 识别结果，按顺序提取标签和纯文本内容
+
     Args:
-        text: ASR识别的原始文本，可能包含多种标签
-        
+        text: ASR 识别的原始文本，可能包含多种标签
+
     Returns:
-        str: 处理后的文本，只保留语言标签（如果存在）
-        
+        dict: {"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好"} 如果有标签
+        str: 纯文本，如果没有标签
+
     Examples:
-        >>> lang_tag_filter("<|zh|><|emotion:happy|>你好")
-        '<|zh|>你好'
-        >>> lang_tag_filter("<|en|>hello world")
-        '<|en|>hello world'
+        FunASR 输出格式：<|语种|><|情绪|><|事件|><|其他选项|>原文
+        >>> lang_tag_filter("<|zh|><|SAD|><|Speech|><|withitn|>你好啊，测试测试。")
+        {"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好啊，测试测试。"}
+        >>> lang_tag_filter("<|en|><|HAPPY|><|Speech|><|withitn|>Hello hello.")
+        {"language": "en", "emotion": "HAPPY", "emoji": "🙂", "content": "Hello hello."}
+        >>> lang_tag_filter("plain text")
+        "plain text"
     """
-    # 定义语言标签模式
-    lang_pattern = r"<\|(zh|en|yue|ja|ko|nospeech)\|>"
-    lang_tags = re.findall(lang_pattern, text)
+    # 提取所有标签（按顺序）
+    tag_pattern = r"<\|([^|]+)\|>"
+    all_tags = re.findall(tag_pattern, text)
 
-    # 移除所有 < | ... | > 格式的标签
-    clean_text = re.sub(r"<\|.*?\|>", "", text)
+    # 移除所有 <|...|> 格式的标签，获取纯文本
+    clean_text = re.sub(tag_pattern, "", text).strip()
 
-    # 在开头添加语言标签（如果存在）
-    if lang_tags:
-        if len(lang_tags) > 1:
-            logger.bind(tag=TAG).warning(
-                f"检测到多个语言标签: {lang_tags}，仅使用第一个: {lang_tags[0]}"
-            )
-        clean_text = f"<|{lang_tags[0]}|>{clean_text}"
+    # 如果没有标签，直接返回纯文本
+    if not all_tags:
+        return clean_text
 
-    return clean_text.strip()
+    # 按照 FunASR 的固定顺序提取标签，返回 dict
+    language = all_tags[0] if len(all_tags) > 0 else "zh"
+    emotion = all_tags[1] if len(all_tags) > 1 else "NEUTRAL"
+    # event = all_tags[2] if len(all_tags) > 2 else "Speech"  # 事件标签暂不使用
+
+    result = {
+        "content": clean_text,
+        "language": language,
+        "emotion": emotion,
+        # "event": event,
+    }
+
+    # 添加 emoji 映射
+    if emotion in EMOTION_EMOJI_MAP:
+        result["emotion"] = EMOTION_EMOJI_MAP[emotion]
+    # 事件标签暂不使用
+    # if event in EVENT_EMOJI_MAP:
+    #     result["event"] = EVENT_EMOJI_MAP[event]
+
+    return result
 

main/xiaozhi-server/core/providers/tts/alibl_stream.py

@@ -39,13 +39,6 @@ class TTSProvider(TTSProviderBase):
         if config.get("private_voice"):
             self.voice = config.get("private_voice")
 
-        # 多语言音色配置
-        self.voice_zh = config.get("voice_zh", self.voice)  # 中文音色
-        self.voice_yue = config.get("voice_yue", self.voice)  # 粤语音色
-        self.voice_en = config.get("voice_en", self.voice)  # 英语音色
-        self.voice_ja = config.get("voice_ja", self.voice)  # 日语音色
-        self.voice_ko = config.get("voice_ko", self.voice)  # 韩语音色
-
         # 音频参数配置
         self.format = config.get("format", "pcm")
         sample_rate = config.get("sample_rate", "24000")
@@ -72,36 +65,6 @@ class TTSProvider(TTSProviderBase):
             sample_rate=self.sample_rate, channels=1, frame_size_ms=60
         )
 
-    def get_voice_by_language(self, language_tag):
-        """根据语言标签返回对应的音色（仅在FunASR语音识别时生效）"""
-        if not language_tag:
-            return self.voice
-
-        # 检查当前ASR是否为FunASR
-        is_funasr = False
-        if hasattr(self, 'conn') and self.conn and hasattr(self.conn, 'asr') and self.conn.asr:
-            asr_module = self.conn.asr.__class__.__module__
-            if 'fun_local' in asr_module or 'fun_server' in asr_module:
-                is_funasr = True
-
-        # 只有在使用FunASR时才应用多语言音色选择
-        if is_funasr:
-            language_tag = language_tag.lower()
-            voice_map = {
-                'zh': self.voice_zh,
-                'yue': self.voice_yue,
-                'en': self.voice_en,
-                'ja': self.voice_ja,
-                'ko': self.voice_ko
-            }
-
-            selected_voice = voice_map.get(language_tag, self.voice)
-            logger.bind(tag=TAG).info(f"FunASR语言标签 '{language_tag}' 选择音色: {selected_voice}")
-            return selected_voice
-        else:
-            # 非FunASR时使用默认音色
-            return self.voice
-
     async def _ensure_connection(self):
         """确保WebSocket连接可用，支持60秒内连接复用"""
         try:
@@ -265,9 +228,6 @@ class TTSProvider(TTSProviderBase):
             # 启动监听任务
             self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
 
-            # 根据当前语言标签选择音色
-            current_voice = self.get_voice_by_language(getattr(self.conn, 'current_language_tag', None))
-
             # 发送run-task消息启动会话
             run_task_message = {
                 "header": {
@@ -282,7 +242,7 @@ class TTSProvider(TTSProviderBase):
                     "model": self.model,
                     "parameters": {
                         "text_type": "PlainText",
-                        "voice": current_voice,
+                        "voice": self.voice,
                         "format": self.format,
                         "sample_rate": self.sample_rate,
                         "volume": self.volume,
@@ -452,12 +412,6 @@ class TTSProvider(TTSProviderBase):
                 )
 
                 try:
-                    # 选择音色：优先使用当前连接的语言标签，否则使用默认音色
-                    if hasattr(self, 'conn') and self.conn and hasattr(self.conn, 'current_language_tag'):
-                        current_voice = self.get_voice_by_language(self.conn.current_language_tag)
-                    else:
-                        current_voice = self.voice
-
                     # 发送run-task消息启动会话
                     run_task_message = {
                         "header": {
@@ -472,7 +426,7 @@ class TTSProvider(TTSProviderBase):
                             "model": self.model,
                             "parameters": {
                                 "text_type": "PlainText",
-                                "voice": current_voice,
+                                "voice": self.voice,
                                 "format": self.format,
                                 "sample_rate": self.sample_rate,
                                 "volume": self.volume,
@@ -565,4 +519,4 @@ class TTSProvider(TTSProviderBase):
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}")
-            return []
+            return []

main/xiaozhi-server/core/utils/util.py

@@ -15,29 +15,6 @@ from pydub import AudioSegment
 from typing import Callable, Any
 
 TAG = __name__
-emoji_map = {
-    "neutral": "😶",
-    "happy": "🙂",
-    "laughing": "😆",
-    "funny": "😂",
-    "sad": "😔",
-    "angry": "😠",
-    "crying": "😭",
-    "loving": "😍",
-    "embarrassed": "😳",
-    "surprised": "😲",
-    "shocked": "😱",
-    "thinking": "🤔",
-    "winking": "😉",
-    "cool": "😎",
-    "relaxed": "😌",
-    "delicious": "🤤",
-    "kissy": "😘",
-    "confident": "😏",
-    "sleepy": "😴",
-    "silly": "😜",
-    "confused": "🙄",
-}
 
 
 def get_local_ip():