Merge pull request #2798 from xinnan-tech/py_asr_emotion_language

Py asr emotion language

main/manager-api/src/main/resources/db/changelog/202601051433.sql

@@ -0,0 +1,11 @@
+-- 更改FunASRServer说明文档模型为SenseVoiceSmall
+UPDATE `ai_model_config` SET 
+`doc_link` = 'https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md',
+`remark` = '独立部署FunASR，使用FunASR的API服务，只需要五句话
+第一句：mkdir -p ./funasr-runtime-resources/models
+第二句：sudo docker run -d -p 10096:10095 --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12
+上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime
+不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir iic/SenseVoiceSmall-onnx  --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx  --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 &
+上一句话执行后会进入到容器，继续第五句：tail -f log.txt
+第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了
+以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md' WHERE `id` = 'ASR_FunASRServer';

main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -480,3 +480,10 @@ databaseChangeLog:
         - sqlFile:
             encoding: utf8
             path: classpath:db/changelog/202512301430.sql
+  - changeSet:
+      id: 202601051433
+      author: RanChen
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202601051433.sql

main/xiaozhi-server/config.yaml

@@ -296,7 +296,7 @@ ASR:
     # 第一句：mkdir -p ./funasr-runtime-resources/models
     # 第二句：sudo docker run -p 10096:10095 -it --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12
     # 上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime
-    # 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx  --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx  --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 &
+    # 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir iic/SenseVoiceSmall-onnx  --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx  --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 &
     # 上一句话执行后会进入到容器，继续第五句：tail -f log.txt
     # 第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了
     # 以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md

main/xiaozhi-server/core/connection.py

@@ -130,6 +130,8 @@ class ConnectionHandler:
         # 所以涉及到ASR的变量，需要在这里定义，属于connection的私有变量
         self.asr_audio = []
         self.asr_audio_queue = queue.Queue()
+        self.current_speaker = None  # 存储当前说话人
+        self.current_language_tag = None  # 存储当前ASR识别的语言标签
 
         # llm相关变量
         self.llm_finish_task = True

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -40,6 +40,7 @@ async def resume_vad_detection(conn):
 async def startToChat(conn, text):
     # 检查输入是否是JSON格式（包含说话人信息）
     speaker_name = None
+    language_tag = None
     actual_text = text
 
     try:
@@ -48,6 +49,7 @@ async def startToChat(conn, text):
             data = json.loads(text)
             if "speaker" in data and "content" in data:
                 speaker_name = data["speaker"]
+                language_tag = data["language"]
                 actual_text = data["content"]
                 conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}")
 
@@ -62,6 +64,11 @@ async def startToChat(conn, text):
         conn.current_speaker = speaker_name
     else:
         conn.current_speaker = None
+    # 保存语种信息到连接对象
+    if language_tag:
+        conn.current_language_tag = language_tag
+    else:
+        conn.current_language_tag = "zh"
 
     if conn.need_bind:
         await check_bind_device(conn)

main/xiaozhi-server/core/providers/asr/base.py

@@ -118,23 +118,45 @@ class ASRProviderBase(ABC):
             else:
                 speaker_name = voiceprint_result
 
-            if raw_text:
+            # 判断 ASR 结果类型
-                logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
+            if isinstance(raw_text, dict):
-            if speaker_name:
+                # FunASR 返回的 dict 格式
-                logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
+                if speaker_name:
+                    raw_text["speaker"] = speaker_name
+
+                # 记录识别结果
+                if raw_text.get("language"):
+                    logger.bind(tag=TAG).info(f"识别语言: {raw_text['language']}")
+                if raw_text.get("emotion"):
+                    logger.bind(tag=TAG).info(f"识别情绪: {raw_text['emotion']}")
+                if raw_text.get("content"):
+                    logger.bind(tag=TAG).info(f"识别文本: {raw_text['content']}")
+                if speaker_name:
+                    logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
+
+                # 转换为 JSON 字符串用于下游
+                enhanced_text = json.dumps(raw_text, ensure_ascii=False)
+                content_for_length_check = raw_text.get("content", "")
+            else:
+                # 其他 ASR 返回的纯文本
+                if raw_text:
+                    logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
+                if speaker_name:
+                    logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
+
+                # 构建包含说话人信息的JSON字符串
+                enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
+                content_for_length_check = raw_text
 
             # 性能监控
             total_time = time.monotonic() - total_start_time
             logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s")
 
             # 检查文本长度
-            text_len, _ = remove_punctuation_and_length(raw_text)
+            text_len, _ = remove_punctuation_and_length(content_for_length_check)
             self.stop_ws_connection()
 
             if text_len > 0:
-                # 构建包含说话人信息的JSON字符串
-                enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
-
                 # 使用自定义模块进行上报
                 await startToChat(conn, enhanced_text)
                 enqueue_asr_report(conn, enhanced_text, asr_audio_task)
@@ -145,7 +167,7 @@ class ASRProviderBase(ABC):
             logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
 
     def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
-        """构建包含说话人信息的文本"""
+        """构建包含说话人信息的文本（仅用于纯文本ASR）"""
         if speaker_name and speaker_name.strip():
             return json.dumps({
                 "speaker": speaker_name,

main/xiaozhi-server/core/providers/asr/fun_local.py

@@ -6,10 +6,10 @@ import shutil
 import psutil
 import asyncio
 
+from funasr import AutoModel
 from config.logger import setup_logging
 from typing import Optional, Tuple, List
-from funasr import AutoModel
+from core.providers.asr.utils import lang_tag_filter
-from funasr.utils.postprocess_utils import rich_transcription_postprocess
 from core.providers.asr.base import ASRProviderBase
 from core.providers.asr.dto.dto import InterfaceType
 
@@ -102,9 +102,9 @@ class ASRProvider(ASRProviderBase):
                     use_itn=True,
                     batch_size_s=60,
                 )
-                text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"])
+                text = lang_tag_filter(result[0]["text"])
                 logger.bind(tag=TAG).debug(
-                    f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
+                    f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}"
                 )
 
                 return text, file_path

main/xiaozhi-server/core/providers/asr/fun_server.py

@@ -1,12 +1,13 @@
-from typing import Optional, Tuple, List
-from core.providers.asr.base import ASRProviderBase
-from core.providers.asr.dto.dto import InterfaceType
 import ssl
 import json
-import websockets
-from config.logger import setup_logging
 import asyncio
-import re
+import websockets
+
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.utils import lang_tag_filter
+from core.providers.asr.dto.dto import InterfaceType
 
 TAG = __name__
 logger = setup_logging()
@@ -151,9 +152,13 @@ class ASRProvider(ASRProviderBase):
 
                 # Get the result from the receive task
                 result = receive_task.result()
-                match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result)
+                
-                if match:
+                # match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result)
-                    result = match.group(4).strip()
+                # if match:
+                #     result = match.group(4).strip()
+
+                # Handle tags
+                result = lang_tag_filter(result)
                 return (
                     result,
                     file_path,

main/xiaozhi-server/core/providers/asr/utils.py

@@ -0,0 +1,79 @@
+import re
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+EMOTION_EMOJI_MAP = {
+    "HAPPY": "🙂",
+    "SAD": "😔",
+    "ANGRY": "😡",
+    "NEUTRAL": "😶",
+    "FEARFUL": "😰",
+    "DISGUSTED": "🤢",
+    "SURPRISED": "😲",
+    "EMO_UNKNOWN": "😶",  # 未知情绪默认用中性表情
+}
+# EVENT_EMOJI_MAP = {
+#     "<|BGM|>": "🎼",
+#     "<|Speech|>": "",
+#     "<|Applause|>": "👏",
+#     "<|Laughter|>": "😀",
+#     "<|Cry|>": "😭",
+#     "<|Sneeze|>": "🤧",
+#     "<|Breath|>": "",
+#     "<|Cough|>": "🤧",
+# }
+
+def lang_tag_filter(text: str) -> dict | str:
+    """
+    解析 FunASR 识别结果，按顺序提取标签和纯文本内容
+
+    Args:
+        text: ASR 识别的原始文本，可能包含多种标签
+
+    Returns:
+        dict: {"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好"} 如果有标签
+        str: 纯文本，如果没有标签
+
+    Examples:
+        FunASR 输出格式：<|语种|><|情绪|><|事件|><|其他选项|>原文
+        >>> lang_tag_filter("<|zh|><|SAD|><|Speech|><|withitn|>你好啊，测试测试。")
+        {"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好啊，测试测试。"}
+        >>> lang_tag_filter("<|en|><|HAPPY|><|Speech|><|withitn|>Hello hello.")
+        {"language": "en", "emotion": "HAPPY", "emoji": "🙂", "content": "Hello hello."}
+        >>> lang_tag_filter("plain text")
+        "plain text"
+    """
+    # 提取所有标签（按顺序）
+    tag_pattern = r"<\|([^|]+)\|>"
+    all_tags = re.findall(tag_pattern, text)
+
+    # 移除所有 <|...|> 格式的标签，获取纯文本
+    clean_text = re.sub(tag_pattern, "", text).strip()
+
+    # 如果没有标签，直接返回纯文本
+    if not all_tags:
+        return clean_text
+
+    # 按照 FunASR 的固定顺序提取标签，返回 dict
+    language = all_tags[0] if len(all_tags) > 0 else "zh"
+    emotion = all_tags[1] if len(all_tags) > 1 else "NEUTRAL"
+    # event = all_tags[2] if len(all_tags) > 2 else "Speech"  # 事件标签暂不使用
+
+    result = {
+        "content": clean_text,
+        "language": language,
+        "emotion": emotion,
+        # "event": event,
+    }
+
+    # 添加 emoji 映射
+    if emotion in EMOTION_EMOJI_MAP:
+        result["emotion"] = EMOTION_EMOJI_MAP[emotion]
+    # 事件标签暂不使用
+    # if event in EVENT_EMOJI_MAP:
+    #     result["event"] = EVENT_EMOJI_MAP[event]
+
+    return result
+

main/xiaozhi-server/core/utils/util.py

@@ -15,29 +15,6 @@ from pydub import AudioSegment
 from typing import Callable, Any
 
 TAG = __name__
-emoji_map = {
-    "neutral": "😶",
-    "happy": "🙂",
-    "laughing": "😆",
-    "funny": "😂",
-    "sad": "😔",
-    "angry": "😠",
-    "crying": "😭",
-    "loving": "😍",
-    "embarrassed": "😳",
-    "surprised": "😲",
-    "shocked": "😱",
-    "thinking": "🤔",
-    "winking": "😉",
-    "cool": "😎",
-    "relaxed": "😌",
-    "delicious": "🤤",
-    "kissy": "😘",
-    "confident": "😏",
-    "sleepy": "😴",
-    "silly": "😜",
-    "confused": "🙄",
-}
 
 
 def get_local_ip():