diff --git a/main/manager-api/src/main/resources/db/changelog/202601051433.sql b/main/manager-api/src/main/resources/db/changelog/202601051433.sql new file mode 100644 index 00000000..15f6d572 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202601051433.sql @@ -0,0 +1,11 @@ +-- 更改FunASRServer说明文档模型为SenseVoiceSmall +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md', +`remark` = '独立部署FunASR，使用FunASR的API服务，只需要五句话 +第一句：mkdir -p ./funasr-runtime-resources/models +第二句：sudo docker run -d -p 10096:10095 --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12 +上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime +不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir iic/SenseVoiceSmall-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 & +上一句话执行后会进入到容器，继续第五句：tail -f log.txt +第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了 +以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md' WHERE `id` = 'ASR_FunASRServer'; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index dd2bdb10..9a8b23cc 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -480,3 +480,10 @@ databaseChangeLog: - sqlFile: encoding: utf8 path: classpath:db/changelog/202512301430.sql + - changeSet: + id: 202601051433 + author: RanChen + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202601051433.sql diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 9009eef8..a3c54c17 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -296,7 +296,7 @@ ASR: # 第一句：mkdir -p ./funasr-runtime-resources/models # 第二句：sudo docker run -p 10096:10095 -it --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12 # 上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime - # 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 & + # 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir iic/SenseVoiceSmall-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 & # 上一句话执行后会进入到容器，继续第五句：tail -f log.txt # 第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了 # 以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index ad8b4385..449fbd2d 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -130,6 +130,8 @@ class ConnectionHandler: # 所以涉及到ASR的变量，需要在这里定义，属于connection的私有变量 self.asr_audio = [] self.asr_audio_queue = queue.Queue() + self.current_speaker = None # 存储当前说话人 + self.current_language_tag = None # 存储当前ASR识别的语言标签 # llm相关变量 self.llm_finish_task = True diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 8879564f..ec1d3bc7 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -40,6 +40,7 @@ async def resume_vad_detection(conn): async def startToChat(conn, text): # 检查输入是否是JSON格式（包含说话人信息） speaker_name = None + language_tag = None actual_text = text try: @@ -48,6 +49,7 @@ async def startToChat(conn, text): data = json.loads(text) if "speaker" in data and "content" in data: speaker_name = data["speaker"] + language_tag = data["language"] actual_text = data["content"] conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}") @@ -62,6 +64,11 @@ async def startToChat(conn, text): conn.current_speaker = speaker_name else: conn.current_speaker = None + # 保存语种信息到连接对象 + if language_tag: + conn.current_language_tag = language_tag + else: + conn.current_language_tag = "zh" if conn.need_bind: await check_bind_device(conn) diff --git a/main/xiaozhi-server/core/providers/asr/base.py b/main/xiaozhi-server/core/providers/asr/base.py index 6c6969fd..0eb23d76 100644 --- a/main/xiaozhi-server/core/providers/asr/base.py +++ b/main/xiaozhi-server/core/providers/asr/base.py @@ -118,23 +118,45 @@ class ASRProviderBase(ABC): else: speaker_name = voiceprint_result - if raw_text: - logger.bind(tag=TAG).info(f"识别文本: {raw_text}") - if speaker_name: - logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}") + # 判断 ASR 结果类型 + if isinstance(raw_text, dict): + # FunASR 返回的 dict 格式 + if speaker_name: + raw_text["speaker"] = speaker_name + + # 记录识别结果 + if raw_text.get("language"): + logger.bind(tag=TAG).info(f"识别语言: {raw_text['language']}") + if raw_text.get("emotion"): + logger.bind(tag=TAG).info(f"识别情绪: {raw_text['emotion']}") + if raw_text.get("content"): + logger.bind(tag=TAG).info(f"识别文本: {raw_text['content']}") + if speaker_name: + logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}") + + # 转换为 JSON 字符串用于下游 + enhanced_text = json.dumps(raw_text, ensure_ascii=False) + content_for_length_check = raw_text.get("content", "") + else: + # 其他 ASR 返回的纯文本 + if raw_text: + logger.bind(tag=TAG).info(f"识别文本: {raw_text}") + if speaker_name: + logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}") + + # 构建包含说话人信息的JSON字符串 + enhanced_text = self._build_enhanced_text(raw_text, speaker_name) + content_for_length_check = raw_text # 性能监控 total_time = time.monotonic() - total_start_time logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s") # 检查文本长度 - text_len, _ = remove_punctuation_and_length(raw_text) + text_len, _ = remove_punctuation_and_length(content_for_length_check) self.stop_ws_connection() if text_len > 0: - # 构建包含说话人信息的JSON字符串 - enhanced_text = self._build_enhanced_text(raw_text, speaker_name) - # 使用自定义模块进行上报 await startToChat(conn, enhanced_text) enqueue_asr_report(conn, enhanced_text, asr_audio_task) @@ -145,7 +167,7 @@ class ASRProviderBase(ABC): logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}") def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str: - """构建包含说话人信息的文本""" + """构建包含说话人信息的文本（仅用于纯文本ASR）""" if speaker_name and speaker_name.strip(): return json.dumps({ "speaker": speaker_name, diff --git a/main/xiaozhi-server/core/providers/asr/fun_local.py b/main/xiaozhi-server/core/providers/asr/fun_local.py index 2fd49c36..bba53f8a 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_local.py +++ b/main/xiaozhi-server/core/providers/asr/fun_local.py @@ -6,10 +6,10 @@ import shutil import psutil import asyncio +from funasr import AutoModel from config.logger import setup_logging from typing import Optional, Tuple, List -from funasr import AutoModel -from funasr.utils.postprocess_utils import rich_transcription_postprocess +from core.providers.asr.utils import lang_tag_filter from core.providers.asr.base import ASRProviderBase from core.providers.asr.dto.dto import InterfaceType @@ -102,9 +102,9 @@ class ASRProvider(ASRProviderBase): use_itn=True, batch_size_s=60, ) - text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"]) + text = lang_tag_filter(result[0]["text"]) logger.bind(tag=TAG).debug( - f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}" + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}" ) return text, file_path diff --git a/main/xiaozhi-server/core/providers/asr/fun_server.py b/main/xiaozhi-server/core/providers/asr/fun_server.py index 7eb9e784..c6674df9 100644 --- a/main/xiaozhi-server/core/providers/asr/fun_server.py +++ b/main/xiaozhi-server/core/providers/asr/fun_server.py @@ -1,12 +1,13 @@ -from typing import Optional, Tuple, List -from core.providers.asr.base import ASRProviderBase -from core.providers.asr.dto.dto import InterfaceType import ssl import json -import websockets -from config.logger import setup_logging import asyncio -import re +import websockets + +from config.logger import setup_logging +from typing import Optional, Tuple, List +from core.providers.asr.base import ASRProviderBase +from core.providers.asr.utils import lang_tag_filter +from core.providers.asr.dto.dto import InterfaceType TAG = __name__ logger = setup_logging() @@ -151,9 +152,13 @@ class ASRProvider(ASRProviderBase): # Get the result from the receive task result = receive_task.result() - match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result) - if match: - result = match.group(4).strip() + + # match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result) + # if match: + # result = match.group(4).strip() + + # Handle tags + result = lang_tag_filter(result) return ( result, file_path, diff --git a/main/xiaozhi-server/core/providers/asr/utils.py b/main/xiaozhi-server/core/providers/asr/utils.py new file mode 100644 index 00000000..d1f538ae --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/utils.py @@ -0,0 +1,79 @@ +import re +from config.logger import setup_logging + +TAG = __name__ +logger = setup_logging() + +EMOTION_EMOJI_MAP = { + "HAPPY": "🙂", + "SAD": "😔", + "ANGRY": "😡", + "NEUTRAL": "😶", + "FEARFUL": "😰", + "DISGUSTED": "🤢", + "SURPRISED": "😲", + "EMO_UNKNOWN": "😶", # 未知情绪默认用中性表情 +} +# EVENT_EMOJI_MAP = { +# "<|BGM|>": "🎼", +# "<|Speech|>": "", +# "<|Applause|>": "👏", +# "<|Laughter|>": "😀", +# "<|Cry|>": "😭", +# "<|Sneeze|>": "🤧", +# "<|Breath|>": "", +# "<|Cough|>": "🤧", +# } + +def lang_tag_filter(text: str) -> dict | str: + """ + 解析 FunASR 识别结果，按顺序提取标签和纯文本内容 + + Args: + text: ASR 识别的原始文本，可能包含多种标签 + + Returns: + dict: {"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好"} 如果有标签 + str: 纯文本，如果没有标签 + + Examples: + FunASR 输出格式：<|语种|><|情绪|><|事件|><|其他选项|>原文 + >>> lang_tag_filter("<|zh|><|SAD|><|Speech|><|withitn|>你好啊，测试测试。") + {"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好啊，测试测试。"} + >>> lang_tag_filter("<|en|><|HAPPY|><|Speech|><|withitn|>Hello hello.") + {"language": "en", "emotion": "HAPPY", "emoji": "🙂", "content": "Hello hello."} + >>> lang_tag_filter("plain text") + "plain text" + """ + # 提取所有标签（按顺序） + tag_pattern = r"<\|([^|]+)\|>" + all_tags = re.findall(tag_pattern, text) + + # 移除所有 <|...|> 格式的标签，获取纯文本 + clean_text = re.sub(tag_pattern, "", text).strip() + + # 如果没有标签，直接返回纯文本 + if not all_tags: + return clean_text + + # 按照 FunASR 的固定顺序提取标签，返回 dict + language = all_tags[0] if len(all_tags) > 0 else "zh" + emotion = all_tags[1] if len(all_tags) > 1 else "NEUTRAL" + # event = all_tags[2] if len(all_tags) > 2 else "Speech" # 事件标签暂不使用 + + result = { + "content": clean_text, + "language": language, + "emotion": emotion, + # "event": event, + } + + # 添加 emoji 映射 + if emotion in EMOTION_EMOJI_MAP: + result["emotion"] = EMOTION_EMOJI_MAP[emotion] + # 事件标签暂不使用 + # if event in EVENT_EMOJI_MAP: + # result["event"] = EVENT_EMOJI_MAP[event] + + return result + diff --git a/main/xiaozhi-server/core/providers/tts/alibl_stream.py b/main/xiaozhi-server/core/providers/tts/alibl_stream.py index c0d187f0..3130cb57 100644 --- a/main/xiaozhi-server/core/providers/tts/alibl_stream.py +++ b/main/xiaozhi-server/core/providers/tts/alibl_stream.py @@ -519,4 +519,4 @@ class TTSProvider(TTSProviderBase): except Exception as e: logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}") - return [] + return [] \ No newline at end of file diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index 4547ad8f..9339da7e 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -15,29 +15,6 @@ from pydub import AudioSegment from typing import Callable, Any TAG = __name__ -emoji_map = { - "neutral": "😶", - "happy": "🙂", - "laughing": "😆", - "funny": "😂", - "sad": "😔", - "angry": "😠", - "crying": "😭", - "loving": "😍", - "embarrassed": "😳", - "surprised": "😲", - "shocked": "😱", - "thinking": "🤔", - "winking": "😉", - "cool": "😎", - "relaxed": "😌", - "delicious": "🤤", - "kissy": "😘", - "confident": "😏", - "sleepy": "😴", - "silly": "😜", - "confused": "🙄", -} def get_local_ip():