Merge pull request #1226 from xinnan-tech/fix-fun-server

fix:修复智控台fish-speech无法按声音播放bug

docs/fish-speech-integration.md

@@ -0,0 +1,72 @@
+登录AutoDL，租赁镜像
+选择镜像:
+```
+PyTorch / 2.1.0 / 3.10(ubuntu22.04) / cuda 12.1
+```
+
+机器开机后，设置学术加速
+```
+source /etc/network_turbo
+```
+
+进入工作目录
+```
+cd autodl-tmp/
+```
+
+拉取项目
+```
+git clone https://gitclone.com/github.com/fishaudio/fish-speech.git ; cd fish-speech
+```
+
+安装依赖
+```
+pip install -e.
+```
+
+如果报错，安装portaudio
+```
+apt-get install portaudio19-dev -y
+```
+
+安装后执行
+```
+pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cu121
+```
+
+下载模型
+```
+cd tools
+python download_models.py 
+```
+
+下载完模型后运行接口
+```
+python -m tools.api_server --listen 0.0.0.0:6006 
+```
+
+然后用浏览器去到aotodl实例页面
+```
+https://autodl.com/console/instance/list
+```
+
+如下图点击你刚才机器的`自定义服务`按钮，开启端口转发服务
+![自定义服务](images/fishspeech/autodl-01.png)
+
+端口转发服务设置完成后，你本地电脑打开网址`http://localhost:6006/`，就可以访问fish-speech的接口了
+![服务预览](images/fishspeech/autodl-02.png)
+
+
+如果你是单模块部署，核心配置如下
+```
+selected_module:
+  TTS: FishSpeech
+TTS:
+  FishSpeech:
+    reference_audio: ["config/assets/wakeup_words.wav",]
+    reference_text: ["哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",]
+    api_key: "123"
+    api_url: "http://127.0.0.1:6006/v1/tts"
+```
+
+然后重启服务

main/manager-api/src/main/resources/db/changelog/202505091555.sql

@@ -12,4 +12,14 @@ config_json = '{"host": "127.0.0.1", "port": 10096, "type": "fun_server", "is_ss
 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx  --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx  --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 &
 上一句话执行后会进入到容器，继续第五句：tail -f log.txt
 第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了
-以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md' WHERE `id` = 'ASR_FunASRServer';
+以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md' WHERE `id` = 'ASR_FunASRServer';
+
+-- FishSpeech配置说明
+UPDATE `ai_model_config` SET 
+`doc_link` = 'https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md',
+`remark` = 'FishSpeech配置说明：
+1. 需要本地部署FishSpeech服务
+2. 支持自定义音色
+3. 本地推理，无需网络连接
+4. 输出文件保存在tmp/目录
+5. 可参照教程https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md' WHERE `id` = 'TTS_FishSpeech';

main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -115,12 +115,12 @@ databaseChangeLog:
             encoding: utf8
             path: classpath:db/changelog/202505091409.sql
   - changeSet:
-      id: 202505091554
+      id: 202505091555
       author: whosmyqueen
       changes:
         - sqlFile:
             encoding: utf8
-            path: classpath:db/changelog/202505091554.sql
+            path: classpath:db/changelog/202505091555.sql
   - changeSet:
       id: 202505111914
       author: hrz

main/xiaozhi-server/config.yaml

@@ -476,20 +476,13 @@ TTS:
     speed: 1
     output_dir: tmp/
   FishSpeech:
-    # 定义TTS API类型
+    # 参照教程：https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md
-    #启动tts方法：
-    #python -m tools.api_server
-    #--listen 0.0.0.0:8080
-    #--llama-checkpoint-path "checkpoints/fish-speech-1.5"
-    #--decoder-checkpoint-path "checkpoints/fish-speech-1.5/firefly-gan-vq-fsq-8x1024-21hz-generator.pth"
-    #--decoder-config-name firefly_gan_vq
-    #--compile
     type: fishspeech
     output_dir: tmp/
     response_format: wav
     reference_id: null
-    reference_audio: ["/tmp/test.wav",]
+    reference_audio: ["config/assets/wakeup_words.wav",]
-    reference_text: ["你弄来这些吟词宴曲来看，还是这些混话来欺负我。",]
+    reference_text: ["哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",]
     normalize: true
     max_new_tokens: 1024
     chunk_length: 200

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -85,7 +85,9 @@ class TTSProvider(TTSProviderBase):
     def __init__(self, config, delete_audio_file):
         super().__init__(config, delete_audio_file)
 
-        self.reference_id = config.get("reference_id")
+        self.reference_id = (
+            None if not config.get("reference_id") else config.get("reference_id")
+        )
         self.reference_audio = parse_string_to_list(config.get("reference_audio"))
         self.reference_text = parse_string_to_list(config.get("reference_text"))
         self.format = config.get("response_format", "wav")

main/xiaozhi-server/core/utils/util.py

@@ -930,7 +930,6 @@ def check_vad_update(before_config, new_config):
         if "type" not in new_config["VAD"][new_vad_module]
         else new_config["VAD"][new_vad_module]["type"]
     )
-    print(f"前vad:{current_vad_type}，后vad:{new_vad_type}")
     update_vad = current_vad_type != new_vad_type
     return update_vad
 
@@ -954,6 +953,5 @@ def check_asr_update(before_config, new_config):
         if "type" not in new_config["ASR"][new_asr_module]
         else new_config["ASR"][new_asr_module]["type"]
     )
-    print(f"前asr:{current_asr_type}，后asr:{new_asr_type}")
     update_asr = current_asr_type != new_asr_type
     return update_asr