Merge remote-tracking branch 'origin/main'

# Conflicts:
#	main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

docs/fish-speech-integration.md

@@ -0,0 +1,72 @@
+登录AutoDL，租赁镜像
+选择镜像:
+```
+PyTorch / 2.1.0 / 3.10(ubuntu22.04) / cuda 12.1
+```
+
+机器开机后，设置学术加速
+```
+source /etc/network_turbo
+```
+
+进入工作目录
+```
+cd autodl-tmp/
+```
+
+拉取项目
+```
+git clone https://gitclone.com/github.com/fishaudio/fish-speech.git ; cd fish-speech
+```
+
+安装依赖
+```
+pip install -e.
+```
+
+如果报错，安装portaudio
+```
+apt-get install portaudio19-dev -y
+```
+
+安装后执行
+```
+pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cu121
+```
+
+下载模型
+```
+cd tools
+python download_models.py 
+```
+
+下载完模型后运行接口
+```
+python -m tools.api_server --listen 0.0.0.0:6006 
+```
+
+然后用浏览器去到aotodl实例页面
+```
+https://autodl.com/console/instance/list
+```
+
+如下图点击你刚才机器的`自定义服务`按钮，开启端口转发服务
+![自定义服务](images/fishspeech/autodl-01.png)
+
+端口转发服务设置完成后，你本地电脑打开网址`http://localhost:6006/`，就可以访问fish-speech的接口了
+![服务预览](images/fishspeech/autodl-02.png)
+
+
+如果你是单模块部署，核心配置如下
+```
+selected_module:
+  TTS: FishSpeech
+TTS:
+  FishSpeech:
+    reference_audio: ["config/assets/wakeup_words.wav",]
+    reference_text: ["哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",]
+    api_key: "123"
+    api_url: "http://127.0.0.1:6006/v1/tts"
+```
+
+然后重启服务

main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentDTO.java

@@ -27,6 +27,9 @@ public class AgentDTO {
     @Schema(description = "大语言模型名称", example = "llm_model_01")
     private String llmModelName;
 
+    @Schema(description = "记忆模型ID", example = "mem_model_01")
+    private String memModelId;
+
     @Schema(description = "角色设定参数", example = "你是一个专业的客服助手，负责回答用户问题并提供帮助")
     private String systemPrompt;
 

main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentServiceImpl.java

@@ -102,6 +102,9 @@ public class AgentServiceImpl extends BaseServiceImpl<AgentDao, AgentEntity> imp
             // 获取 LLM 模型名称
             dto.setLlmModelName(modelConfigService.getModelNameById(agent.getLlmModelId()));
 
+            // 获取记忆模型名称
+            dto.setMemModelId(agent.getMemModelId());
+
             // 获取 TTS 音色名称
             dto.setTtsVoiceName(timbreModelService.getTimbreNameById(agent.getTtsVoiceId()));
 

main/manager-api/src/main/resources/db/changelog/202505091555.sql

@@ -0,0 +1,25 @@
+-- 更新模型供应器表
+UPDATE `ai_model_provider` SET fields = '[{"key": "host", "type": "string", "label": "服务地址"}, {"key": "port", "type": "number", "label": "端口号"}, {"key": "type", "type": "string", "label": "服务类型"}, {"key": "is_ssl", "type": "boolean", "label": "是否使用SSL"}, {"key": "api_key", "type": "string", "label": "API密钥"}, {"key": "output_dir", "type": "string", "label": "输出目录"}]' WHERE id = 'SYSTEM_ASR_FunASRServer';
+
+-- 更新模型配置表
+UPDATE `ai_model_config` SET 
+config_json = '{"host": "127.0.0.1", "port": 10096, "type": "fun_server", "is_ssl": true, "api_key": "none", "output_dir": "tmp/"}',
+`doc_link` = 'https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md',
+`remark` = '独立部署FunASR，使用FunASR的API服务，只需要五句话
+第一句：mkdir -p ./funasr-runtime-resources/models
+第二句：sudo docker run -p 10096:10095 -it --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12
+上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime
+不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx  --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx  --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 &
+上一句话执行后会进入到容器，继续第五句：tail -f log.txt
+第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了
+以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md' WHERE `id` = 'ASR_FunASRServer';
+
+-- FishSpeech配置说明
+UPDATE `ai_model_config` SET 
+`doc_link` = 'https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md',
+`remark` = 'FishSpeech配置说明：
+1. 需要本地部署FishSpeech服务
+2. 支持自定义音色
+3. 本地推理，无需网络连接
+4. 输出文件保存在tmp/目录
+5. 可参照教程https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md' WHERE `id` = 'TTS_FishSpeech';

main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -114,6 +114,13 @@ databaseChangeLog:
         - sqlFile:
             encoding: utf8
             path: classpath:db/changelog/202505091409.sql
+  - changeSet:
+      id: 202505091555
+      author: whosmyqueen
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202505091555.sql
   - changeSet:
       id: 202505111914
       author: hrz
@@ -127,4 +134,4 @@ databaseChangeLog:
       changes:
         - sqlFile:
             encoding: utf8
-            path: classpath:db/changelog/202505122348.sql
+            path: classpath:db/changelog/202505122348.sql

main/manager-web/src/components/DeviceItem.vue

@@ -26,8 +26,12 @@
       <div class="settings-btn" @click="handleDeviceManage">
         设备管理({{ device.deviceCount }})
       </div>
-      <div class="settings-btn" @click="handleChatHistory">
+      <div class="settings-btn" @click="handleChatHistory"
-        聊天记录
+        :class="{ 'disabled-btn': device.memModelId === 'Memory_nomem' }">
+        <el-tooltip v-if="device.memModelId === 'Memory_nomem'" content="未开启记忆" placement="top">
+          <span>聊天记录</span>
+        </el-tooltip>
+        <span v-else>聊天记录</span>
       </div>
     </div>
     <div class="version-info">
@@ -77,6 +81,9 @@ export default {
       this.$router.push({ path: '/device-management', query: { agentId: this.device.agentId } });
     },
     handleChatHistory() {
+      if (this.device.memModelId === 'Memory_nomem') {
+        return
+      }
       this.$emit('chat-history', { agentId: this.device.agentId, agentName: this.device.agentName })
     }
   }
@@ -120,6 +127,12 @@ export default {
   color: #979db1;
   font-weight: 400;
 }
+
+.disabled-btn {
+  background: #e6e6e6;
+  color: #999;
+  cursor: not-allowed;
+}
 </style>
 
 <style>

main/xiaozhi-server/config.yaml

@@ -221,7 +221,7 @@ ASR:
   FunASRServer:
     # 独立部署FunASR，使用FunASR的API服务，只需要五句话
     # 第一句：mkdir -p ./funasr-runtime-resources/models
-    # 第二句：sudo docker run -d -p 10096:10095 --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12
+    # 第二句：sudo docker run -p 10096:10095 -it --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12
     # 上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime
     # 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx  --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx  --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 &
     # 上一句话执行后会进入到容器，继续第五句：tail -f log.txt
@@ -231,6 +231,7 @@ ASR:
     host: 127.0.0.1
     port: 10096
     is_ssl: true
+    api_key: none
     output_dir: tmp/
   SherpaASR:
     type: sherpa_onnx_local
@@ -475,20 +476,13 @@ TTS:
     speed: 1
     output_dir: tmp/
   FishSpeech:
-    # 定义TTS API类型
+    # 参照教程：https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md
-    #启动tts方法：
-    #python -m tools.api_server
-    #--listen 0.0.0.0:8080
-    #--llama-checkpoint-path "checkpoints/fish-speech-1.5"
-    #--decoder-checkpoint-path "checkpoints/fish-speech-1.5/firefly-gan-vq-fsq-8x1024-21hz-generator.pth"
-    #--decoder-config-name firefly_gan_vq
-    #--compile
     type: fishspeech
     output_dir: tmp/
     response_format: wav
     reference_id: null
-    reference_audio: ["/tmp/test.wav",]
+    reference_audio: ["config/assets/wakeup_words.wav",]
-    reference_text: ["你弄来这些吟词宴曲来看，还是这些混话来欺负我。",]
+    reference_text: ["哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",]
     normalize: true
     max_new_tokens: 1024
     chunk_length: 200

main/xiaozhi-server/core/connection.py

@@ -33,7 +33,7 @@ from core.mcp.manager import MCPManager
 from config.config_loader import get_private_config_from_api
 from config.manage_api_client import DeviceNotFoundException, DeviceBindException
 from core.utils.output_counter import add_device_output
-from core.handle.ttsReportHandle import enqueue_tts_report, report_tts
+from core.handle.reportHandle import enqueue_tts_report, report
 
 TAG = __name__
 
@@ -89,8 +89,11 @@ class ConnectionHandler:
         self.executor = ThreadPoolExecutor(max_workers=10)
 
         # 上报线程
-        self.tts_report_queue = queue.Queue()
+        self.report_queue = queue.Queue()
-        self.tts_report_thread = None
+        self.report_thread = None
+        # TODO(haotian): 2025/5/12 可以通过修改此处，调节asr的上报和tts的上报
+        self.report_asr_enable = self.read_config_from_api
+        self.report_tts_enable = self.read_config_from_api
 
         # 依赖的组件
         self.vad = None
@@ -317,11 +320,11 @@ class ConnectionHandler:
             return
         if self.chat_history_conf == 0:
             return
-        if self.tts_report_thread is None or not self.tts_report_thread.is_alive():
+        if self.report_thread is None or not self.report_thread.is_alive():
-            self.tts_report_thread = threading.Thread(
+            self.report_thread = threading.Thread(
-                target=self._tts_report_worker, daemon=True
+                target=self._report_worker, daemon=True
             )
-            self.tts_report_thread.start()
+            self.report_thread.start()
             self.logger.bind(tag=TAG).info("TTS上报线程已启动")
 
     def _initialize_private_config(self):
@@ -878,8 +881,8 @@ class ConnectionHandler:
                                 audio_datas, _ = self.tts.audio_to_pcm_data(tts_file)
                             else:
                                 audio_datas, _ = self.tts.audio_to_opus_data(tts_file)
-                            # 在这里上报TTS数据（使用文件路径）
+                            # 在这里上报TTS数据
-                            enqueue_tts_report(self, 2, text, audio_datas)
+                            enqueue_tts_report(self, text, audio_datas)
                         else:
                             self.logger.bind(tag=TAG).error(
                                 f"TTS出错：文件不存在{tts_file}"
@@ -935,13 +938,12 @@ class ConnectionHandler:
                     f"audio_play_priority priority_thread: {text} {e}"
                 )
 
-    def _tts_report_worker(self):
+    def _report_worker(self):
-        """TTS上报工作线程"""
+        """聊天记录上报工作线程"""
-
         while not self.stop_event.is_set():
             try:
                 # 从队列获取数据，设置超时以便定期检查停止事件
-                item = self.tts_report_queue.get(timeout=1)
+                item = self.report_queue.get(timeout=1)
                 if item is None:  # 检测毒丸对象
                     break
 
@@ -949,18 +951,18 @@ class ConnectionHandler:
 
                 try:
                     # 执行上报（传入二进制数据）
-                    report_tts(self, type, text, audio_data)
+                    report(self, type, text, audio_data)
                 except Exception as e:
-                    self.logger.bind(tag=TAG).error(f"TTS上报线程异常: {e}")
+                    self.logger.bind(tag=TAG).error(f"聊天记录上报线程异常: {e}")
                 finally:
                     # 标记任务完成
-                    self.tts_report_queue.task_done()
+                    self.report_queue.task_done()
             except queue.Empty:
                 continue
             except Exception as e:
-                self.logger.bind(tag=TAG).error(f"TTS上报工作线程异常: {e}")
+                self.logger.bind(tag=TAG).error(f"聊天记录上报工作线程异常: {e}")
 
-        self.logger.bind(tag=TAG).info("TTS上报线程已退出")
+        self.logger.bind(tag=TAG).info("聊天记录上报线程已退出")
 
     def speak_and_play(self, text, text_index=0):
         if text is None or len(text) <= 0:
@@ -1008,7 +1010,7 @@ class ConnectionHandler:
             self.executor = None
 
         # 添加毒丸对象到上报队列确保线程退出
-        self.tts_report_queue.put(None)
+        self.report_queue.put(None)
 
         # 清空任务队列
         self.clear_queues()

main/xiaozhi-server/core/handle/receiveAudioHandle.py

@@ -4,7 +4,7 @@ from core.utils.util import remove_punctuation_and_length
 from core.handle.sendAudioHandle import send_stt_message
 from core.handle.intentHandler import handle_user_intent
 from core.utils.output_counter import check_device_output_limit
-from core.handle.ttsReportHandle import enqueue_tts_report
+from core.handle.reportHandle import enqueue_asr_report
 from core.utils.util import audio_to_data
 
 TAG = __name__
@@ -44,7 +44,7 @@ async def handleAudioMessage(conn, audio):
             text_len, _ = remove_punctuation_and_length(text)
             if text_len > 0:
                 # 使用自定义模块进行上报
-                enqueue_tts_report(conn, 1, text, copy.deepcopy(conn.asr_audio))
+                enqueue_asr_report(conn, text, copy.deepcopy(conn.asr_audio))
 
                 await startToChat(conn, text)
             else:

main/xiaozhi-server/core/handle/reportHandle.py

@@ -11,13 +11,13 @@ TTS上报功能已集成到ConnectionHandler类中。
 
 import opuslib_next
 
-from config.manage_api_client import report
+from config.manage_api_client import report as manage_report
 
 TAG = __name__
 
 
-def report_tts(conn, type, text, opus_data):
+def report(conn, type, text, opus_data):
-    """执行TTS上报操作
+    """执行聊天记录上报操作
 
     Args:
         conn: 连接对象
@@ -31,7 +31,7 @@ def report_tts(conn, type, text, opus_data):
         else:
             audio_data = None
         # 执行上报
-        report(
+        manage_report(
             mac_address=conn.device_id,
             session_id=conn.session_id,
             chat_type=type,
@@ -39,7 +39,7 @@ def report_tts(conn, type, text, opus_data):
             audio=audio_data,
         )
     except Exception as e:
-        conn.logger.bind(tag=TAG).error(f"TTS上报失败: {e}")
+        conn.logger.bind(tag=TAG).error(f"聊天记录上报失败: {e}")
 
 
 def opus_to_wav(conn, opus_data):
@@ -89,8 +89,8 @@ def opus_to_wav(conn, opus_data):
     return bytes(wav_header) + pcm_data_bytes
 
 
-def enqueue_tts_report(conn, type, text, opus_data):
+def enqueue_tts_report(conn, text, opus_data):
-    if not conn.read_config_from_api or conn.need_bind:
+    if not conn.read_config_from_api or conn.need_bind or not conn.report_tts_enable:
         return
     if conn.chat_history_conf == 0:
         return
@@ -104,14 +104,42 @@ def enqueue_tts_report(conn, type, text, opus_data):
     try:
         # 使用连接对象的队列，传入文本和二进制数据而非文件路径
         if conn.chat_history_conf == 2:
-            conn.tts_report_queue.put((type, text, opus_data))
+            conn.report_queue.put((2, text, opus_data))
             conn.logger.bind(tag=TAG).debug(
                 f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
             )
         else:
-            conn.tts_report_queue.put((type, text, None))
+            conn.report_queue.put((2, text, None))
             conn.logger.bind(tag=TAG).debug(
                 f"TTS数据已加入上报队列: {conn.device_id}, 不上报音频"
             )
     except Exception as e:
         conn.logger.bind(tag=TAG).error(f"加入TTS上报队列失败: {text}, {e}")
+
+
+def enqueue_asr_report(conn, text, opus_data):
+    if not conn.read_config_from_api or conn.need_bind or not conn.report_asr_enable:
+        return
+    if conn.chat_history_conf == 0:
+        return
+    """将ASR数据加入上报队列
+
+    Args:
+        conn: 连接对象
+        text: 合成文本
+        opus_data: opus音频数据
+    """
+    try:
+        # 使用连接对象的队列，传入文本和二进制数据而非文件路径
+        if conn.chat_history_conf == 2:
+            conn.report_queue.put((1, text, opus_data))
+            conn.logger.bind(tag=TAG).debug(
+                f"ASR数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
+            )
+        else:
+            conn.report_queue.put((1, text, None))
+            conn.logger.bind(tag=TAG).debug(
+                f"ASR数据已加入上报队列: {conn.device_id}, 不上报音频"
+            )
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"加入ASR上报队列失败: {text}, {e}")

main/xiaozhi-server/core/handle/textHandle.py

@@ -5,7 +5,7 @@ from core.utils.util import remove_punctuation_and_length
 from core.handle.receiveAudioHandle import startToChat, handleAudioMessage
 from core.handle.sendAudioHandle import send_stt_message, send_tts_message
 from core.handle.iotHandle import handleIotDescriptors, handleIotStatus
-from core.handle.ttsReportHandle import enqueue_tts_report
+from core.handle.reportHandle import enqueue_asr_report
 import asyncio
 
 TAG = __name__
@@ -56,11 +56,11 @@ async def handleTextMessage(conn, message):
                         await send_tts_message(conn, "stop", None)
                     elif is_wakeup_words:
                         # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
-                        enqueue_tts_report(conn, 1, "嘿，你好呀", [])
+                        enqueue_asr_report(conn, "嘿，你好呀", [])
                         await startToChat(conn, "嘿，你好呀")
                     else:
                         # 上报纯文字数据（复用ASR上报功能，但不提供音频数据）
-                        enqueue_tts_report(conn, 1, text, [])
+                        enqueue_asr_report(conn, text, [])
                         # 否则需要LLM对文字内容进行答复
                         await startToChat(conn, text)
         elif msg_json["type"] == "iot":

main/xiaozhi-server/core/providers/asr/fun_server.py

@@ -9,6 +9,7 @@ import wave
 import websockets
 from config.logger import setup_logging
 import asyncio
+import re
 
 TAG = __name__
 logger = setup_logging()
@@ -24,7 +25,12 @@ class ASRProvider(ASRProviderBase):
         super().__init__()
         self.host = config.get("host", "localhost")
         self.port = config.get("port", 10095)
-        self.is_ssl = config.get("is_ssl", True)
+        self.api_key = config.get("api_key", "none")
+        self.is_ssl = str(config.get("is_ssl", True)).lower() in (
+            "true",
+            "1",
+            "yes",
+        )
         self.output_dir = config.get("output_dir")
         self.delete_audio_file = delete_audio_file
         self.uri = (
@@ -130,9 +136,13 @@ class ASRProvider(ASRProviderBase):
             pass
         else:
             file_path = self.save_audio_to_file(pcm_data, session_id)
-
+        auth_header = {"Authorization": "Bearer; {}".format(self.api_key)}
         async with websockets.connect(
-            self.uri, subprotocols=["binary"], ping_interval=None, ssl=self.ssl_context
+            self.uri,
+            additional_headers=auth_header,
+            subprotocols=["binary"],
+            ping_interval=None,
+            ssl=self.ssl_context,
         ) as ws:
             try:
                 # Use asyncio to handle WebSocket communication
@@ -157,6 +167,9 @@ class ASRProvider(ASRProviderBase):
 
                 # Get the result from the receive task
                 result = receive_task.result()
+                match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result)
+                if match:
+                    result = match.group(4).strip()
                 return (
                     result,
                     file_path,

main/xiaozhi-server/core/providers/llm/gemini/gemini.py

@@ -1,140 +1,180 @@
-import google.generativeai as genai
+import os, json, uuid
-from core.utils.util import check_model_key
+from types import SimpleNamespace
-from core.providers.llm.base import LLMProviderBase
+from typing import Any, Dict, List
-from config.logger import setup_logging
-import requests
-import json
 
+import requests
+from google import generativeai as genai
+from google.generativeai import types, GenerationConfig
+
+from core.providers.llm.base import LLMProviderBase
+from core.utils.util import check_model_key
+from config.logger import setup_logging
+from google.generativeai.types import GenerateContentResponse
+from requests import RequestException
+
+log = setup_logging()
 TAG = __name__
-logger = setup_logging()
+
+
+def test_proxy(proxy_url: str, test_url: str) -> bool:
+    try:
+        resp = requests.get(test_url, proxies={"http": proxy_url, "https": proxy_url})
+        return 200 <= resp.status_code < 400
+    except RequestException:
+        return False
+
+
+def setup_proxy_env(http_proxy: str | None, https_proxy: str | None):
+    """
+    分别测试 HTTP 和 HTTPS 代理是否可用，并设置环境变量。
+    如果 HTTPS 代理不可用但 HTTP 可用，会将 HTTPS_PROXY 也指向 HTTP。
+    """
+    test_http_url = "http://www.google.com"
+    test_https_url = "https://www.google.com"
+
+    ok_http = ok_https = False
+
+    if http_proxy:
+        ok_http = test_proxy(http_proxy, test_http_url)
+        if ok_http:
+            os.environ["HTTP_PROXY"] = http_proxy
+            log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {http_proxy}")
+        else:
+            log.bind(tag=TAG).warn(f"配置提供的Gemini HTTP代理不可用: {http_proxy}")
+
+    if https_proxy:
+        ok_https = test_proxy(https_proxy, test_https_url)
+        if ok_https:
+            os.environ["HTTPS_PROXY"] = https_proxy
+            log.bind(tag=TAG).info(f"配置提供的Gemini HTTPS代理连通成功: {https_proxy}")
+        else:
+            log.bind(tag=TAG).warning(f"配置提供的Gemini HTTPS代理不可用: {https_proxy}")
+
+    # 如果https_proxy不可用，但http_proxy可用且能走通https，则复用http_proxy作为https_proxy
+    if ok_http and not ok_https:
+        if test_proxy(http_proxy, test_https_url):
+            os.environ["HTTPS_PROXY"] = http_proxy
+            ok_https = True
+            log.bind(tag=TAG).info(f"复用HTTP代理作为HTTPS代理: {http_proxy}")
+
+    if not ok_http and not ok_https:
+        log.bind(tag=TAG).error(f"Gemini 代理设置失败: HTTP 和 HTTPS 代理都不可用，请检查配置")
+        raise RuntimeError("HTTP 和 HTTPS 代理都不可用，请检查配置")
 
 
 class LLMProvider(LLMProviderBase):
-    def __init__(self, config):
+    def __init__(self, cfg: Dict[str, Any]):
-        """初始化Gemini LLM Provider"""
+        self.model_name = cfg.get("model_name", "gemini-2.0-flash")
-        self.model_name = config.get("model_name", "gemini-1.5-pro")
+        self.api_key = cfg["api_key"]
-        self.api_key = config.get("api_key")
+        http_proxy = cfg.get("http_proxy")
-        self.http_proxy = config.get("http_proxy")
+        https_proxy = cfg.get("https_proxy")
-        self.https_proxy = config.get("https_proxy")
-        have_key = check_model_key("LLM", self.api_key)
 
-        if not have_key:
+        if not check_model_key("LLM", self.api_key):
-            return
+            raise ValueError("无效的Gemini API Key，请检查是否配置正确")
 
-        try:
+        if http_proxy or https_proxy:
-            # 初始化Gemini客户端
+            log.bind(tag=TAG).info(f"检测到Gemini代理配置，开始测试代理连通性和设置代理环境...")
-            # 配置代理（如果提供了代理配置）
+            setup_proxy_env(http_proxy, https_proxy)
-            self.proxies = None
+            log.bind(tag=TAG).info(f"Gemini 代理设置成功 - HTTP: {http_proxy}, HTTPS: {https_proxy}")
-            if self.http_proxy is not "" or self.https_proxy is not "":
+        genai.configure(api_key=self.api_key)
+        self.model = genai.GenerativeModel(self.model_name)
 
-                self.proxies = {
+        self.gen_cfg = GenerationConfig(
-                    "http": self.http_proxy,
+            temperature=0.7,
-                    "https": self.https_proxy,
+            top_p=0.9,
-                }
+            top_k=40,
-                logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}")
+            max_output_tokens=2048,
-                # 使用猴子补丁修改 google-generativeai 库的请求会话
+        )
 
-                # 使用 session 对象配置 genai
+    @staticmethod
-
+    def _build_tools(funcs: List[Dict[str, Any]] | None):
-            genai.configure(api_key=self.api_key)
+        if not funcs:
-            self.model = genai.GenerativeModel(self.model_name)
+            return None
-
+        return [types.Tool(function_declarations=[
-            # 设置生成参数
+            types.FunctionDeclaration(
-            self.generation_config = {
+                name=f["function"]["name"],
-                "temperature": 0.7,
+                description=f["function"]["description"],
-                "top_p": 0.9,
+                parameters=f["function"]["parameters"],
-                "top_k": 40,
+            )
-                "max_output_tokens": 2048,
+            for f in funcs
-            }
+        ])]
-            self.chat = None
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Gemini初始化失败: {e}")
-            self.model = None
 
+    # Gemini文档提到，无需维护session-id，直接用dialogue拼接而成
     def response(self, session_id, dialogue):
-        """生成Gemini对话响应"""
+        yield from self._generate(dialogue, None)
-        if not self.model:
-            yield "【Gemini服务未正确初始化】"
-            return
-
-        try:
-            # 处理对话历史
-            chat_history = []
-            for msg in dialogue[:-1]:  # 历史对话
-                role = "model" if msg["role"] == "assistant" else "user"
-                content = msg["content"].strip()
-                if content:
-                    chat_history.append({"role": role, "parts": [{"text": content}]})
-
-            # 获取当前消息
-            current_msg = dialogue[-1]["content"]
-
-            # 构建请求体
-            request_body = {
-                "contents": chat_history
-                + [{"role": "user", "parts": [{"text": current_msg}]}],
-                "generationConfig": self.generation_config,
-            }
-
-            # 构建请求URL
-            url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}"
-
-            # 构建请求头
-            headers = {
-                "Content-Type": "application/json",
-            }
-
-            # 发送POST请求,经测试手动 request 无法使用 stream 模式
-            if self.proxies:
-                response = requests.post(
-                    url,
-                    headers=headers,
-                    json=request_body,
-                    stream=False,
-                    proxies=self.proxies,
-                )
-                try:
-                    data = response.json()  # 直接解析JSON
-                    if "candidates" in data and data["candidates"]:
-                        yield data["candidates"][0]["content"]["parts"][0]["text"]
-                    else:
-                        yield "未找到候选回复。"
-                except json.JSONDecodeError as e:
-                    yield f"JSON解码错误：{e}"
-                except Exception as e:
-                    yield f"发生错误：{e}"
-            else:
-                logger.bind(tag=TAG).info(f"Gemini stream mode ")
-                chat = self.model.start_chat(history=chat_history)
-
-                # 发送消息并获取流式响应
-                response = chat.send_message(
-                    current_msg, stream=True, generation_config=self.generation_config
-                )
-                # 处理流式响应
-                for chunk in response:
-                    if hasattr(chunk, "text") and chunk.text:
-                        yield chunk.text
-
-        except Exception as e:
-            error_msg = str(e)
-            logger.bind(tag=TAG).error(f"Gemini响应生成错误: {error_msg}")
-
-            # 针对不同错误返回友好提示
-            if "Rate limit" in error_msg:
-                yield "【Gemini服务请求太频繁,请稍后再试】"
-            elif "Invalid API key" in error_msg:
-                yield "【Gemini API key无效】"
-            else:
-                yield f"【Gemini服务响应异常: {error_msg}】"
-
-        except requests.exceptions.RequestException as e:
-            yield f"请求失败：{e}"
-        except json.JSONDecodeError as e:
-            yield f"JSON解码错误：{e}"
-        except Exception as e:
-            yield f"发生错误：{e}"
 
     def response_with_functions(self, session_id, dialogue, functions=None):
-        logger.bind(tag=TAG).info(f"gemini暂未实现完整的工具调用（function call）")
+        yield from self._generate(dialogue, self._build_tools(functions))
-        return self.response(session_id, dialogue)
+
+    def _generate(self, dialogue, tools):
+        role_map = {"assistant": "model", "user": "user"}
+        contents: list = []
+        # 拼接对话
+        for m in dialogue:
+            r = m["role"]
+
+            if r == "assistant" and "tool_calls" in m:
+                tc = m["tool_calls"][0]
+                contents.append({
+                    "role": "model",
+                    "parts": [{"function_call": {
+                        "name": tc["function"]["name"],
+                        "args": json.loads(tc["function"]["arguments"]),
+                    }}],
+                })
+                continue
+
+            if r == "tool":
+                contents.append({
+                    "role": "model",
+                    "parts": [{"text": str(m.get("content", ""))}],
+                })
+                continue
+
+            contents.append({
+                "role": role_map.get(r, "user"),
+                "parts": [{"text": str(m.get("content", ""))}],
+            })
+
+        stream: GenerateContentResponse = self.model.generate_content(
+            contents=contents,
+            generation_config=self.gen_cfg,
+            tools=tools,
+            stream=True,
+        )
+
+        try:
+            for chunk in stream:
+                cand = chunk.candidates[0]
+                for part in cand.content.parts:
+                    # a) 函数调用-通常是最后一段话才是函数调用
+                    if getattr(part, "function_call", None):
+                        fc = part.function_call
+                        yield None, [SimpleNamespace(
+                            id=uuid.uuid4().hex,
+                            type="function",
+                            function=SimpleNamespace(
+                                name=fc.name,
+                                arguments=json.dumps(dict(fc.args),
+                                                     ensure_ascii=False),
+                            ),
+                        )]
+                        return
+                    # b) 普通文本
+                    if getattr(part, "text", None):
+                        yield part.text if tools is None else (part.text, None)
+
+        finally:
+            if tools is not None:
+                yield None, None  # function‑mode 结束，返回哑包
+
+    # 关闭stream，预留后续打断对话功能的功能方法，官方文档推荐打断对话要关闭上一个流，可以有效减少配额计费和资源占用
+    @staticmethod
+    def _safe_finish_stream(stream: GenerateContentResponse):
+        if hasattr(stream, "resolve"):
+            stream.resolve()  # Gemini SDK version ≥ 0.5.0
+        elif hasattr(stream, "close"):
+            stream.close()  # Gemini SDK version < 0.5.0
+        else:
+            for _ in stream:  # 兜底耗尽
+                pass

main/xiaozhi-server/core/providers/tts/fishspeech.py

@@ -85,16 +85,22 @@ class TTSProvider(TTSProviderBase):
     def __init__(self, config, delete_audio_file):
         super().__init__(config, delete_audio_file)
 
-        self.reference_id = config.get("reference_id")
+        self.reference_id = (
+            None if not config.get("reference_id") else config.get("reference_id")
+        )
         self.reference_audio = parse_string_to_list(config.get("reference_audio"))
         self.reference_text = parse_string_to_list(config.get("reference_text"))
-        self.format = config.get("format", "wav")
+        self.format = config.get("response_format", "wav")
 
         self.api_key = config.get("api_key", "YOUR_API_KEY")
         have_key = check_model_key("FishSpeech TTS", self.api_key)
         if not have_key:
             return
-        self.normalize = config.get("normalize", True)
+        self.normalize = str(config.get("normalize", True)).lower() in (
+            "true",
+            "1",
+            "yes",
+        )
 
         # 处理空字符串的情况
         channels = config.get("channels", "1")

main/xiaozhi-server/core/utils/util.py

@@ -931,7 +931,6 @@ def check_vad_update(before_config, new_config):
         if "type" not in new_config["VAD"][new_vad_module]
         else new_config["VAD"][new_vad_module]["type"]
     )
-    print(f"前vad:{current_vad_type}，后vad:{new_vad_type}")
     update_vad = current_vad_type != new_vad_type
     return update_vad
 
@@ -955,6 +954,5 @@ def check_asr_update(before_config, new_config):
         if "type" not in new_config["ASR"][new_asr_module]
         else new_config["ASR"][new_asr_module]["type"]
     )
-    print(f"前asr:{current_asr_type}，后asr:{new_asr_type}")
     update_asr = current_asr_type != new_asr_type
     return update_asr