Merge pull request #443 from HiCoderMonkey/tts-response

Tts 改造

.gitignore

@@ -141,8 +141,8 @@ music/
 # Cython debug symbols
 cython_debug/
 *.iml
-model.pt
 tmp
+.history
 .DS_Store
 main/xiaozhi-server/data
 main/manager-web/node_modules
@@ -151,5 +151,8 @@ main/manager-web/node_modules
 .private_config.yaml
 .env.development
 
+# model files
+main/xiaozhi-server/models/SenseVoiceSmall/model.pt
+main/xiaozhi-server/models/sherpa-onnx*
 /main/xiaozhi-server/audio_ref/
 /audio_ref/

README.md

@@ -201,6 +201,7 @@ server:
 | TTS |  CosyVoiceSiliconflow  | 接口调用 | 消耗 token |                         需申请硅基流动 API 密钥；输出格式为 wav                          |
 | TTS |        TTS302AI        | 接口调用 | 消耗 token |                  [点击创建密钥](https://dash.302.ai/apis/list)                  |
 | TTS |       CozeCnTTS        | 接口调用 | 消耗 token |                        需提供 Coze API key；输出格式为 wav                         |
+| TTS |       GizwitsTTS       | 接口调用 | 消耗 token |                  [点击创建密钥](https://agentrouter.gizwitsapi.com)                  |
 | TTS |        ACGNTTS         | 接口调用 | 消耗 token |                        [联系网站管理员购买密钥](www.ttson.cn)                        |
 | TTS |       OpenAITTS        | 接口调用 | 消耗 token |                                 境外使用，境外购买                                 |
 | TTS |       FishSpeech       | 接口调用 |  免费/自定义  |                         本地启动 TTS 服务；启动方法见配置文件内说明                          |
@@ -223,8 +224,10 @@ server:
 | 类型  |   平台名称    | 使用方式 | 收费模式 | 备注 |
 |:---:|:---------:|:----:|:----:|:--:|
 | ASR |  FunASR   | 本地使用 |  免费  |    |
+| ASR | SherpaASR | 本地使用 |  免费  |    |
 | ASR | DoubaoASR | 接口调用 |  收费  |    |
 
+
 ---
 
 ### Memory 记忆存储

README_en.md

@@ -165,6 +165,7 @@ In fact, any LLM that supports OpenAI API calls can be integrated.
 | Type |    Platform Name    | Usage Method | Pricing Model | Remarks |
 |:----:|:-------------------:|:------------:|:-------------:|:-------:|
 | ASR  | FunASR              | Local        | Free          |         |
+| ASR  | SherpaASR           | Local        | Free          |         |
 | ASR  | DoubaoASR           | API call     | Paid          |         |
 
 ---

main/manager-web/src/apis/module/user.js

@@ -18,20 +18,6 @@ export default {
                 })
             }).send()
     },
-    // 获取用户信息
-    getUserInfo(callback) {
-        RequestService.sendRequest().url(`${getServiceUrl()}/api/v1/user/info`)
-            .method('GET')
-            .success((res) => {
-                RequestService.clearRequestTime()
-                callback(res)
-            })
-            .fail(() => {
-                RequestService.reAjaxFun(() => {
-                    this.getUserInfo()
-                })
-            }).send()
-    },
     // 获取设备信息
     getHomeList(callback) {
         RequestService.sendRequest().url(`${getServiceUrl()}/api/v1/user/device/bind`)
@@ -173,5 +159,52 @@ export default {
                 });
             }).send();
     },
+    // 获取智能体列表
+    getAgentList(callback) {
+        RequestService.sendRequest()
+            .url(`${getServiceUrl()}/api/v1/user/agent`)
+            .method('GET')
+            .success((res) => {
+                RequestService.clearRequestTime();
+                callback(res);
+            })
+            .fail(() => {
+                RequestService.reAjaxFun(() => {
+                    this.getAgentList(callback);
+                });
+            }).send();
+        },
+
+    getUserInfo(callback) {
+        RequestService.sendRequest()
+            .url(`${getServiceUrl()}/api/v1/user/info`)
+            .method('GET')
+            .success((res) => {
+                RequestService.clearRequestTime()
+                callback(res)
+            })
+            .fail((err) => {
+                console.error('接口请求失败:', err)
+                RequestService.reAjaxFun(() => {
+                    this.getUserInfo(callback)
+                })
+            }).send()
+    },
+    // 添加智能体
+    addAgent(agentName, callback) {
+        RequestService.sendRequest()
+            .url(`${getServiceUrl()}/api/v1/user/agent`)
+            .method('POST')
+            .data({ name: agentName })
+            .success((res) => {
+                RequestService.clearRequestTime();
+                callback(res);
+            })
+            .fail(() => {
+                RequestService.reAjaxFun(() => {
+                    this.addAgent(agentName, callback);
+                });
+            }).send();
+    },
 
 }

main/manager-web/src/components/AddWisdomBodyDialog.vue

@@ -29,6 +29,9 @@
 </template>
 
 <script>
+import userApi from '@/apis/module/user';
+
+
 export default {
   name: 'AddWisdomBodyDialog',
   props: {
@@ -39,9 +42,16 @@ export default {
   },
   methods: {
     confirm() {
-      this.$emit('update:visible', false)
+      if (!this.wisdomBodyName.trim()) {
-      this.$emit('confirmed', this.wisdomBodyName)
+        this.$message.error('请输入智慧体名称');
-      this.wisdomBodyName = ""
+        return;
+      }
+      userApi.addAgent(this.wisdomBodyName, (res) => {
+        this.$message.success('添加成功');
+        this.$emit('confirm', res);
+        this.$emit('update:visible', false);
+        this.wisdomBodyName = "";
+      });
     },
     cancel() {
       this.$emit('update:visible', false)

main/manager-web/src/components/DeviceItem.vue

@@ -1,8 +1,8 @@
 <template>
   <div class="device-item">
     <div style="display: flex;justify-content: space-between;">
-      <div style="font-weight: 700;font-size: 24px;text-align: left;color: #3d4566;">
+      <div style="font-weight: 700;font-size: 18px;text-align: left;color: #3d4566;">
-        {{ device.mac }}
+         {{ device.agentName }}
       </div>
       <div>
         <img src="@/assets/home/delete.png" alt=""
@@ -11,10 +11,10 @@
       </div>
     </div>
     <div class="device-name">
-      设备型号：{{ device.model }}
+      设备型号：{{ device.ttsModelName }}
     </div>
     <div class="device-name">
-      音色模型：{{ device.voiceModel }}
+      音色模型：{{ device.ttsVoiceName }}
     </div>
     <div style="display: flex;gap: 10px;align-items: center;">
       <div class="settings-btn" @click="$emit('configure')">
@@ -31,7 +31,7 @@
       </div>
     </div>
     <div class="version-info">
-      <div>最近对话：{{ device.lastConversation }}</div>
+      <div>最近对话：{{ device.lastConnectedAt }}</div>
     </div>
   </div>
 </template>
@@ -58,7 +58,7 @@ export default {
 .device-name {
   margin: 7px 0 10px;
   font-weight: 400;
-  font-size: 10px;
+  font-size: 11px;
   color: #3d4566;
   text-align: left;
 }

main/manager-web/src/components/HeaderBar.vue

@@ -19,13 +19,13 @@
       </div>
       <div style="display: flex;align-items: center;gap: 7px; margin-top: 2px;">
         <div class="serach-box">
-          <el-input placeholder="输入名称搜索.." v-model="serach" style="border: none; background: transparent;" />
+          <el-input placeholder="输入名称搜索.." v-model="serach" style="border: none; background: transparent;" @keyup.enter.native="handleSearch" />
           <img src="@/assets/home/search.png" alt=""
-               style="width: 14px;height: 14px;margin-right: 11px;cursor: pointer;" />
+               style="width: 14px;height: 14px;margin-right: 11px;cursor: pointer;" @click="handleSearch" />
         </div>
         <img src="@/assets/home/avatar.png" alt="" style="width: 21px;height: 21px;" />
         <div class="user-info">
-          158 3632 4642
+           {{ userInfo.mobile || '加载中...' }}
         </div>
       </div>
     </div>
@@ -33,17 +33,57 @@
 </template>
 
 <script>
+import userApi from '@/apis/module/user'
+
+
 export default {
   name: 'HeaderBar',
+  props: ['devices'],  // 接收父组件设备列表
   data() {
-    return { serach: '' }
+    return {
+      serach: '',
+      userInfo: {
+        mobile: ''
+      }
+    }
+  },
+  mounted() {
+    this.fetchUserInfo()
   },
   methods: {
     goHome() {
       // 跳转到首页
       this.$router.push('/')
+    },
+    // 获取用户信息
+    fetchUserInfo() {
+      userApi.getUserInfo(({data}) => {
+        this.userInfo = data.data
+      })
+    },
+
+    // 处理搜索
+    handleSearch() {
+      const searchValue = this.serach.trim();
+      let filteredDevices;
+
+      if (!searchValue) {
+        // 当搜索内容为空时，显示原始完整列表
+        filteredDevices = this.$parent.originalDevices;
+      } else {
+         // 过滤逻辑
+        filteredDevices = this.devices.filter(device => {
+          return device.agentName.includes(searchValue) ||
+              device.ttsModelName.includes(searchValue) ||
+              device.ttsVoiceName.includes(searchValue);
+        });
+      }
+
+      this.$emit('search-result', filteredDevices);
     }
+
   }
+
 }
 </script>
 

main/manager-web/src/views/home.vue

@@ -1,7 +1,7 @@
 <template>
   <div class="welcome">
       <!-- 公共头部 -->
-      <HeaderBar />
+      <HeaderBar :devices="devices" @search-result="handleSearchResult" />
       <el-main style="padding: 20px;display: flex;flex-direction: column;">
         <div>
           <!-- 首页内容 -->
@@ -30,7 +30,7 @@
               </div>
             </div>
           </div>
-          <div style="display: flex;flex-wrap: wrap;margin-top: 20px;gap: 20px;justify-content: space-between;box-sizing: border-box;">
+          <div style="display: flex;flex-wrap: wrap;margin-top: 20px;gap: 20px;justify-content: flex-start;box-sizing: border-box;">
             <DeviceItem v-for="(item,index) in devices" :key="index" :device="item" @configure="goToRoleConfig" @deviceManage="handleDeviceManage" />
           </div>
         </div>
@@ -47,22 +47,22 @@
 import DeviceItem from '@/components/DeviceItem.vue'
 import AddWisdomBodyDialog from '@/components/AddWisdomBodyDialog.vue'
 import HeaderBar from '@/components/HeaderBar.vue'
+
 export default {
   name: 'HomePage',
   components: { DeviceItem, AddWisdomBodyDialog, HeaderBar },
   data() {
     return {
       addDeviceDialogVisible: false,
-      // 此处模拟设备列表（10条数据）
+      devices: [],
-      devices: Array.from({ length: 10 }, (_, i) => ({
+      originalDevices: [],
-        id: i,
-        mac: 'CC:ba:97:11:a6:ac',
-        model: 'esp32-s3-touch-amoled-1.8',
-        voiceModel: 'esp32-s3-touch-amoled-1.8',
-        lastConversation: '6天前',
-      }))
     }
   },
+
+  mounted() {
+    this.fetchAgentList();
+  },
+
   methods: {
     showAddDialog() {
       this.addDeviceDialogVisible = true
@@ -71,13 +71,28 @@ export default {
       // 点击配置角色后跳转到角色配置页
       this.$router.push('/role-config')
     },
-    handleWisdomBodyAdded(name) {
+    handleWisdomBodyAdded(res) {
-      console.log('新增智慧体名称：', name)
+      console.log('新增智能体响应：', res);
-      this.addDeviceDialogVisible = false
+      this.fetchAgentList();
+      this.addDeviceDialogVisible = false;
     },
     handleDeviceManage() {
       this.$router.push('/device-management');
     },
+    // 获取智能体列表
+    fetchAgentList() {
+      import('@/apis/module/user').then(({ default: userApi }) => {
+        userApi.getAgentList(({data}) => {
+          this.originalDevices = data.data;
+          this.devices = data.data;
+        });
+      });
+    },
+    // 搜索更新智能体列表
+    handleSearchResult(filteredList) {
+      this.devices = filteredList; // 更新设备列表
+    }
+
   }
 }
 </script>

main/xiaozhi-server/config.yaml

@@ -125,6 +125,10 @@ ASR:
     type: fun_local
     model_dir: models/SenseVoiceSmall
     output_dir: tmp/
+  SherpaASR:
+    type: sherpa_onnx_local
+    model_dir: models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17
+    output_dir: tmp/
   DoubaoASR:
     type: doubao
     appid: 你的火山引擎语音合成服务appid
@@ -138,6 +142,7 @@ VAD:
     min_silence_duration_ms: 700  # 如果说话停顿比较长，可以把这个值设置大一些
 
 LLM:
+  # 所有openai类型均可以修改超参，以AliLLM为例
   # 当前支持的type为openai、dify、ollama，可自行适配
   AliLLM:
     # 定义LLM API类型
@@ -146,6 +151,11 @@ LLM:
     base_url: https://dashscope.aliyuncs.com/compatible-mode/v1
     model_name: qwen-turbo
     api_key: 你的deepseek web key
+    temperature: 0.7  # 温度值
+    max_tokens: 500   # 最大生成token数
+    top_p: 1         
+    top_k: 50        
+    frequency_penalty: 0  # 频率惩罚
   DoubaoLLM:
     # 定义LLM API类型
     type: openai
@@ -181,8 +191,12 @@ LLM:
     type: dify
     # 建议使用本地部署的dify接口，国内部分区域访问dify公有云接口可能会受限
     # 如果使用DifyLLM，配置文件里prompt(提示词)是无效的，需要在dify控制台设置提示词
-    base_url: https://api.dify.cn/v1
+    base_url: https://api.dify.ai/v1
     api_key: 你的DifyLLM web key
+    # 使用的对话模式 可以选择工作流 workflows/run 对话模式 chat-messages  文本生成 completion-messages
+    # 使用workflows进行返回的时候输入参数为 query 返回参数的名字要设置为 answer
+    # 文本生成的默认输入参数也是query
+    mode: chat-messages
   GeminiLLM:
     type: gemini
     # 谷歌Gemini API，需要先在Google Cloud控制台创建API密钥并获取api_key
@@ -190,7 +204,9 @@ LLM:
     # token申请地址： https://aistudio.google.com/apikey
     # 若部署地无法访问接口，需要开启科学上网
     api_key: 你的gemini web key
-    model_name: "gemini-1.5-pro"  #  gemini-1.5-pro 是免费的
+    model_name: "gemini-2.0-flash"
+    http_proxy: ""  #"http://127.0.0.1:10808"
+    https_proxy: "" #http://127.0.0.1:10808"
   CozeLLM:
     # 定义LLM API类型
     type: coze
@@ -396,9 +412,21 @@ TTS:
     type: doubao
     api_url: https://api.302ai.cn/doubao/tts_hd
     authorization: "Bearer "
+    # 湾湾小何音色
     voice: "zh_female_wanwanxiaohe_moon_bigtts"
     output_dir: tmp/
     access_token: "你的302API密钥"
+  GizwitsTTS:
+    type: doubao
+    # 火山引擎作为基座，可以完全使用企业级火山引擎语音合成服务
+    # 前一万名注册的用户，将送5元体验金额
+    # 获取API Key地址：https://agentrouter.gizwitsapi.com/panel/token
+    api_url: https://bytedance.gizwitsapi.com/api/v1/tts
+    authorization: "Bearer "
+    # 湾湾小何音色
+    voice: "zh_female_wanwanxiaohe_moon_bigtts"
+    output_dir: tmp/
+    access_token: "你的机智云API key"
   ACGNTTS:
     #在线网址：https://acgn.ttson.cn/
     #token购买：www.ttson.cn

main/xiaozhi-server/core/providers/asr/sherpa_onnx_local.py

@@ -0,0 +1,164 @@
+import time
+import wave
+import os
+import sys
+import io
+from config.logger import setup_logging
+from typing import Optional, Tuple, List
+import uuid
+import opuslib_next
+from core.providers.asr.base import ASRProviderBase
+
+import numpy as np
+import sherpa_onnx
+
+from modelscope.hub.file_download import model_file_download
+
+TAG = __name__
+logger = setup_logging()
+
+
+# 捕获标准输出
+class CaptureOutput:
+    def __enter__(self):
+        self._output = io.StringIO()
+        self._original_stdout = sys.stdout
+        sys.stdout = self._output
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        sys.stdout = self._original_stdout
+        self.output = self._output.getvalue()
+        self._output.close()
+
+        # 将捕获到的内容通过 logger 输出
+        if self.output:
+            logger.bind(tag=TAG).info(self.output.strip())
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config: dict, delete_audio_file: bool):
+        self.model_dir = config.get("model_dir")
+        self.output_dir = config.get("output_dir")
+        self.delete_audio_file = delete_audio_file
+
+        # 确保输出目录存在
+        os.makedirs(self.output_dir, exist_ok=True)
+        
+        # 初始化模型文件路径
+        model_files = {
+            "model.int8.onnx": os.path.join(self.model_dir, "model.int8.onnx"),
+            "tokens.txt": os.path.join(self.model_dir, "tokens.txt")
+        }
+
+        # 下载并检查模型文件
+        try:
+            for file_name, file_path in model_files.items():
+                if not os.path.isfile(file_path):
+                    logger.bind(tag=TAG).info(f"正在下载模型文件: {file_name}")
+                    model_file_download(
+                        model_id="pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue",
+                        file_path=file_name,
+                        local_dir=self.model_dir
+                    )
+                    
+                    if not os.path.isfile(file_path):
+                        raise FileNotFoundError(f"模型文件下载失败: {file_path}")
+                        
+            self.model_path = model_files["model.int8.onnx"]
+            self.tokens_path = model_files["tokens.txt"]
+            
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"模型文件处理失败: {str(e)}")
+            raise
+
+        with CaptureOutput():
+            self.model = sherpa_onnx.OfflineRecognizer.from_sense_voice(
+                model=self.model_path,
+                tokens=self.tokens_path,
+                num_threads=2,
+                sample_rate=16000,
+                feature_dim=80,
+                decoding_method="greedy_search",
+                debug=False,
+                use_itn=True,
+            )
+
+    def save_audio_to_file(self, opus_data: List[bytes], session_id: str) -> str:
+        """将Opus音频数据解码并保存为WAV文件"""
+        file_name = f"asr_{session_id}_{uuid.uuid4()}.wav"
+        file_path = os.path.join(self.output_dir, file_name)
+
+        decoder = opuslib_next.Decoder(16000, 1)  # 16kHz, 单声道
+        pcm_data = []
+
+        for opus_packet in opus_data:
+            try:
+                pcm_frame = decoder.decode(opus_packet, 960)  # 960 samples = 60ms
+                pcm_data.append(pcm_frame)
+            except opuslib_next.OpusError as e:
+                logger.bind(tag=TAG).error(f"Opus解码错误: {e}", exc_info=True)
+
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)  # 2 bytes = 16-bit
+            wf.setframerate(16000)
+            wf.writeframes(b"".join(pcm_data))
+
+        return file_path
+
+    def read_wave(self, wave_filename: str) -> Tuple[np.ndarray, int]:
+        """
+        Args:
+        wave_filename:
+            Path to a wave file. It should be single channel and each sample should
+            be 16-bit. Its sample rate does not need to be 16kHz.
+        Returns:
+        Return a tuple containing:
+        - A 1-D array of dtype np.float32 containing the samples, which are
+        normalized to the range [-1, 1].
+        - sample rate of the wave file
+        """
+
+        with wave.open(wave_filename) as f:
+            assert f.getnchannels() == 1, f.getnchannels()
+            assert f.getsampwidth() == 2, f.getsampwidth()  # it is in bytes
+            num_samples = f.getnframes()
+            samples = f.readframes(num_samples)
+            samples_int16 = np.frombuffer(samples, dtype=np.int16)
+            samples_float32 = samples_int16.astype(np.float32)
+
+            samples_float32 = samples_float32 / 32768
+            return samples_float32, f.getframerate()
+
+    async def speech_to_text(self, opus_data: List[bytes], session_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """语音转文本主处理逻辑"""
+        file_path = None
+        try:
+            # 保存音频文件
+            start_time = time.time()
+            file_path = self.save_audio_to_file(opus_data, session_id)
+            logger.bind(tag=TAG).debug(f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}")
+
+            # 语音识别
+            start_time = time.time()
+            s = self.model.create_stream()
+            samples, sample_rate = self.read_wave(file_path)
+            s.accept_waveform(sample_rate, samples)
+            self.model.decode_stream(s)
+            text = s.result.text
+            logger.bind(tag=TAG).debug(f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}")
+
+            return text, file_path
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"语音识别失败: {e}", exc_info=True)
+            return "", None
+
+        finally:
+            # 文件清理逻辑
+            if self.delete_audio_file and file_path and os.path.exists(file_path):
+                try:
+                    os.remove(file_path)
+                    logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")

main/xiaozhi-server/core/providers/llm/dify/dify.py

@@ -9,6 +9,7 @@ logger = setup_logging()
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         self.api_key = config["api_key"]
+        self.mode = config.get("mode", "chat-messages")
         self.base_url = config.get("base_url", "https://api.dify.ai/v1").rstrip('/')
         self.session_conversation_map = {}  # 存储session_id和conversation_id的映射
 
@@ -19,27 +20,59 @@ class LLMProvider(LLMProviderBase):
             conversation_id = self.session_conversation_map.get(session_id)
 
             # 发起流式请求
-            with requests.post(
+            if self.mode == "chat-messages":
-                    f"{self.base_url}/chat-messages",
+                request_json = {
-                    headers={"Authorization": f"Bearer {self.api_key}"},
-                    json={
                         "query": last_msg["content"],
                         "response_mode": "streaming",
                         "user": session_id,
                         "inputs": {},
                         "conversation_id": conversation_id
-                    },
+                    }
+            elif self.mode == "workflows/run":
+                request_json = {
+                    "inputs": {"query": last_msg["content"]},
+                    "response_mode": "streaming",
+                    "user": session_id
+                }
+            elif self.mode == "completion-messages":
+                request_json = {
+                    "inputs": {"query": last_msg["content"]},
+                    "response_mode": "streaming",
+                    "user": session_id
+                }
+
+            with requests.post(
+                    f"{self.base_url}/{self.mode}",
+                    headers={"Authorization": f"Bearer {self.api_key}"},
+                    json=request_json,
                     stream=True
             ) as r:
-                for line in r.iter_lines():
+                if self.mode == "chat-messages":
-                    if line.startswith(b'data: '):
+                    for line in r.iter_lines():
-                        event = json.loads(line[6:])
+                        if line.startswith(b'data: '):
-                        # 如果没有找到conversation_id，则获取此次conversation_id
+                            event = json.loads(line[6:])
-                        if not conversation_id:
+                            # 如果没有找到conversation_id，则获取此次conversation_id
-                            conversation_id = event.get('conversation_id')
+                            if not conversation_id:
-                            self.session_conversation_map[session_id] = conversation_id  # 更新映射
+                                conversation_id = event.get('conversation_id')
-                        if event.get('answer'):
+                                self.session_conversation_map[session_id] = conversation_id  # 更新映射
-                            yield event['answer']
+                            if event.get('answer'):
+                                yield event['answer']
+                elif self.mode == "workflows/run":
+                    for line in r.iter_lines():
+                        # logger.bind(tag=TAG).info(f"chat message response: {line}")
+                        if line.startswith(b'data: '):
+                            event = json.loads(line[6:])
+                            if event.get('event') == "workflow_finished":
+                                if event['data']['status'] == "succeeded":
+                                    yield event['data']['outputs']['answer']
+                                else:
+                                    yield "【服务响应异常】"
+                elif self.mode == "completion-messages":
+                    for line in r.iter_lines():
+                        if line.startswith(b'data: '):
+                            event = json.loads(line[6:])
+                            if event.get('answer'):
+                                yield event['answer'] 
 
         except Exception as e:
             logger.bind(tag=TAG).error(f"Error in response generation: {e}")

main/xiaozhi-server/core/providers/llm/gemini/gemini.py

@@ -1,14 +1,19 @@
 import google.generativeai as genai
 from core.utils.util import check_model_key
 from core.providers.llm.base import LLMProviderBase
-
+from config.logger import setup_logging
+import requests
+import json
+TAG = __name__
+logger = setup_logging()
 
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
         """初始化Gemini LLM Provider"""
         self.model_name = config.get("model_name", "gemini-1.5-pro")
         self.api_key = config.get("api_key")
-
+        self.http_proxy=config.get("http_proxy")
+        self.https_proxy = config.get("https_proxy")
         have_key = check_model_key("LLM", self.api_key)
 
         if not have_key:
@@ -16,6 +21,19 @@ class LLMProvider(LLMProviderBase):
 
         try:
             # 初始化Gemini客户端
+            # 配置代理（如果提供了代理配置）
+            self.proxies=None
+            if self.http_proxy is not "" or self.https_proxy is not "":
+
+                self.proxies = {
+                    "http": self.http_proxy,
+                    "https": self.https_proxy,
+                }
+                logger.bind(tag=TAG).info(f"Gemini set proxys:{self.proxies}")
+                # 使用猴子补丁修改 google-generativeai 库的请求会话
+
+                # 使用 session 对象配置 genai
+
             genai.configure(api_key=self.api_key)
             self.model = genai.GenerativeModel(self.model_name)
 
@@ -46,26 +64,54 @@ class LLMProvider(LLMProviderBase):
                 if content:
                     chat_history.append({
                         "role": role,
-                        "parts": [content]
+                        "parts": [{"text":content}]
+
                     })
 
             # 获取当前消息
             current_msg = dialogue[-1]["content"]
 
-            # 创建新的聊天会话
+            # 构建请求体
-            chat = self.model.start_chat(history=chat_history)
+            request_body = {
+                "contents": chat_history + [{"role": "user", "parts": [{"text":current_msg}]}],
+                "generationConfig": self.generation_config
+            }
 
-            # 发送消息并获取流式响应
+            # 构建请求URL
-            response = chat.send_message(
+            url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_name}:generateContent?key={self.api_key}"
-                current_msg,
-                stream=True,
-                generation_config=self.generation_config
-            )
 
-            # 处理流式响应
+            # 构建请求头
-            for chunk in response:
+            headers = {
-                if hasattr(chunk, 'text') and chunk.text:
+                "Content-Type": "application/json",
-                    yield chunk.text
+            }
+
+            # 发送POST请求,经测试手动 request 无法使用 stream 模式
+            if self.proxies:
+                response = requests.post(url, headers=headers, json=request_body, stream=False, proxies=self.proxies)
+                try:
+                    data = response.json()  # 直接解析JSON
+                    if 'candidates' in data and data['candidates']:
+                        yield data['candidates'][0]['content']['parts'][0]['text']
+                    else:
+                        yield "未找到候选回复。"
+                except json.JSONDecodeError as e:
+                    yield f"JSON解码错误：{e}"
+                except Exception as e:
+                    yield f"发生错误：{e}"
+            else:
+                logger.bind(tag=TAG).info(f"Gemini stream mode ")
+                chat = self.model.start_chat(history=chat_history)
+
+                # 发送消息并获取流式响应
+                response = chat.send_message(
+                    current_msg,
+                    stream=True,
+                    generation_config=self.generation_config
+                )
+                # 处理流式响应
+                for chunk in response:
+                    if hasattr(chunk, 'text') and chunk.text:
+                        yield chunk.text
 
         except Exception as e:
             error_msg = str(e)
@@ -78,3 +124,13 @@ class LLMProvider(LLMProviderBase):
                 yield "【Gemini API key无效】"
             else:
                 yield f"【Gemini服务响应异常: {error_msg}】"
+
+
+
+
+        except requests.exceptions.RequestException as e:
+            yield f"请求失败：{e}"
+        except json.JSONDecodeError as e:
+            yield f"JSON解码错误：{e}"
+        except Exception as e:
+            yield f"发生错误：{e}"

main/xiaozhi-server/core/providers/llm/openai/openai.py

@@ -1,8 +1,11 @@
 import openai
+from config.logger import setup_logging
 from core.utils.util import check_model_key
 from core.providers.llm.base import LLMProviderBase
 
 TAG = __name__
+logger = setup_logging()
+
 
 class LLMProvider(LLMProviderBase):
     def __init__(self, config):
@@ -12,6 +15,8 @@ class LLMProvider(LLMProviderBase):
             self.base_url = config.get("base_url")
         else:
             self.base_url = config.get("url")
+        self.max_tokens = config.get("max_tokens", 500)
+
         check_model_key("LLM", self.api_key)
         self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
 
@@ -20,9 +25,10 @@ class LLMProvider(LLMProviderBase):
             responses = self.client.chat.completions.create(
                 model=self.model_name,
                 messages=dialogue,
-                stream=True
+                stream=True,
+                max_tokens=self.max_tokens,
             )
-            
+
             is_active = True
             for chunk in responses:
                 try:
@@ -43,7 +49,7 @@ class LLMProvider(LLMProviderBase):
                         yield content
 
         except Exception as e:
-            self.logger.bind(tag=TAG).error(f"Error in response generation: {e}")
+            logger.bind(tag=TAG).error(f"Error in response generation: {e}")
 
     def response_with_functions(self, session_id, dialogue, functions=None):
         try:
@@ -51,12 +57,12 @@ class LLMProvider(LLMProviderBase):
                 model=self.model_name,
                 messages=dialogue,
                 stream=True,
-                tools=functions,
+                tools=functions
             )
-            
+
             for chunk in stream:
                 yield chunk.choices[0].delta.content, chunk.choices[0].delta.tool_calls
-                
+
         except Exception as e:
             self.logger.bind(tag=TAG).error(f"Error in function call streaming: {e}")
-            yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}
+            yield {"type": "content", "content": f"【OpenAI服务响应异常: {e}】"}

main/xiaozhi-server/requirements.txt

@@ -20,3 +20,5 @@ requests==2.32.3
 cozepy==0.12.0
 mem0ai==0.1.62
 bs4==0.0.2
+modelscope==1.23.2
+sherpa_onnx==1.11.0