Merge branch 'main' into mem0ai

2026-07-22 07:03:53 +08:00 · 2025-03-04 09:13:14 +08:00
parent 343ecb3ad4 72fa13b0e7
commit cf7acb12d8
12 changed files with 104 additions and 163 deletions
@@ -16,6 +16,14 @@ jobs:
      issues: write

    steps:
+      - name: Check Disk Space
+        run: |
+          df -h
+          docker system df
+      - name: Clean up Docker resources
+        run: |
+          docker system prune -af
+          docker builder prune -af
      - name: Check out the repo
        uses: actions/checkout@v4

@@ -43,4 +51,4 @@ jobs:
          tags: |
            ghcr.io/${{ github.repository }}:${{ env.VERSION }}
            ghcr.io/${{ github.repository }}:latest
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm64
@@ -146,3 +146,4 @@ tmp
 .private_config.yaml
 .env.development
 docker-compose.yml
+web/vue/node_modules
@@ -1,20 +1,4 @@
-# 第一阶段：前端构建
-
-FROM kalicyh/node:v18-alpine AS frontend-builder
-
-WORKDIR /app/ZhiKongTaiWeb
-
-# RUN corepack enable && yarn config set registry https://registry.npmmirror.com
-
-COPY ZhiKongTaiWeb/package.json ZhiKongTaiWeb/yarn.lock ./
-
-RUN yarn install --frozen-lockfile
-
-COPY ZhiKongTaiWeb . 
-RUN yarn build
-
-# 第二阶段：构建 Python 依赖
-
+# 第一阶段：构建 Python 依赖
 FROM kalicyh/poetry:v3.10_xiaozhi AS builder

 WORKDIR /app
@@ -24,19 +8,6 @@ COPY . .
 # 检查是否有缺失
 RUN poetry install --no-root

-# 使用清华源加速apt安装，该镜像内置所以注释
-# RUN rm -rf /etc/apt/sources.list.d/* && \
-#     echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list && \
-#     echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \
-#     echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-backports main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \
-#     echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bookworm-security main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \
-#     apt-get update && \
-#     apt-get install -y --no-install-recommends libopus0 ffmpeg && \
-#     apt-get clean
-
-# 从构建阶段复制虚拟环境和前端构建产物
-COPY --from=frontend-builder /app/ZhiKongTaiWeb/dist /app/manager/static/webui
-
 # 设置虚拟环境路径
 ENV PATH="/app/.venv/bin:$PATH"

@@ -57,6 +57,22 @@
         </picture>
        </a>
    </td>
+  </tr>
+  <tr>
+    <td>
+        <a href="https://www.bilibili.com/video/BV1Vy96YCE3R" target="_blank">
+         <picture>
+           <img alt="自定义音色" src="docs/images/demo6.png" />
+         </picture>
+        </a>
+    </td>
+    <td>
+        <a href="https://www.bilibili.com/video/BV1VC96Y5EMH" target="_blank">
+         <picture>
+           <img alt="播放音乐" src="docs/images/demo7.png" />
+         </picture>
+        </a>
+    </td>
    <td>
        <a href="https://www.bilibili.com/video/BV1kgA2eYEQ9" target="_blank">
         <picture>
@@ -66,7 +82,7 @@
    </td>
    <td>
    </td>
-    </tr>
+  </tr>
 </table>

 ---
@@ -154,7 +154,6 @@ LLM:
    type: coze
    bot_id: 你的bot_id
    user_id: 你的user_id
-    base_url: "https://api.coze.cn/open_api/v2/chat" # 服务地址
    personal_access_token: 你的coze个人令牌
  LMStudioLLM:
    # 定义LLM API类型
@@ -22,6 +22,10 @@ from core.utils.auth_code_gen import AuthCodeGenerator
 TAG = __name__


+class TTSException(RuntimeError):
+    pass
+
+
 class ConnectionHandler:
    def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory):
        self.config = config
@@ -68,10 +72,8 @@ class ConnectionHandler:
        self.dialogue = Dialogue()

        # tts相关变量
-        self.tts_first_text = None
-        self.tts_last_text = None
-        self.tts_start_speak_time = None
-        self.tts_duration = 0
+        self.tts_first_text_index = -1
+        self.tts_last_text_index = -1

        # iot相关变量
        self.iot_descriptors = {}
@@ -229,6 +231,7 @@ class ConnectionHandler:
            return None

        self.llm_finish_task = False
+        text_index = 0
        for content in llm_responses:
            response_message.append(content)
            if self.client_abort:
@@ -242,7 +245,7 @@ class ConnectionHandler:
            current_text = full_text[processed_chars:]  # 从未处理的位置开始

            # 查找最后一个有效标点
-            punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：", "，")
+            punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：")
            last_punct_pos = -1
            for punct in punctuations:
                pos = current_text.rfind(punct)
@@ -254,8 +257,12 @@ class ConnectionHandler:
                segment_text_raw = current_text[:last_punct_pos + 1]
                segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
                if segment_text:
-                    self.recode_first_last_text(segment_text)
-                    future = self.executor.submit(self.speak_and_play, segment_text)
+                    # 强制设置空字符，测试TTS出错返回语音的健壮性
+                    # if text_index % 2 == 0:
+                    #     segment_text = " "
+                    text_index += 1
+                    self.recode_first_last_text(segment_text, text_index)
+                    future = self.executor.submit(self.speak_and_play, segment_text, text_index)
                    self.tts_queue.put(future)
                    processed_chars += len(segment_text_raw)  # 更新已处理字符位置

@@ -265,8 +272,9 @@ class ConnectionHandler:
        if remaining_text:
            segment_text = get_string_no_punctuation_or_emoji(remaining_text)
            if segment_text:
-                self.recode_first_last_text(segment_text)
-                future = self.executor.submit(self.speak_and_play, segment_text)
+                text_index += 1
+                self.recode_first_last_text(segment_text, text_index)
+                future = self.executor.submit(self.speak_and_play, segment_text, text_index)
                self.tts_queue.put(future)

        self.llm_finish_task = True
@@ -282,30 +290,28 @@ class ConnectionHandler:
                if future is None:
                    continue
                text = None
+                opus_datas, text_index, tts_file = [], 0, None
                try:
                    self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
-                    tts_file, text = future.result(timeout=10)
+                    tts_file, text, text_index = future.result(timeout=10)
                    if text is None or len(text) <= 0:
-                        continue
-                    if tts_file is None:
-                        self.logger.bind(tag=TAG).error(f"TTS文件生成失败: {text}")
-                        continue
-                    self.logger.bind(tag=TAG).debug(f"TTS文件生成完毕，文件路径: {tts_file}")
-                    if os.path.exists(tts_file):
-                        opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
+                        self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
+                    elif tts_file is None:
+                        self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}")
                    else:
-                        self.logger.bind(tag=TAG).error(f"TTS文件不存在: {tts_file}")
-                        opus_datas = []
+                        self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
+                        if os.path.exists(tts_file):
+                            opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
+                        else:
+                            self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
                except TimeoutError:
-                    self.logger.bind(tag=TAG).error("TTS 任务超时")
-                    continue
+                    self.logger.bind(tag=TAG).error("TTS超时")
                except Exception as e:
-                    self.logger.bind(tag=TAG).error(f"TTS 任务出错: {e}")
-                    continue
+                    self.logger.bind(tag=TAG).error(f"TTS出错: {e}")
                if not self.client_abort:
                    # 如果没有中途打断就发送语音
-                    self.audio_play_queue.put((opus_datas, text))
-                if self.tts.delete_audio_file and os.path.exists(tts_file):
+                    self.audio_play_queue.put((opus_datas, text, text_index))
+                if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
                    os.remove(tts_file)
            except Exception as e:
                self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
@@ -314,42 +320,41 @@ class ConnectionHandler:
                    self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
                    self.loop
                )
-                self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text}{e}")
+                self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")

    def _audio_play_priority_thread(self):
        while not self.stop_event.is_set():
            text = None
            try:
-                opus_datas, text = self.audio_play_queue.get()
-                future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text), self.loop)
+                opus_datas, text, text_index = self.audio_play_queue.get()
+                future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
+                                                          self.loop)
                future.result()
            except Exception as e:
-                self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text}{e}")
+                self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")

-    def speak_and_play(self, text):
+    def speak_and_play(self, text, text_index=0):
        if text is None or len(text) <= 0:
            self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
-            return None, text
+            return None, text, text_index
        tts_file = self.tts.to_tts(text)
        if tts_file is None:
            self.logger.bind(tag=TAG).error(f"tts转换失败，{text}")
-            return None, text
+            return None, text, text_index
        self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}")
-        return tts_file, text
+        return tts_file, text, text_index

    def clearSpeakStatus(self):
        self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
        self.asr_server_receive = True
-        self.tts_last_text = None
-        self.tts_first_text = None
-        self.tts_duration = 0
-        self.tts_start_speak_time = None
+        self.tts_last_text_index = -1
+        self.tts_first_text_index = -1

-    def recode_first_last_text(self, text):
-        if not self.tts_first_text:
+    def recode_first_last_text(self, text, text_index=0):
+        if self.tts_first_text_index == -1:
            self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}")
-            self.tts_first_text = text
-        self.tts_last_text = text
+            self.tts_first_text_index = text_index
+        self.tts_last_text_index = text_index

    async def close(self):
        """资源清理方法"""
@@ -139,14 +139,14 @@ class MusicHandler:
                    return
            text = f"正在播放{selected_music}"
            await send_stt_message(conn, text)
-            conn.tts_first_text = selected_music
-            conn.tts_last_text = selected_music
+            conn.tts_first_text_index = 0
+            conn.tts_last_text_index = 0
            conn.llm_finish_task = True
            if music_path.endswith(".p3"):
                opus_packets, duration = p3.decode_opus_from_file(music_path)
            else:
                opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
-            conn.audio_play_queue.put((opus_packets, selected_music))
+            conn.audio_play_queue.put((opus_packets, selected_music, 0))

        except Exception as e:
            logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
@@ -15,11 +15,10 @@ async def isLLMWantToFinish(last_text):
    return False


-async def sendAudioMessage(conn, audios, text):
+async def sendAudioMessage(conn, audios, text, text_index=0):
    # 发送句子开始消息
-    if text == conn.tts_first_text:
+    if text_index == conn.tts_first_text_index:
        logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
-        conn.tts_start_speak_time = time.perf_counter()
    await send_tts_message(conn, "sentence_start", text)

    # 初始化流控参数
@@ -45,7 +44,7 @@ async def sendAudioMessage(conn, audios, text):
        play_position += frame_duration  # 更新播放位置
    await send_tts_message(conn, "sentence_end", text)
    # 发送结束消息（如果是最后一个文本）
-    if conn.llm_finish_task and text == conn.tts_last_text:
+    if conn.llm_finish_task and text_index == conn.tts_last_text_index:
        await send_tts_message(conn, 'stop', None)
        if await isLLMWantToFinish(text):
            await conn.close()
@@ -3,94 +3,35 @@ import requests
 import json
 import re
 from core.providers.llm.base import LLMProviderBase
+import os
+# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
+from cozepy import COZE_CN_BASE_URL
+from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType  # noqa

 TAG = __name__
 logger = setup_logging()

-# 定义用于匹配中文标点符号的正则表达式（包括句号、感叹号、问号、分号）
-punctuation_pattern = re.compile(r'([。！？；])')
-
 class LLMProvider(LLMProviderBase):
    def __init__(self, config):
        self.personal_access_token = config.get("personal_access_token")
        self.bot_id = config.get("bot_id")
-        self.user_id = config.get("user_id")  # 默认用户 ID
-        self.base_url = config.get("base_url")
+        self.user_id = config.get("user_id")

    def response(self, session_id, dialogue):
-        try:
-            # 从对话中取出最新的用户消息
-            last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
-            data = {
-                "conversation_id": session_id,
-                "bot_id": self.bot_id,
-                "user": self.user_id,
-                "query": last_msg["content"],
-                "stream": True
-            }
-            logger.bind(tag=TAG).info(f"发送到 Coze API 的请求数据: {json.dumps(data, ensure_ascii=False)}")
-            
-            headers = {
-                'Authorization': f'Bearer {self.personal_access_token}',
-                'Content-Type': 'application/json',
-                'Accept': '*/*',
-                'Host': 'api.coze.cn',
-                'Connection': 'keep-alive'
-            }
-            
-            response = requests.post(
-                self.base_url,
-                headers=headers,
-                json=data,
-                stream=True
-            )
-            logger.bind(tag=TAG).info(f"请求状态: {response.status_code}")
-            
-            if response.status_code == 200:
-                # 对每一行流数据进行处理，不做跨块累积
-                for line_bytes in response.iter_lines(decode_unicode=False):
-                    if not line_bytes:
-                        continue
-                    try:
-                        # 使用 utf-8 解码，错误部分用替换符
-                        line = line_bytes.decode('utf-8', errors='replace')
-                    except Exception as e:
-                        logger.bind(tag=TAG).error(f"解码失败: {e}")
-                        continue
-                    if line.startswith("data:"):
-                        data_str = line[len("data:"):].strip()
-                        if data_str == "[DONE]":
-                            break
-                        try:
-                            data_chunk = json.loads(data_str)
-                        except json.JSONDecodeError as e:
-                            logger.bind(tag=TAG).error(f"JSON解析失败: {e} 数据: {line}")
-                            continue
-                        msg = data_chunk.get("message", {})
-                        if msg.get("role") == "assistant" and msg.get("type") == "answer":
-                            content = msg.get("content", "")
-                            # 如果返回内容中包含标点符号，则按标点拆分，立即返回每个片段
-                            if punctuation_pattern.search(content):
-                                # 利用 finditer 找到每个标点，并返回以标点结尾的片段
-                                start = 0
-                                for match in punctuation_pattern.finditer(content):
-                                    end = match.end()
-                                    sentence = content[start:end].strip()
-                                    if sentence:
-                                        yield sentence
-                                    start = end
-                                # 如果拆分后剩余内容也返回（不含标点），直接返回
-                                if start < len(content):
-                                    remainder = content[start:].strip()
-                                    if remainder:
-                                        yield remainder
-                            else:
-                                # 如果没有标点，则直接返回这块内容
-                                if content.strip():
-                                    yield content.strip()
-            else:
-                logger.bind(tag=TAG).error(f"请求失败，状态码: {response.status_code}")
-                yield f"【Coze服务响应异常：请求失败，状态码 {response.status_code}】"
-        except Exception as e:
-            logger.bind(tag=TAG).error(f"Error in Coze response generation: {e}")
-            yield "【Coze服务响应异常】"
+        coze_api_token = self.personal_access_token
+        coze_api_base = COZE_CN_BASE_URL
+
+        last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
+
+        coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
+
+        for event in coze.chat.stream(
+            bot_id=self.bot_id,
+            user_id=self.user_id,
+            additional_messages=[
+                Message.build_user_question_text(last_msg["content"]),
+            ],
+        ):
+            if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
+                print(event.message.content, end="", flush=True)
+                yield event.message.content
@@ -16,4 +16,5 @@ aiohttp_cors==0.7.0
 ormsgpack==1.7.0
 ruamel.yaml==0.18.10
 loguru==0.7.3
-requests==2.32.3
+requests==2.32.3
+cozepy==0.12.0