diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index b957ee2e..1ed76391 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -16,6 +16,14 @@ jobs: issues: write steps: + - name: Check Disk Space + run: | + df -h + docker system df + - name: Clean up Docker resources + run: | + docker system prune -af + docker builder prune -af - name: Check out the repo uses: actions/checkout@v4 @@ -43,4 +51,4 @@ jobs: tags: | ghcr.io/${{ github.repository }}:${{ env.VERSION }} ghcr.io/${{ github.repository }}:latest - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64 \ No newline at end of file diff --git a/.gitignore b/.gitignore index c09248c5..cff679fb 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,4 @@ tmp .private_config.yaml .env.development docker-compose.yml +web/vue/node_modules diff --git a/Dockerfile b/Dockerfile index 9648b7b5..cd1fef9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,4 @@ -# 第一阶段：前端构建 - -FROM kalicyh/node:v18-alpine AS frontend-builder - -WORKDIR /app/ZhiKongTaiWeb - -# RUN corepack enable && yarn config set registry https://registry.npmmirror.com - -COPY ZhiKongTaiWeb/package.json ZhiKongTaiWeb/yarn.lock ./ - -RUN yarn install --frozen-lockfile - -COPY ZhiKongTaiWeb . -RUN yarn build - -# 第二阶段：构建 Python 依赖 - +# 第一阶段：构建 Python 依赖 FROM kalicyh/poetry:v3.10_xiaozhi AS builder WORKDIR /app @@ -24,19 +8,6 @@ COPY . . # 检查是否有缺失 RUN poetry install --no-root -# 使用清华源加速apt安装，该镜像内置所以注释 -# RUN rm -rf /etc/apt/sources.list.d/* && \ -# echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list && \ -# echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \ -# echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-backports main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \ -# echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bookworm-security main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \ -# apt-get update && \ -# apt-get install -y --no-install-recommends libopus0 ffmpeg && \ -# apt-get clean - -# 从构建阶段复制虚拟环境和前端构建产物 -COPY --from=frontend-builder /app/ZhiKongTaiWeb/dist /app/manager/static/webui - # 设置虚拟环境路径 ENV PATH="/app/.venv/bin:$PATH" diff --git a/README.md b/README.md index a0f6c4f1..23424ae7 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,22 @@ + + + + + + 自定义音色 + + + + + + + 播放音乐 + + + @@ -66,7 +82,7 @@ - + --- diff --git a/config.yaml b/config.yaml index 157a23e8..4185640b 100644 --- a/config.yaml +++ b/config.yaml @@ -154,7 +154,6 @@ LLM: type: coze bot_id: 你的bot_id user_id: 你的user_id - base_url: "https://api.coze.cn/open_api/v2/chat" # 服务地址 personal_access_token: 你的coze个人令牌 LMStudioLLM: # 定义LLM API类型 diff --git a/core/connection.py b/core/connection.py index 2f3e1f76..588a3a2c 100644 --- a/core/connection.py +++ b/core/connection.py @@ -22,6 +22,10 @@ from core.utils.auth_code_gen import AuthCodeGenerator TAG = __name__ +class TTSException(RuntimeError): + pass + + class ConnectionHandler: def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory): self.config = config @@ -68,10 +72,8 @@ class ConnectionHandler: self.dialogue = Dialogue() # tts相关变量 - self.tts_first_text = None - self.tts_last_text = None - self.tts_start_speak_time = None - self.tts_duration = 0 + self.tts_first_text_index = -1 + self.tts_last_text_index = -1 # iot相关变量 self.iot_descriptors = {} @@ -229,6 +231,7 @@ class ConnectionHandler: return None self.llm_finish_task = False + text_index = 0 for content in llm_responses: response_message.append(content) if self.client_abort: @@ -242,7 +245,7 @@ class ConnectionHandler: current_text = full_text[processed_chars:] # 从未处理的位置开始 # 查找最后一个有效标点 - punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：", "，") + punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：") last_punct_pos = -1 for punct in punctuations: pos = current_text.rfind(punct) @@ -254,8 +257,12 @@ class ConnectionHandler: segment_text_raw = current_text[:last_punct_pos + 1] segment_text = get_string_no_punctuation_or_emoji(segment_text_raw) if segment_text: - self.recode_first_last_text(segment_text) - future = self.executor.submit(self.speak_and_play, segment_text) + # 强制设置空字符，测试TTS出错返回语音的健壮性 + # if text_index % 2 == 0: + # segment_text = " " + text_index += 1 + self.recode_first_last_text(segment_text, text_index) + future = self.executor.submit(self.speak_and_play, segment_text, text_index) self.tts_queue.put(future) processed_chars += len(segment_text_raw) # 更新已处理字符位置 @@ -265,8 +272,9 @@ class ConnectionHandler: if remaining_text: segment_text = get_string_no_punctuation_or_emoji(remaining_text) if segment_text: - self.recode_first_last_text(segment_text) - future = self.executor.submit(self.speak_and_play, segment_text) + text_index += 1 + self.recode_first_last_text(segment_text, text_index) + future = self.executor.submit(self.speak_and_play, segment_text, text_index) self.tts_queue.put(future) self.llm_finish_task = True @@ -282,30 +290,28 @@ class ConnectionHandler: if future is None: continue text = None + opus_datas, text_index, tts_file = [], 0, None try: self.logger.bind(tag=TAG).debug("正在处理TTS任务...") - tts_file, text = future.result(timeout=10) + tts_file, text, text_index = future.result(timeout=10) if text is None or len(text) <= 0: - continue - if tts_file is None: - self.logger.bind(tag=TAG).error(f"TTS文件生成失败: {text}") - continue - self.logger.bind(tag=TAG).debug(f"TTS文件生成完毕，文件路径: {tts_file}") - if os.path.exists(tts_file): - opus_datas, duration = self.tts.wav_to_opus_data(tts_file) + self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty") + elif tts_file is None: + self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}") else: - self.logger.bind(tag=TAG).error(f"TTS文件不存在: {tts_file}") - opus_datas = [] + self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}") + if os.path.exists(tts_file): + opus_datas, duration = self.tts.wav_to_opus_data(tts_file) + else: + self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}") except TimeoutError: - self.logger.bind(tag=TAG).error("TTS 任务超时") - continue + self.logger.bind(tag=TAG).error("TTS超时") except Exception as e: - self.logger.bind(tag=TAG).error(f"TTS 任务出错: {e}") - continue + self.logger.bind(tag=TAG).error(f"TTS出错: {e}") if not self.client_abort: # 如果没有中途打断就发送语音 - self.audio_play_queue.put((opus_datas, text)) - if self.tts.delete_audio_file and os.path.exists(tts_file): + self.audio_play_queue.put((opus_datas, text, text_index)) + if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file): os.remove(tts_file) except Exception as e: self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}") @@ -314,42 +320,41 @@ class ConnectionHandler: self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})), self.loop ) - self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text}{e}") + self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}") def _audio_play_priority_thread(self): while not self.stop_event.is_set(): text = None try: - opus_datas, text = self.audio_play_queue.get() - future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text), self.loop) + opus_datas, text, text_index = self.audio_play_queue.get() + future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index), + self.loop) future.result() except Exception as e: - self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text}{e}") + self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}") - def speak_and_play(self, text): + def speak_and_play(self, text, text_index=0): if text is None or len(text) <= 0: self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}") - return None, text + return None, text, text_index tts_file = self.tts.to_tts(text) if tts_file is None: self.logger.bind(tag=TAG).error(f"tts转换失败，{text}") - return None, text + return None, text, text_index self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}") - return tts_file, text + return tts_file, text, text_index def clearSpeakStatus(self): self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态") self.asr_server_receive = True - self.tts_last_text = None - self.tts_first_text = None - self.tts_duration = 0 - self.tts_start_speak_time = None + self.tts_last_text_index = -1 + self.tts_first_text_index = -1 - def recode_first_last_text(self, text): - if not self.tts_first_text: + def recode_first_last_text(self, text, text_index=0): + if self.tts_first_text_index == -1: self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}") - self.tts_first_text = text - self.tts_last_text = text + self.tts_first_text_index = text_index + self.tts_last_text_index = text_index async def close(self): """资源清理方法""" diff --git a/core/handle/musicHandler.py b/core/handle/musicHandler.py index f1eb37ed..53ec49c4 100644 --- a/core/handle/musicHandler.py +++ b/core/handle/musicHandler.py @@ -139,14 +139,14 @@ class MusicHandler: return text = f"正在播放{selected_music}" await send_stt_message(conn, text) - conn.tts_first_text = selected_music - conn.tts_last_text = selected_music + conn.tts_first_text_index = 0 + conn.tts_last_text_index = 0 conn.llm_finish_task = True if music_path.endswith(".p3"): opus_packets, duration = p3.decode_opus_from_file(music_path) else: opus_packets, duration = conn.tts.wav_to_opus_data(music_path) - conn.audio_play_queue.put((opus_packets, selected_music)) + conn.audio_play_queue.put((opus_packets, selected_music, 0)) except Exception as e: logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}") diff --git a/core/handle/sendAudioHandle.py b/core/handle/sendAudioHandle.py index 07b24fb9..d0dcb82f 100644 --- a/core/handle/sendAudioHandle.py +++ b/core/handle/sendAudioHandle.py @@ -15,11 +15,10 @@ async def isLLMWantToFinish(last_text): return False -async def sendAudioMessage(conn, audios, text): +async def sendAudioMessage(conn, audios, text, text_index=0): # 发送句子开始消息 - if text == conn.tts_first_text: + if text_index == conn.tts_first_text_index: logger.bind(tag=TAG).info(f"发送第一段语音: {text}") - conn.tts_start_speak_time = time.perf_counter() await send_tts_message(conn, "sentence_start", text) # 初始化流控参数 @@ -45,7 +44,7 @@ async def sendAudioMessage(conn, audios, text): play_position += frame_duration # 更新播放位置 await send_tts_message(conn, "sentence_end", text) # 发送结束消息（如果是最后一个文本） - if conn.llm_finish_task and text == conn.tts_last_text: + if conn.llm_finish_task and text_index == conn.tts_last_text_index: await send_tts_message(conn, 'stop', None) if await isLLMWantToFinish(text): await conn.close() diff --git a/core/providers/llm/coze/coze.py b/core/providers/llm/coze/coze.py index 8abdfbc5..feb62ea6 100644 --- a/core/providers/llm/coze/coze.py +++ b/core/providers/llm/coze/coze.py @@ -3,94 +3,35 @@ import requests import json import re from core.providers.llm.base import LLMProviderBase +import os +# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py) +from cozepy import COZE_CN_BASE_URL +from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType # noqa TAG = __name__ logger = setup_logging() -# 定义用于匹配中文标点符号的正则表达式（包括句号、感叹号、问号、分号） -punctuation_pattern = re.compile(r'([。！？；])') - class LLMProvider(LLMProviderBase): def __init__(self, config): self.personal_access_token = config.get("personal_access_token") self.bot_id = config.get("bot_id") - self.user_id = config.get("user_id") # 默认用户 ID - self.base_url = config.get("base_url") + self.user_id = config.get("user_id") def response(self, session_id, dialogue): - try: - # 从对话中取出最新的用户消息 - last_msg = next(m for m in reversed(dialogue) if m["role"] == "user") - data = { - "conversation_id": session_id, - "bot_id": self.bot_id, - "user": self.user_id, - "query": last_msg["content"], - "stream": True - } - logger.bind(tag=TAG).info(f"发送到 Coze API 的请求数据: {json.dumps(data, ensure_ascii=False)}") - - headers = { - 'Authorization': f'Bearer {self.personal_access_token}', - 'Content-Type': 'application/json', - 'Accept': '*/*', - 'Host': 'api.coze.cn', - 'Connection': 'keep-alive' - } - - response = requests.post( - self.base_url, - headers=headers, - json=data, - stream=True - ) - logger.bind(tag=TAG).info(f"请求状态: {response.status_code}") - - if response.status_code == 200: - # 对每一行流数据进行处理，不做跨块累积 - for line_bytes in response.iter_lines(decode_unicode=False): - if not line_bytes: - continue - try: - # 使用 utf-8 解码，错误部分用替换符 - line = line_bytes.decode('utf-8', errors='replace') - except Exception as e: - logger.bind(tag=TAG).error(f"解码失败: {e}") - continue - if line.startswith("data:"): - data_str = line[len("data:"):].strip() - if data_str == "[DONE]": - break - try: - data_chunk = json.loads(data_str) - except json.JSONDecodeError as e: - logger.bind(tag=TAG).error(f"JSON解析失败: {e} 数据: {line}") - continue - msg = data_chunk.get("message", {}) - if msg.get("role") == "assistant" and msg.get("type") == "answer": - content = msg.get("content", "") - # 如果返回内容中包含标点符号，则按标点拆分，立即返回每个片段 - if punctuation_pattern.search(content): - # 利用 finditer 找到每个标点，并返回以标点结尾的片段 - start = 0 - for match in punctuation_pattern.finditer(content): - end = match.end() - sentence = content[start:end].strip() - if sentence: - yield sentence - start = end - # 如果拆分后剩余内容也返回（不含标点），直接返回 - if start < len(content): - remainder = content[start:].strip() - if remainder: - yield remainder - else: - # 如果没有标点，则直接返回这块内容 - if content.strip(): - yield content.strip() - else: - logger.bind(tag=TAG).error(f"请求失败，状态码: {response.status_code}") - yield f"【Coze服务响应异常：请求失败，状态码 {response.status_code}】" - except Exception as e: - logger.bind(tag=TAG).error(f"Error in Coze response generation: {e}") - yield "【Coze服务响应异常】" + coze_api_token = self.personal_access_token + coze_api_base = COZE_CN_BASE_URL + + last_msg = next(m for m in reversed(dialogue) if m["role"] == "user") + + coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base) + + for event in coze.chat.stream( + bot_id=self.bot_id, + user_id=self.user_id, + additional_messages=[ + Message.build_user_question_text(last_msg["content"]), + ], + ): + if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA: + print(event.message.content, end="", flush=True) + yield event.message.content \ No newline at end of file diff --git a/docs/images/demo6.png b/docs/images/demo6.png new file mode 100644 index 00000000..18d8f5f3 Binary files /dev/null and b/docs/images/demo6.png differ diff --git a/docs/images/demo7.png b/docs/images/demo7.png new file mode 100644 index 00000000..ed7a3557 Binary files /dev/null and b/docs/images/demo7.png differ diff --git a/requirements.txt b/requirements.txt index 2ccc0584..4d75889d 100755 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,5 @@ aiohttp_cors==0.7.0 ormsgpack==1.7.0 ruamel.yaml==0.18.10 loguru==0.7.3 -requests==2.32.3 \ No newline at end of file +requests==2.32.3 +cozepy==0.12.0 \ No newline at end of file