diff --git a/README.md b/README.md index 639c1e62..1fbe5db1 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,13 @@ 本项目需要配合esp32硬件设备使用，如果童鞋们已经购买了esp32相关硬件，且成功对接虾哥部署的后端，并且想自己独立搭建 `xiaozhi-esp32`后端服务，可学习本项目。 -想看使用效果,请猛戳这个视频[小智esp32连接自己的后台模型](https://www.bilibili.com/video/BV1FMFyejExX) +想看使用效果,请猛戳这个视频 + + + + 小智esp32连接自己的后台模型 + + 要想完整体验本项目，需要以下总体步骤： @@ -22,6 +28,10 @@ - 下载`xiaozhi-esp32`项目，把`接口地址`修改成本项目地址，然后编译，把新固件烧录到硬件设备上。 - 启动设备，查看电脑或服务器的控制台，如果看到日志，说明成功连到本项目的接口了。 +## 警告 + +本项目成立时间较短，还未通过网络安全测评，请勿在生产环境中使用。 + ## 功能清单 ## 已实现 @@ -166,7 +176,7 @@ docker rmi ccr.ccs.tencentyun.com/xinnan/xiaozhi-esp32-server:latest ### 1.安装基础环境 -本项目使用`conda`管理依赖环境，安装好后开始执行以下命令。 +本项目使用`conda`管理依赖环境，安装好后，开始执行以下命令。 ``` conda remove -n xiaozhi-esp32-server --all -y @@ -174,7 +184,7 @@ conda create -n xiaozhi-esp32-server python=3.10 -y conda activate xiaozhi-esp32-server ``` -如果你的电脑是windows，执行： +执行以上命令后， 如果你的电脑是Windows或Mac，执行下面的语句： ``` conda activate xiaozhi-esp32-server @@ -182,7 +192,7 @@ conda install conda-forge::libopus conda install conda-forge::ffmpeg ``` -如果你的电脑是ubuntu，执行： +如果你的电脑是ubuntu，执行下面的语句： ``` apt-get install libopus0 ffmpeg diff --git a/README_en.md b/README_en.md index 5ccc2865..8d038e58 100644 --- a/README_en.md +++ b/README_en.md @@ -14,8 +14,13 @@ This project requires compatible esp32 hardware devices. If you have purchased e Xiage's deployed backend, and wish to independently set up the `xiaozhi-esp32` backend service, this project is for you. -To see a demo, watch this -video: [Xiaozhi ESP32 Connecting to Custom Backend Model](https://www.bilibili.com/video/BV1FMFyejExX) +To see a demo, watch this video: + + + + 小智esp32连接自己的后台模型 + + To fully experience this project, follow these steps: @@ -27,6 +32,11 @@ To fully experience this project, follow these steps: firmware to your device. - Start the device and check your server console logs to verify successful connection. +## Warning + +This project has been established for a short time and has not passed the network security assessment, so please do not +use it in the production environment. + ## Feature List ## Implemented @@ -64,7 +74,8 @@ is recommended to run the local source code. ## Method 1: Quick deployment of docker -The docker image has supported the CPU of x86 architecture and arm64 architecture, and supports running on Chinese operating systems. +The docker image has supported the CPU of x86 architecture and arm64 architecture, and supports running on Chinese +operating systems. 1. Install docker @@ -174,8 +185,7 @@ is very useful, and it is required to `compile esp32 firmware` later. ### 1.Install Prerequisites -This project uses `conda` to manage the dependent environment, and after installation, it starts executing the following -commands. +This project uses 'conda' to manage dependencies, and after installation, start executing the following commands: ``` conda remove -n xiaozhi-esp32-server --all -y @@ -183,7 +193,7 @@ conda create -n xiaozhi-esp32-server python=3.10 -y conda activate xiaozhi-esp32-server ``` -If your computer is Windows, execute: +After executing the above command, if your computer is Windows or Mac, execute the following statement: ``` conda activate xiaozhi-esp32-server @@ -191,7 +201,7 @@ conda install conda-forge::libopus conda install conda-forge::ffmpeg ``` -If your computer is ubuntu, execute: +If your computer is ubuntu, execute the following statement: ``` apt-get install libopus0 ffmpeg diff --git a/core/connection.py b/core/connection.py index cb2d06d0..3f59762f 100644 --- a/core/connection.py +++ b/core/connection.py @@ -153,12 +153,17 @@ class ConnectionHandler: text = None try: future = self.tts_queue.get() + if future is None: + continue text = None try: + self.logger.debug("正在处理TTS任务...") tts_file, text = future.result(timeout=10) + self.logger.debug(f"TTS文件生成完毕，文件路径: {tts_file}") if os.path.exists(tts_file): opus_datas, duration = self.tts.wav_to_opus_data(tts_file) else: + self.logger.error(f"TTS文件不存在: {tts_file}") opus_datas = [] duration = 0 except TimeoutError: @@ -175,6 +180,7 @@ class ConnectionHandler: if self.tts.delete_audio_file and os.path.exists(tts_file): os.remove(tts_file) except Exception as e: + self.logger.error(f"TTS任务处理错误: {e}") self.clearSpeakStatus() asyncio.run_coroutine_threadsafe( self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})), @@ -190,7 +196,7 @@ class ConnectionHandler: if tts_file is None: self.logger.error(f"tts转换失败，{text}") return None - self.logger.debug(f"TTS 文件生成完毕") + self.logger.debug(f"TTS 文件生成完毕: {tts_file}") return tts_file, text def clearSpeakStatus(self): diff --git a/core/handle/audioHandle.py b/core/handle/audioHandle.py index d15d1ff5..7217178c 100644 --- a/core/handle/audioHandle.py +++ b/core/handle/audioHandle.py @@ -36,6 +36,57 @@ async def handleAudioMessage(conn, audio): conn.reset_vad_states() async def startToChat(conn, text): + # 异步发送 stt 信息 + asyncio.create_task( + schedule_with_interrupt(0, send_stt_message(conn, text)) + ) + conn.executor.submit(conn.chat, text) + + +async def sendAudioMessage(conn, audios, duration, text): + base_delay = conn.tts_duration + + # 发送 tts.start + if text == conn.tts_first_text: + logger.info(f"发送第一段语音: {text}") + conn.tts_start_speak_time = time.time() + + # 发送 sentence_start（每个音频文件之前发送一次） + await send_tts_message(conn, "sentence_start", text) + + conn.tts_duration += duration + + # 发送音频数据 + for idx, opus_packet in enumerate(audios): + await conn.websocket.send(opus_packet) + + # 每个音频文件发送结束时，发送 sentence_end + if idx == len(audios) - 1: + await send_tts_message(conn, "sentence_end", text) + + if conn.llm_finish_task and text == conn.tts_last_text: + stop_duration = conn.tts_duration - (time.time() - conn.tts_start_speak_time) + stop_task = asyncio.create_task( + schedule_with_interrupt(stop_duration, send_tts_message(conn, 'stop')) + ) + conn.scheduled_tasks.append(stop_task) + +async def send_tts_message(conn, state, text=None): + """发送 TTS 状态消息""" + message = { + "type": "tts", + "state": state, + "session_id": conn.session_id + } + if text is not None: + message["text"] = text + + await conn.websocket.send(json.dumps(message)) + if state == "stop": + conn.clearSpeakStatus() + +async def send_stt_message(conn, text): + """发送 STT 状态消息""" stt_text = get_string_no_punctuation_or_emoji(text) await conn.websocket.send(json.dumps({ "type": "stt", @@ -49,66 +100,7 @@ async def startToChat(conn, text): "emotion": "happy", "session_id": conn.session_id} )) - conn.executor.submit(conn.chat, text) - - -async def sendAudioMessage(conn, audios, duration, text): - base_delay = conn.tts_duration - - if text == conn.tts_first_text: - logger.info(f"发送第一段语音: {text}") - conn.tts_start_speak_time = time.time() - await conn.websocket.send(json.dumps({ - "type": "tts", - "state": "start", - "session_id": conn.session_id - })) - - # 调度文字显示任务 - text_task = asyncio.create_task( - schedule_with_interrupt( - base_delay - 0.5, - send_sentence_start(conn, text) - ) - ) - conn.scheduled_tasks.append(text_task) - - conn.tts_duration = conn.tts_duration + duration - - # 发送音频数据 - for opus_packet in audios: - await conn.websocket.send(opus_packet) - - if conn.llm_finish_task and text == conn.tts_last_text: - stop_duration = conn.tts_duration - (time.time() - conn.tts_start_speak_time) - stop_task = asyncio.create_task( - schedule_with_interrupt(stop_duration, send_tts_stop(conn, text)) - ) - conn.scheduled_tasks.append(stop_task) - - -async def send_sentence_start(conn, text): - await conn.websocket.send(json.dumps({ - "type": "tts", - "state": "sentence_start", - "text": text, - "session_id": conn.session_id - })) - - -async def send_tts_stop(conn, text): - await conn.websocket.send(json.dumps({ - "type": "tts", - "state": "sentence_end", - "text": text, - "session_id": conn.session_id - })) - await conn.websocket.send(json.dumps({ - "type": "tts", - "state": "stop", - "session_id": conn.session_id - })) - conn.clearSpeakStatus() + await send_tts_message(conn, "start") async def schedule_with_interrupt(delay, coro): diff --git a/docs/images/demo.png b/docs/images/demo.png new file mode 100644 index 00000000..0fe4ebce Binary files /dev/null and b/docs/images/demo.png differ diff --git a/docs/images/wechat.jpg b/docs/images/wechat.jpg index fee5f051..105232b6 100644 Binary files a/docs/images/wechat.jpg and b/docs/images/wechat.jpg differ diff --git a/img.png b/img.png new file mode 100644 index 00000000..364c8b25 Binary files /dev/null and b/img.png differ