From 992eb89d1ad4f92d89097575c619a661a8c3d6b3 Mon Sep 17 00:00:00 2001 From: brucelee Date: Fri, 6 Jun 2025 15:39:25 +0800 Subject: [PATCH 01/10] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BE=9D=E8=B5=96?= =?UTF-8?q?=E6=96=87=E4=BB=B6=20requirements=5Fbruce.txt=EF=BC=8C=E5=8C=85?= =?UTF-8?q?=E5=90=AB=E5=A4=9A=E4=B8=AA=E5=BA=93=E7=9A=84=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E4=BB=A5=E6=94=AF=E6=8C=81=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/requirements_bruce.txt | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 main/xiaozhi-server/requirements_bruce.txt diff --git a/main/xiaozhi-server/requirements_bruce.txt b/main/xiaozhi-server/requirements_bruce.txt new file mode 100755 index 00000000..0c8840cd --- /dev/null +++ b/main/xiaozhi-server/requirements_bruce.txt @@ -0,0 +1,35 @@ +pyyml==0.0.2 +torch==2.2.2 +silero_vad==5.1.2 +websockets==14.2 +opuslib_next==1.1.2 +numpy==1.26.4 +pydub==0.25.1 +funasr==1.2.3 +torchaudio==2.2.2 +openai==1.61.0 +google-generativeai==0.8.4 +edge_tts==7.0.0 +httpx==0.27.2 +aiohttp==3.9.3 +aiohttp_cors==0.7.0 +ormsgpack==1.7.0 +ruamel.yaml==0.18.10 +loguru==0.7.3 +requests==2.32.3 +cozepy==0.12.0 +mem0ai==0.1.62 +bs4==0.0.2 +modelscope==1.23.2 +#sherpa_onnx==1.11.0 +sherpa_onnx==1.12.1 +mcp==1.8.1 +cnlunar==0.2.0 +PySocks==1.7.1 +dashscope==1.23.1 +baidu-aip==4.16.13 +chardet==5.2.0 +aioconsole==0.8.1 +markitdown==0.1.1 +mcp-proxy==0.6.0 +PyJWT==2.8.0 From a8f559b22cdbafec6925d5ca80e566533018b80a Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Fri, 6 Jun 2025 16:29:57 +0800 Subject: [PATCH 02/10] =?UTF-8?q?update:=20tts=5Fone=5Fsentence=E5=8D=95?= =?UTF-8?q?=E5=8F=A5=E6=96=87=E6=9C=AC=E5=88=86=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/providers/tts/base.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 70c68ee7..fc103bed 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -1,4 +1,5 @@ import os +import re import queue import uuid import asyncio @@ -141,15 +142,18 @@ class TTSProviderBase(ABC): content_type=ContentType.ACTION, ) ) - self.tts_text_queue.put( - TTSMessageDTO( - sentence_id=sentence_id, - sentence_type=SentenceType.MIDDLE, - content_type=content_type, - content_detail=content_detail, - content_file=content_file, + # 对于单句的文本，进行分段处理 + segments = re.split(r'([。！？!?；;\n])', content_detail) + for seg in segments: + self.tts_text_queue.put( + TTSMessageDTO( + sentence_id=sentence_id, + sentence_type=SentenceType.MIDDLE, + content_type=content_type, + content_detail=seg, + content_file=content_file, + ) ) - ) self.tts_text_queue.put( TTSMessageDTO( sentence_id=sentence_id, From b273181ba65a9968e9c8ee1001d98b8badaa4df3 Mon Sep 17 00:00:00 2001 From: Sakura-RanChen <1908198662@qq.com> Date: Mon, 9 Jun 2025 15:17:37 +0800 Subject: [PATCH 03/10] =?UTF-8?q?=E9=9A=90=E8=97=8F=E6=BB=91=E5=8A=A8?= =?UTF-8?q?=E6=9D=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/manager-web/src/views/DictManagement.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/manager-web/src/views/DictManagement.vue b/main/manager-web/src/views/DictManagement.vue index f4de8350..9f6c80ac 100644 --- a/main/manager-web/src/views/DictManagement.vue +++ b/main/manager-web/src/views/DictManagement.vue @@ -538,7 +538,7 @@ export default { .data-table { border-radius: 6px; - overflow-y: auto; + overflow-y: hidden; background-color: transparent !important; --table-max-height: calc(100vh - 40vh); max-height: var(--table-max-height); From cef5813318cce8bde86d6781e8fc7d242fc56c64 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 10 Jun 2025 11:19:30 +0800 Subject: [PATCH 04/10] =?UTF-8?q?update:=E7=BA=A0=E6=AD=A3=E6=9C=80?= =?UTF-8?q?=E7=AE=80=E5=8C=96=E9=83=A8=E7=BD=B2=E7=9A=84=E7=AB=AF=E5=8F=A3?= =?UTF-8?q?=E5=8F=B7=E6=98=AF8003?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/Deployment.md | 4 ++-- docs/firmware-build.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Deployment.md b/docs/Deployment.md index 01f0afeb..99aa8682 100644 --- a/docs/Deployment.md +++ b/docs/Deployment.md @@ -269,7 +269,7 @@ LLM: 如果你能看到，类似以下日志,则是本项目服务启动成功的标志。 ``` -250427 13:04:20[0.3.11_SiFuChTTnofu][__main__]-INFO-OTA接口是 http://192.168.4.123:8002/xiaozhi/ota/ +250427 13:04:20[0.3.11_SiFuChTTnofu][__main__]-INFO-OTA接口是 http://192.168.4.123:8003/xiaozhi/ota/ 250427 13:04:20[0.3.11_SiFuChTTnofu][__main__]-INFO-Websocket地址是 ws://192.168.4.123:8000/xiaozhi/v1/ 250427 13:04:20[0.3.11_SiFuChTTnofu][__main__]-INFO-=======上面的地址是websocket协议地址，请勿用浏览器访问======= 250427 13:04:20[0.3.11_SiFuChTTnofu][__main__]-INFO-如想测试websocket请用谷歌浏览器打开test目录下的test_page.html @@ -280,7 +280,7 @@ LLM: 但是如果你用docker部署，那么你的日志里给出的接口地址信息就不是真实的接口地址。 最正确的方法，是根据电脑的局域网IP来确定你的接口地址。 -如果你的电脑的局域网IP比如是`192.168.1.25`，那么你的接口地址就是：`ws://192.168.1.25:8000/xiaozhi/v1/`，对应的OTA地址就是：`http://192.168.1.25:8002/xiaozhi/ota/`。 +如果你的电脑的局域网IP比如是`192.168.1.25`，那么你的接口地址就是：`ws://192.168.1.25:8000/xiaozhi/v1/`，对应的OTA地址就是：`http://192.168.1.25:8003/xiaozhi/ota/`。 这个信息很有用的，后面`编译esp32固件`需要用到。 diff --git a/docs/firmware-build.md b/docs/firmware-build.md index fffac288..64584b27 100644 --- a/docs/firmware-build.md +++ b/docs/firmware-build.md @@ -9,7 +9,7 @@ ### 如果你用的是简单Server部署 此刻，请你用浏览器打开你的ota地址，例如我的ota地址 ``` -http://192.168.1.25:8002/xiaozhi/ota/ +http://192.168.1.25:8003/xiaozhi/ota/ ``` 如果显示“OTA接口运行正常，向设备发送的websocket地址是：ws://xxx:8000/xiaozhi/v1/ From 33faeee75d8ce3580f24853226b34e0318f2dd46 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Tue, 10 Jun 2025 14:14:02 +0800 Subject: [PATCH 05/10] =?UTF-8?q?update:=E9=80=9A=E8=BF=87=E8=BF=87?= =?UTF-8?q?=E6=BB=A4=E2=80=9C=E6=9B=B4=E6=96=B0=E5=94=A4=E9=86=92=E8=AF=8D?= =?UTF-8?q?=E5=93=8D=E5=BA=94=E9=85=8D=E7=BD=AE=E2=80=9D=E6=97=B6=E6=96=87?= =?UTF-8?q?=E6=9C=AC=E4=B8=AD=E7=9A=84=E8=A1=A8=E6=83=85=E7=AC=A6=E5=8F=B7?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3GBK=E7=BC=96=E7=A0=81=E4=BF=9D?= =?UTF-8?q?=E5=AD=98=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/utils/wakeup_word.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/main/xiaozhi-server/core/utils/wakeup_word.py b/main/xiaozhi-server/core/utils/wakeup_word.py index 29929081..fe07b41c 100644 --- a/main/xiaozhi-server/core/utils/wakeup_word.py +++ b/main/xiaozhi-server/core/utils/wakeup_word.py @@ -1,4 +1,5 @@ import os +import re import yaml import time import hashlib @@ -108,13 +109,16 @@ class WakeupWordsConfig: def update_wakeup_response(self, voice: str, file_path: str, text: str): """更新唤醒词回复配置""" try: + # 过滤表情字符 + filtered_text = re.sub(r'[\U0001F600-\U0001F64F]', '', text) + config = self._load_config() voice_hash = hashlib.md5(voice.encode()).hexdigest() config[voice_hash] = { "voice": voice, "file_path": file_path, "time": time.time(), - "text": text, + "text": filtered_text, } self._save_config(config) except Exception as e: From d49462152b65c3ea91801706d57c0d2ae37df4c4 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Tue, 10 Jun 2025 14:36:31 +0800 Subject: [PATCH 06/10] =?UTF-8?q?update:=E9=9F=B3=E9=A2=91=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/providers/tts/cozecn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/xiaozhi-server/core/providers/tts/cozecn.py b/main/xiaozhi-server/core/providers/tts/cozecn.py index ede08928..6c5d0547 100644 --- a/main/xiaozhi-server/core/providers/tts/cozecn.py +++ b/main/xiaozhi-server/core/providers/tts/cozecn.py @@ -11,8 +11,8 @@ class TTSProvider(TTSProviderBase): self.voice = config.get("private_voice") else: self.voice = config.get("voice") - self.response_format = config.get("response_format", "mp3") - self.audio_file_type = config.get("response_format", "mp3") + self.response_format = config.get("response_format", "wav") + self.audio_file_type = config.get("response_format", "wav") self.host = "api.coze.cn" self.api_url = f"https://{self.host}/v1/audio/speech" From aa7303c70725723e7765e9ceca2c7f5a93c4bcf8 Mon Sep 17 00:00:00 2001 From: brucelee Date: Tue, 10 Jun 2025 15:53:53 +0800 Subject: [PATCH 07/10] =?UTF-8?q?fix(agent):=20=E4=BF=AE=E5=A4=8D=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E6=99=BA=E8=83=BD=E4=BD=93=E6=97=B6=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E4=BF=9D=E5=AD=98=E8=A7=86=E8=A7=89=E6=A8=A1=E5=9E=8B=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent/controller/AgentController.java | 3 ++ .../modules/agent/dto/AgentUpdateDTO.java | 3 ++ main/xiaozhi-server/requirements_bruce.txt | 35 ------------------- 3 files changed, 6 insertions(+), 35 deletions(-) delete mode 100755 main/xiaozhi-server/requirements_bruce.txt diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java index 54f3fa8c..d2ed6f22 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java @@ -171,6 +171,9 @@ public class AgentController { if (dto.getLlmModelId() != null) { existingEntity.setLlmModelId(dto.getLlmModelId()); } + if (dto.getVllmModelId() != null) { + existingEntity.setVllmModelId(dto.getVllmModelId()); + } if (dto.getTtsModelId() != null) { existingEntity.setTtsModelId(dto.getTtsModelId()); } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentUpdateDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentUpdateDTO.java index ccfbd53f..ccfd14a9 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentUpdateDTO.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentUpdateDTO.java @@ -30,6 +30,9 @@ public class AgentUpdateDTO implements Serializable { @Schema(description = "大语言模型标识", example = "llm_model_02", required = false) private String llmModelId; + @Schema(description = "VLLM模型标识", example = "vllm_model_01", required = false) + private String vllmModelId; + @Schema(description = "语音合成模型标识", example = "tts_model_02", required = false) private String ttsModelId; diff --git a/main/xiaozhi-server/requirements_bruce.txt b/main/xiaozhi-server/requirements_bruce.txt deleted file mode 100755 index 0c8840cd..00000000 --- a/main/xiaozhi-server/requirements_bruce.txt +++ /dev/null @@ -1,35 +0,0 @@ -pyyml==0.0.2 -torch==2.2.2 -silero_vad==5.1.2 -websockets==14.2 -opuslib_next==1.1.2 -numpy==1.26.4 -pydub==0.25.1 -funasr==1.2.3 -torchaudio==2.2.2 -openai==1.61.0 -google-generativeai==0.8.4 -edge_tts==7.0.0 -httpx==0.27.2 -aiohttp==3.9.3 -aiohttp_cors==0.7.0 -ormsgpack==1.7.0 -ruamel.yaml==0.18.10 -loguru==0.7.3 -requests==2.32.3 -cozepy==0.12.0 -mem0ai==0.1.62 -bs4==0.0.2 -modelscope==1.23.2 -#sherpa_onnx==1.11.0 -sherpa_onnx==1.12.1 -mcp==1.8.1 -cnlunar==0.2.0 -PySocks==1.7.1 -dashscope==1.23.1 -baidu-aip==4.16.13 -chardet==5.2.0 -aioconsole==0.8.1 -markitdown==0.1.1 -mcp-proxy==0.6.0 -PyJWT==2.8.0 From fedb6f2da2f2e7edb479ec479a2f9698b8e6da74 Mon Sep 17 00:00:00 2001 From: CGD <3030332422@qq.com> Date: Tue, 10 Jun 2025 16:38:21 +0800 Subject: [PATCH 08/10] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E2=80=9C?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=94=A4=E9=86=92=E8=AF=8D=E5=9B=9E=E5=A4=8D?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E2=80=9D=E4=B8=AD=E7=9A=84=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E8=A1=A8=E6=83=85=E5=AD=97=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/utils/wakeup_word.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/xiaozhi-server/core/utils/wakeup_word.py b/main/xiaozhi-server/core/utils/wakeup_word.py index fe07b41c..8a7a96a0 100644 --- a/main/xiaozhi-server/core/utils/wakeup_word.py +++ b/main/xiaozhi-server/core/utils/wakeup_word.py @@ -109,8 +109,8 @@ class WakeupWordsConfig: def update_wakeup_response(self, voice: str, file_path: str, text: str): """更新唤醒词回复配置""" try: - # 过滤表情字符 - filtered_text = re.sub(r'[\U0001F600-\U0001F64F]', '', text) + # 过滤表情符号 + filtered_text = re.sub(r'[\U0001F600-\U0001F64F\U0001F900-\U0001F9FF]', '', text) config = self._load_config() voice_hash = hashlib.md5(voice.encode()).hexdigest() From 1c4e5f0eedf5e775c09a2ae24b7f71399ae77d7d Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 10 Jun 2025 22:12:39 +0800 Subject: [PATCH 09/10] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20#1538?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi-server/core/handle/helloHandle.py | 21 +++++++++++++------ main/xiaozhi-server/core/utils/wakeup_word.py | 10 ++------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/main/xiaozhi-server/core/handle/helloHandle.py b/main/xiaozhi-server/core/handle/helloHandle.py index 95b16109..f5a3b0cd 100644 --- a/main/xiaozhi-server/core/handle/helloHandle.py +++ b/main/xiaozhi-server/core/handle/helloHandle.py @@ -68,23 +68,32 @@ async def checkWakeupWords(conn, text): # 获取当前音色 voice = getattr(conn.tts, "voice", "default") + if not voice: + voice = "default" # 获取唤醒词回复配置 response = wakeup_words_config.get_wakeup_response(voice) + if not response or not response.get("file_path"): + response = { + "voice": "default", + "file_path": "config/assets/wakeup_words.wav", + "time": 0, + "text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦", + } # 播放唤醒词回复 conn.client_abort = False - opus_packets, _ = audio_to_data(response["file_path"]) + opus_packets, _ = audio_to_data(response.get("file_path")) - conn.logger.bind(tag=TAG).info(f"播放唤醒词回复: {response['text']}") - await sendAudioMessage(conn, SentenceType.FIRST, opus_packets, response["text"]) + conn.logger.bind(tag=TAG).info(f"播放唤醒词回复: {response.get('text')}") + await sendAudioMessage(conn, SentenceType.FIRST, opus_packets, response.get("text")) await sendAudioMessage(conn, SentenceType.LAST, [], None) # 补充对话 - conn.dialogue.put(Message(role="assistant", content=response["text"])) + conn.dialogue.put(Message(role="assistant", content=response.get("text"))) # 检查是否需要更新唤醒词回复 - if time.time() - response["time"] > WAKEUP_CONFIG["refresh_time"]: + if time.time() - response.get("time", 0) > WAKEUP_CONFIG["refresh_time"]: if not _wakeup_response_lock.locked(): asyncio.create_task(wakeupWordsResponse(conn)) return True @@ -104,7 +113,7 @@ async def wakeupWordsResponse(conn): question = ( "此刻用户正在和你说```" + wakeup_word - + "```。\n请你根据以上用户的内容进行简短回复。要像一个人正常人一样说话，不要像机器人一样说话。\n" + + "```。\n请你根据以上用户的内容进行20-30字回复。要符合系统设置的角色情感和态度，不要像机器人一样说话。\n" + "请勿对这条内容本身进行任何解释和回应，请勿返回表情符号，仅返回对用户的内容的回复。" ) diff --git a/main/xiaozhi-server/core/utils/wakeup_word.py b/main/xiaozhi-server/core/utils/wakeup_word.py index fe07b41c..dc767af8 100644 --- a/main/xiaozhi-server/core/utils/wakeup_word.py +++ b/main/xiaozhi-server/core/utils/wakeup_word.py @@ -89,20 +89,14 @@ class WakeupWordsConfig: voice = hashlib.md5(voice.encode()).hexdigest() """获取唤醒词回复配置""" config = self._load_config() - default_response = { - "voice": "default", - "file_path": "config/assets/wakeup_words.wav", - "time": 0, - "text": "哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦", - } if not config or voice not in config: - return default_response + return None # 检查文件大小 file_path = config[voice]["file_path"] if not os.path.exists(file_path) or os.stat(file_path).st_size < (15 * 1024): - return default_response + return None return config[voice] From 742db377f64efd84373999a6ec1d56713e5101b8 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Tue, 10 Jun 2025 23:01:31 +0800 Subject: [PATCH 10/10] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E6=99=BA=E8=83=BD=E4=BD=93=E7=9A=84=E9=BB=98=E8=AE=A4=E8=A7=86?= =?UTF-8?q?=E8=A7=89=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/xiaozhi/modules/agent/controller/AgentController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java b/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java index d2ed6f22..8aa912a4 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/agent/controller/AgentController.java @@ -106,6 +106,7 @@ public class AgentController { entity.setAsrModelId(template.getAsrModelId()); entity.setVadModelId(template.getVadModelId()); entity.setLlmModelId(template.getLlmModelId()); + entity.setVllmModelId(template.getVllmModelId()); entity.setTtsModelId(template.getTtsModelId()); entity.setTtsVoiceId(template.getTtsVoiceId()); entity.setMemModelId(template.getMemModelId());