From 1c823c4255cc9dd04d752a27bfd3bd67a8c035d2 Mon Sep 17 00:00:00 2001 From: JavaZeroo <2487163254@qq.com> Date: Sat, 26 Apr 2025 00:43:40 +0800 Subject: [PATCH 01/11] feat: add news retrieval functionality and register new function --- Dockerfile-server | 8 + .../core/handle/functionHandler.py | 1 + .../core/handle/intentHandler.py | 75 ++++++-- .../providers/intent/intent_llm/intent_llm.py | 9 + .../get_news_from_different_source.py | 179 ++++++++++++++++++ 5 files changed, 255 insertions(+), 17 deletions(-) create mode 100644 main/xiaozhi-server/plugins_func/functions/get_news_from_different_source.py diff --git a/Dockerfile-server b/Dockerfile-server index 7eb77114..86f4d666 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -3,11 +3,18 @@ FROM python:3.10-slim AS builder WORKDIR /app +RUN apt-get update && \ + apt-get install -y --no-install-recommends git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + COPY main/xiaozhi-server/requirements.txt . # 安装Python依赖 RUN pip install --no-cache-dir -r requirements.txt +RUN git clone https://github.com/microsoft/markitdown.git && cd markitdown && pip install -e 'packages/markitdown[all]' + # 第二阶段：生产镜像 FROM python:3.10-slim @@ -21,6 +28,7 @@ RUN apt-get update && \ # 从构建阶段复制Python包和前端构建产物 COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages +COPY --from=builder /app/markitdown/ /app/markitdown/ # 复制应用代码 COPY main/xiaozhi-server . diff --git a/main/xiaozhi-server/core/handle/functionHandler.py b/main/xiaozhi-server/core/handle/functionHandler.py index b09a369c..10c2bf1a 100644 --- a/main/xiaozhi-server/core/handle/functionHandler.py +++ b/main/xiaozhi-server/core/handle/functionHandler.py @@ -55,6 +55,7 @@ class FunctionHandler: self.function_registry.register_function("get_time") self.function_registry.register_function("get_lunar") self.function_registry.register_function("handle_device") + self.function_registry.register_function("get_news_from_different_source") def register_config_functions(self): """注册配置中的函数,可以不同客户端使用不同的配置""" diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index 0174bc43..5b00f2ef 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -5,6 +5,8 @@ from core.handle.sendAudioHandle import send_stt_message from core.handle.helloHandle import checkWakeupWords from core.utils.util import remove_punctuation_and_length from core.utils.dialogue import Message +from plugins_func.register import Action +from loguru import logger TAG = __name__ @@ -100,24 +102,63 @@ async def process_intent_result(conn, intent_result, original_text): result = conn.func_handler.handle_llm_function_call( conn, function_call_data ) - if result and function_name != "play_music": - # 获取当前最新的文本索引 - text = result.response - if text is None: + logger.bind(tag=TAG).debug(f"检测到Action : {result.action}") + + if result: + if result.action == Action.RESPONSE: # 直接回复前端 + text = result.response + if text is not None: + text_index = ( + conn.tts_last_text_index + 1 + if hasattr(conn, "tts_last_text_index") + else 0 + ) + conn.recode_first_last_text(text, text_index) + future = conn.executor.submit( + conn.speak_and_play, text, text_index + ) + conn.llm_finish_task = True + conn.tts_queue.put(future) + conn.dialogue.put(Message(role="assistant", content=text)) + elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 text = result.result - if text is not None: - text_index = ( - conn.tts_last_text_index + 1 - if hasattr(conn, "tts_last_text_index") - else 0 - ) - conn.recode_first_last_text(text, text_index) - future = conn.executor.submit( - conn.speak_and_play, text, text_index - ) - conn.llm_finish_task = True - conn.tts_queue.put((future, text_index)) - conn.dialogue.put(Message(role="assistant", content=text)) + if text is not None and len(text) > 0: + conn.dialogue.put(Message(role="tool", content=text)) + conn.executor.submit(conn.chat_with_function_calling, text, True) + elif result.action == Action.NOTFOUND or result.action == Action.ERROR: + text = result.result + if text is not None: + text_index = ( + conn.tts_last_text_index + 1 + if hasattr(conn, "tts_last_text_index") + else 0 + ) + conn.recode_first_last_text(text, text_index) + future = conn.executor.submit( + conn.speak_and_play, text, text_index + ) + conn.llm_finish_task = True + conn.tts_queue.put((future, text_index)) + conn.dialogue.put(Message(role="assistant", content=text)) + elif function_name != "play_music": + # For backward compatibility with original code + # 获取当前最新的文本索引 + text = result.response + if text is None: + text = result.result + if text is not None: + text_index = ( + conn.tts_last_text_index + 1 + if hasattr(conn, "tts_last_text_index") + else 0 + ) + conn.recode_first_last_text(text, text_index) + future = conn.executor.submit( + conn.speak_and_play, text, text_index + ) + conn.llm_finish_task = True + conn.tts_queue.put(future) + conn.dialogue.put(Message(role="assistant", content=text)) # 将函数执行放在线程池中 conn.executor.submit(process_function_call) diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py index 05d01c10..9915be33 100644 --- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py +++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py @@ -41,6 +41,7 @@ class IntentProvider(IntentProviderBase): '2. 结束对话意图: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n' '3. 获取当天日期时间: {"function_call": {"name": "get_time"}}\n' '4. 继续聊天意图: {"function_call": {"name": "continue_chat"}}\n' + '5. 查询新闻意图: {"function_call": {"name": "get_news_from_different_source", "arguments":{"source": {"type": "string","description": "新闻源，可选项有{"thepaper": "澎湃新闻","wallstreetcn-quick": "华尔街见闻","ithome": "IT之家","zhihu": "知乎"}。可选参数，如果不提供则使用默认新闻源"},"detail": {"type": "boolean","description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容"}}}\n' "\n" "注意:\n" '- 播放音乐：无歌名时，song_name设为"random"\n' @@ -53,6 +54,14 @@ class IntentProvider(IntentProviderBase): '返回: {"function_call": {"name": "continue_chat"}}\n' "```\n" "```\n" + "用户: 最近有什么新闻吗\n" + '返回: {"function_call": {"name": "get_news_from_different_source", "arguments": {"source":"thepaper", "detail":"False"}}}\n' + "```\n" + "```\n" + "用户: 详细说说这个新闻\n" + '返回: {"function_call": {"name": "get_news_from_different_source", "arguments": {"source":"thepaper", "detail":"True"}}}\n' + "```\n" + "```\n" "用户: 现在是几号了?现在几点了？\n" '返回: {"function_call": {"name": "get_time"}}\n' "```\n" diff --git a/main/xiaozhi-server/plugins_func/functions/get_news_from_different_source.py b/main/xiaozhi-server/plugins_func/functions/get_news_from_different_source.py new file mode 100644 index 00000000..c6440f8f --- /dev/null +++ b/main/xiaozhi-server/plugins_func/functions/get_news_from_different_source.py @@ -0,0 +1,179 @@ +import random +import requests +import json +from config.logger import setup_logging +from plugins_func.register import register_function, ToolType, ActionResponse, Action +from markitdown import MarkItDown + +TAG = __name__ +logger = setup_logging() + +# 新闻来源字典，包含名称和对应的API ID +NEWS_SOURCES = { + "thepaper": "澎湃新闻", + "cls-depth": "财联社", +} + +# 动态生成新闻源描述 +def generate_news_sources_description(): + sources_desc = [] + for source_id, source_name in NEWS_SOURCES.items(): + sources_desc.append(f"{source_name}({source_id})") + return "、".join(sources_desc) + +GET_NEWS_FROM_DIFFERENT_SOURCE_FUNCTION_DESC = { + "type": "function", + "function": { + "name": "get_news_from_different_source", + "description": ( + "获取最新新闻，随机选择一条新闻进行播报。" + f"用户可以选择不同的新闻源，如{generate_news_sources_description()}等。" + "如果没有指定，默认从澎湃新闻获取。" + "用户可以要求获取详细内容，此时会获取新闻的详细内容。" + ), + "parameters": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": f"新闻源，例如{generate_news_sources_description()}等。可选参数，如果不提供则使用默认新闻源" + }, + "detail": { + "type": "boolean", + "description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容" + }, + "lang": { + "type": "string", + "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN" + } + }, + "required": ["lang"] + } + } +} + + +def fetch_news_from_api(source="thepaper"): + """从API获取新闻列表""" + try: + api_url = f"https://newsnow.busiyi.world/api/s?id={source}" + response = requests.get(api_url, timeout=10) + response.raise_for_status() + + data = response.json() + + if "items" in data: + return data["items"] + else: + logger.bind(tag=TAG).error(f"获取新闻API响应格式错误: {data}") + return [] + + except Exception as e: + logger.bind(tag=TAG).error(f"获取新闻API失败: {e}") + return [] + + +def fetch_news_detail(url): + """获取新闻详情页内容并使用MarkItDown清理HTML""" + try: + response = requests.get(url, timeout=10) + response.raise_for_status() + + # 使用MarkItDown清理HTML内容 + md = MarkItDown(enable_plugins=False) + result = md.convert(response) + + # 获取清理后的文本内容 + clean_text = result.text_content + + # 如果清理后的内容为空，返回提示信息 + if not clean_text or len(clean_text.strip()) == 0: + logger.bind(tag=TAG).warning(f"清理后的新闻内容为空: {url}") + return "无法解析新闻详情内容，可能是网站结构特殊或内容受限。" + + return clean_text + except Exception as e: + logger.bind(tag=TAG).error(f"获取新闻详情失败: {e}") + return "无法获取详细内容" + + +@register_function('get_news_from_different_source', GET_NEWS_FROM_DIFFERENT_SOURCE_FUNCTION_DESC, ToolType.SYSTEM_CTL) +def get_news_from_different_source(conn, source: str = "thepaper", detail: bool = False, lang: str = "zh_CN"): + """获取新闻并随机选择一条进行播报，或获取上一条新闻的详细内容""" + try: + # 如果detail为True，获取上一条新闻的详细内容 + detail = str(detail).lower() == 'true' + if detail: + if not hasattr(conn, 'last_news_link_different') or not conn.last_news_link_different or 'url' not in conn.last_news_link_different: + return ActionResponse(Action.REQLLM, "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", None) + + url = conn.last_news_link_different.get('url') + title = conn.last_news_link_different.get('title', '未知标题') + source_id = conn.last_news_link_different.get('source_id', 'thepaper') + source_name = NEWS_SOURCES.get(source_id, '未知来源') + + if not url or url == '#': + return ActionResponse(Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None) + + logger.bind(tag=TAG).debug(f"获取新闻详情: {title}, 来源: {source_name}, URL={url}") + + # 获取新闻详情 + detail_content = fetch_news_detail(url) + + if not detail_content or detail_content == "无法获取详细内容": + return ActionResponse(Action.REQLLM, + f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", None) + + # 构建详情报告 + detail_report = ( + f"根据下列数据，用{lang}回应用户的新闻详情查询请求：\n\n" + f"新闻标题: {title}\n" + # f"新闻来源: {source_name}\n" + f"详细内容: {detail_content}\n\n" + f"(请对上述新闻内容进行总结，提取关键信息，以自然、流畅的方式向用户播报，" + f"不要提及这是总结，就像是在讲述一个完整的新闻故事)" + ) + + return ActionResponse(Action.REQLLM, detail_report, None) + + # 否则，获取新闻列表并随机选择一条 + # 验证新闻源是否有效，如果无效则使用默认源 + if source not in NEWS_SOURCES: + logger.bind(tag=TAG).warning(f"无效的新闻源: {source}，使用默认源thepaper") + source = "thepaper" + + source_name = NEWS_SOURCES.get(source, "澎湃新闻") + logger.bind(tag=TAG).info(f"获取新闻: 新闻源={source}({source_name})") + + # 获取新闻列表 + news_items = fetch_news_from_api(source) + + if not news_items: + return ActionResponse(Action.REQLLM, f"抱歉，未能从{source_name}获取到新闻信息，请稍后再试或尝试其他新闻源。", None) + + # 随机选择一条新闻 + selected_news = random.choice(news_items) + + # 保存当前新闻链接到连接对象，以便后续查询详情 + if not hasattr(conn, 'last_news_link_different'): + conn.last_news_link_different = {} + conn.last_news_link_different = { + 'url': selected_news.get('url', '#'), + 'title': selected_news.get('title', '未知标题'), + 'source_id': source + } + + # 构建新闻报告 + news_report = ( + f"根据下列数据，用{lang}回应用户的新闻查询请求：\n\n" + f"新闻标题: {selected_news['title']}\n" + # f"新闻来源: {source_name}\n" + f"(请以自然、流畅的方式向用户播报这条新闻标题，" + f"提示用户可以要求获取详细内容，此时会获取新闻的详细内容。)" + ) + + return ActionResponse(Action.REQLLM, news_report, None) + + except Exception as e: + logger.bind(tag=TAG).error(f"获取新闻出错: {e}") + return ActionResponse(Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None) \ No newline at end of file From 297c9e00857d5073559562065baeb91e6967a6a6 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 8 May 2025 18:23:06 +0800 Subject: [PATCH 02/11] =?UTF-8?q?update:=E9=87=8D=E5=91=BD=E5=90=8D?= =?UTF-8?q?=E6=96=B0=E9=97=BB=E6=8F=92=E4=BB=B6=EF=BC=8C=E4=BB=A5=E5=B9=B3?= =?UTF-8?q?=E5=8F=B0=E6=9D=A5=E6=BA=90=E6=9D=A5=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...get_news.py => get_news_from_chinanews.py} | 135 ++++++++++++------ ...ent_source.py => get_news_from_newsnow.py} | 119 +++++++++------ 2 files changed, 167 insertions(+), 87 deletions(-) rename main/xiaozhi-server/plugins_func/functions/{get_news.py => get_news_from_chinanews.py} (62%) rename main/xiaozhi-server/plugins_func/functions/{get_news_from_different_source.py => get_news_from_newsnow.py} (65%) diff --git a/main/xiaozhi-server/plugins_func/functions/get_news.py b/main/xiaozhi-server/plugins_func/functions/get_news_from_chinanews.py similarity index 62% rename from main/xiaozhi-server/plugins_func/functions/get_news.py rename to main/xiaozhi-server/plugins_func/functions/get_news_from_chinanews.py index 85b203dc..64d499b1 100644 --- a/main/xiaozhi-server/plugins_func/functions/get_news.py +++ b/main/xiaozhi-server/plugins_func/functions/get_news_from_chinanews.py @@ -8,10 +8,10 @@ from plugins_func.register import register_function, ToolType, ActionResponse, A TAG = __name__ logger = setup_logging() -GET_NEWS_FUNCTION_DESC = { +GET_NEWS_FROM_CHINANEWS_FUNCTION_DESC = { "type": "function", "function": { - "name": "get_news", + "name": "get_news_from_chinanews", "description": ( "获取最新新闻，随机选择一条新闻进行播报。" "用户可以指定新闻类型，如社会新闻、科技新闻、国际新闻等。" @@ -23,20 +23,20 @@ GET_NEWS_FUNCTION_DESC = { "properties": { "category": { "type": "string", - "description": "新闻类别，例如社会、科技、国际。可选参数，如果不提供则使用默认类别" + "description": "新闻类别，例如社会、科技、国际。可选参数，如果不提供则使用默认类别", }, "detail": { "type": "boolean", - "description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容" + "description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容", }, "lang": { "type": "string", - "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN" - } + "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN", + }, }, - "required": ["lang"] - } - } + "required": ["lang"], + }, + }, } @@ -51,18 +51,30 @@ def fetch_news_from_rss(rss_url): # 查找所有item元素（新闻条目） news_items = [] - for item in root.findall('.//item'): - title = item.find('title').text if item.find('title') is not None else "无标题" - link = item.find('link').text if item.find('link') is not None else "#" - description = item.find('description').text if item.find('description') is not None else "无描述" - pubDate = item.find('pubDate').text if item.find('pubDate') is not None else "未知时间" + for item in root.findall(".//item"): + title = ( + item.find("title").text if item.find("title") is not None else "无标题" + ) + link = item.find("link").text if item.find("link") is not None else "#" + description = ( + item.find("description").text + if item.find("description") is not None + else "无描述" + ) + pubDate = ( + item.find("pubDate").text + if item.find("pubDate") is not None + else "未知时间" + ) - news_items.append({ - 'title': title, - 'link': link, - 'description': description, - 'pubDate': pubDate - }) + news_items.append( + { + "title": title, + "link": link, + "description": description, + "pubDate": pubDate, + } + ) return news_items except Exception as e: @@ -76,18 +88,24 @@ def fetch_news_detail(url): response = requests.get(url) response.raise_for_status() - soup = BeautifulSoup(response.content, 'html.parser') + soup = BeautifulSoup(response.content, "html.parser") # 尝试提取正文内容 (这里的选择器需要根据实际网站结构调整) - content_div = soup.select_one('.content_desc, .content, article, .article-content') + content_div = soup.select_one( + ".content_desc, .content, article, .article-content" + ) if content_div: - paragraphs = content_div.find_all('p') - content = '\n'.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()]) + paragraphs = content_div.find_all("p") + content = "\n".join( + [p.get_text().strip() for p in paragraphs if p.get_text().strip()] + ) return content else: # 如果找不到特定的内容区域，尝试获取所有段落 - paragraphs = soup.find_all('p') - content = '\n'.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()]) + paragraphs = soup.find_all("p") + content = "\n".join( + [p.get_text().strip() for p in paragraphs if p.get_text().strip()] + ) return content[:2000] # 限制长度 except Exception as e: logger.bind(tag=TAG).error(f"获取新闻详情失败: {e}") @@ -111,7 +129,7 @@ def map_category(category_text): "财经": "finance", "财经新闻": "finance", "金融": "finance", - "经济": "finance" + "经济": "finance", } # 转换为小写并去除空格 @@ -121,20 +139,36 @@ def map_category(category_text): return category_map.get(normalized_category, category_text) -@register_function('get_news', GET_NEWS_FUNCTION_DESC, ToolType.SYSTEM_CTL) -def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_CN"): +@register_function( + "get_news_from_chinanews", + GET_NEWS_FROM_CHINANEWS_FUNCTION_DESC, + ToolType.SYSTEM_CTL, +) +def get_news_from_chinanews( + conn, category: str = None, detail: bool = False, lang: str = "zh_CN" +): """获取新闻并随机选择一条进行播报，或获取上一条新闻的详细内容""" try: # 如果detail为True，获取上一条新闻的详细内容 if detail: - if not hasattr(conn, 'last_news_link') or not conn.last_news_link or 'link' not in conn.last_news_link: - return ActionResponse(Action.REQLLM, "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", None) + if ( + not hasattr(conn, "last_news_link") + or not conn.last_news_link + or "link" not in conn.last_news_link + ): + return ActionResponse( + Action.REQLLM, + "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", + None, + ) - link = conn.last_news_link.get('link') - title = conn.last_news_link.get('title', '未知标题') + link = conn.last_news_link.get("link") + title = conn.last_news_link.get("title", "未知标题") - if link == '#': - return ActionResponse(Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None) + if link == "#": + return ActionResponse( + Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None + ) logger.bind(tag=TAG).debug(f"获取新闻详情: {title}, URL={link}") @@ -142,8 +176,11 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C detail_content = fetch_news_detail(link) if not detail_content or detail_content == "无法获取详细内容": - return ActionResponse(Action.REQLLM, - f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", None) + return ActionResponse( + Action.REQLLM, + f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", + None, + ) # 构建详情报告 detail_report = ( @@ -158,8 +195,10 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C # 否则，获取新闻列表并随机选择一条 # 从配置中获取RSS URL - rss_config = conn.config["plugins"]["get_news"] - default_rss_url = rss_config.get("default_rss_url", "https://www.chinanews.com.cn/rss/society.xml") + rss_config = conn.config["plugins"]["get_news_from_chinanews"] + default_rss_url = rss_config.get( + "default_rss_url", "https://www.chinanews.com.cn/rss/society.xml" + ) # 将用户输入的类别映射到配置中的类别键 mapped_category = map_category(category) @@ -169,23 +208,27 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C if mapped_category and mapped_category in rss_config.get("category_urls", {}): rss_url = rss_config["category_urls"][mapped_category] - logger.bind(tag=TAG).info(f"获取新闻: 原始类别={category}, 映射类别={mapped_category}, URL={rss_url}") + logger.bind(tag=TAG).info( + f"获取新闻: 原始类别={category}, 映射类别={mapped_category}, URL={rss_url}" + ) # 获取新闻列表 news_items = fetch_news_from_rss(rss_url) if not news_items: - return ActionResponse(Action.REQLLM, "抱歉，未能获取到新闻信息，请稍后再试。", None) + return ActionResponse( + Action.REQLLM, "抱歉，未能获取到新闻信息，请稍后再试。", None + ) # 随机选择一条新闻 selected_news = random.choice(news_items) # 保存当前新闻链接到连接对象，以便后续查询详情 - if not hasattr(conn, 'last_news_link'): + if not hasattr(conn, "last_news_link"): conn.last_news_link = {} conn.last_news_link = { - 'link': selected_news.get('link', '#'), - 'title': selected_news.get('title', '未知标题') + "link": selected_news.get("link", "#"), + "title": selected_news.get("title", "未知标题"), } # 构建新闻报告 @@ -203,4 +246,6 @@ def get_news(conn, category: str = None, detail: bool = False, lang: str = "zh_C except Exception as e: logger.bind(tag=TAG).error(f"获取新闻出错: {e}") - return ActionResponse(Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None) \ No newline at end of file + return ActionResponse( + Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None + ) diff --git a/main/xiaozhi-server/plugins_func/functions/get_news_from_different_source.py b/main/xiaozhi-server/plugins_func/functions/get_news_from_newsnow.py similarity index 65% rename from main/xiaozhi-server/plugins_func/functions/get_news_from_different_source.py rename to main/xiaozhi-server/plugins_func/functions/get_news_from_newsnow.py index c6440f8f..1b98539d 100644 --- a/main/xiaozhi-server/plugins_func/functions/get_news_from_different_source.py +++ b/main/xiaozhi-server/plugins_func/functions/get_news_from_newsnow.py @@ -11,9 +11,11 @@ logger = setup_logging() # 新闻来源字典，包含名称和对应的API ID NEWS_SOURCES = { "thepaper": "澎湃新闻", + "baidu": "百度热搜", "cls-depth": "财联社", } + # 动态生成新闻源描述 def generate_news_sources_description(): sources_desc = [] @@ -21,10 +23,11 @@ def generate_news_sources_description(): sources_desc.append(f"{source_name}({source_id})") return "、".join(sources_desc) -GET_NEWS_FROM_DIFFERENT_SOURCE_FUNCTION_DESC = { + +GET_NEWS_FROM_NEWSNOW_FUNCTION_DESC = { "type": "function", "function": { - "name": "get_news_from_different_source", + "name": "get_news_from_newsnow", "description": ( "获取最新新闻，随机选择一条新闻进行播报。" f"用户可以选择不同的新闻源，如{generate_news_sources_description()}等。" @@ -36,38 +39,43 @@ GET_NEWS_FROM_DIFFERENT_SOURCE_FUNCTION_DESC = { "properties": { "source": { "type": "string", - "description": f"新闻源，例如{generate_news_sources_description()}等。可选参数，如果不提供则使用默认新闻源" + "description": f"新闻源，例如{generate_news_sources_description()}等。可选参数，如果不提供则使用默认新闻源", }, "detail": { "type": "boolean", - "description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容" + "description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容", }, "lang": { "type": "string", - "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN" - } + "description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN", + }, }, - "required": ["lang"] - } - } + "required": ["lang"], + }, + }, } -def fetch_news_from_api(source="thepaper"): +def fetch_news_from_api(conn, source="thepaper"): """从API获取新闻列表""" try: api_url = f"https://newsnow.busiyi.world/api/s?id={source}" + if conn.config["plugins"].get("get_news_from_newsnow") and conn.config[ + "plugins" + ]["get_news_from_newsnow"].get("url"): + api_url = conn.config["plugins"]["get_news_from_newsnow"]["url"] + source + response = requests.get(api_url, timeout=10) response.raise_for_status() - + data = response.json() - + if "items" in data: return data["items"] else: logger.bind(tag=TAG).error(f"获取新闻API响应格式错误: {data}") return [] - + except Exception as e: logger.bind(tag=TAG).error(f"获取新闻API失败: {e}") return [] @@ -78,51 +86,72 @@ def fetch_news_detail(url): try: response = requests.get(url, timeout=10) response.raise_for_status() - + # 使用MarkItDown清理HTML内容 md = MarkItDown(enable_plugins=False) result = md.convert(response) - + # 获取清理后的文本内容 clean_text = result.text_content - + # 如果清理后的内容为空，返回提示信息 if not clean_text or len(clean_text.strip()) == 0: logger.bind(tag=TAG).warning(f"清理后的新闻内容为空: {url}") return "无法解析新闻详情内容，可能是网站结构特殊或内容受限。" - + return clean_text except Exception as e: logger.bind(tag=TAG).error(f"获取新闻详情失败: {e}") return "无法获取详细内容" -@register_function('get_news_from_different_source', GET_NEWS_FROM_DIFFERENT_SOURCE_FUNCTION_DESC, ToolType.SYSTEM_CTL) -def get_news_from_different_source(conn, source: str = "thepaper", detail: bool = False, lang: str = "zh_CN"): +@register_function( + "get_news_from_newsnow", + GET_NEWS_FROM_NEWSNOW_FUNCTION_DESC, + ToolType.SYSTEM_CTL, +) +def get_news_from_newsnow( + conn, source: str = "thepaper", detail: bool = False, lang: str = "zh_CN" +): """获取新闻并随机选择一条进行播报，或获取上一条新闻的详细内容""" try: # 如果detail为True，获取上一条新闻的详细内容 - detail = str(detail).lower() == 'true' + detail = str(detail).lower() == "true" if detail: - if not hasattr(conn, 'last_news_link_different') or not conn.last_news_link_different or 'url' not in conn.last_news_link_different: - return ActionResponse(Action.REQLLM, "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", None) + if ( + not hasattr(conn, "last_newsnow_link") + or not conn.last_newsnow_link + or "url" not in conn.last_newsnow_link + ): + return ActionResponse( + Action.REQLLM, + "抱歉，没有找到最近查询的新闻，请先获取一条新闻。", + None, + ) - url = conn.last_news_link_different.get('url') - title = conn.last_news_link_different.get('title', '未知标题') - source_id = conn.last_news_link_different.get('source_id', 'thepaper') - source_name = NEWS_SOURCES.get(source_id, '未知来源') + url = conn.last_newsnow_link.get("url") + title = conn.last_newsnow_link.get("title", "未知标题") + source_id = conn.last_newsnow_link.get("source_id", "thepaper") + source_name = NEWS_SOURCES.get(source_id, "未知来源") - if not url or url == '#': - return ActionResponse(Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None) + if not url or url == "#": + return ActionResponse( + Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None + ) - logger.bind(tag=TAG).debug(f"获取新闻详情: {title}, 来源: {source_name}, URL={url}") + logger.bind(tag=TAG).debug( + f"获取新闻详情: {title}, 来源: {source_name}, URL={url}" + ) # 获取新闻详情 detail_content = fetch_news_detail(url) if not detail_content or detail_content == "无法获取详细内容": - return ActionResponse(Action.REQLLM, - f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", None) + return ActionResponse( + Action.REQLLM, + f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。", + None, + ) # 构建详情报告 detail_report = ( @@ -141,26 +170,30 @@ def get_news_from_different_source(conn, source: str = "thepaper", detail: bool if source not in NEWS_SOURCES: logger.bind(tag=TAG).warning(f"无效的新闻源: {source}，使用默认源thepaper") source = "thepaper" - + source_name = NEWS_SOURCES.get(source, "澎湃新闻") logger.bind(tag=TAG).info(f"获取新闻: 新闻源={source}({source_name})") # 获取新闻列表 - news_items = fetch_news_from_api(source) + news_items = fetch_news_from_api(conn, source) if not news_items: - return ActionResponse(Action.REQLLM, f"抱歉，未能从{source_name}获取到新闻信息，请稍后再试或尝试其他新闻源。", None) + return ActionResponse( + Action.REQLLM, + f"抱歉，未能从{source_name}获取到新闻信息，请稍后再试或尝试其他新闻源。", + None, + ) # 随机选择一条新闻 selected_news = random.choice(news_items) - + # 保存当前新闻链接到连接对象，以便后续查询详情 - if not hasattr(conn, 'last_news_link_different'): - conn.last_news_link_different = {} - conn.last_news_link_different = { - 'url': selected_news.get('url', '#'), - 'title': selected_news.get('title', '未知标题'), - 'source_id': source + if not hasattr(conn, "last_newsnow_link"): + conn.last_newsnow_link = {} + conn.last_newsnow_link = { + "url": selected_news.get("url", "#"), + "title": selected_news.get("title", "未知标题"), + "source_id": source, } # 构建新闻报告 @@ -176,4 +209,6 @@ def get_news_from_different_source(conn, source: str = "thepaper", detail: bool except Exception as e: logger.bind(tag=TAG).error(f"获取新闻出错: {e}") - return ActionResponse(Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None) \ No newline at end of file + return ActionResponse( + Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None + ) From be1ff832977fdade9f4123d209af2002b97056d6 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 8 May 2025 18:23:37 +0800 Subject: [PATCH 03/11] =?UTF-8?q?update:=E9=80=9A=E8=BF=87pip=E6=9D=A5?= =?UTF-8?q?=E5=AE=89=E8=A3=85markitdown=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile-server | 8 -------- main/xiaozhi-server/requirements.txt | 3 ++- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/Dockerfile-server b/Dockerfile-server index 86f4d666..7eb77114 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -3,18 +3,11 @@ FROM python:3.10-slim AS builder WORKDIR /app -RUN apt-get update && \ - apt-get install -y --no-install-recommends git && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - COPY main/xiaozhi-server/requirements.txt . # 安装Python依赖 RUN pip install --no-cache-dir -r requirements.txt -RUN git clone https://github.com/microsoft/markitdown.git && cd markitdown && pip install -e 'packages/markitdown[all]' - # 第二阶段：生产镜像 FROM python:3.10-slim @@ -28,7 +21,6 @@ RUN apt-get update && \ # 从构建阶段复制Python包和前端构建产物 COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages -COPY --from=builder /app/markitdown/ /app/markitdown/ # 复制应用代码 COPY main/xiaozhi-server . diff --git a/main/xiaozhi-server/requirements.txt b/main/xiaozhi-server/requirements.txt index 02fbfcdb..0a6b9b75 100755 --- a/main/xiaozhi-server/requirements.txt +++ b/main/xiaozhi-server/requirements.txt @@ -27,4 +27,5 @@ cnlunar==0.2.0 PySocks==1.7.1 dashscope==1.23.1 baidu-aip==4.16.13 -chardet==5.2.0 \ No newline at end of file +chardet==5.2.0 +markitdown==0.1.1 \ No newline at end of file From 47f409246db1a285543dcc06a2a9195ffb44c58c Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Thu, 8 May 2025 18:25:02 +0800 Subject: [PATCH 04/11] =?UTF-8?q?update:intent=5Fllm=E6=9A=82=E6=97=B6?= =?UTF-8?q?=E4=B8=8D=E5=BC=95=E7=94=A8=E6=96=B0=E9=97=BB=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=EF=BC=8C=E4=B8=8B=E4=B8=80=E4=B8=AA=E7=89=88=E6=9C=AC=E6=94=B9?= =?UTF-8?q?=E9=80=A0intent=5Fllm=EF=BC=8C=E8=AE=A9=E5=AE=83=E8=83=BD?= =?UTF-8?q?=E5=8A=A8=E6=80=81=E8=8E=B7=E5=8F=96functioncall?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 6 ++++-- main/xiaozhi-server/core/handle/functionHandler.py | 1 - .../core/providers/intent/intent_llm/intent_llm.py | 9 --------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index cef214ac..ad794965 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -104,12 +104,13 @@ plugins: get_weather: { "api_key": "a861d0d5e7bf4ee1a83d9a9e4f96d4da", "default_location": "广州" } # 获取新闻插件的配置，这里根据需要的新闻类型传入对应的url链接，默认支持社会、科技、财经新闻 # 更多类型的新闻列表查看 https://www.chinanews.com.cn/rss/ - get_news: + get_news_from_chinanews: default_rss_url: "https://www.chinanews.com.cn/rss/society.xml" category_urls: society: "https://www.chinanews.com.cn/rss/society.xml" world: "https://www.chinanews.com.cn/rss/world.xml" finance: "https://www.chinanews.com.cn/rss/finance.xml" + get_news_from_newsnow: {"url": "https://newsnow.busiyi.world/api/s?id="} home_assistant: devices: - 客厅,玩具灯,switch.cuco_cn_460494544_cp1_on_p_2_1 @@ -183,7 +184,8 @@ Intent: functions: - change_role - get_weather - - get_news + # - get_news_from_chinanews + - get_news_from_newsnow # play_music是服务器自带的音乐播放，hass_play_music是通过home assistant控制的独立外部程序音乐播放 # 如果用了hass_play_music，就不要开启play_music，两者只留一个 - play_music diff --git a/main/xiaozhi-server/core/handle/functionHandler.py b/main/xiaozhi-server/core/handle/functionHandler.py index 10c2bf1a..b09a369c 100644 --- a/main/xiaozhi-server/core/handle/functionHandler.py +++ b/main/xiaozhi-server/core/handle/functionHandler.py @@ -55,7 +55,6 @@ class FunctionHandler: self.function_registry.register_function("get_time") self.function_registry.register_function("get_lunar") self.function_registry.register_function("handle_device") - self.function_registry.register_function("get_news_from_different_source") def register_config_functions(self): """注册配置中的函数,可以不同客户端使用不同的配置""" diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py index 9915be33..05d01c10 100644 --- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py +++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py @@ -41,7 +41,6 @@ class IntentProvider(IntentProviderBase): '2. 结束对话意图: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n' '3. 获取当天日期时间: {"function_call": {"name": "get_time"}}\n' '4. 继续聊天意图: {"function_call": {"name": "continue_chat"}}\n' - '5. 查询新闻意图: {"function_call": {"name": "get_news_from_different_source", "arguments":{"source": {"type": "string","description": "新闻源，可选项有{"thepaper": "澎湃新闻","wallstreetcn-quick": "华尔街见闻","ithome": "IT之家","zhihu": "知乎"}。可选参数，如果不提供则使用默认新闻源"},"detail": {"type": "boolean","description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容"}}}\n' "\n" "注意:\n" '- 播放音乐：无歌名时，song_name设为"random"\n' @@ -54,14 +53,6 @@ class IntentProvider(IntentProviderBase): '返回: {"function_call": {"name": "continue_chat"}}\n' "```\n" "```\n" - "用户: 最近有什么新闻吗\n" - '返回: {"function_call": {"name": "get_news_from_different_source", "arguments": {"source":"thepaper", "detail":"False"}}}\n' - "```\n" - "```\n" - "用户: 详细说说这个新闻\n" - '返回: {"function_call": {"name": "get_news_from_different_source", "arguments": {"source":"thepaper", "detail":"True"}}}\n' - "```\n" - "```\n" "用户: 现在是几号了?现在几点了？\n" '返回: {"function_call": {"name": "get_time"}}\n' "```\n" From ff98f84f187513edda0bcc5f3ba962e0bf362c83 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Fri, 9 May 2025 11:39:32 +0800 Subject: [PATCH 05/11] =?UTF-8?q?update:intent=5Fllm=E5=8A=A0=E8=BD=BD?= =?UTF-8?q?=E5=8A=A8=E6=80=81=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 7 ++ main/xiaozhi-server/core/connection.py | 26 +++-- .../core/handle/intentHandler.py | 17 ++- main/xiaozhi-server/core/handle/iotHandle.py | 2 +- .../core/handle/receiveAudioHandle.py | 2 +- .../core/providers/intent/base.py | 12 -- .../providers/intent/intent_llm/intent_llm.py | 106 +++++++++++------- main/xiaozhi-server/core/utils/dialogue.py | 8 +- .../plugins_func/functions/hass_init.py | 4 +- 9 files changed, 113 insertions(+), 71 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index ad794965..98c3d058 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -175,6 +175,13 @@ Intent: # 如果这里不填，则会默认使用selected_module.LLM的模型作为意图识别的思考模型 # 如果你的不想使用selected_module.LLM意图识别，这里最好使用独立的LLM作为意图识别，例如使用免费的ChatGLMLLM llm: ChatGLMLLM + # plugins_func/functions下的模块，可以通过配置，选择加载哪个模块，加载后对话支持相应的function调用 + # 系统默认已经记载“handle_exit_intent(退出识别)”、“play_music(音乐播放)”插件，请勿重复加载 + # 下面是加载查天气、角色切换、加载查新闻的插件示例 + functions: + - get_weather + - get_news_from_newsnow + - play_music function_call: # 不需要动type type: function_call diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index b49917ba..0a5cd8e4 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -58,9 +58,9 @@ class ConnectionHandler: self.config = copy.deepcopy(config) self.session_id = str(uuid.uuid4()) self.logger = setup_logging() - self.auth = AuthMiddleware(config) self.server = server # 保存server实例的引用 + self.auth = AuthMiddleware(config) self.need_bind = False self.bind_code = None self.read_config_from_api = self.config.get("read_config_from_api", False) @@ -128,8 +128,10 @@ class ConnectionHandler: if len(cmd) > self.max_cmd_length: self.max_cmd_length = len(cmd) - self.close_after_chat = False # 是否在聊天结束后关闭连接 - self.use_function_call_mode = False + # 是否在聊天结束后关闭连接 + self.close_after_chat = False + self.load_function_plugin = False + self.intent_type = "nointent" self.timeout_task = None self.timeout_seconds = ( @@ -365,11 +367,11 @@ class ConnectionHandler: self.memory.init_memory(self.device_id, self.llm) def _initialize_intent(self): - if ( - self.config["Intent"][self.config["selected_module"]["Intent"]]["type"] - == "function_call" - ): - self.use_function_call_mode = True + self.intent_type = self.config["Intent"][ + self.config["selected_module"]["Intent"] + ]["type"] + if self.intent_type == "function_call" or self.intent_type == "intent_llm": + self.load_function_plugin = True """初始化意图识别模块""" # 获取意图识别配置 intent_config = self.config["Intent"] @@ -757,7 +759,13 @@ class ConnectionHandler: ) self.dialogue.put( - Message(role="tool", tool_call_id=function_id, content=text) + Message( + role="tool", + tool_call_id=( + str(uuid.uuid4()) if function_id is None else function_id + ), + content=text, + ) ) self.chat_with_function_calling(text, tool_call=True) elif result.action == Action.NOTFOUND or result.action == Action.ERROR: diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index 5b00f2ef..383e4e65 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -19,7 +19,7 @@ async def handle_user_intent(conn, text): if await checkWakeupWords(conn, text): return True - if conn.use_function_call_mode: + if conn.intent_type == "function_call": # 使用支持function calling的聊天方法,不再进行意图分析 return False # 使用LLM进行意图分析 @@ -103,7 +103,7 @@ async def process_intent_result(conn, intent_result, original_text): conn, function_call_data ) logger.bind(tag=TAG).debug(f"检测到Action : {result.action}") - + if result: if result.action == Action.RESPONSE: # 直接回复前端 text = result.response @@ -118,14 +118,19 @@ async def process_intent_result(conn, intent_result, original_text): conn.speak_and_play, text, text_index ) conn.llm_finish_task = True - conn.tts_queue.put(future) + conn.tts_queue.put((future, text_index)) conn.dialogue.put(Message(role="assistant", content=text)) elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 text = result.result if text is not None and len(text) > 0: conn.dialogue.put(Message(role="tool", content=text)) - conn.executor.submit(conn.chat_with_function_calling, text, True) - elif result.action == Action.NOTFOUND or result.action == Action.ERROR: + conn.executor.submit( + conn.chat_with_function_calling, text, True + ) + elif ( + result.action == Action.NOTFOUND + or result.action == Action.ERROR + ): text = result.result if text is not None: text_index = ( @@ -157,7 +162,7 @@ async def process_intent_result(conn, intent_result, original_text): conn.speak_and_play, text, text_index ) conn.llm_finish_task = True - conn.tts_queue.put(future) + conn.tts_queue.put((future, text_index)) conn.dialogue.put(Message(role="assistant", content=text)) # 将函数执行放在线程池中 diff --git a/main/xiaozhi-server/core/handle/iotHandle.py b/main/xiaozhi-server/core/handle/iotHandle.py index 12432102..5b1bb9bb 100644 --- a/main/xiaozhi-server/core/handle/iotHandle.py +++ b/main/xiaozhi-server/core/handle/iotHandle.py @@ -317,7 +317,7 @@ async def handleIotDescriptors(conn, descriptors): ) conn.iot_descriptors[descriptor["name"]] = iot_descriptor - if conn.use_function_call_mode: + if conn.load_function_plugin: # 注册或获取设备类型 type_id = register_device_type(descriptor) device_functions = device_type_registry.get_device_functions(type_id) diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 383616c3..48ec24b2 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -76,7 +76,7 @@ async def startToChat(conn, text): # 意图未被处理，继续常规聊天流程 await send_stt_message(conn, text) - if conn.use_function_call_mode: + if conn.intent_type == "function_call": # 使用支持function calling的聊天方法 conn.executor.submit(conn.chat_with_function_calling, text) else: diff --git a/main/xiaozhi-server/core/providers/intent/base.py b/main/xiaozhi-server/core/providers/intent/base.py index 040dd025..f6df10dd 100644 --- a/main/xiaozhi-server/core/providers/intent/base.py +++ b/main/xiaozhi-server/core/providers/intent/base.py @@ -9,18 +9,6 @@ logger = setup_logging() class IntentProviderBase(ABC): def __init__(self, config): self.config = config - self.intent_options = [ - { - "name": "handle_exit_intent", - "desc": "结束聊天, 用户发来如再见之类的表示结束的话, 不想再进行对话的时候", - }, - { - "name": "play_music", - "desc": "播放音乐, 用户希望你可以播放音乐, 只用于播放音乐的意图", - }, - {"name": "get_time", "desc": "获取今天日期或者当前时间信息"}, - {"name": "continue_chat", "desc": "继续聊天"}, - ] def set_llm(self, llm): self.llm = llm diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py index 05d01c10..d8b18916 100644 --- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py +++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py @@ -15,57 +15,71 @@ class IntentProvider(IntentProviderBase): def __init__(self, config): super().__init__(config) self.llm = None - self.promot = self.get_intent_system_prompt() + self.promot = "" # 添加缓存管理 self.intent_cache = {} # 缓存意图识别结果 self.cache_expiry = 600 # 缓存有效期10分钟 self.cache_max_size = 100 # 最多缓存100个意图 - def get_intent_system_prompt(self) -> str: + def get_intent_system_prompt(self, functions_list: str) -> str: """ - 根据配置的意图选项动态生成系统提示词 + 根据配置的意图选项和可用函数动态生成系统提示词 + Args: + functions: 可用的函数列表，JSON格式字符串 Returns: 格式化后的系统提示词 """ + # 构建函数说明部分 + functions_desc = "可用的函数列表：\n" + for func in functions_list: + func_info = func.get("function", {}) + name = func_info.get("name", "") + desc = func_info.get("description", "") + params = func_info.get("parameters", {}) + + functions_desc += f"\n函数名: {name}\n" + functions_desc += f"描述: {desc}\n" + + if params: + functions_desc += "参数:\n" + for param_name, param_info in params.get("properties", {}).items(): + param_desc = param_info.get("description", "") + param_type = param_info.get("type", "") + functions_desc += f"- {param_name} ({param_type}): {param_desc}\n" + + functions_desc += "---\n" + prompt = ( - "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图属于以下哪一类：\n" - "" - f"{str(self.intent_options)}" - "\n" - "处理步骤:" - "1. 思考意图类型，生成function_call格式" - "\n\n" - "返回格式示例：\n" - '1. 播放音乐意图: {"function_call": {"name": "play_music", "arguments": {"song_name": "音乐名称"}}}\n' - '2. 结束对话意图: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n' - '3. 获取当天日期时间: {"function_call": {"name": "get_time"}}\n' - '4. 继续聊天意图: {"function_call": {"name": "continue_chat"}}\n' - "\n" - "注意:\n" - '- 播放音乐：无歌名时，song_name设为"random"\n' - "- 如果没有明显的意图，应按照继续聊天意图处理\n" - "- 只返回纯JSON，不要任何其他内容\n" - "\n" - "示例分析:\n" + "你是一个意图识别助手。请分析用户的最后一句话，判断用户意图并调用相应的函数。\n\n" + f"{functions_desc}\n" + "处理步骤:\n" + "1. 分析用户输入，确定用户意图\n" + "2. 从可用函数列表中选择最匹配的函数\n" + "3. 如果找到匹配的函数，生成对应的function_call 格式\n" + '4. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n\n' + "返回格式要求：\n" + "1. 必须返回纯JSON格式\n" + "2. 必须包含function_call字段\n" + "3. function_call必须包含name字段\n" + "4. 如果函数需要参数，必须包含arguments字段\n\n" + "示例：\n" "```\n" - "用户: 你也太搞笑了\n" - '返回: {"function_call": {"name": "continue_chat"}}\n' - "```\n" - "```\n" - "用户: 现在是几号了?现在几点了？\n" + "用户: 现在几点了？\n" '返回: {"function_call": {"name": "get_time"}}\n' "```\n" "```\n" - "用户: 我们明天再聊吧\n" - '返回: {"function_call": {"name": "handle_exit_intent"}}\n' + "用户: 我想结束对话\n" + '返回: {"function_call": {"name": "handle_exit_intent", "arguments": {"say_goodbye": "goodbye"}}}\n' "```\n" "```\n" - "用户: 播放中秋月\n" - '返回: {"function_call": {"name": "play_music", "arguments": {"song_name": "中秋月"}}}\n' - "```\n" - "```\n" - "可用的音乐名称:\n" + "用户: 你好啊\n" + '返回: {"function_call": {"name": "continue_chat"}}\n' + "```\n\n" + "注意：\n" + "1. 只返回JSON格式，不要包含任何其他文字\n" + '2. 如果没有找到匹配的函数，返回{"function_call": {"name": "continue_chat"}}\n' + "3. 确保返回的JSON格式正确，包含所有必要的字段\n" ) return prompt @@ -118,6 +132,24 @@ class IntentProvider(IntentProviderBase): # 清理缓存 self.clean_cache() + if self.promot == "": + if hasattr(conn, "func_handler"): + functions = conn.func_handler.get_functions() + self.promot = self.get_intent_system_prompt(functions) + + music_config = initialize_music_handler(conn) + music_file_names = music_config["music_file_names"] + prompt_music = f"{self.promot}\n{music_file_names}\n" + + devices = conn.config["plugins"]["home_assistant"].get("devices", []) + if len(devices) > 0: + hass_prompt = "\n下面是我家智能设备列表（位置，设备名，entity_id），可以通过homeassistant控制\n" + for device in devices: + hass_prompt += device + "\n" + prompt_music += hass_prompt + + logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}") + # 构建用户最后一句话的提示 msgStr = "" @@ -128,11 +160,7 @@ class IntentProvider(IntentProviderBase): msgStr += f"{dialogue_history[-1].role}: {dialogue_history[-1].content}\n" msgStr += f"User: {text}\n" - user_prompt = f"当前的对话如下：\n{msgStr}" - music_config = initialize_music_handler(conn) - music_file_names = music_config["music_file_names"] - prompt_music = f"{self.promot}\n{music_file_names}\n" - logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}") + user_prompt = f"current dialogue:\n{msgStr}" # 记录预处理完成时间 preprocess_time = time.time() - total_start_time diff --git a/main/xiaozhi-server/core/utils/dialogue.py b/main/xiaozhi-server/core/utils/dialogue.py index 8b79a35a..b6fa541b 100644 --- a/main/xiaozhi-server/core/utils/dialogue.py +++ b/main/xiaozhi-server/core/utils/dialogue.py @@ -33,7 +33,13 @@ class Dialogue: dialogue.append({"role": m.role, "tool_calls": m.tool_calls}) elif m.role == "tool": dialogue.append( - {"role": m.role, "tool_call_id": m.tool_call_id, "content": m.content} + { + "role": m.role, + "tool_call_id": ( + str(uuid.uuid4()) if m.tool_call_id is None else m.tool_call_id + ), + "content": m.content, + } ) else: dialogue.append({"role": m.role, "content": m.content}) diff --git a/main/xiaozhi-server/plugins_func/functions/hass_init.py b/main/xiaozhi-server/plugins_func/functions/hass_init.py index 06d7c8bc..3a5458d3 100644 --- a/main/xiaozhi-server/plugins_func/functions/hass_init.py +++ b/main/xiaozhi-server/plugins_func/functions/hass_init.py @@ -8,7 +8,7 @@ HASS_CACHE = {} def append_devices_to_prompt(conn): - if conn.use_function_call_mode: + if conn.intent_type == "function_call": funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get( "functions", [] ) @@ -27,7 +27,7 @@ def append_devices_to_prompt(conn): def initialize_hass_handler(conn): global HASS_CACHE if HASS_CACHE == {}: - if conn.use_function_call_mode: + if conn.load_function_plugin: funcs = conn.config["Intent"][conn.config["selected_module"]["Intent"]].get( "functions", [] ) From e76a8d547bcff668318d2fcfdfd41c945c7664e6 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Fri, 9 May 2025 13:45:01 +0800 Subject: [PATCH 06/11] =?UTF-8?q?update=EF=BC=9A=E4=BD=BF=E7=94=A8intent?= =?UTF-8?q?=5Fllm=E4=B8=AD=E7=9A=84LLM=E8=BF=9B=E8=A1=8C=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E5=86=85=E5=AE=B9=E5=9B=9E=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 2 +- .../core/handle/intentHandler.py | 76 +++++-------------- .../providers/intent/intent_llm/intent_llm.py | 19 +++-- 3 files changed, 31 insertions(+), 66 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 98c3d058..3b82dba0 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -157,7 +157,7 @@ selected_module: Memory: nomem # 意图识别模块开启后，可以播放音乐、控制音量、识别退出指令。 # 不想开通意图识别，就设置成：nointent - # 意图识别可使用intent_llm。优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间，这个意图识别暂时不支持控制音量大小等iot操作 + # 意图识别可使用intent_llm。优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间，支持控制音量大小等iot操作 # 意图识别可使用function_call，缺点：需要所选择的LLM支持function_call，优点：按需调用工具、速度快，理论上能全部操作所有iot指令 # 默认免费的ChatGLMLLM就已经支持function_call，但是如果像追求稳定建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-1-5-pro-32k-250115 Intent: function_call diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index 383e4e65..c9cfeeb0 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -108,43 +108,21 @@ async def process_intent_result(conn, intent_result, original_text): if result.action == Action.RESPONSE: # 直接回复前端 text = result.response if text is not None: - text_index = ( - conn.tts_last_text_index + 1 - if hasattr(conn, "tts_last_text_index") - else 0 - ) - conn.recode_first_last_text(text, text_index) - future = conn.executor.submit( - conn.speak_and_play, text, text_index - ) - conn.llm_finish_task = True - conn.tts_queue.put((future, text_index)) - conn.dialogue.put(Message(role="assistant", content=text)) + speak_and_play(conn, text) elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 text = result.result - if text is not None and len(text) > 0: - conn.dialogue.put(Message(role="tool", content=text)) - conn.executor.submit( - conn.chat_with_function_calling, text, True - ) + conn.dialogue.put(Message(role="tool", content=text)) + llm_result = conn.intent.replyResult(text) + if llm_result is None: + llm_result = text + speak_and_play(conn, llm_result) elif ( result.action == Action.NOTFOUND or result.action == Action.ERROR ): text = result.result if text is not None: - text_index = ( - conn.tts_last_text_index + 1 - if hasattr(conn, "tts_last_text_index") - else 0 - ) - conn.recode_first_last_text(text, text_index) - future = conn.executor.submit( - conn.speak_and_play, text, text_index - ) - conn.llm_finish_task = True - conn.tts_queue.put((future, text_index)) - conn.dialogue.put(Message(role="assistant", content=text)) + speak_and_play(conn, text) elif function_name != "play_music": # For backward compatibility with original code # 获取当前最新的文本索引 @@ -152,18 +130,7 @@ async def process_intent_result(conn, intent_result, original_text): if text is None: text = result.result if text is not None: - text_index = ( - conn.tts_last_text_index + 1 - if hasattr(conn, "tts_last_text_index") - else 0 - ) - conn.recode_first_last_text(text, text_index) - future = conn.executor.submit( - conn.speak_and_play, text, text_index - ) - conn.llm_finish_task = True - conn.tts_queue.put((future, text_index)) - conn.dialogue.put(Message(role="assistant", content=text)) + speak_and_play(conn, text) # 将函数执行放在线程池中 conn.executor.submit(process_function_call) @@ -174,21 +141,12 @@ async def process_intent_result(conn, intent_result, original_text): return False -def extract_text_in_brackets(s): - """ - 从字符串中提取中括号内的文字 - - :param s: 输入字符串 - :return: 中括号内的文字，如果不存在则返回空字符串 - """ - left_bracket_index = s.find("[") - right_bracket_index = s.find("]") - - if ( - left_bracket_index != -1 - and right_bracket_index != -1 - and left_bracket_index < right_bracket_index - ): - return s[left_bracket_index + 1 : right_bracket_index] - else: - return "" +def speak_and_play(conn, text): + text_index = ( + conn.tts_last_text_index + 1 if hasattr(conn, "tts_last_text_index") else 0 + ) + conn.recode_first_last_text(text, text_index) + future = conn.executor.submit(conn.speak_and_play, text, text_index) + conn.llm_finish_task = True + conn.tts_queue.put((future, text_index)) + conn.dialogue.put(Message(role="assistant", content=text)) diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py index d8b18916..2398ca9a 100644 --- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py +++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py @@ -20,6 +20,7 @@ class IntentProvider(IntentProviderBase): self.intent_cache = {} # 缓存意图识别结果 self.cache_expiry = 600 # 缓存有效期10分钟 self.cache_max_size = 100 # 最多缓存100个意图 + self.history_count = 4 # 默认使用最近4条对话记录 def get_intent_system_prompt(self, functions_list: str) -> str: """ @@ -104,6 +105,13 @@ class IntentProvider(IntentProviderBase): for key, _ in sorted_items[: len(sorted_items) - self.cache_max_size]: del self.intent_cache[key] + def replyResult(self, text: str): + llm_result = self.llm.response_no_stream( + system_prompt=text, + user_prompt="请总结以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。", + ) + return llm_result + async def detect_intent(self, conn, dialogue_history: List[Dict], text: str) -> str: if not self.llm: raise ValueError("LLM provider not set") @@ -150,14 +158,13 @@ class IntentProvider(IntentProviderBase): logger.bind(tag=TAG).debug(f"User prompt: {prompt_music}") - # 构建用户最后一句话的提示 + # 构建用户对话历史的提示 msgStr = "" - # 只使用最后两句即可 - if len(dialogue_history) >= 2: - # 保证最少有两句话的时候处理 - msgStr += f"{dialogue_history[-2].role}: {dialogue_history[-2].content}\n" - msgStr += f"{dialogue_history[-1].role}: {dialogue_history[-1].content}\n" + # 获取最近的对话历史 + start_idx = max(0, len(dialogue_history) - self.history_count) + for i in range(start_idx, len(dialogue_history)): + msgStr += f"{dialogue_history[i].role}: {dialogue_history[i].content}\n" msgStr += f"User: {text}\n" user_prompt = f"current dialogue:\n{msgStr}" From 777265c73035b2a822f1944ef8a2fdf7d30af9ba Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Fri, 9 May 2025 14:00:42 +0800 Subject: [PATCH 07/11] =?UTF-8?q?fix:=E5=8F=82=E6=95=B0=E7=AE=A1=E7=90=86?= =?UTF-8?q?=E4=BF=9D=E5=AD=98json=E7=B1=BB=E5=9E=8B=E5=87=BA=E9=94=99bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/xiaozhi/modules/sys/dto/SysParamsDTO.java | 2 +- .../modules/sys/service/impl/SysParamsServiceImpl.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/sys/dto/SysParamsDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/sys/dto/SysParamsDTO.java index ad816b35..a6f27b72 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/sys/dto/SysParamsDTO.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/sys/dto/SysParamsDTO.java @@ -39,7 +39,7 @@ public class SysParamsDTO implements Serializable { @Schema(description = "值类型") @NotBlank(message = "{sysparams.valuetype.require}", groups = DefaultGroup.class) - @Pattern(regexp = "^(string|number|boolean|array)$", message = "{sysparams.valuetype.pattern}", groups = DefaultGroup.class) + @Pattern(regexp = "^(string|number|boolean|array|json)$", message = "{sysparams.valuetype.pattern}", groups = DefaultGroup.class) private String valueType; @Schema(description = "备注") diff --git a/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysParamsServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysParamsServiceImpl.java index 15f62816..2187a8b4 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysParamsServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/sys/service/impl/SysParamsServiceImpl.java @@ -127,6 +127,12 @@ public class SysParamsServiceImpl extends BaseServiceImpl Date: Fri, 9 May 2025 14:07:46 +0800 Subject: [PATCH 08/11] =?UTF-8?q?update:=E5=8F=96=E6=B6=88=E8=AE=BE?= =?UTF-8?q?=E5=A4=87=E4=B8=80=E8=BF=9E=E6=8E=A5=E5=B0=B1=E5=BC=BA=E5=88=B6?= =?UTF-8?q?=E5=8D=87=E7=BA=A7=EF=BC=8C=E6=94=B9=E6=88=90=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E5=90=8E=E5=86=8D=E6=A0=B9=E6=8D=AE=E5=8D=87=E7=BA=A7=E7=AD=96?= =?UTF-8?q?=E7=95=A5=E5=8D=87=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/xiaozhi/modules/device/dao/DeviceDao.java | 6 +++--- .../modules/device/service/impl/DeviceServiceImpl.java | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java b/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java index 6570b89b..f3fdb7a3 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/dao/DeviceDao.java @@ -1,18 +1,18 @@ package xiaozhi.modules.device.dao; +import java.util.Date; + import org.apache.ibatis.annotations.Mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import xiaozhi.modules.device.entity.DeviceEntity; -import java.util.Date; -import java.util.List; - @Mapper public interface DeviceDao extends BaseMapper { /** * 获取此智能体全部设备的最后连接时间 + * * @param agentId 智能体id * @return */ diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index 515aea7f..19ab7178 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -118,7 +118,8 @@ public class DeviceServiceImpl extends BaseServiceImpl DeviceEntity deviceById = getDeviceByMacAddress(macAddress); - if (deviceById == null || deviceById.getAutoUpdate() != 0) { + // 只有在设备已绑定且autoUpdate不为0的情况下才返回固件升级信息 + if (deviceById != null && deviceById.getAutoUpdate() != 0) { String type = deviceReport.getBoard() == null ? null : deviceReport.getBoard().getType(); DeviceReportRespDTO.Firmware firmware = buildFirmwareInfo(type, deviceReport.getApplication() == null ? null : deviceReport.getApplication().getVersion()); From 15263b5223549d8a49e13f33b691922ea440a77b Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Fri, 9 May 2025 14:26:49 +0800 Subject: [PATCH 09/11] =?UTF-8?q?update:=E6=9B=B4=E6=96=B0=E8=8B=B1?= =?UTF-8?q?=E6=96=87=E7=89=88=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- README_en.md | 273 ++++++++++++++++++ main/README.md | 2 +- .../resources/db/changelog/202505091409.sql | 27 ++ 4 files changed, 302 insertions(+), 2 deletions(-) create mode 100644 README_en.md create mode 100644 main/manager-api/src/main/resources/db/changelog/202505091409.sql diff --git a/README.md b/README.md index b1bf6dae..4b3215f6 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@

-English +English · 常见问题 · 反馈问题 · 部署文档 diff --git a/README_en.md b/README_en.md new file mode 100644 index 00000000..9b4ed8ce --- /dev/null +++ b/README_en.md @@ -0,0 +1,273 @@ +[![Banners](docs/images/banner1.png)](https://github.com/xinnan-tech/xiaozhi-esp32-server) + +

Xiaozhi Backend Service xiaozhi-esp32-server

+ +

+This project provides backend services for the open-source smart hardware project +xiaozhi-esp32
+Implemented using Python, Java, and Vue according to the Xiaozhi Communication Protocol
+Helping you quickly set up your Xiaozhi server +

+ +

+中文 +· FAQ +· Report Issues +· Deployment Guide +· Release Notes +

+

+ + GitHub Contributors + + + GitHub Contributors + + + Issues + + + GitHub pull requests + + + GitHub pull requests + + + stars + +

+ +--- + +## Target Users 👥 + +This project requires ESP32 hardware devices. If you have purchased ESP32-related hardware, successfully connected to Brother Xia's backend service, and want to set up your own `xiaozhi-esp32` backend service, then this project is perfect for you. + +Want to see it in action? Check out these videos 🎥 + + + + + + + + + + + + + + + + +
+ + + Xiaozhi esp32 connecting to custom backend model + + + + + + Custom voice + + + + + + Cantonese communication + + + + + + Home appliance control + + + + + + Lowest cost configuration + + +
+ + + Custom voice + + + + + + Music playback + + + + + + Weather plugin + + + + + + IOT device control + + + + + + News broadcast + + +
+ +--- + +## Warning ⚠️ + +1. This project is open-source software. This software has no commercial relationship with any third-party API service providers (including but not limited to speech recognition, large models, speech synthesis, and other platforms) and does not provide any form of guarantee for their service quality or financial security. +It is recommended that users prioritize service providers with relevant business licenses and carefully read their service agreements and privacy policies. This software does not host any account keys, does not participate in fund transfers, and does not bear the risk of recharge fund losses. + +2. This project's functionality is not complete and has not passed network security testing. Please do not use it in production environments. If you deploy this project for learning in a public network environment, please ensure necessary protection measures are in place. + +--- + +## Deployment Documentation + +![Banners](docs/images/banner2.png) + +This project offers two deployment methods. Please choose based on your specific needs: + +#### 🚀 Deployment Method Selection + +| Deployment Method | Features | Use Case | Docker Deployment Guide | Source Code Deployment Guide | +|---------|------|---------|---------|---------| +| **Simplified Installation** | Smart dialogue, IOT functionality, data stored in configuration files | Low-configuration environment, no database required | [Docker Server Only](./docs/Deployment.md#%E6%96%B9%E5%BC%8F%E4%B8%80docker%E5%8F%AA%E8%BF%90%E8%A1%8Cserver) | [Local Source Code Server Only](./docs/Deployment.md#%E6%96%B9%E5%BC%8F%E4%BA%8C%E6%9C%AC%E5%9C%B0%E6%BA%90%E7%A0%81%E5%8F%AA%E8%BF%90%E8%A1%8Cserver)| +| **Full Module Installation** | Smart dialogue, IOT, OTA, Control Panel, data stored in database | Complete functionality experience |[Docker Full Module](./docs/Deployment_all.md#%E6%96%B9%E5%BC%8F%E4%B8%80docker%E8%BF%90%E8%A1%8C%E5%85%A8%E6%A8%A1%E5%9D%97) | [Local Source Code Full Module](./docs/Deployment_all.md#%E6%96%B9%E5%BC%8F%E4%BA%8C%E6%9C%AC%E5%9C%B0%E6%BA%90%E7%A0%81%E8%BF%90%E8%A1%8C%E5%85%A8%E6%A8%A1%E5%9D%97) | + +> 💡 Note: Below is the test platform deployed with the latest code. You can flash and test if needed. Concurrent users: 6, data cleared daily + +``` +Control Panel: https://2662r3426b.vicp.fun + +Service Test Tool: https://2662r3426b.vicp.fun/test/ +OTA Interface: https://2662r3426b.vicp.fun/xiaozhi/ota/ +Websocket Interface: wss://2662r3426b.vicp.fun/xiaozhi/v1/ +``` + +--- +## Feature List ✨ + +### Implemented ✅ + +| Feature Module | Description | +|---------|------| +| Communication Protocol | Based on `xiaozhi-esp32` protocol, implements data interaction through WebSocket | +| Dialogue Interaction | Supports wake-up dialogue, manual dialogue, and real-time interruption. Auto-sleep after long periods of inactivity | +| Intent Recognition | Supports LLM intent recognition, function call, reducing hard-coded intent judgment | +| Multi-language Recognition | Supports Mandarin, Cantonese, English, Japanese, Korean (default using FunASR) | +| LLM Module | Supports flexible LLM module switching, default using ChatGLMLLM, also supports Ali Bailing, DeepSeek, Ollama, etc. | +| TTS Module | Supports EdgeTTS (default), Volcano Engine Doubao TTS, and other TTS interfaces for speech synthesis | +| Memory Function | Supports ultra-long memory, local summary memory, and no memory modes for different scenarios | +| IOT Function | Supports managing registered device IOT functionality, intelligent IoT control based on dialogue context | +| Control Panel | Provides web management interface, supports agent management, user management, system configuration, etc. | + +### In Development 🚧 + +To learn about specific development progress, [click here](https://github.com/users/xinnan-tech/projects/3) + +If you're a software developer, here's an [Open Letter to Developers](docs/contributor_open_letter.md). Welcome to join! + +--- + +## Product Ecosystem 👬 +Xiaozhi is an ecosystem. When using this product, you might want to check out other excellent projects in this ecosystem: + +| Project Name | Project Link | Description | +|:---------------------|:--------|:--------| +| Xiaozhi Android Client | [xiaozhi-android-client](https://github.com/TOM88812/xiaozhi-android-client) | A Flutter-based Android and iOS voice dialogue application supporting real-time voice interaction and text dialogue | +| Xiaozhi PC Client | [py-xiaozhi](https://github.com/Huang-junsen/py-xiaozhi) | A Python-based AI client that allows you to experience Xiaozhi AI functionality through code without physical hardware | +| Xiaozhi Java Server | [xiaozhi-esp32-server-java](https://github.com/joey-zhou/xiaozhi-esp32-server-java) | A Java-based open-source project providing complete backend service solutions | + +--- + +## Supported Platforms/Components 📋 + +### LLM Language Models + +| Usage Method | Supported Platforms | Free Platforms | +|:---:|:---:|:---:| +| openai API | Ali Bailing, Volcano Engine Doubao, DeepSeek, ChatGLM, Gemini | ChatGLM, Gemini | +| ollama API | Ollama | - | +| dify API | Dify | - | +| fastgpt API | Fastgpt | - | +| coze API | Coze | - | + +Actually, any LLM supporting openai API calls can be integrated. + +--- + +### TTS Speech Synthesis + +| Usage Method | Supported Platforms | Free Platforms | +|:---:|:---:|:---:| +| API Calls | EdgeTTS, Volcano Engine Doubao TTS, Tencent Cloud, Aliyun TTS, CosyVoiceSiliconflow, TTS302AI, CozeCnTTS, GizwitsTTS, ACGNTTS, OpenAITTS | EdgeTTS, CosyVoiceSiliconflow(partial) | +| Local Service | FishSpeech, GPT_SOVITS_V2, GPT_SOVITS_V3, MinimaxTTS | FishSpeech, GPT_SOVITS_V2, GPT_SOVITS_V3, MinimaxTTS | + +--- + +### VAD Voice Activity Detection + +| Type | Platform Name | Usage Method | Pricing | Notes | +|:---:|:---------:|:----:|:----:|:--:| +| VAD | SileroVAD | Local Use | Free | | + +--- + +### ASR Speech Recognition + +| Usage Method | Supported Platforms | Free Platforms | +|:---:|:---:|:---:| +| Local Use | FunASR, SherpaASR | FunASR, SherpaASR | +| API Calls | DoubaoASR, FunASRServer, TencentASR, AliyunASR | FunASRServer | + +--- + +### Memory Storage + +| Type | Platform Name | Usage Method | Pricing | Notes | +|:------:|:---------------:|:----:|:---------:|:--:| +| Memory | mem0ai | API Calls | 1000 calls/month quota | | +| Memory | mem_local_short | Local Summary | Free | | + +--- + +### Intent Recognition + +| Type | Platform Name | Usage Method | Pricing | Notes | +|:------:|:-------------:|:----:|:-------:|:---------------------:| +| Intent | intent_llm | API Calls | Based on LLM pricing | Uses large model for intent recognition, highly versatile | +| Intent | function_call | API Calls | Based on LLM pricing | Uses large model function calls for intent, fast and effective | + +--- + +## Acknowledgments 🙏 + +| Logo | Project/Company | Description | +|:---:|:---:|:---| +| | [Bailing Voice Dialogue Robot](https://github.com/wwbin2017/bailing) | This project was inspired by [Bailing Voice Dialogue Robot](https://github.com/wwbin2017/bailing) and implemented based on it | +| | [Tenclass](https://www.tenclass.com/) | Thanks to [Tenclass](https://www.tenclass.com/) for developing standard communication protocols, multi-device compatibility solutions, and high-concurrency scenario practices for the Xiaozhi ecosystem; providing comprehensive technical documentation support for this project | +| | [Xuanfeng Technology](https://github.com/Eric0308) | Thanks to [Xuanfeng Technology](https://github.com/Eric0308) for contributing function call framework, MCP communication protocol, and plugin call mechanism implementation code, significantly improving front-end device (IoT) interaction efficiency and functional extensibility through standardized instruction scheduling system and dynamic expansion capabilities | +| | [Huiyuan Design](http://ui.kwd988.net/) | Thanks to [Huiyuan Design](http://ui.kwd988.net/) for providing professional visual solutions for this project, empowering product user experience with their design experience serving over a thousand enterprises | +| | [Xi'an Qinren Information Technology](https://www.029app.com/) | Thanks to [Xi'an Qinren Information Technology](https://www.029app.com/) for deepening this project's visual system, ensuring consistency and extensibility of overall design style in multi-scenario applications | + + + + + + + + Star History Chart + + \ No newline at end of file diff --git a/main/README.md b/main/README.md index a91c9fdc..4255ab14 100644 --- a/main/README.md +++ b/main/README.md @@ -16,4 +16,4 @@ https://ccnphfhqs21z.feishu.cn/wiki/M0XiwldO9iJwHikpXD5cEx71nKh # manager-web 、manager-api接口协议 -https://2662r3426b.vicp.fun/xiaozhi/v1/doc.html +https://2662r3426b.vicp.fun/xiaozhi/doc.html diff --git a/main/manager-api/src/main/resources/db/changelog/202505091409.sql b/main/manager-api/src/main/resources/db/changelog/202505091409.sql new file mode 100644 index 00000000..ccbaaaee --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202505091409.sql @@ -0,0 +1,27 @@ +-- 初始化智能体聊天记录 +DROP TABLE IF EXISTS ai_chat_history; +DROP TABLE IF EXISTS ai_chat_message; +DROP TABLE IF EXISTS ai_agent_chat_history; +CREATE TABLE ai_agent_chat_history +( + id BIGINT AUTO_INCREMENT COMMENT '主键ID' PRIMARY KEY, + mac_address VARCHAR(50) COMMENT 'MAC地址', + agent_id VARCHAR(32) COMMENT '智能体id', + session_id VARCHAR(50) COMMENT '会话ID', + chat_type TINYINT(3) COMMENT '消息类型: 1-用户, 2-智能体', + content VARCHAR(1024) COMMENT '聊天内容', + audio_id VARCHAR(32) COMMENT '音频ID', + created_at DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3) NOT NULL COMMENT '创建时间', + updated_at DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3) NOT NULL ON UPDATE CURRENT_TIMESTAMP(3) COMMENT '更新时间', + INDEX idx_ai_agent_chat_history_mac (mac_address), + INDEX idx_ai_agent_chat_history_session_id (session_id), + INDEX idx_ai_agent_chat_history_agent_id (agent_id), + INDEX idx_ai_agent_chat_history_agent_session_created (agent_id, session_id, created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT '智能体聊天记录表'; + +DROP TABLE IF EXISTS ai_agent_chat_audio; +CREATE TABLE ai_agent_chat_audio +( + id VARCHAR(32) COMMENT '主键ID' PRIMARY KEY, + audio LONGBLOB COMMENT '音频opus数据' +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT '智能体聊天音频数据表'; \ No newline at end of file From 61c278249140a15b7b3f5d9d39befffd0d81069a Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Fri, 9 May 2025 15:23:12 +0800 Subject: [PATCH 10/11] =?UTF-8?q?update:=E6=99=BA=E6=8E=A7=E5=8F=B0?= =?UTF-8?q?=EF=BC=8Cintent=5FllmM=E4=BE=9B=E5=BA=94=E5=99=A8=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0functions=E8=BE=93=E5=85=A5=E6=A1=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/db/changelog/202505091409.sql | 33 ++++--------------- .../db/changelog/db.changelog-master.yaml | 9 ++++- .../core/handle/intentHandler.py | 2 +- .../providers/intent/intent_llm/intent_llm.py | 5 +-- 4 files changed, 18 insertions(+), 31 deletions(-) diff --git a/main/manager-api/src/main/resources/db/changelog/202505091409.sql b/main/manager-api/src/main/resources/db/changelog/202505091409.sql index ccbaaaee..76ea8ae0 100644 --- a/main/manager-api/src/main/resources/db/changelog/202505091409.sql +++ b/main/manager-api/src/main/resources/db/changelog/202505091409.sql @@ -1,27 +1,6 @@ --- 初始化智能体聊天记录 -DROP TABLE IF EXISTS ai_chat_history; -DROP TABLE IF EXISTS ai_chat_message; -DROP TABLE IF EXISTS ai_agent_chat_history; -CREATE TABLE ai_agent_chat_history -( - id BIGINT AUTO_INCREMENT COMMENT '主键ID' PRIMARY KEY, - mac_address VARCHAR(50) COMMENT 'MAC地址', - agent_id VARCHAR(32) COMMENT '智能体id', - session_id VARCHAR(50) COMMENT '会话ID', - chat_type TINYINT(3) COMMENT '消息类型: 1-用户, 2-智能体', - content VARCHAR(1024) COMMENT '聊天内容', - audio_id VARCHAR(32) COMMENT '音频ID', - created_at DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3) NOT NULL COMMENT '创建时间', - updated_at DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3) NOT NULL ON UPDATE CURRENT_TIMESTAMP(3) COMMENT '更新时间', - INDEX idx_ai_agent_chat_history_mac (mac_address), - INDEX idx_ai_agent_chat_history_session_id (session_id), - INDEX idx_ai_agent_chat_history_agent_id (agent_id), - INDEX idx_ai_agent_chat_history_agent_session_created (agent_id, session_id, created_at) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT '智能体聊天记录表'; - -DROP TABLE IF EXISTS ai_agent_chat_audio; -CREATE TABLE ai_agent_chat_audio -( - id VARCHAR(32) COMMENT '主键ID' PRIMARY KEY, - audio LONGBLOB COMMENT '音频opus数据' -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT '智能体聊天音频数据表'; \ No newline at end of file +-- 更新intent_llmM供应器 +update `ai_model_provider` set fields = '[{"key":"llm","label":"LLM模型","type":"string"},{"key":"functions","label":"函数列表","type":"dict","dict_name":"functions"}]' where id = 'SYSTEM_Intent_intent_llm'; +-- 更新ChatGLMLLM的意图识别配置 +update `ai_model_config` set config_json = '{\"type\": \"intent_llm\", \"llm\": \"LLM_ChatGLMLLM\", \"functions\": \"get_weather;get_news_from_newsnow;play_music\"}' where id = 'Intent_intent_llm'; +-- 更新函数调用意图识别配置 +UPDATE `ai_model_config` SET config_json = REPLACE(config_json, ';get_news;', ';get_news_from_newsnow;') WHERE id = 'Intent_function_call'; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index ffbef2ce..eb4e69c3 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -99,4 +99,11 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202505022134.sql \ No newline at end of file + path: classpath:db/changelog/202505022134.sql + - changeSet: + id: 202505091409 + author: hrz + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202505091409.sql \ No newline at end of file diff --git a/main/xiaozhi-server/core/handle/intentHandler.py b/main/xiaozhi-server/core/handle/intentHandler.py index c9cfeeb0..1cae8bab 100644 --- a/main/xiaozhi-server/core/handle/intentHandler.py +++ b/main/xiaozhi-server/core/handle/intentHandler.py @@ -112,7 +112,7 @@ async def process_intent_result(conn, intent_result, original_text): elif result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 text = result.result conn.dialogue.put(Message(role="tool", content=text)) - llm_result = conn.intent.replyResult(text) + llm_result = conn.intent.replyResult(text, original_text) if llm_result is None: llm_result = text speak_and_play(conn, llm_result) diff --git a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py index 2398ca9a..b847ac73 100644 --- a/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py +++ b/main/xiaozhi-server/core/providers/intent/intent_llm/intent_llm.py @@ -105,10 +105,11 @@ class IntentProvider(IntentProviderBase): for key, _ in sorted_items[: len(sorted_items) - self.cache_max_size]: del self.intent_cache[key] - def replyResult(self, text: str): + def replyResult(self, text: str, original_text: str): llm_result = self.llm.response_no_stream( system_prompt=text, - user_prompt="请总结以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。", + user_prompt="请根据以上内容，像人类一样说话的口吻回复用户，要求简洁，请直接返回结果。用户现在说：" + + original_text, ) return llm_result From 3a5bbe32b9bdd6f80462cf4a794e54baafb2eaa3 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Fri, 9 May 2025 16:15:26 +0800 Subject: [PATCH 11/11] =?UTF-8?q?fix:intent=5Fllm=E7=9A=84functions?= =?UTF-8?q?=E4=B9=9F=E9=9C=80=E8=A6=81=E8=BF=94=E5=9B=9E=E6=88=90=E6=95=B0?= =?UTF-8?q?=E7=BB=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi/modules/config/service/impl/ConfigServiceImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java index a15df002..ee42c934 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java @@ -257,7 +257,8 @@ public class ConfigServiceImpl implements ConfigService { if (intentLLMModelId != null && intentLLMModelId.equals(llmModelId)) { intentLLMModelId = null; } - } else if ("function_call".equals(map.get("type"))) { + } + if (map.get("functions") != null) { String functionStr = (String) map.get("functions"); if (StringUtils.isNotBlank(functionStr)) { String[] functions = functionStr.split("\\;");