refactor(get_news_from_newsnow): 改用流式处理解析网页内容

将原有的直接转换响应内容改为流式读取字节流并传入参数,适配MarkItDown的convert_stream接口,优化大内容处理时的内存占用
This commit is contained in:
wengzh
2026-07-20 15:51:23 +08:00
parent ed89c05245
commit 6e92d169ec
@@ -1,6 +1,7 @@
import random import random
import httpx import httpx
from markitdown import MarkItDown from io import BytesIO
from markitdown import MarkItDown, StreamInfo
from config.logger import setup_logging from config.logger import setup_logging
from plugins_func.register import register_function, ToolType, ActionResponse, Action from plugins_func.register import register_function, ToolType, ActionResponse, Action
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
@@ -144,7 +145,14 @@ async def fetch_news_detail(url):
# 使用MarkItDown清理HTML内容 # 使用MarkItDown清理HTML内容
md = MarkItDown(enable_plugins=False) md = MarkItDown(enable_plugins=False)
result = md.convert(response) result = md.convert_stream(
BytesIO(response.content),
stream_info=StreamInfo(
mimetype="text/html",
extension=".html",
charset=response.encoding or "utf-8",
),
)
# 获取清理后的文本内容 # 获取清理后的文本内容
clean_text = result.text_content clean_text = result.text_content