Jacobandhrz
15d9b0fba1
* fix: harden weather and news plugins * 修掉天气新闻插件的三处鲁棒性缺陷 城市查询不再写死国内范围，境外城市也能查到。实时天气和七天预报要都拿到才走 JSON，缺一个就整体退回网页路径，不产出残缺报告。网页抓取加上跟随跳转，旧链接的 302 也能拿到内容。城市查不到时不再让用户去核对地名，多数情况是密钥失效。报告补上气压和降水量。新闻 URL 拼接抽成纯函数，配了 23 项单测。 --------- Co-authored-by: hrz <1710360675@qq.com>
329 lines
12 KiB
Python
329 lines
12 KiB
Python
import random
|
||
import httpx
|
||
from io import BytesIO
|
||
from urllib.parse import quote, urlsplit, urlunsplit, parse_qsl
|
||
from markitdown import MarkItDown, StreamInfo
|
||
from config.logger import setup_logging
|
||
from plugins_func.register import register_function, ToolType, ActionResponse, Action
|
||
from typing import TYPE_CHECKING
|
||
|
||
if TYPE_CHECKING:
|
||
from core.connection import ConnectionHandler
|
||
|
||
|
||
TAG = __name__
|
||
logger = setup_logging()
|
||
|
||
CHANNEL_MAP = {
|
||
"V2EX": "v2ex-share",
|
||
"知乎": "zhihu",
|
||
"微博": "weibo",
|
||
"联合早报": "zaobao",
|
||
"酷安": "coolapk",
|
||
"MKTNews": "mktnews-flash",
|
||
"华尔街见闻": "wallstreetcn-quick",
|
||
"36氪": "36kr-quick",
|
||
"抖音": "douyin",
|
||
"虎扑": "hupu",
|
||
"百度贴吧": "tieba",
|
||
"今日头条": "toutiao",
|
||
"IT之家": "ithome",
|
||
"澎湃新闻": "thepaper",
|
||
"卫星通讯社": "sputniknewscn",
|
||
"参考消息": "cankaoxiaoxi",
|
||
"远景论坛": "pcbeta-windows11",
|
||
"财联社": "cls-depth",
|
||
"雪球": "xueqiu-hotstock",
|
||
"格隆汇": "gelonghui",
|
||
"法布财经": "fastbull-express",
|
||
"Solidot": "solidot",
|
||
"Hacker News": "hackernews",
|
||
"Product Hunt": "producthunt",
|
||
"Github": "github-trending-today",
|
||
"哔哩哔哩": "bilibili-hot-search",
|
||
"快手": "kuaishou",
|
||
"靠谱新闻": "kaopu",
|
||
"金十数据": "jin10",
|
||
"百度热搜": "baidu",
|
||
"牛客": "nowcoder",
|
||
"少数派": "sspai",
|
||
"稀土掘金": "juejin",
|
||
"凤凰网": "ifeng",
|
||
"虫部落": "chongbuluo-latest",
|
||
}
|
||
|
||
# 默认新闻来源字典，当配置中没有指定时使用
|
||
DEFAULT_NEWS_SOURCES = "澎湃新闻;百度热搜;财联社"
|
||
|
||
DEFAULT_NEWS_API_URL = "https://newsnow.busiyi.world/api/s?id="
|
||
|
||
|
||
def _build_news_api_url(source, configured_url=None):
|
||
"""拼出新闻接口的请求地址和查询参数。
|
||
|
||
配置里带 {source} 占位符就替换占位符；否则把 id 设成新闻源 id，
|
||
这样配置里已经写了 id 时也不会拼出脏串。
|
||
"""
|
||
api_url = str(configured_url).strip() if configured_url else DEFAULT_NEWS_API_URL
|
||
if "{source}" in api_url:
|
||
return api_url.replace("{source}", quote(source, safe="")), None
|
||
parts = urlsplit(api_url)
|
||
query = dict(parse_qsl(parts.query, keep_blank_values=True))
|
||
query["id"] = source
|
||
return (
|
||
urlunsplit((parts.scheme, parts.netloc, parts.path, "", parts.fragment)),
|
||
query,
|
||
)
|
||
|
||
|
||
def _get_newsnow_config(conn):
|
||
# 从连接配置获取
|
||
plugins = conn.config.get("plugins", {})
|
||
newsnow = plugins.get("get_news_from_newsnow", {})
|
||
sources = newsnow.get("news_sources", "")
|
||
if isinstance(sources, str) and sources.strip():
|
||
return sources
|
||
if isinstance(sources, (list, tuple)):
|
||
return ";".join(str(item).strip() for item in sources if str(item).strip())
|
||
|
||
return ""
|
||
|
||
def get_news_sources_from_config(conn):
|
||
"""从配置中获取新闻源字符串"""
|
||
try:
|
||
result = _get_newsnow_config(conn)
|
||
if result:
|
||
logger.bind(tag=TAG).debug(f"使用配置的新闻源: {result}")
|
||
return result
|
||
|
||
logger.bind(tag=TAG).debug("未找到新闻源配置，使用默认配置")
|
||
return DEFAULT_NEWS_SOURCES
|
||
|
||
except Exception as e:
|
||
logger.bind(tag=TAG).error(f"获取新闻源配置失败: {e}，使用默认配置")
|
||
return DEFAULT_NEWS_SOURCES
|
||
|
||
|
||
# 从默认配置获取可用的新闻源名称（运行时由get_news_sources_from_config动态获取）
|
||
example_sources_str = DEFAULT_NEWS_SOURCES.replace(";","、")
|
||
|
||
GET_NEWS_FROM_NEWSNOW_FUNCTION_DESC = {
|
||
"type": "function",
|
||
"function": {
|
||
"name": "get_news_from_newsnow",
|
||
"description": "当用户要求查看或收听新闻时调用（如'来条新闻''今天有什么新闻'）。",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"source": {
|
||
"type": "string",
|
||
"description": f"新闻源的标准中文名称，例如{example_sources_str}等。可选参数，如果不提供则使用默认新闻源",
|
||
},
|
||
"detail": {
|
||
"type": "boolean",
|
||
"description": "是否获取详细内容，默认为false。如果为true，则获取上一条新闻的详细内容",
|
||
},
|
||
"lang": {
|
||
"type": "string",
|
||
"description": "返回用户使用的语言code，例如zh_CN/zh_HK/en_US/ja_JP等，默认zh_CN",
|
||
},
|
||
},
|
||
"required": ["lang"],
|
||
},
|
||
},
|
||
}
|
||
|
||
|
||
async def fetch_news_from_api(conn: "ConnectionHandler", source="thepaper"):
|
||
"""从API获取新闻列表"""
|
||
try:
|
||
news_config = conn.config.get("plugins", {}).get("get_news_from_newsnow", {})
|
||
configured_url = news_config.get("url") if isinstance(news_config, dict) else None
|
||
api_url, request_params = _build_news_api_url(source, configured_url)
|
||
|
||
headers = {"User-Agent": "Mozilla/5.0"}
|
||
async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=3.0)) as client:
|
||
response = await client.get(api_url, params=request_params, headers=headers)
|
||
|
||
if response.status_code != 200:
|
||
logger.bind(tag=TAG).error(
|
||
f"获取新闻API失败：HTTP {response.status_code}，source={source}"
|
||
)
|
||
return []
|
||
data = response.json()
|
||
|
||
items = data.get("items") if isinstance(data, dict) else None
|
||
if not isinstance(items, list):
|
||
logger.bind(tag=TAG).error(f"获取新闻API响应格式错误: {data}")
|
||
return []
|
||
return [item for item in items if isinstance(item, dict) and item.get("title")]
|
||
|
||
except Exception as e:
|
||
logger.bind(tag=TAG).error(f"获取新闻API失败: {e}")
|
||
return []
|
||
|
||
|
||
async def fetch_news_detail(url):
|
||
"""获取新闻详情页内容并使用MarkItDown清理HTML"""
|
||
try:
|
||
headers = {"User-Agent": "Mozilla/5.0"}
|
||
async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, connect=3.0)) as client:
|
||
response = await client.get(url, headers=headers)
|
||
|
||
# 使用MarkItDown清理HTML内容
|
||
md = MarkItDown(enable_plugins=False)
|
||
result = md.convert_stream(
|
||
BytesIO(response.content),
|
||
stream_info=StreamInfo(
|
||
mimetype="text/html",
|
||
extension=".html",
|
||
charset=response.encoding or "utf-8",
|
||
),
|
||
)
|
||
|
||
# 获取清理后的文本内容
|
||
clean_text = result.text_content
|
||
|
||
# 如果清理后的内容为空，返回提示信息
|
||
if not clean_text or len(clean_text.strip()) == 0:
|
||
logger.bind(tag=TAG).warning(f"清理后的新闻内容为空: {url}")
|
||
return "无法解析新闻详情内容，可能是网站结构特殊或内容受限。"
|
||
|
||
return clean_text
|
||
except Exception as e:
|
||
logger.bind(tag=TAG).error(f"获取新闻详情失败: {e}")
|
||
return "无法获取详细内容"
|
||
|
||
|
||
@register_function(
|
||
"get_news_from_newsnow",
|
||
GET_NEWS_FROM_NEWSNOW_FUNCTION_DESC,
|
||
ToolType.SYSTEM_CTL,
|
||
)
|
||
async def get_news_from_newsnow(
|
||
conn: "ConnectionHandler",
|
||
source: str = "澎湃新闻",
|
||
detail: bool = False,
|
||
lang: str = "zh_CN",
|
||
):
|
||
"""获取新闻并随机选择一条进行播报，或获取上一条新闻的详细内容"""
|
||
try:
|
||
# 获取当前配置的新闻源
|
||
news_sources = get_news_sources_from_config(conn)
|
||
|
||
# 如果detail为True，获取上一条新闻的详细内容
|
||
detail = str(detail).lower() == "true"
|
||
if detail:
|
||
if (
|
||
not hasattr(conn, "last_newsnow_link")
|
||
or not conn.last_newsnow_link
|
||
or "url" not in conn.last_newsnow_link
|
||
):
|
||
return ActionResponse(
|
||
Action.REQLLM,
|
||
"抱歉，没有找到最近查询的新闻，请先获取一条新闻。",
|
||
None,
|
||
)
|
||
|
||
url = conn.last_newsnow_link.get("url")
|
||
title = conn.last_newsnow_link.get("title", "未知标题")
|
||
source_id = conn.last_newsnow_link.get("source_id", "thepaper")
|
||
source_name = CHANNEL_MAP.get(source_id, "未知来源")
|
||
|
||
if not url or url == "#":
|
||
return ActionResponse(
|
||
Action.REQLLM, "抱歉，该新闻没有可用的链接获取详细内容。", None
|
||
)
|
||
|
||
logger.bind(tag=TAG).debug(
|
||
f"获取新闻详情: {title}, 来源: {source_name}, URL={url}"
|
||
)
|
||
|
||
# 获取新闻详情
|
||
detail_content = await fetch_news_detail(url)
|
||
|
||
if not detail_content or detail_content == "无法获取详细内容":
|
||
return ActionResponse(
|
||
Action.REQLLM,
|
||
f"抱歉，无法获取《{title}》的详细内容，可能是链接已失效或网站结构发生变化。",
|
||
None,
|
||
)
|
||
|
||
# 构建详情报告
|
||
detail_report = (
|
||
f"根据下列数据，用{lang}回应用户的新闻详情查询请求：\n\n"
|
||
f"新闻标题: {title}\n"
|
||
# f"新闻来源: {source_name}\n"
|
||
f"详细内容: {detail_content}\n\n"
|
||
f"(请对上述新闻内容进行总结，提取关键信息，以自然、流畅的方式向用户播报，"
|
||
f"不要提及这是总结，就像是在讲述一个完整的新闻)"
|
||
)
|
||
|
||
return ActionResponse(Action.REQLLM, detail_report, None)
|
||
|
||
# 否则，获取新闻列表并随机选择一条
|
||
# 将中文名称转换为英文ID
|
||
english_source_id = None
|
||
|
||
# 检查输入的中文名称是否在配置的新闻源中
|
||
news_sources_list = [name.strip() for name in news_sources.split(";") if name.strip()]
|
||
if source in news_sources_list:
|
||
english_source_id = CHANNEL_MAP.get(source)
|
||
elif not source and news_sources_list:
|
||
source = news_sources_list[0]
|
||
english_source_id = CHANNEL_MAP.get(source)
|
||
|
||
# 如果找不到对应的英文ID，使用默认源
|
||
if not english_source_id:
|
||
fallback_source = next(
|
||
(name for name in news_sources_list if CHANNEL_MAP.get(name)),
|
||
"澎湃新闻",
|
||
)
|
||
logger.bind(tag=TAG).warning(
|
||
f"无效的新闻源: {source}，使用配置中的默认源{fallback_source}"
|
||
)
|
||
english_source_id = CHANNEL_MAP[fallback_source]
|
||
source = fallback_source
|
||
|
||
logger.bind(tag=TAG).info(f"获取新闻: 新闻源={source}({english_source_id})")
|
||
|
||
# 获取新闻列表
|
||
news_items = await fetch_news_from_api(conn, english_source_id)
|
||
|
||
if not news_items:
|
||
return ActionResponse(
|
||
Action.REQLLM,
|
||
f"抱歉，未能从{source}获取到新闻信息，请稍后再试或尝试其他新闻源。",
|
||
None,
|
||
)
|
||
|
||
# 随机选择一条新闻
|
||
selected_news = random.choice(news_items)
|
||
title = selected_news.get("title", "未知标题")
|
||
|
||
# 保存当前新闻链接到连接对象，以便后续查询详情
|
||
if not hasattr(conn, "last_newsnow_link"):
|
||
conn.last_newsnow_link = {}
|
||
conn.last_newsnow_link = {
|
||
"url": selected_news.get("url", "#"),
|
||
"title": title,
|
||
"source_id": english_source_id,
|
||
}
|
||
|
||
# 构建新闻报告
|
||
news_report = (
|
||
f"根据下列数据，用{lang}回应用户的新闻查询请求：\n\n"
|
||
f"新闻标题: {title}\n"
|
||
# f"新闻来源: {source}\n"
|
||
f"(请以自然、流畅的方式向用户播报这条新闻标题，"
|
||
f"提示用户可以要求获取详细内容，此时会获取新闻的详细内容。)"
|
||
)
|
||
|
||
return ActionResponse(Action.REQLLM, news_report, None)
|
||
|
||
except Exception as e:
|
||
logger.bind(tag=TAG).error(f"获取新闻出错: {e}")
|
||
return ActionResponse(
|
||
Action.REQLLM, "抱歉，获取新闻时发生错误，请稍后再试。", None
|
||
)
|