221 lines
7.8 KiB
Python
221 lines
7.8 KiB
Python
|
import os.path
|
|||
|
|
import threading
|
|||
|
|
import urllib
|
|||
|
|
from urllib.parse import urlparse
|
|||
|
|
|
|||
|
|
import requests
|
|||
|
|
from DrissionPage._pages.mix_tab import MixTab
|
|||
|
|
from loguru import logger
|
|||
|
|
from sqlmodel import Session, select
|
|||
|
|
|
|||
|
|
from app.config.config import AppCtx
|
|||
|
|
from app.models.report_urls import ReportUrlModel
|
|||
|
|
from app.utils.common import md5
|
|||
|
|
from app.utils.dp import DPEngine
|
|||
|
|
|
|||
|
|
|
|||
|
|
class EvidenceEngine:
|
|||
|
|
"""固定色站证据,搜索 URL 后截图,并生成举报链接存入数据库"""
|
|||
|
|
|
|||
|
|
def __init__(self):
|
|||
|
|
# 开启一个浏览器窗口
|
|||
|
|
self.dp_engine = DPEngine()
|
|||
|
|
self.wap_dp_engine = DPEngine(is_wap=True)
|
|||
|
|
|
|||
|
|
# 控制运行状态的数据
|
|||
|
|
self.ev = threading.Event()
|
|||
|
|
self.status = 1
|
|||
|
|
|
|||
|
|
# 工作线程
|
|||
|
|
self.worker_thread = None
|
|||
|
|
|
|||
|
|
# 数据库连接
|
|||
|
|
self.database = AppCtx.g_db_engine
|
|||
|
|
|
|||
|
|
def start(self):
|
|||
|
|
"""启动线程"""
|
|||
|
|
self.worker_thread = threading.Thread(target=self.worker, name="evidence_engine", daemon=True)
|
|||
|
|
self.worker_thread.start()
|
|||
|
|
|
|||
|
|
def cli_start(self):
|
|||
|
|
"""以CLI模式开启，就是只执行一次，不循环"""
|
|||
|
|
# 从数据库中获取所有待收集证据的 URL 列表
|
|||
|
|
targets = self.get_surl_from_db()
|
|||
|
|
logger.debug(f"共获取到 {len(targets)} 条待处理数据")
|
|||
|
|
|
|||
|
|
# 依次处理
|
|||
|
|
for target in targets:
|
|||
|
|
logger.debug(f"开始获取 {target['surl']} 的举报数据")
|
|||
|
|
self.get_screenshot_and_report_link(target)
|
|||
|
|
|
|||
|
|
def worker(self):
|
|||
|
|
"""工作函数"""
|
|||
|
|
while self.status:
|
|||
|
|
# 从数据库中获取所有待收集证据的 URL 列表
|
|||
|
|
targets = self.get_surl_from_db()
|
|||
|
|
|
|||
|
|
# 依次处理
|
|||
|
|
for target in targets:
|
|||
|
|
logger.debug(f"开始获取 {target['surl']} 的举报数据")
|
|||
|
|
self.get_screenshot_and_report_link(target)
|
|||
|
|
|
|||
|
|
# 每分钟跑一次
|
|||
|
|
self.ev.wait(60)
|
|||
|
|
|
|||
|
|
def stop(self):
|
|||
|
|
"""结束线程"""
|
|||
|
|
self.status = 0
|
|||
|
|
self.ev.set()
|
|||
|
|
self.dp_engine.close()
|
|||
|
|
self.wap_dp_engine.close()
|
|||
|
|
|
|||
|
|
def get_surl_from_db(self):
|
|||
|
|
"""从数据库中获取数据"""
|
|||
|
|
result: list = []
|
|||
|
|
with Session(self.database) as session:
|
|||
|
|
stmt = select(ReportUrlModel).where(ReportUrlModel.has_evidence == False)
|
|||
|
|
surl = session.exec(stmt).all()
|
|||
|
|
for url in surl:
|
|||
|
|
result.append({"id": url.id, "surl": url.surl, "domain": url.domain})
|
|||
|
|
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
def get_screenshot_and_report_link(self, target: dict):
|
|||
|
|
"""获取证据截图和举报链接"""
|
|||
|
|
try:
|
|||
|
|
surl = target["surl"]
|
|||
|
|
|
|||
|
|
# Part1 获取证据截图
|
|||
|
|
logger.debug(f"开始获取 {surl} 在百度搜索中的截图")
|
|||
|
|
img_path, tab = self.get_screenshot(target)
|
|||
|
|
if not img_path:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
# Part2 截一张surl本身的图
|
|||
|
|
logger.debug(f"开始获取 {surl} 的截图")
|
|||
|
|
img_path, wap_tab = self.get_wap_screenshot(target)
|
|||
|
|
wap_tab.close()
|
|||
|
|
|
|||
|
|
# Part3 获取举报链接
|
|||
|
|
logger.debug(f"开始获取 {surl} 的举报链接")
|
|||
|
|
report_link = self.get_report_link(tab)
|
|||
|
|
logger.debug(f"获取到举报链接为: {report_link}")
|
|||
|
|
if not report_link:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
# Part4 获取举报链接的信息
|
|||
|
|
logger.debug(f"开始获取举报链接的参数信息")
|
|||
|
|
params = self.resolve_report_link(report_link)
|
|||
|
|
if not params:
|
|||
|
|
logger.error(f"解析举报链接失败，surl: {surl}")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
token = params["token"][0]
|
|||
|
|
title = params["title"][0]
|
|||
|
|
q = params["q"][0]
|
|||
|
|
surl = params["surl"][0]
|
|||
|
|
|
|||
|
|
# 更新数据库
|
|||
|
|
with Session(self.database) as session:
|
|||
|
|
stmt = select(ReportUrlModel).where(ReportUrlModel.id == target["id"])
|
|||
|
|
model: ReportUrlModel = session.exec(stmt).first()
|
|||
|
|
if not model:
|
|||
|
|
logger.error(f"{target['id']} 记录不存在，跳过...")
|
|||
|
|
return None
|
|||
|
|
# 更新数据
|
|||
|
|
model.token = token
|
|||
|
|
model.title = title
|
|||
|
|
model.q = q
|
|||
|
|
model.has_evidence = True
|
|||
|
|
session.add(model)
|
|||
|
|
session.commit()
|
|||
|
|
logger.debug(f"{surl} 处理完成")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"获取证据截图和举报链接失败: {e}")
|
|||
|
|
|
|||
|
|
def get_screenshot(self, target: dict) -> tuple[str | None, MixTab]:
|
|||
|
|
"""获取搜索页面的截图，返回 img_path """
|
|||
|
|
search_keyword = target["surl"].lstrip("https://").lstrip("http://")
|
|||
|
|
tab = self.dp_engine.browser.new_tab()
|
|||
|
|
tab.get("https://www.baidu.com")
|
|||
|
|
tab.ele("#kw").input(f"{search_keyword}\n", clear=True)
|
|||
|
|
tab.wait.eles_loaded([".content_none", "#content_left"], any_one=True)
|
|||
|
|
|
|||
|
|
if "未找到相关结果" in tab.html:
|
|||
|
|
logger.info(f"没有关于 {search_keyword} 的数据")
|
|||
|
|
return None, tab
|
|||
|
|
|
|||
|
|
# 图片的存储路径
|
|||
|
|
# 截完图先不要关闭 tab，别的地方还要用
|
|||
|
|
img_path = f"./imgs/{target['domain']}/{md5(target['surl'])}.png"
|
|||
|
|
return self.do_screenshot(tab, img_path)
|
|||
|
|
|
|||
|
|
def get_wap_screenshot(self, target: dict) -> tuple[str | None, MixTab]:
|
|||
|
|
"""用 wap dp 再截一张 surl 本身的图"""
|
|||
|
|
tab = self.wap_dp_engine.browser.new_tab()
|
|||
|
|
tab.get(target["surl"])
|
|||
|
|
tab.wait(5) # 这里只能硬等，不知道surl的结构，没办法精确判断
|
|||
|
|
|
|||
|
|
img_path = f"./imgs/{target['domain']}/{md5(target['surl'])}-wap.png"
|
|||
|
|
return self.do_screenshot(tab, img_path)
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def do_screenshot(tab: MixTab, img_path: str, force=False) -> tuple[str | None, MixTab]:
|
|||
|
|
"""截图函数"""
|
|||
|
|
if os.path.exists(img_path):
|
|||
|
|
if force:
|
|||
|
|
os.remove(img_path)
|
|||
|
|
else:
|
|||
|
|
logger.debug(f"截图路径 {img_path} 已经存在，跳过截图")
|
|||
|
|
return img_path, tab
|
|||
|
|
|
|||
|
|
tab.get_screenshot(path=img_path)
|
|||
|
|
logger.debug(f"截图成功: {img_path}")
|
|||
|
|
|
|||
|
|
return img_path, tab
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def get_report_link(tab: MixTab):
|
|||
|
|
"""获取举报链接，这个时候页面应该停留在搜索结果页"""
|
|||
|
|
tools = tab.eles(".:c-tools")
|
|||
|
|
tab.wait(0.5)
|
|||
|
|
if tools:
|
|||
|
|
tool = tools[0]
|
|||
|
|
tool.hover(0, 0)
|
|||
|
|
tool.click(True)
|
|||
|
|
|
|||
|
|
tips = tab.eles(".c-tip-menu")
|
|||
|
|
if tips:
|
|||
|
|
tip = tips[0]
|
|||
|
|
report = tip.ele("t:a@@text()=举报")
|
|||
|
|
if report:
|
|||
|
|
return report.attr("href")
|
|||
|
|
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def resolve_report_link(report_link):
|
|||
|
|
try:
|
|||
|
|
proxy_link = AppCtx.g_app_config.chrome.proxy
|
|||
|
|
proxies = {
|
|||
|
|
"http": proxy_link,
|
|||
|
|
"https": proxy_link,
|
|||
|
|
}
|
|||
|
|
response = requests.get(report_link, proxies=proxies, timeout=5, allow_redirects=False)
|
|||
|
|
location = response.headers["Location"]
|
|||
|
|
if not location:
|
|||
|
|
logger.warning("没有获取到举报链接的 Location")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
parsed_url = urllib.parse.urlparse(response.headers["Location"])
|
|||
|
|
query_params = urllib.parse.parse_qs(parsed_url.query)
|
|||
|
|
decoded_params = {
|
|||
|
|
key: [urllib.parse.unquote(value) for value in values] for key, values in query_params.items()
|
|||
|
|
}
|
|||
|
|
if len(decoded_params) == 0:
|
|||
|
|
return None
|
|||
|
|
return decoded_params
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"解析举报链接失败，错误: {e}")
|
|||
|
|
pass
|