Compare commits
No commits in common. "master" and "v0.1.0" have entirely different histories.
2
.gitignore
vendored
2
.gitignore
vendored
@ -50,7 +50,7 @@ __pycache__/
|
||||
# Distribution / packaging
|
||||
build/
|
||||
develop-eggs/
|
||||
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
|
||||
14
README.md
14
README.md
@ -2,20 +2,6 @@
|
||||
|
||||
## 使用方式
|
||||
|
||||
```shell
|
||||
# WEB 模式，只启动 web 控制台，不启动任何引擎
|
||||
python main.py --web
|
||||
# 等价于
|
||||
python main.py --web-only
|
||||
|
||||
# 启动 web 的时候启动引擎
|
||||
python main.py --web --crawl --evidence --report wap,pc,site
|
||||
|
||||
# 这几个选项可以任意组合，例如只启动采集引擎和证据收集引擎
|
||||
python main.py --web --crawl --evidence
|
||||
```
|
||||
|
||||
|
||||
```shell
|
||||
# 采集模式，采集指定关键字的URL列表，直接存入数据库
|
||||
python main.py --crawl www.yunzhiju.net
|
||||
|
||||
@ -1 +1 @@
|
||||
from .app import MainApp
|
||||
from .app import *
|
||||
72
app/app.py
72
app/app.py
@ -1,9 +1,7 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
import signal
|
||||
|
||||
from app.engines.report_engine import Reporter
|
||||
|
||||
@ -16,8 +14,6 @@ from .models.base import connect_db, create_database
|
||||
from loguru import logger
|
||||
import sqlalchemy.exc
|
||||
|
||||
from .web.web import WebApp
|
||||
|
||||
|
||||
class MainApp:
|
||||
"""主应用"""
|
||||
@ -27,11 +23,6 @@ class MainApp:
|
||||
self.config: AppConfig = None
|
||||
self.db_engine = None
|
||||
|
||||
# 所有的engine
|
||||
self.crawl_engine = None
|
||||
self.evidence_engine = None
|
||||
self.report_engine = None
|
||||
|
||||
def parse_args(self):
|
||||
"""解析命令行参数"""
|
||||
parser = argparse.ArgumentParser(description="Baidu Reporter")
|
||||
@ -45,10 +36,7 @@ class MainApp:
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--crawl",
|
||||
nargs="?",
|
||||
const="",
|
||||
help="采集模式，根据域名批量采集 SURL，多个域名可使用英文逗号分割，也可通过 --crawl-file 传入文件",
|
||||
"--crawl", help="采集模式，根据域名批量采集 SURL，多个域名可使用英文逗号分割，也可通过 --crawl-file 传入文件",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--crawl-file", help="目标域名文件，批量传入待采集的域名，每行一个"
|
||||
@ -69,10 +57,7 @@ class MainApp:
|
||||
|
||||
# 添加 web 服务器参数
|
||||
parser.add_argument(
|
||||
"--web", action="store_true", help="启动 web 服务器"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--web-only", action="store_true", help="启动 web 服务器，但是不启动引擎"
|
||||
"--web", action="store_true", help="启动 web 服务器，启动后将忽略其他选项"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
@ -97,7 +82,7 @@ class MainApp:
|
||||
if invalid_modes:
|
||||
parser.error(f'无效的运行模式: {", ".join(invalid_modes)}')
|
||||
args.report = reports
|
||||
elif args.report is not None:
|
||||
else:
|
||||
args.report = ["pc", "site", "wap"]
|
||||
|
||||
# 检查输入的文件是否存在
|
||||
@ -112,10 +97,6 @@ class MainApp:
|
||||
|
||||
def start_cli(self):
|
||||
"""开启 CLI 模式"""
|
||||
|
||||
# 注册 ctrl+c 处理程序，正常结束所有的 engine
|
||||
signal.signal(signal.SIGINT, self.exit_handler)
|
||||
|
||||
if self.args.crawl or self.args.crawl_file:
|
||||
crawl = CrawlEngine()
|
||||
crawl.cli_start(self.args.crawl, self.args.crawl_file)
|
||||
@ -135,31 +116,7 @@ class MainApp:
|
||||
|
||||
def start_web(self):
|
||||
"""开启 Web 模式"""
|
||||
|
||||
# 注册 ctrl+c 处理程序，正常结束所有的 engine
|
||||
signal.signal(signal.SIGINT, self.exit_handler)
|
||||
|
||||
# 启动所有的 engine
|
||||
if self.args.crawl is not None:
|
||||
self.crawl_engine = CrawlEngine()
|
||||
self.crawl_engine.start()
|
||||
logger.info("crawl 启动")
|
||||
|
||||
if self.args.evidence:
|
||||
self.evidence_engine = EvidenceEngine()
|
||||
self.evidence_engine.start()
|
||||
logger.info("evidence 启动")
|
||||
|
||||
if self.args.report:
|
||||
self.report_engine = Reporter(self.args.report)
|
||||
self.report_engine.start()
|
||||
logger.info("report 启动")
|
||||
|
||||
# 启动 web 页面
|
||||
web_app = WebApp()
|
||||
asyncio.run(web_app.start())
|
||||
|
||||
logger.info("web stop.")
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
"""运行应用"""
|
||||
@ -194,28 +151,9 @@ class MainApp:
|
||||
sys.exit(1)
|
||||
|
||||
# 如果指定了 --web 参数，启动 web 服务器，忽略其他选项
|
||||
if self.args.web or self.args.web_only:
|
||||
if self.args.web:
|
||||
logger.info("启动 Web 模式")
|
||||
return self.start_web()
|
||||
else:
|
||||
logger.info("启动 CLI 模式")
|
||||
return self.start_cli()
|
||||
|
||||
def exit_handler(self, signum, frame):
|
||||
# 在这里结束各个 engine
|
||||
logger.debug("CTRL+C called.")
|
||||
|
||||
if self.crawl_engine:
|
||||
self.crawl_engine.stop()
|
||||
self.crawl_engine.cli_wait()
|
||||
logger.info("crawl 退出")
|
||||
|
||||
if self.evidence_engine:
|
||||
self.evidence_engine.stop()
|
||||
self.evidence_engine.wait()
|
||||
logger.info("evidence 退出")
|
||||
|
||||
if self.report_engine:
|
||||
self.report_engine.stop()
|
||||
self.report_engine.wait()
|
||||
logger.info("report 退出")
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
import enum
|
||||
|
||||
|
||||
class ApiCode(enum.Enum):
|
||||
OK = 20000
|
||||
PARAM_ERROR = 30000
|
||||
DB_ERROR = 40000
|
||||
RUNTIME_ERROR = 50000
|
||||
@ -1,9 +0,0 @@
|
||||
import enum
|
||||
|
||||
|
||||
class DomainStatus(enum.Enum):
|
||||
READY = 1 # 采集结束之后回到这个状态，新添加的默认也是这个状态
|
||||
QUEUEING = 2 # 排队中，已经压入任务队列了，但是还没轮到处理
|
||||
CRAWLING = 3 # 采集中
|
||||
|
||||
PAUSE = 999 # 暂停采集
|
||||
@ -1,21 +1,15 @@
|
||||
import queue
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
|
||||
import execjs
|
||||
import requests
|
||||
from DrissionPage.errors import ElementNotFoundError
|
||||
from loguru import logger
|
||||
from sqlmodel import Session, select, or_, and_
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from app.config.config import AppCtx
|
||||
from app.constants.domain import DomainStatus
|
||||
from app.models.domain import DomainModel
|
||||
from app.models.report_urls import ReportUrlModel
|
||||
from app.utils.common import get_proxies
|
||||
from app.utils.dp import DPEngine
|
||||
from app.utils.ydm_verify import YdmVerify
|
||||
|
||||
|
||||
class CrawlEngine:
|
||||
@ -33,7 +27,7 @@ class CrawlEngine:
|
||||
|
||||
# 线程池
|
||||
self.pool: list[threading.Thread] = []
|
||||
self.worker_count = 1
|
||||
self.worker_count = 2
|
||||
|
||||
# 工作队列
|
||||
self.target_queue = queue.Queue(1024)
|
||||
@ -41,6 +35,8 @@ class CrawlEngine:
|
||||
# 创建一个浏览器
|
||||
self.dp_engine = DPEngine()
|
||||
|
||||
self.database = AppCtx.g_db_engine
|
||||
|
||||
def cli_start(self, target_domains: str, target_domain_filepath: str):
|
||||
"""CLI 模式启动
|
||||
target_domains: 英文逗号分割的字符串
|
||||
@ -94,7 +90,7 @@ class CrawlEngine:
|
||||
continue
|
||||
|
||||
# 存入数据库
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
with Session(self.database) as session:
|
||||
self.save_surl(session, domain, surl)
|
||||
except queue.Empty:
|
||||
# 队列空了，等1秒再取一次
|
||||
@ -123,7 +119,7 @@ class CrawlEngine:
|
||||
|
||||
# 检查在数据库中是否有重复的
|
||||
for domain in domains:
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
with Session(self.database) as session:
|
||||
stmt = select(DomainModel).where(DomainModel.domain == domain)
|
||||
result = session.exec(stmt).first()
|
||||
if not result:
|
||||
@ -152,42 +148,29 @@ class CrawlEngine:
|
||||
def worker(self):
|
||||
"""真正的工作函数，后续以Web模式启动的时候，走这个"""
|
||||
logger.info("crawl worker start!")
|
||||
while self.worker_status:
|
||||
while self.worker_status == 1:
|
||||
# 检查数据库，从中获取需要爬取的域名
|
||||
current_timestamp = int(time.time())
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
|
||||
stmt = select(DomainModel).where(
|
||||
or_(
|
||||
DomainModel.status == 2, # 条件1: status = 2
|
||||
and_(
|
||||
DomainModel.latest_crawl_time + DomainModel.crawl_interval * 60 <= current_timestamp, # 条件2
|
||||
DomainModel.status == 1 # 条件2
|
||||
)
|
||||
)
|
||||
DomainModel.latest_crawl_time + DomainModel.crawl_interval <= current_timestamp
|
||||
)
|
||||
domains = session.exec(stmt).all()
|
||||
|
||||
for domain_model in domains:
|
||||
|
||||
# 采集前修改状态
|
||||
domain_model.status = DomainStatus.CRAWLING.value
|
||||
session.add(domain_model)
|
||||
session.commit()
|
||||
|
||||
# 采集
|
||||
surl_set = self.crawl(domain_model.domain)
|
||||
|
||||
# 存储
|
||||
if surl_set:
|
||||
self.save_surl(session, domain_model.domain, surl_set)
|
||||
self.save_surl(session, domain_model, surl_set)
|
||||
|
||||
domain_model.latest_crawl_time = int(time.time())
|
||||
domain_model.status = DomainStatus.READY.value
|
||||
session.add(domain_model)
|
||||
session.commit()
|
||||
|
||||
self.ev.wait(10)
|
||||
self.ev.wait(60)
|
||||
|
||||
logger.info("crawl worker stop!")
|
||||
|
||||
@ -199,8 +182,8 @@ class CrawlEngine:
|
||||
|
||||
try:
|
||||
# 初始数据
|
||||
# end_time = int(time.time())
|
||||
# start_time = end_time - 3600 * 24 * 30 # 获取最近一个月的数据
|
||||
end_time = int(time.time())
|
||||
start_time = end_time - 3600 * 24 * 30 # 获取最近一个月的数据
|
||||
|
||||
# 依次每一页处理
|
||||
max_page = 10 # 最大页码数量，0表示不限制最大数量
|
||||
@ -233,26 +216,6 @@ class CrawlEngine:
|
||||
# f"https://www.baidu.com/s?wd=site%3A{domain}&gpc=stf%3D{start_time}%2C{end_time}%7Cstftype%3D1&pn={(current_page - 1) * 10}")
|
||||
# tab.get(f"https://www.baidu.com/s?wd=site%3A{domain}&pn={(current_page - 1) * 10}")
|
||||
|
||||
# 检查一下当前的URL是不是跳到验证码的页面
|
||||
if "//wappass.baidu.com/static/captcha/tuxing_v2.html" in tab.url:
|
||||
logger.warning("触发验证码了，尝试识别")
|
||||
idx = 0
|
||||
while idx < 3:
|
||||
idx += 1
|
||||
logger.debug(f"开始第{idx}次识别...")
|
||||
captcha_result = self.verify_captcha(tab.url)
|
||||
if not captcha_result:
|
||||
tab.refresh()
|
||||
continue
|
||||
else:
|
||||
tab.get(captcha_result)
|
||||
break
|
||||
else:
|
||||
logger.error("验证码打码失败，放弃本次采集，等待3分钟后继续")
|
||||
self.ev.wait(180)
|
||||
break
|
||||
|
||||
|
||||
# 终止条件
|
||||
if current_page > max_page and max_page:
|
||||
logger.debug(f"{threading.current_thread().name} 达到指定页码，退出")
|
||||
@ -268,19 +231,12 @@ class CrawlEngine:
|
||||
tab.wait.eles_loaded("@id=content_left")
|
||||
results = tab.ele("@id=content_left").eles("@class:result")
|
||||
# temp = [result.attr("mu") for result in results if result.attr("mu") is not None]
|
||||
# logger.debug(f"{len(results)=}")
|
||||
for result in results:
|
||||
# logger.debug(f"{result=}")
|
||||
surl = result.attr("mu")
|
||||
if not surl:
|
||||
continue
|
||||
|
||||
# 添加结果的时候，也检查一下抓到的 surl 是否和目标域名有关
|
||||
if domain not in surl:
|
||||
logger.debug(f"{threading.current_thread().name} URL {surl} 与目标域名 {domain} 无关，跳过")
|
||||
else:
|
||||
surl_set.add(surl)
|
||||
logger.debug(f"{threading.current_thread().name} 找到 {surl}")
|
||||
logger.debug(f"{threading.current_thread().name} 找到 URL : {surl}")
|
||||
surl_set.add(surl)
|
||||
|
||||
# 翻页的时候等一下，别太快了
|
||||
self.ev.wait(0.3)
|
||||
@ -305,12 +261,6 @@ class CrawlEngine:
|
||||
def save_surl(session: Session, domain: str, surl_set: set[str]):
|
||||
"""保存采集到的URL"""
|
||||
for surl in surl_set:
|
||||
|
||||
# 简单的判断一下 surl 中是否包含目标域名
|
||||
if domain not in surl:
|
||||
logger.debug(f"跳过保存 {surl} 因为与目标域名 {domain} 不符合")
|
||||
continue
|
||||
|
||||
# 先检查是否存在
|
||||
stmt = select(ReportUrlModel).where(ReportUrlModel.surl == surl)
|
||||
exist = session.exec(stmt).first()
|
||||
@ -333,170 +283,3 @@ class CrawlEngine:
|
||||
)
|
||||
session.add(example)
|
||||
session.commit()
|
||||
|
||||
# def captcha_listener(self):
|
||||
# for pkg in self.tab.listen.steps():
|
||||
# if "/cap/init" in pkg.url:
|
||||
# self.captcha_data["init"] = pkg.response.body
|
||||
# if "/cap/style" in pkg.url:
|
||||
# self.captcha_data["style"] = pkg.response.body
|
||||
# self.captcha_data["referer"] = pkg.request.headers.get("Referer")
|
||||
# logger.debug(f"触发验证码的 referer: {self.captcha_data["referer"]}")
|
||||
#
|
||||
# self.captcha_data["cookie"] = pkg.request.headers.get("Cookie")
|
||||
# logger.debug(f"触发验证码的 cookie: {self.captcha_data['cookie']}")
|
||||
# if "/cap/log" in pkg.url:
|
||||
# self.captcha_data["log"] = pkg.response.body
|
||||
|
||||
def verify_captcha(self, current_url: str):
|
||||
"""尝试识别验证码，因为和 pc_reporter 的逻辑有点区别，所以单独写一遍"""
|
||||
headers = {
|
||||
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-US;q=0.7,zh-TW;q=0.6',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Pragma': 'no-cache',
|
||||
'Referer': current_url,
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0",
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
'sec-ch-ua_wap': '"Not(A:Brand";v="99", "Google Chrome";v="133", "Chromium";v="133"',
|
||||
'sec-ch-ua_wap-mobile': '?0',
|
||||
'sec-ch-ua_wap-platform': '"Windows"',
|
||||
# "Cookie": self.captcha_data["cookie"],
|
||||
}
|
||||
|
||||
# 解出AS / TK
|
||||
ts = time.time()
|
||||
ts1 = int(ts)
|
||||
ts2 = int(ts * 1000)
|
||||
response = requests.post(
|
||||
"https://passport.baidu.com/cap/init",
|
||||
data={
|
||||
"_": ts2,
|
||||
"refer": re.sub(r'timestamp=\d+', f'timestamp={ts1}', current_url),
|
||||
"ak": "c27bbc89afca0463650ac9bde68ebe06",
|
||||
"ver": "2",
|
||||
"scene": "",
|
||||
"ds": "",
|
||||
"tk": "",
|
||||
"as": "",
|
||||
"reinit": 0
|
||||
},
|
||||
headers=headers,
|
||||
proxies=get_proxies()
|
||||
).json()
|
||||
as_value = response["data"]["as"]
|
||||
tk_value = response["data"]["tk"]
|
||||
|
||||
# 解出 style
|
||||
response = requests.post(
|
||||
"https://passport.baidu.com/cap/style",
|
||||
data={
|
||||
"_": int(time.time() * 1000),
|
||||
"refer": re.sub(r'timestamp=\d+', f'timestamp={ts1}', current_url),
|
||||
"ak": "c27bbc89afca0463650ac9bde68ebe06",
|
||||
"tk": tk_value,
|
||||
"scene": "",
|
||||
"isios": "0",
|
||||
"type": "spin",
|
||||
"ver": "2"
|
||||
},
|
||||
headers=headers,
|
||||
proxies=get_proxies()
|
||||
)
|
||||
response = response.json()
|
||||
backstr = response["data"]["backstr"]
|
||||
captcha_link = response["data"]["captchalist"][0]["source"]["back"]["path"]
|
||||
logger.debug(f"{backstr=}, {captcha_link=}")
|
||||
|
||||
# 下载验证码图片
|
||||
image_response = requests.get(captcha_link, headers=headers, proxies=get_proxies())
|
||||
with open("captcha.png", "wb") as f:
|
||||
f.write(image_response.content)
|
||||
logger.debug("download captcha.png")
|
||||
|
||||
# 识别验证码
|
||||
ydm = YdmVerify()
|
||||
with open("captcha.png", "rb") as fp:
|
||||
picture = fp.read()
|
||||
|
||||
slide_distance = ydm.rotate(picture)
|
||||
logger.debug(f"{slide_distance=}")
|
||||
if not slide_distance:
|
||||
logger.error("识别验证码失败")
|
||||
return None
|
||||
rotate_angle_rate = round(slide_distance / 360, 2)
|
||||
logger.debug(f"{rotate_angle_rate=}")
|
||||
|
||||
if not rotate_angle_rate:
|
||||
logger.debug("识别验证码失败")
|
||||
return None
|
||||
|
||||
# 发送验证码请求
|
||||
time_log = str(int(time.time() * 1000))
|
||||
with open("./js/mkd_v2_link_submit.js", 'r', encoding='utf-8') as f:
|
||||
ds_js = f.read()
|
||||
fs = execjs.compile(ds_js).call('getFs2', backstr, rotate_angle_rate, as_value)
|
||||
data = {
|
||||
"_": time_log,
|
||||
"refer": current_url,
|
||||
"ak": "c27bbc89afca0463650ac9bde68ebe06",
|
||||
"as": as_value,
|
||||
"scene": "",
|
||||
"tk": tk_value,
|
||||
"ver": "2",
|
||||
"cv": "submit",
|
||||
"typeid": "spin-0",
|
||||
"fuid": "FOCoIC3q5fKa8fgJnwzbE67EJ49BGJeplOzf+4l4EOvDuu2RXBRv6R3A1AZMa49I27C0gDDLrJyxcIIeAeEhD8JYsoLTpBiaCXhLqvzbzmvy3SeAW17tKgNq/Xx+RgOdb8TWCFe62MVrDTY6lMf2GrfqL8c87KLF2qFER3obJGnfaJTn/Ne60I9LwR04t6XmGEimjy3MrXEpSuItnI4KD0FJKzTbw1AN69fBnzR2FuvMmmQZ+1zgJ72wdcVU+mcQxiE2ir0+TEYgjPJt1Qa3K1mLi+P4IWJeag2lvxB4yJ/GgLbz7OSojK1zRbqBESR5Pdk2R9IA3lxxOVzA+Iw1TWLSgWjlFVG9Xmh1+20oPSbrzvDjYtVPmZ+9/6evcXmhcO1Y58MgLozKnaQIaLfWRPAn9I0uOqAMff6fuUeWcH1mRYoTw2Nhr4J4agZi377iM/izL6cVCGRy2F8c0VpEvM5FjnYxYstXg/9EfB3EVmKAfzNRIeToJ5YV9twMcgdmlV1Uhhp5FAe6gNJIUptp7EMAaXYKm11G+JVPszQFdp9AJLcm4YSsYUXkaPI2Tl66J246cmjWQDTahAOINR5rXR5r/7VVI1RMZ8gb40q7az7vCK56XLooKT5a+rsFrf5Zu0yyCiiagElhrTEOtNdBJJq8eHwEHuFBni9ahSwpC7lbKkUwaKH69tf0DFV7hJROiLETSFloIVkHdy3+I2JUr1LsplAz0hMkWt/tE4tXVUV7QcTDTZWS/2mCoS/GV3N9awQ6iM6hs/BWjlgnEa1+5iP7WSc7RJ34FaE5PsyGXyoCWdXwNRGSZPSvVtB/Ea6w5FKazXcZ/j40FJv+iLGBn3nkkgHlne61I8I7KhtQgIkmBMJIjPMkS/L051MeqdGScsKYTJuSucgI5c3+79eVH+y2TvbOTuuHv1uGxwXFb2atIU1ZYPbmmXculmizKcKI/s44qf8uM8iBZLGkKeVyL74aPyLkg7Gk359g98BIGN/ZzJR/h+Y6AyFx+HlMoYJnS06dVmqFbvlCtSdGylKQ5f8eWtxPkJGqOFtWjIVteQYMsH/AaSJonqw+WLiZvGjYfm9p0alEyujapoTy77HzDcUoU1wUSXa5xS/Z6hXEr2OnLi0LdPVcGjz8lpLcdVeSfm9p0alEyujapoTy77HzDWf5PERRSTFqLd9BTUHLyY4Ji3EQLGQPaM1aeHxG1bJZH0s1Si/KwzTaTYzu6ziQiqwcr2kaYUiH+fMOxn69/BhNJVMhpQkhprc1KZuJRvXjppq0gKweencPxgS/jd0rjw==",
|
||||
# "fuid": "FOCoIC3q5fKa8fgJnwzbE67EJ49BGJeplOzf+4l4EOvDuu2RXBRv6R3A1AZMa49I27C0gDDLrJyxcIIeAeEhD8JYsoLTpBiaCXhLqvzbzmvy3SeAW17tKgNq/Xx+RgOdb8TWCFe62MVrDTY6lMf2GrfqL8c87KLF2qFER3obJGm51EODDlnqgz44AdUN5VVLGEimjy3MrXEpSuItnI4KD4X6JLdk9kt5JRR+RlJ66q1+4kQEivhwAoCrm3oUNdYdi+yNJadLMQy5pqjmiW757BJsVwXkGdF24AsEQ3K5XBbh9EHAWDOg2T1ejpq0s2eFy9ar/j566XqWDobGoNNfmfpaEhZpob9le2b5QIEdiQcF+6iOKqU/r67N8lf+wxW6FCMUN0p4SXVVUMsKNJv2TwEq3+MvKTlPBjfdM81CMPq4LkPV+7TROLMG0V6r0A++zkWOdjFiy1eD/0R8HcRWYsUPXjDqADgs+Xs31pnSHeup+HBavJhpxl858h16cMtKQmxzisHOxsE/KMoDNYYE7ucLE22Bi0Ojbor7y6SXfVj7+B4iuZO+f7FUDWABtt/WWQqHKVfXMaw5WUmKnfSR5wwQa+N01amx6X+p+x97kkGmoNOSwxWgGvuezNFuiJQdt51yrWaL9Re9fZveXFsIu/gzGjL50VLcWv2NICayyI8BE9m62pdBPySuv4pVqQ9Sl1uTC//wIcO7QL9nm+0N6JgtCkSAWOZCh7Lr0XP6QztjlyD3bkwYJ4FTiNanaDaDJMNOONUIptCYaHTS+UC6IlHE1MUFHThGQXNkGIX8AdBh0GvEV9dnyTGKy8XFjCQiSGk66HDxtjKMU4HPNa0dtuC6f3Qc1BA80dVENIrm5fvupUvtUx+t4D1r3M6jRrNCFDmi5MpkOxe5k51gshb/lV68JOKfsQeXT2p7EM9kdbZAphQDW3ajjXdDRh/L4vMDrWe1PKtUcuW/fWn+hZVZzw+X5dQWsFNhWzqaDLLTRZQpSBdWrMIHd5mkoSCb/UJmNfWI9UswFst29h1Heb04lgaYXvleBbteLbUi5NoCAChP5oZfoCeoKKuvUEAPXXTPVjO0TTi0sVqFSdG+GFyi03wlrm3wCRN8QsWhT10pXJL0RhcLTagDnxauF9flnVwiWaq+daLSn0MEazavBACRErAMWXEI9EFQPGJKv0Ijpq+0VDw8xeJloxMf4I+yn8oxuqFuBSz8I0Kfe0QZwk5OQW6lRvv5iBU4fcPzWWTZ9FnzQ2GA5eh8aiV0nDOGmtfhiYNjbs2NxP0acAgApNd0ew==",
|
||||
"fs": fs
|
||||
}
|
||||
response = requests.post(
|
||||
"https://passport.baidu.com/cap/log",
|
||||
headers=headers,
|
||||
data=data,
|
||||
proxies=get_proxies(),
|
||||
).json()
|
||||
try:
|
||||
result = {
|
||||
"ds": response["data"]["ds"],
|
||||
"op": response["data"]["op"],
|
||||
"tk": response["data"]["tk"]
|
||||
}
|
||||
except KeyError:
|
||||
logger.error(f"验证码没转成功, response: {response=}")
|
||||
time.sleep(1)
|
||||
return None
|
||||
logger.debug(f"{result=}")
|
||||
|
||||
# 检查验证码是否正确
|
||||
if result["op"] != 1:
|
||||
logger.error(f"op != 1, 重试")
|
||||
return None
|
||||
|
||||
# 发送验证码请求 /cap/c 请求，获取待跳转的URL
|
||||
response = requests.post(
|
||||
"https://passport.baidu.com/cap/c?ak=c27bbc89afca0463650ac9bde68ebe06",
|
||||
headers=headers,
|
||||
json={
|
||||
"tk": result["tk"],
|
||||
"ds": result["ds"],
|
||||
"qrsign": "",
|
||||
"refer": current_url
|
||||
},
|
||||
proxies=get_proxies()
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
if data["data"].get("f"):
|
||||
logger.error(f"验证码失败: {data['data'].get('f')}")
|
||||
return None
|
||||
if data["data"].get("s"):
|
||||
logger.debug("验证成功，URL：" + data["data"].get("s").get("url"))
|
||||
url = data["data"].get("s").get("url")
|
||||
url = url.encode("utf-8").decode("unicode-escape")
|
||||
logger.success("解码后的URL：" + url)
|
||||
return url
|
||||
|
||||
@ -59,8 +59,8 @@ class EvidenceEngine:
|
||||
logger.debug(f"开始获取 {target['surl']} 的举报数据")
|
||||
self.get_screenshot_and_report_link(target)
|
||||
|
||||
# 每10秒跑一次
|
||||
self.ev.wait(10)
|
||||
# 每分钟跑一次
|
||||
self.ev.wait(60)
|
||||
|
||||
def stop(self):
|
||||
"""结束线程"""
|
||||
@ -69,9 +69,6 @@ class EvidenceEngine:
|
||||
self.dp_engine.close()
|
||||
self.wap_dp_engine.close()
|
||||
|
||||
def wait(self):
|
||||
self.worker_thread.join()
|
||||
|
||||
def get_surl_from_db(self):
|
||||
"""从数据库中获取数据"""
|
||||
result: list = []
|
||||
@ -90,20 +87,7 @@ class EvidenceEngine:
|
||||
|
||||
# Part1 获取证据截图
|
||||
logger.debug(f"开始获取 {surl} 在百度搜索中的截图")
|
||||
img_path, tab, has_result = self.get_screenshot(target)
|
||||
if not has_result:
|
||||
# 如果没有搜到结果，直接把 has_evidence 标记为 true 就行了
|
||||
with Session(self.database) as session:
|
||||
stmt = select(ReportUrlModel).where(ReportUrlModel.id == target["id"])
|
||||
model: ReportUrlModel = session.exec(stmt).first()
|
||||
if not model:
|
||||
logger.error(f"{target['id']} 记录不存在，跳过...")
|
||||
return None
|
||||
# 更新数据
|
||||
model.has_evidence = True
|
||||
session.add(model)
|
||||
session.commit()
|
||||
return None
|
||||
img_path, tab = self.get_screenshot(target)
|
||||
if not img_path:
|
||||
return None
|
||||
|
||||
@ -150,7 +134,7 @@ class EvidenceEngine:
|
||||
except Exception as e:
|
||||
logger.error(f"获取证据截图和举报链接失败: {e}")
|
||||
|
||||
def get_screenshot(self, target: dict) -> tuple[str | None, MixTab, bool]:
|
||||
def get_screenshot(self, target: dict) -> tuple[str | None, MixTab]:
|
||||
"""获取搜索页面的截图，返回 img_path """
|
||||
search_keyword = target["surl"].lstrip("https://").lstrip("http://")
|
||||
tab = self.dp_engine.browser.new_tab()
|
||||
@ -160,13 +144,12 @@ class EvidenceEngine:
|
||||
|
||||
if "未找到相关结果" in tab.html:
|
||||
logger.info(f"没有关于 {search_keyword} 的数据")
|
||||
return None, tab, False
|
||||
return None, tab
|
||||
|
||||
# 图片的存储路径
|
||||
# 截完图先不要关闭 tab，别的地方还要用
|
||||
img_path = f"./imgs/{target['domain']}/{md5(target['surl'])}.png"
|
||||
img_path, tab = self.do_screenshot(tab, img_path)
|
||||
return img_path, tab, True
|
||||
return self.do_screenshot(tab, img_path)
|
||||
|
||||
def get_wap_screenshot(self, target: dict) -> tuple[str | None, MixTab]:
|
||||
"""用 wap dp 再截一张 surl 本身的图"""
|
||||
|
||||
@ -33,7 +33,6 @@ class Reporter:
|
||||
def wait(self):
|
||||
self.worker_thread.join()
|
||||
|
||||
# noinspection DuplicatedCode
|
||||
def cli_start(self):
|
||||
for mode in self.mode:
|
||||
if mode == "pc":
|
||||
@ -47,33 +46,20 @@ class Reporter:
|
||||
continue
|
||||
|
||||
def stop(self):
|
||||
|
||||
for mode in self.mode:
|
||||
if mode == "pc":
|
||||
self.reporters["pc"].stop()
|
||||
elif mode == "wap":
|
||||
self.reporters["wap"].stop()
|
||||
elif mode == "site":
|
||||
self.reporters["site"].stop()
|
||||
else:
|
||||
logger.error(f"参数错误: {mode}")
|
||||
continue
|
||||
|
||||
self.status = 0
|
||||
self.ev.set()
|
||||
|
||||
# noinspection DuplicatedCode
|
||||
def worker(self):
|
||||
while self.status:
|
||||
for mode in self.mode:
|
||||
if mode == "pc" and self.status:
|
||||
if mode == "pc":
|
||||
self.reporters["pc"].run()
|
||||
elif mode == "wap" and self.status:
|
||||
elif mode == "wap":
|
||||
self.reporters["wap"].run()
|
||||
elif mode == "site" and self.status:
|
||||
elif mode == "site":
|
||||
self.reporters["site"].run()
|
||||
else:
|
||||
logger.error(f"参数错误: {mode}")
|
||||
continue
|
||||
|
||||
self.ev.wait(10)
|
||||
self.ev.wait(60)
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class BaseReporter(ABC):
|
||||
"""所有 reporter 的基类"""
|
||||
|
||||
@ -8,7 +7,3 @@ class BaseReporter(ABC):
|
||||
def run(self):
|
||||
"""运行 reporter，子类必须实现此方法"""
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
"""控制结束"""
|
||||
pass
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
import os.path
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
||||
@ -20,8 +19,6 @@ from ...utils.ydm_verify import YdmVerify
|
||||
class PcReporter(BaseReporter):
|
||||
def __init__(self):
|
||||
self.engine_name = "PC_REPORTER"
|
||||
self.status = 1
|
||||
self.ev = threading.Event()
|
||||
self.database = AppCtx.g_db_engine
|
||||
|
||||
self.upload_pic_url = "http://jubao.baidu.com/jubao/accu/upload"
|
||||
@ -48,23 +45,14 @@ class PcReporter(BaseReporter):
|
||||
"Cookie": "",
|
||||
}
|
||||
|
||||
def stop(self):
|
||||
self.status = 0
|
||||
self.ev.set()
|
||||
logger.warning(f"{self.engine_name} 收到退出消息，等待当前任务完成后退出")
|
||||
|
||||
def run(self):
|
||||
with Session(self.database) as session:
|
||||
stmt = select(ReportUrlModel).where(ReportUrlModel.is_report_by_one == False).where(ReportUrlModel.has_evidence == True)
|
||||
stmt = select(ReportUrlModel).where(ReportUrlModel.is_report_by_one == False)
|
||||
rows: list[ReportUrlModel] = session.exec(stmt).all()
|
||||
|
||||
logger.info(f"[{self.engine_name}] 共计 {len(rows)} 条记录需要举报")
|
||||
|
||||
for row in rows:
|
||||
|
||||
if not self.status:
|
||||
break
|
||||
|
||||
# 选个 cookie
|
||||
report_cookie = random.choice(get_all_cookies())
|
||||
self.headers["Cookie"] = report_cookie
|
||||
@ -78,8 +66,8 @@ class PcReporter(BaseReporter):
|
||||
domain = row.domain
|
||||
# timestamp_s = {int(time.time() * 1000)} # 这里为啥要用 set ?
|
||||
timestamp_s = int(time.time() * 1000)
|
||||
referer = f"https://jubao.baidu.com/jubao/accu/?surl={surl}token={token}&title={title}&q={q}&has_gw=0&has_v=0&_t8={timestamp_s}"
|
||||
# referer = "https://jubao.baidu.com/"
|
||||
# referer = f"https://jubao.baidu.com/jubao/accu/?surl={surl}token={token}&title={title}&q={q}&has_gw=0&has_v=0&_t8={timestamp_s}"
|
||||
referer = "https://jubao.baidu.com/"
|
||||
logger.debug(f"referer: {referer}, type of referer: {type(referer)}")
|
||||
self.headers["Referer"] = referer
|
||||
|
||||
@ -132,8 +120,6 @@ class PcReporter(BaseReporter):
|
||||
|
||||
retry += 1
|
||||
|
||||
self.ev.wait(5)
|
||||
|
||||
def do_report(self, ds, tk, surl, token, title, q, upload=''):
|
||||
try:
|
||||
phone = generate_random_phone_number()
|
||||
@ -215,26 +201,20 @@ class PcReporter(BaseReporter):
|
||||
# 获取 as、tk 值
|
||||
try:
|
||||
get_as_tk = self.post_init(surl, token, title, q, timestamp_s)
|
||||
# logger.debug(f"{get_as_tk=}")
|
||||
get_as = get_as_tk['as']
|
||||
get_tk = get_as_tk['tk']
|
||||
|
||||
# 获取验证码图片下载链接、backstr
|
||||
get_style_result = self.get_style(get_tk, surl, token, title, q, timestamp_s)
|
||||
# logger.debug(f"{get_style_result=}")
|
||||
get_backstr = get_style_result['backstr']
|
||||
pic_download_link = get_style_result['captcha']
|
||||
|
||||
# 下载验证码图片
|
||||
self.download_captcha(pic_download_link)
|
||||
rotate_angle_rate = self.get_rotate_angle_rate()
|
||||
logger.debug(f"{rotate_angle_rate=}")
|
||||
# key = self.get_key(get_as)
|
||||
if not rotate_angle_rate:
|
||||
return {'op': 3}
|
||||
|
||||
get_ds_tk = self.post_log(get_as, get_tk, get_backstr, rotate_angle_rate)
|
||||
logger.debug(f"{get_ds_tk=}")
|
||||
log_ds = get_ds_tk['ds']
|
||||
log_tk = get_ds_tk['tk']
|
||||
log_op = get_ds_tk['op']
|
||||
@ -245,7 +225,7 @@ class PcReporter(BaseReporter):
|
||||
}
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.exception(f'{e}')
|
||||
logger.error(f'{e}')
|
||||
return {'op': 3}
|
||||
|
||||
def post_init(self, surl, token, title, q, timestamp_s):
|
||||
@ -323,13 +303,9 @@ class PcReporter(BaseReporter):
|
||||
with open('./captcha/captcha.png', 'rb') as p:
|
||||
picture = p.read()
|
||||
slide_distance = identify_distance.rotate(image=picture)
|
||||
logger.debug(f"{slide_distance=}")
|
||||
if not slide_distance:
|
||||
return None
|
||||
# 旋转角度为
|
||||
# logger.info('rotate angle: ' + str(slide_distance))
|
||||
rotate_angle_rate = round(slide_distance / 360, 2)
|
||||
logger.debug(f"{rotate_angle_rate=}")
|
||||
# logger.info('rotate angle rate: ' + str(rotate_angle_rate))
|
||||
return rotate_angle_rate
|
||||
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
import os.path
|
||||
import random
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
|
||||
import requests
|
||||
@ -20,8 +19,6 @@ from ...utils.ua import random_ua
|
||||
class SiteReporter(BaseReporter):
|
||||
def __init__(self):
|
||||
self.engine_name = "SITE_REPORTER"
|
||||
self.status = 1
|
||||
self.ev = threading.Event()
|
||||
|
||||
self.upload_pic_url = "https://help.baidu.com/api/mpic"
|
||||
self.report_url = "https://help.baidu.com/jubaosubmit"
|
||||
@ -45,25 +42,15 @@ class SiteReporter(BaseReporter):
|
||||
|
||||
self.token_pattern = r'name="submit_token" value="(.*?)"'
|
||||
|
||||
def stop(self):
|
||||
# logger.debug(f"{self.engine_name} stop called.")
|
||||
self.status = 0
|
||||
self.ev.set()
|
||||
logger.warning(f"{self.engine_name} 收到退出消息，等待当前任务完成后退出")
|
||||
|
||||
def run(self):
|
||||
"""实现 PC 端的举报逻辑"""
|
||||
with Session(self.database) as session:
|
||||
stmt = select(ReportUrlModel).where(ReportUrlModel.is_report_by_site == False).where(ReportUrlModel.has_evidence == True)
|
||||
stmt = select(ReportUrlModel).where(ReportUrlModel.is_report_by_site == False)
|
||||
rows: list[ReportUrlModel] = session.exec(stmt).all()
|
||||
|
||||
logger.info(f"[{self.engine_name}] 共计 {len(rows)} 条需要举报")
|
||||
|
||||
for row in rows:
|
||||
|
||||
if not self.status:
|
||||
break
|
||||
|
||||
# 生成举报需要的基础数据
|
||||
surl = row.surl
|
||||
q = row.q
|
||||
@ -90,8 +77,8 @@ class SiteReporter(BaseReporter):
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
# 等待5秒继续举报
|
||||
self.ev.wait(5)
|
||||
# 等待5秒继续举报
|
||||
time.sleep(5)
|
||||
|
||||
def upload_pic(self, img_path: str):
|
||||
try:
|
||||
|
||||
@ -2,7 +2,6 @@ import base64
|
||||
import json
|
||||
import os.path
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
|
||||
import requests
|
||||
@ -21,8 +20,6 @@ class WapReporter(BaseReporter):
|
||||
def __init__(self):
|
||||
|
||||
self.engine_name = "WAP_REPORTER"
|
||||
self.status = 1
|
||||
self.ev = threading.Event()
|
||||
|
||||
self.report_url = "https://ufosdk.baidu.com/api?m=Client&a=postMsg"
|
||||
self.request = requests.session()
|
||||
@ -43,27 +40,16 @@ class WapReporter(BaseReporter):
|
||||
self.database = AppCtx.g_db_engine
|
||||
self.all_cookies = get_all_cookies()
|
||||
|
||||
def stop(self):
|
||||
self.status = 0
|
||||
self.ev.set()
|
||||
logger.warning(f"{self.engine_name} 收到退出消息，等待当前任务完成后退出")
|
||||
|
||||
def run(self):
|
||||
"""实现 WAP 端的举报逻辑"""
|
||||
with Session(self.database) as session:
|
||||
stmt = select(ReportUrlModel).where(ReportUrlModel.is_report_by_wap == False).where(
|
||||
ReportUrlModel.has_evidence == True)
|
||||
stmt = select(ReportUrlModel).where(ReportUrlModel.is_report_by_wap == False)
|
||||
rows: list[ReportUrlModel] = session.exec(stmt).all()
|
||||
|
||||
logger.debug(f"[{self.engine_name}] 共找到 {len(rows)} 条待举报记录")
|
||||
|
||||
for row in rows:
|
||||
|
||||
if not self.status:
|
||||
break
|
||||
|
||||
self.ev.wait(1)
|
||||
|
||||
# 选个 cookie
|
||||
report_cookie = random.choice(get_all_cookies())
|
||||
report_site_cookie = GenCookie.run(report_cookie)
|
||||
@ -88,7 +74,7 @@ class WapReporter(BaseReporter):
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
self.ev.wait(5)
|
||||
time.sleep(5)
|
||||
|
||||
def get_user_info(self):
|
||||
try:
|
||||
@ -96,10 +82,9 @@ class WapReporter(BaseReporter):
|
||||
# wapUserAgent = random.choice(self.wapUserAgent)
|
||||
response = self.request.get(
|
||||
"https://ufosdk.baidu.com/api?m=Web&a=getUserInfo&appid=293852",
|
||||
headers=self.headers, proxies=self.proxies, allow_redirects=False, timeout=10
|
||||
headers=self.headers, proxies=self.proxies, allow_redirects=False, timeout=10, verify=False
|
||||
)
|
||||
json_data = response.json()
|
||||
logger.debug(f"{self.engine_name} get_user_info response: {json_data}")
|
||||
uid = json_data['result']['uid']
|
||||
un = json_data['result']['un']
|
||||
userinfo["uid"] = uid
|
||||
@ -155,14 +140,11 @@ class WapReporter(BaseReporter):
|
||||
proxies=self.proxies,
|
||||
allow_redirects=False,
|
||||
timeout=10,
|
||||
verify=False
|
||||
)
|
||||
# logger.debug(req.json())
|
||||
data = response.json()
|
||||
logger.debug(data)
|
||||
if data['errno'] == 0:
|
||||
logger.debug(response.json())
|
||||
if response.json()['errno'] == 0:
|
||||
logger.success(f"[{self.engine_name}] {fb_url} 举报成功")
|
||||
return True
|
||||
if "请勿重复提交" in data["errmsg"]:
|
||||
logger.success(f"[{self.engine_name}] {fb_url} 重复提交，标记为成功")
|
||||
return True
|
||||
return False
|
||||
|
||||
@ -26,16 +26,13 @@ def update_updated_at(mapper, connection, target):
|
||||
target.updated_at = get_timestamp()
|
||||
|
||||
|
||||
# noinspection PyUnresolvedReferences
|
||||
def connect_db(config: AppConfig):
|
||||
"""连接数据库"""
|
||||
|
||||
# 导入所有模型，为了自动创建数据表
|
||||
from .domain import DomainModel
|
||||
from .report_urls import ReportUrlModel
|
||||
|
||||
dsn = f"mysql+pymysql://{config.database.user}:{config.database.password}@{config.database.host}:{config.database.port}/{config.database.database}"
|
||||
engine = create_engine(dsn, echo=False, pool_size=4, max_overflow=10, pool_recycle=60, pool_pre_ping=True)
|
||||
engine = create_engine(dsn, echo=False)
|
||||
|
||||
SQLModel.metadata.create_all(engine)
|
||||
AppCtx.g_db_engine = engine
|
||||
|
||||
@ -12,7 +12,7 @@ class DomainModel(BaseModel, table=True):
|
||||
# 域名
|
||||
domain: str = Field(alias="domain", default="", sa_type=VARCHAR(1024))
|
||||
|
||||
# 爬取状态，@see constants.DomainStatus
|
||||
# 爬取状态，TODO：先空着，后续有任务控制之后，用这个字段表示这个域名的任务状态
|
||||
status: int = Field(alias="status", default=0)
|
||||
|
||||
# 爬取间隔，默认间隔为1周
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
import hashlib
|
||||
import random
|
||||
|
||||
from app.config.config import AppCtx
|
||||
|
||||
|
||||
def md5(s: str) -> str:
|
||||
m = hashlib.md5()
|
||||
@ -11,24 +9,17 @@ def md5(s: str) -> str:
|
||||
|
||||
|
||||
def get_proxies():
|
||||
# username = "t14131310374591"
|
||||
# password = "qg6xwmrq"
|
||||
# tunnel = "d432.kdltps.com:15818"
|
||||
# proxies = {
|
||||
# "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
|
||||
# "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
|
||||
# }
|
||||
username = "t14131310374591"
|
||||
password = "qg6xwmrq"
|
||||
tunnel = "d432.kdltps.com:15818"
|
||||
proxies = {
|
||||
"http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
|
||||
"https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
|
||||
}
|
||||
# proxies = {
|
||||
# "http": "http://127.0.0.1:8080",
|
||||
# "https": "http://127.0.0.1:8080"
|
||||
# }
|
||||
|
||||
proxy = AppCtx.g_app_config.chrome.proxy
|
||||
proxies = {
|
||||
"http": proxy,
|
||||
"https": proxy
|
||||
}
|
||||
|
||||
return proxies
|
||||
|
||||
|
||||
|
||||
@ -2,12 +2,10 @@ import base64
|
||||
import json
|
||||
|
||||
import requests
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class YdmVerify(object):
|
||||
_custom_url = "https://www.jfbym.com/api/YmServer/customApi"
|
||||
_token = "2HNCDBee_JFmXAZZanQm9I7x1sqQln9BggF1xaGtMX0"
|
||||
_token = "HhUGwpI6AtQGoux36i1ZpsDv7hwGSbr1hQ0RX-HXSZE"
|
||||
_headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
@ -19,11 +17,4 @@ class YdmVerify(object):
|
||||
"type": "90009"
|
||||
}
|
||||
resp = requests.post(self._custom_url, headers=self._headers, data=json.dumps(payload))
|
||||
logger.debug(f"{resp.json()=}")
|
||||
|
||||
response_data = resp.json()
|
||||
if response_data.get("code") == 10002:
|
||||
logger.error(f'{response_data.get("msg")}')
|
||||
return None
|
||||
|
||||
return resp.json()['data']['data']
|
||||
@ -1,116 +0,0 @@
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, UploadFile, Form, Query
|
||||
|
||||
from app.constants.api_result import ApiCode
|
||||
from app.constants.domain import DomainStatus
|
||||
from app.web.request.domain_request import AddDomainRequest, DeleteDomainRequest, UpdateDomainRequest, \
|
||||
GetDomainListRequest, CrawlNowRequest, ToggleDomainRequest
|
||||
from app.web.results import ApiResult
|
||||
from app.web.service.domain_service import DomainService
|
||||
|
||||
router = APIRouter(prefix="/api/domain", tags=["域名管理"])
|
||||
|
||||
|
||||
@router.get("/v1/list")
|
||||
def get_all_domains(request: Annotated[GetDomainListRequest, Query()]):
|
||||
"""获取所有的域名信息，支持根据域名、状态进行搜索，不传则返回全部数据，支持分页"""
|
||||
return DomainService.get_list(request.page, request.size, request.domain, request.status)
|
||||
|
||||
|
||||
@router.post("/v1/add")
|
||||
def add_domains(request: AddDomainRequest):
|
||||
"""添加域名"""
|
||||
|
||||
# 检查是否有重复的
|
||||
result = DomainService.get_by_domains(request.domains)
|
||||
if not result.success:
|
||||
return result
|
||||
|
||||
existed_domains = [item.domain for item in result.data]
|
||||
new_domains = [x for x in request.domains if x not in existed_domains]
|
||||
if not new_domains:
|
||||
return ApiResult.ok(0)
|
||||
|
||||
# 添加并返回
|
||||
return DomainService.add_domains(request.crawl_interval, request.crawl_now, new_domains)
|
||||
|
||||
|
||||
@router.post("/v1/import")
|
||||
def import_domains(
|
||||
# 同时提交文件和参数的时候，没办法使用 FormModel 的形式，必须一个一个定义
|
||||
file: UploadFile,
|
||||
crawl_interval: int = Form(),
|
||||
crawl_now: bool = Form(),
|
||||
):
|
||||
"""通过上传文件添加域名，如果单个文件很大，以后改成开新协程/线程处理"""
|
||||
|
||||
# 把文件内容读出来
|
||||
domains = []
|
||||
for line in file.file:
|
||||
line = line.strip()
|
||||
domains.append(line.decode("UTF-8"))
|
||||
|
||||
# 创建协程任务
|
||||
# asyncio.create_task(DomainService.add_domains(crawl_interval, crawl_now, domains))
|
||||
|
||||
# 检查是否有重复域名
|
||||
result = DomainService.get_by_domains(domains)
|
||||
if not result.success:
|
||||
return result
|
||||
existed_domains = [item.domain for item in result.data]
|
||||
new_domains = [x for x in domains if x not in existed_domains]
|
||||
|
||||
# 添加并返回
|
||||
return DomainService.add_domains(crawl_interval, crawl_now, new_domains)
|
||||
|
||||
|
||||
# noinspection DuplicatedCode
|
||||
@router.post("/v1/update")
|
||||
def update_domain(request: UpdateDomainRequest):
|
||||
"""更新域名的数据，主要是采集间隔，支持批量修改，传入多个 id"""
|
||||
|
||||
# 检查待更新的域名是否存在
|
||||
result = DomainService.get_by_ids(request.domain_ids)
|
||||
if not result.success:
|
||||
return result
|
||||
|
||||
existed_domain_ids = [item.id for item in result.data]
|
||||
for domain_id in request.domain_ids:
|
||||
if domain_id not in existed_domain_ids:
|
||||
return ApiResult.error(ApiCode.PARAM_ERROR.value, f"域名 ID {domain_id} 不存在")
|
||||
|
||||
# 更新刷新时间
|
||||
return DomainService.update_domain_interval(request.domain_ids, request.crawl_interval)
|
||||
|
||||
|
||||
# noinspection DuplicatedCode
|
||||
@router.post("/v1/delete")
|
||||
def delete_domain(request: DeleteDomainRequest):
|
||||
"""删除域名，支持批量删除，传入多个 id"""
|
||||
|
||||
# 检查待删除的域名是否存在
|
||||
result = DomainService.get_by_ids(request.domain_ids)
|
||||
if not result.success:
|
||||
return result
|
||||
|
||||
existed_domain_ids = [item.id for item in result.data]
|
||||
for domain_id in request.domain_ids:
|
||||
if domain_id not in existed_domain_ids:
|
||||
return ApiResult.error(ApiCode.PARAM_ERROR.value, f"域名 ID {domain_id} 不存在")
|
||||
|
||||
# 删除域名
|
||||
return DomainService.delete_domains(request.domain_ids, request.remove_surl)
|
||||
|
||||
|
||||
@router.post("/v1/crawl")
|
||||
def crawl_now(request: CrawlNowRequest):
|
||||
"""立即爬取，实际上是把 status 置为 2"""
|
||||
result = DomainService.update_domain_status(request.domain_ids, DomainStatus.QUEUEING.value)
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/v1/toggle")
|
||||
def toggle_domain(request: ToggleDomainRequest):
|
||||
"""暂停爬取某个域名"""
|
||||
return DomainService.update_domain_status(request.domain_ids, DomainStatus.PAUSE.value)
|
||||
@ -1,85 +0,0 @@
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
from loguru import logger
|
||||
|
||||
from app.web.request.report_request import AddUrlsRequest, CollectEvidenceRequest, ReportRequest, GetUrlListRequest
|
||||
from app.web.service.domain_service import DomainService
|
||||
from app.web.service.report_service import ReportURLService
|
||||
|
||||
router = APIRouter(prefix="/api/urls", tags=["URL管理"])
|
||||
|
||||
|
||||
@router.get("/v1/list")
|
||||
def get_all_urls(request: Annotated[GetUrlListRequest, Query()]):
|
||||
"""获取所有的URL，支持根据域名、状态进行过滤，不传则返回全部数据，支持分页"""
|
||||
|
||||
logger.debug(f"{request=}")
|
||||
|
||||
return ReportURLService.get_list(
|
||||
request.domain,
|
||||
request.surl,
|
||||
request.is_report_by_one,
|
||||
request.is_report_by_site,
|
||||
request.is_report_by_wap,
|
||||
request.has_evidence,
|
||||
request.page,
|
||||
request.size
|
||||
)
|
||||
|
||||
|
||||
@router.post("/v1/add")
|
||||
def add_urls(request: AddUrlsRequest):
|
||||
"""
|
||||
手动添加 URL 到域名中，支持批量添加
|
||||
格式 [
|
||||
{"domain": "", "surl": ""}, {"domain": "", "surl": ""} ...
|
||||
]
|
||||
添加之前先检查 domain 有没有，没有的话就去创建一个 domain
|
||||
"""
|
||||
# 把所有的域名列表解出来，看看有没有不存在的，如果有就新建一个域名
|
||||
# 这里还需要获取域名的 id
|
||||
input_domains = [item.domain for item in request.urls]
|
||||
result = DomainService.get_by_domains(input_domains)
|
||||
if not result.success:
|
||||
return result
|
||||
|
||||
# 创建新域名
|
||||
new_domains = [x for x in input_domains if x not in result.data]
|
||||
if new_domains:
|
||||
result = DomainService.add_domains(1440, True, new_domains)
|
||||
if not result.success:
|
||||
return result
|
||||
|
||||
# 再获取一遍域名模型
|
||||
result = DomainService.get_by_domains(input_domains)
|
||||
if not result.success:
|
||||
return result
|
||||
|
||||
# 创建 URL
|
||||
domain_map: dict[str, int] = {x.domain: x.id for x in result.data}
|
||||
return ReportURLService.add_urls(domain_map, request.urls)
|
||||
|
||||
|
||||
@router.post("/v1/evidence")
|
||||
def collect_evidence(request: CollectEvidenceRequest):
|
||||
"""
|
||||
强制手动触发证据收集任务，支持批量传入，已经收集过的 URL 也要强制收集
|
||||
TODO:本来应该需要使用任务队列的，为了简单先把数据库的相关标记改为 0 ，也能达到一样的效果
|
||||
又不是不能用 XD
|
||||
"""
|
||||
return ReportURLService.batch_update_evidence_flag(request.ids)
|
||||
|
||||
|
||||
@router.post("/v1/report")
|
||||
def report(request: ReportRequest):
|
||||
"""举报指定的URL，支持批量传入 id 批量举报
|
||||
先通过改数据库，然后等引擎自己调度实现
|
||||
"""
|
||||
logger.debug(f"{request=}")
|
||||
return ReportURLService.batch_update_report_flag(
|
||||
request.ids,
|
||||
request.report_by_one,
|
||||
request.report_by_site,
|
||||
request.report_by_wap
|
||||
)
|
||||
@ -1,10 +0,0 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
|
||||
router = APIRouter(tags=["健康检查"])
|
||||
|
||||
@router.get("/status")
|
||||
async def status():
|
||||
return {
|
||||
"status": "ok"
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class GetDomainListRequest(BaseModel):
|
||||
"""获取域名列表"""
|
||||
|
||||
# 分页参数
|
||||
page: int = Field(default=1, gt=0)
|
||||
size: int = Field(default=50, gt=0)
|
||||
|
||||
# 过滤条件
|
||||
domain: str = ""
|
||||
status: int = 0
|
||||
|
||||
|
||||
class AddDomainRequest(BaseModel):
|
||||
"""添加域名到数据库的请求参数"""
|
||||
crawl_interval: int
|
||||
crawl_now: bool = True
|
||||
domains: list[str]
|
||||
|
||||
|
||||
class ImportDomainFormRequest(BaseModel):
|
||||
"""通过文件导入的"""
|
||||
crawl_interval: int
|
||||
crawl_now: bool = True
|
||||
|
||||
|
||||
class DeleteDomainRequest(BaseModel):
|
||||
"""删除域名的请求"""
|
||||
domain_ids: list[int]
|
||||
remove_surl: bool = False
|
||||
|
||||
|
||||
class UpdateDomainRequest(BaseModel):
|
||||
"""更新域名的请求"""
|
||||
domain_ids: list[int]
|
||||
crawl_interval: int
|
||||
|
||||
|
||||
class CrawlNowRequest(BaseModel):
|
||||
"""立即爬取的请求"""
|
||||
domain_ids: list[int]
|
||||
|
||||
|
||||
class ToggleDomainRequest(BaseModel):
|
||||
"""暂停某个域名的爬取"""
|
||||
domain_ids: list[int]
|
||||
@ -1,38 +0,0 @@
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class GetUrlListRequest(BaseModel):
|
||||
domain: str = ""
|
||||
surl: str = ""
|
||||
is_report_by_one: Optional[int] = 2
|
||||
is_report_by_site: Optional[int] = 2
|
||||
is_report_by_wap: Optional[int] = 2
|
||||
has_evidence: Optional[int] = 2
|
||||
|
||||
page: int = Field(default=1, gt=0)
|
||||
size: int = Field(default=50, gt=0)
|
||||
|
||||
|
||||
class AddUrlItem(BaseModel):
|
||||
domain: str
|
||||
surl: str
|
||||
|
||||
|
||||
class AddUrlsRequest(BaseModel):
|
||||
"""手动添加URL的请求体"""
|
||||
urls: list[AddUrlItem]
|
||||
|
||||
|
||||
class CollectEvidenceRequest(BaseModel):
|
||||
"""手动触发证据收集的请求体"""
|
||||
ids: list[int]
|
||||
|
||||
|
||||
class ReportRequest(BaseModel):
|
||||
"""手动触发证据收集的请求体"""
|
||||
ids: list[int]
|
||||
report_by_one: bool
|
||||
report_by_site: bool
|
||||
report_by_wap: bool
|
||||
@ -1,24 +0,0 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Generic
|
||||
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from app.constants.api_result import ApiCode
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ApiResult(Generic[T]):
|
||||
code: int
|
||||
message: str
|
||||
success: bool
|
||||
data: T | None = None
|
||||
|
||||
@staticmethod
|
||||
def ok(data: T | None = None) -> 'ApiResult[T]':
|
||||
return ApiResult(code=ApiCode.OK.value, message="ok", success=True, data=data)
|
||||
|
||||
@staticmethod
|
||||
def error(code: int, message: str) -> 'ApiResult[None]':
|
||||
return ApiResult(code=code, message=message, success=False, data=None)
|
||||
@ -1,141 +0,0 @@
|
||||
import time
|
||||
from typing import Iterable, Optional
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import delete, func, update
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from app.config.config import AppCtx
|
||||
from app.constants.api_result import ApiCode
|
||||
from app.constants.domain import DomainStatus
|
||||
from app.models.domain import DomainModel
|
||||
from app.models.report_urls import ReportUrlModel
|
||||
from app.web.results import ApiResult
|
||||
|
||||
|
||||
class DomainService:
|
||||
|
||||
@classmethod
|
||||
def get_list(cls, page: int, page_size: int, domain: str, status: int):
|
||||
"""获取域名列表"""
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
stmt = select(DomainModel)
|
||||
stmt_total = select(func.count(DomainModel.id))
|
||||
if domain:
|
||||
stmt = stmt.where(DomainModel.domain.like(f"%{domain}%"))
|
||||
stmt_total = stmt_total.where(DomainModel.domain.like(f"%{domain}%"))
|
||||
if status:
|
||||
stmt = stmt.where(DomainModel.status == status)
|
||||
stmt_total = stmt_total.where(DomainModel.status == status)
|
||||
|
||||
# 设置分页
|
||||
stmt = stmt.offset((page - 1) * page_size).limit(page_size)
|
||||
|
||||
try:
|
||||
# 域名列表
|
||||
rows = session.exec(stmt).all()
|
||||
|
||||
# 查询符合筛选条件的总数量
|
||||
total = session.exec(stmt_total).first()
|
||||
logger.debug(f"{total=}")
|
||||
|
||||
return ApiResult.ok({"total": total, "rows": rows})
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
logger.exception(f"查询域名列表失败，错误:{e}")
|
||||
return ApiResult.error(ApiCode.DB_ERROR.value, f"查询域名列表失败，错误:{e}")
|
||||
|
||||
@classmethod
|
||||
def get_by_domains(cls, domains: list[str]) -> ApiResult[Optional[DomainModel]]:
|
||||
"""根据域名查询"""
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
stmt = select(DomainModel).where(DomainModel.domain.in_(domains))
|
||||
try:
|
||||
rows = session.exec(stmt).all()
|
||||
return ApiResult.ok(rows)
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
return ApiResult.error(ApiCode.DB_ERROR.value, f"查询域名失败，错误:{e}")
|
||||
|
||||
@classmethod
|
||||
def get_by_ids(cls, domain_ids: list[int]) -> ApiResult[Optional[DomainModel]]:
|
||||
"""根据id查询"""
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
stmt = select(DomainModel).where(DomainModel.id.in_(domain_ids))
|
||||
try:
|
||||
rows = session.exec(stmt).all()
|
||||
return ApiResult.ok(rows)
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
return ApiResult.error(ApiCode.DB_ERROR.value, f"查询域名失败，错误:{e}")
|
||||
|
||||
@classmethod
|
||||
def add_domains(cls, interval: int, crawl_now: bool, domains: Iterable[str]):
|
||||
"""批量添加域名"""
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
new_domains = [
|
||||
DomainModel(
|
||||
domain=x,
|
||||
status=DomainStatus.READY.value,
|
||||
crawl_interval=interval,
|
||||
latest_crawl_time=0 if crawl_now else int(time.time())
|
||||
) for x in domains
|
||||
]
|
||||
|
||||
session.add_all(new_domains)
|
||||
try:
|
||||
session.commit()
|
||||
return ApiResult.ok(len(new_domains))
|
||||
except Exception as e:
|
||||
logger.error(f"添加域名到数据库失败，错误：{e}")
|
||||
session.rollback()
|
||||
return ApiResult.error(ApiCode.DB_ERROR.value, f"添加域名失败，错误:{e}")
|
||||
|
||||
@classmethod
|
||||
def delete_domains(cls, domain_ids: list[int], remove_surl: bool = False):
|
||||
"""批量删除域名，remove_surl 表示是否同时删除 report_url 中该域名相关的数据"""
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
stmt = delete(DomainModel).where(DomainModel.id.in_(domain_ids))
|
||||
try:
|
||||
session.exec(stmt)
|
||||
|
||||
# 如果设置了 remove_surl 为 True，则删除 report_url 中该域名相关的数据
|
||||
if remove_surl:
|
||||
stmt = delete(ReportUrlModel).where(ReportUrlModel.domain_id.in_(domain_ids))
|
||||
session.exec(stmt)
|
||||
|
||||
session.commit()
|
||||
|
||||
return ApiResult.ok(len(domain_ids))
|
||||
except Exception as e:
|
||||
logger.error(f"删除域名失败，错误：{e}")
|
||||
session.rollback()
|
||||
return ApiResult.error(ApiCode.DB_ERROR.value, f"删除域名失败，错误:{e}")
|
||||
|
||||
@classmethod
|
||||
def update_domain_interval(cls, domain_ids: list[int], interval: int) -> ApiResult[Optional[int]]:
|
||||
"""批量更新域名的 interval 值"""
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
stmt = update(DomainModel).where(DomainModel.id.in_(domain_ids)).values(crawl_interval=interval)
|
||||
try:
|
||||
session.exec(stmt)
|
||||
session.commit()
|
||||
return ApiResult.ok(len(domain_ids))
|
||||
except Exception as e:
|
||||
logger.error(f"更新域名 interval 失败，错误：{e}")
|
||||
session.rollback()
|
||||
return ApiResult.error(ApiCode.DB_ERROR.value, f"更新域名 interval 失败，错误:{e}")
|
||||
|
||||
@classmethod
|
||||
def update_domain_status(cls, domain_ids: list[int], status: int) -> ApiResult[Optional[int]]:
|
||||
"""批量更新域名的 status 值"""
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
stmt = update(DomainModel).where(DomainModel.id.in_(domain_ids)).values(status=status)
|
||||
try:
|
||||
session.exec(stmt)
|
||||
session.commit()
|
||||
return ApiResult.ok(len(domain_ids))
|
||||
except Exception as e:
|
||||
logger.error(f"更新域名 status 失败，错误：{e}")
|
||||
session.rollback()
|
||||
return ApiResult.error(ApiCode.DB_ERROR.value, f"更新域名 status 失败，错误:{e}")
|
||||
@ -1,119 +0,0 @@
|
||||
from typing import Optional
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import update, func
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from app.config.config import AppCtx
|
||||
from app.constants.api_result import ApiCode
|
||||
from app.models.report_urls import ReportUrlModel
|
||||
from app.web.request.report_request import AddUrlItem
|
||||
from app.web.results import ApiResult
|
||||
|
||||
|
||||
class ReportURLService:
|
||||
|
||||
@classmethod
|
||||
def get_list(
|
||||
cls, domain: str, surl: str, is_report_by_one: Optional[int], is_report_by_site: Optional[int],
|
||||
is_report_by_wap: Optional[int], has_evidence: Optional[int], page: int, size: int
|
||||
):
|
||||
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
stmt = select(ReportUrlModel)
|
||||
total_stmt = select(func.count(ReportUrlModel.id))
|
||||
if domain:
|
||||
stmt = stmt.where(ReportUrlModel.domain.like(f"%{domain}%"))
|
||||
total_stmt = total_stmt.where(ReportUrlModel.domain.like(f"%{domain}%"))
|
||||
if surl:
|
||||
stmt = stmt.where(ReportUrlModel.surl.like(f"%{surl}%"))
|
||||
total_stmt = total_stmt.where(ReportUrlModel.surl.like(f"%{surl}%"))
|
||||
if is_report_by_one and is_report_by_one != 2:
|
||||
stmt = stmt.where(ReportUrlModel.is_report_by_one == is_report_by_one)
|
||||
total_stmt = total_stmt.where(ReportUrlModel.is_report_by_one == is_report_by_one)
|
||||
if is_report_by_site and is_report_by_site != 2:
|
||||
stmt = stmt.where(ReportUrlModel.is_report_by_site == is_report_by_site)
|
||||
total_stmt = total_stmt.where(ReportUrlModel.is_report_by_site == is_report_by_site)
|
||||
if is_report_by_wap and is_report_by_wap != 2:
|
||||
stmt = stmt.where(ReportUrlModel.is_report_by_wap == is_report_by_wap)
|
||||
total_stmt = total_stmt.where(ReportUrlModel.is_report_by_wap == is_report_by_wap)
|
||||
if has_evidence and has_evidence != 2:
|
||||
stmt = stmt.where(ReportUrlModel.has_evidence == has_evidence)
|
||||
total_stmt = total_stmt.where(ReportUrlModel.has_evidence == has_evidence)
|
||||
|
||||
# 设置分页
|
||||
stmt = stmt.offset((page - 1) * size).limit(size)
|
||||
# logger.debug(f"{str(stmt)=}")
|
||||
try:
|
||||
total = session.exec(total_stmt).first()
|
||||
urls = session.exec(stmt).all()
|
||||
return ApiResult.ok({
|
||||
"total": total,
|
||||
"data": urls,
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"获取URL列表失败: {e}")
|
||||
return ApiResult.error(ApiCode.DB_ERROR.value, str(e))
|
||||
|
||||
@classmethod
|
||||
def add_urls(cls, domain_map: dict[str, int], urls: list[AddUrlItem]) -> ApiResult[Optional[int]]:
|
||||
"""添加URL"""
|
||||
if not urls:
|
||||
return ApiResult.ok(0)
|
||||
|
||||
models = []
|
||||
|
||||
for url in urls:
|
||||
domain_id = domain_map.get(url.domain, None)
|
||||
if not domain_id:
|
||||
return ApiResult.error(ApiCode.PARAM_ERROR.value, f"域名 {url.domain} 不存在")
|
||||
models.append(ReportUrlModel(
|
||||
domain_id=domain_id,
|
||||
domain=url.domain,
|
||||
surl=url.surl,
|
||||
))
|
||||
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
try:
|
||||
session.add_all(models)
|
||||
session.commit()
|
||||
return ApiResult.ok(len(models))
|
||||
except Exception as e:
|
||||
logger.error(f"添加URL失败: {e}")
|
||||
session.rollback()
|
||||
return ApiResult.error(ApiCode.DB_ERROR.value, str(e))
|
||||
|
||||
@classmethod
|
||||
def batch_update_evidence_flag(cls, url_ids: list[int]):
|
||||
"""批量更新URL的has_evidence字段"""
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
try:
|
||||
stmt = update(ReportUrlModel).where(ReportUrlModel.id.in_(url_ids)).values(has_evidence=False)
|
||||
session.exec(stmt)
|
||||
session.commit()
|
||||
return ApiResult.ok(len(url_ids))
|
||||
except Exception as e:
|
||||
logger.error(f"批量更新URL的has_evidence字段失败: {e}")
|
||||
session.rollback()
|
||||
return ApiResult.error(ApiCode.DB_ERROR.value, str(e))
|
||||
|
||||
@classmethod
|
||||
def batch_update_report_flag(cls, ids: list[int], report_by_one: bool, report_by_site: bool, report_by_wap: bool):
|
||||
with Session(AppCtx.g_db_engine) as session:
|
||||
try:
|
||||
stmt = update(ReportUrlModel).where(ReportUrlModel.id.in_(ids))
|
||||
if report_by_wap:
|
||||
stmt = stmt.values(is_report_by_wap=False)
|
||||
if report_by_site:
|
||||
stmt = stmt.values(is_report_by_site=False)
|
||||
if report_by_one:
|
||||
stmt = stmt.values(is_report_by_one=False)
|
||||
|
||||
logger.debug(f"{str(stmt)=}")
|
||||
session.exec(stmt)
|
||||
session.commit()
|
||||
return ApiResult.ok(len(ids))
|
||||
except Exception as e:
|
||||
logger.error(f"批量更新URL的has_evidence字段失败: {e}")
|
||||
session.rollback()
|
||||
return ApiResult.error(ApiCode.DB_ERROR.value, str(e))
|
||||
@ -1,43 +0,0 @@
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.responses import FileResponse, Response
|
||||
from loguru import logger
|
||||
from starlette.staticfiles import StaticFiles
|
||||
from starlette.types import Scope
|
||||
|
||||
from .controller.domain import router as domain_router
|
||||
from .controller.report import router as report_router
|
||||
from .controller.status import router as status_router
|
||||
|
||||
|
||||
class SPAStaticFiles(StaticFiles):
|
||||
async def get_response(self, path: str, scope: Scope) -> Response:
|
||||
# 如果是前端路由，直接返回 index.html，否则直接访问的时候会404
|
||||
if path in ("domain", "url"):
|
||||
return FileResponse(os.path.join(self.directory, "index.html"))
|
||||
return await super().get_response(path, scope)
|
||||
|
||||
|
||||
class WebApp:
|
||||
|
||||
def __init__(self):
|
||||
self.app = FastAPI()
|
||||
|
||||
@staticmethod
|
||||
async def start():
|
||||
app = FastAPI()
|
||||
|
||||
# 导入路由
|
||||
app.include_router(status_router)
|
||||
app.include_router(report_router)
|
||||
app.include_router(domain_router)
|
||||
|
||||
# 挂载前端文件
|
||||
app.mount("/", SPAStaticFiles(directory="fe/dist", html=True), name="static")
|
||||
|
||||
# TODO 先写死，后面从配置文件里取
|
||||
cfg = uvicorn.Config(app, host="127.0.0.1", port=3000)
|
||||
server = uvicorn.Server(cfg)
|
||||
await server.serve()
|
||||
@ -15,5 +15,5 @@ database = "baidu_reporter"
|
||||
|
||||
# chrome 配置
|
||||
[chrome]
|
||||
proxy = "http://127.0.0.1:7890"
|
||||
proxy = "http://127.0.0.1:8080"
|
||||
browser_path = "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe"
|
||||
@ -1,9 +0,0 @@
|
||||
[*.{js,jsx,mjs,cjs,ts,tsx,mts,cts,vue,css,scss,sass,less,styl}]
|
||||
charset = utf-8
|
||||
indent_size = 2
|
||||
indent_style = space
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
|
||||
end_of_line = lf
|
||||
max_line_length = 100
|
||||
1
fe/.gitattributes
vendored
1
fe/.gitattributes
vendored
@ -1 +0,0 @@
|
||||
* text=auto eol=lf
|
||||
30
fe/.gitignore
vendored
30
fe/.gitignore
vendored
@ -1,30 +0,0 @@
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
lerna-debug.log*
|
||||
|
||||
node_modules
|
||||
.DS_Store
|
||||
|
||||
dist-ssr
|
||||
coverage
|
||||
*.local
|
||||
|
||||
/cypress/videos/
|
||||
/cypress/screenshots/
|
||||
|
||||
# Editor directories and files
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea
|
||||
*.suo
|
||||
*.ntvs*
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.sw?
|
||||
|
||||
*.tsbuildinfo
|
||||
@ -1,6 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/prettierrc",
|
||||
"semi": false,
|
||||
"singleQuote": true,
|
||||
"printWidth": 100
|
||||
}
|
||||
8
fe/.vscode/extensions.json
vendored
8
fe/.vscode/extensions.json
vendored
@ -1,8 +0,0 @@
|
||||
{
|
||||
"recommendations": [
|
||||
"Vue.volar",
|
||||
"dbaeumer.vscode-eslint",
|
||||
"EditorConfig.EditorConfig",
|
||||
"esbenp.prettier-vscode"
|
||||
]
|
||||
}
|
||||
39
fe/README.md
39
fe/README.md
@ -1,39 +0,0 @@
|
||||
# fe
|
||||
|
||||
This template should help get you started developing with Vue 3 in Vite.
|
||||
|
||||
## Recommended IDE Setup
|
||||
|
||||
[VSCode](https://code.visualstudio.com/) + [Volar](https://marketplace.visualstudio.com/items?itemName=Vue.volar) (and disable Vetur).
|
||||
|
||||
## Type Support for `.vue` Imports in TS
|
||||
|
||||
TypeScript cannot handle type information for `.vue` imports by default, so we replace the `tsc` CLI with `vue-tsc` for type checking. In editors, we need [Volar](https://marketplace.visualstudio.com/items?itemName=Vue.volar) to make the TypeScript language service aware of `.vue` types.
|
||||
|
||||
## Customize configuration
|
||||
|
||||
See [Vite Configuration Reference](https://vite.dev/config/).
|
||||
|
||||
## Project Setup
|
||||
|
||||
```sh
|
||||
pnpm install
|
||||
```
|
||||
|
||||
### Compile and Hot-Reload for Development
|
||||
|
||||
```sh
|
||||
pnpm dev
|
||||
```
|
||||
|
||||
### Type-Check, Compile and Minify for Production
|
||||
|
||||
```sh
|
||||
pnpm build
|
||||
```
|
||||
|
||||
### Lint with [ESLint](https://eslint.org/)
|
||||
|
||||
```sh
|
||||
pnpm lint
|
||||
```
|
||||
75
fe/auto-imports.d.ts
vendored
75
fe/auto-imports.d.ts
vendored
@ -1,75 +0,0 @@
|
||||
/* eslint-disable */
|
||||
/* prettier-ignore */
|
||||
// @ts-nocheck
|
||||
// noinspection JSUnusedGlobalSymbols
|
||||
// Generated by unplugin-auto-import
|
||||
// biome-ignore lint: disable
|
||||
export {}
|
||||
declare global {
|
||||
const EffectScope: typeof import('vue')['EffectScope']
|
||||
const computed: typeof import('vue')['computed']
|
||||
const createApp: typeof import('vue')['createApp']
|
||||
const customRef: typeof import('vue')['customRef']
|
||||
const defineAsyncComponent: typeof import('vue')['defineAsyncComponent']
|
||||
const defineComponent: typeof import('vue')['defineComponent']
|
||||
const effectScope: typeof import('vue')['effectScope']
|
||||
const getCurrentInstance: typeof import('vue')['getCurrentInstance']
|
||||
const getCurrentScope: typeof import('vue')['getCurrentScope']
|
||||
const h: typeof import('vue')['h']
|
||||
const inject: typeof import('vue')['inject']
|
||||
const isProxy: typeof import('vue')['isProxy']
|
||||
const isReactive: typeof import('vue')['isReactive']
|
||||
const isReadonly: typeof import('vue')['isReadonly']
|
||||
const isRef: typeof import('vue')['isRef']
|
||||
const markRaw: typeof import('vue')['markRaw']
|
||||
const nextTick: typeof import('vue')['nextTick']
|
||||
const onActivated: typeof import('vue')['onActivated']
|
||||
const onBeforeMount: typeof import('vue')['onBeforeMount']
|
||||
const onBeforeUnmount: typeof import('vue')['onBeforeUnmount']
|
||||
const onBeforeUpdate: typeof import('vue')['onBeforeUpdate']
|
||||
const onDeactivated: typeof import('vue')['onDeactivated']
|
||||
const onErrorCaptured: typeof import('vue')['onErrorCaptured']
|
||||
const onMounted: typeof import('vue')['onMounted']
|
||||
const onRenderTracked: typeof import('vue')['onRenderTracked']
|
||||
const onRenderTriggered: typeof import('vue')['onRenderTriggered']
|
||||
const onScopeDispose: typeof import('vue')['onScopeDispose']
|
||||
const onServerPrefetch: typeof import('vue')['onServerPrefetch']
|
||||
const onUnmounted: typeof import('vue')['onUnmounted']
|
||||
const onUpdated: typeof import('vue')['onUpdated']
|
||||
const onWatcherCleanup: typeof import('vue')['onWatcherCleanup']
|
||||
const provide: typeof import('vue')['provide']
|
||||
const reactive: typeof import('vue')['reactive']
|
||||
const readonly: typeof import('vue')['readonly']
|
||||
const ref: typeof import('vue')['ref']
|
||||
const resolveComponent: typeof import('vue')['resolveComponent']
|
||||
const shallowReactive: typeof import('vue')['shallowReactive']
|
||||
const shallowReadonly: typeof import('vue')['shallowReadonly']
|
||||
const shallowRef: typeof import('vue')['shallowRef']
|
||||
const toRaw: typeof import('vue')['toRaw']
|
||||
const toRef: typeof import('vue')['toRef']
|
||||
const toRefs: typeof import('vue')['toRefs']
|
||||
const toValue: typeof import('vue')['toValue']
|
||||
const triggerRef: typeof import('vue')['triggerRef']
|
||||
const unref: typeof import('vue')['unref']
|
||||
const useAttrs: typeof import('vue')['useAttrs']
|
||||
const useCssModule: typeof import('vue')['useCssModule']
|
||||
const useCssVars: typeof import('vue')['useCssVars']
|
||||
const useDialog: typeof import('naive-ui')['useDialog']
|
||||
const useId: typeof import('vue')['useId']
|
||||
const useLoadingBar: typeof import('naive-ui')['useLoadingBar']
|
||||
const useMessage: typeof import('naive-ui')['useMessage']
|
||||
const useModel: typeof import('vue')['useModel']
|
||||
const useNotification: typeof import('naive-ui')['useNotification']
|
||||
const useSlots: typeof import('vue')['useSlots']
|
||||
const useTemplateRef: typeof import('vue')['useTemplateRef']
|
||||
const watch: typeof import('vue')['watch']
|
||||
const watchEffect: typeof import('vue')['watchEffect']
|
||||
const watchPostEffect: typeof import('vue')['watchPostEffect']
|
||||
const watchSyncEffect: typeof import('vue')['watchSyncEffect']
|
||||
}
|
||||
// for type re-export
|
||||
declare global {
|
||||
// @ts-ignore
|
||||
export type { Component, ComponentPublicInstance, ComputedRef, DirectiveBinding, ExtractDefaultPropTypes, ExtractPropTypes, ExtractPublicPropTypes, InjectionKey, PropType, Ref, MaybeRef, MaybeRefOrGetter, VNode, WritableComputedRef } from 'vue'
|
||||
import('vue')
|
||||
}
|
||||
35
fe/components.d.ts
vendored
35
fe/components.d.ts
vendored
@ -1,35 +0,0 @@
|
||||
/* eslint-disable */
|
||||
// @ts-nocheck
|
||||
// Generated by unplugin-vue-components
|
||||
// Read more: https://github.com/vuejs/core/pull/3399
|
||||
// biome-ignore lint: disable
|
||||
export {}
|
||||
|
||||
/* prettier-ignore */
|
||||
declare module 'vue' {
|
||||
export interface GlobalComponents {
|
||||
AddDomainDialog: typeof import('./src/components/AddDomainDialog.vue')['default']
|
||||
EditDomainDialog: typeof import('./src/components/EditDomainDialog.vue')['default']
|
||||
ImportDomainDialog: typeof import('./src/components/ImportDomainDialog.vue')['default']
|
||||
NButton: typeof import('naive-ui')['NButton']
|
||||
NCard: typeof import('naive-ui')['NCard']
|
||||
NCheckbox: typeof import('naive-ui')['NCheckbox']
|
||||
NDataTable: typeof import('naive-ui')['NDataTable']
|
||||
NDialogProvider: typeof import('naive-ui')['NDialogProvider']
|
||||
NDropdown: typeof import('naive-ui')['NDropdown']
|
||||
NForm: typeof import('naive-ui')['NForm']
|
||||
NFormItem: typeof import('naive-ui')['NFormItem']
|
||||
NFormItemGi: typeof import('naive-ui')['NFormItemGi']
|
||||
NGrid: typeof import('naive-ui')['NGrid']
|
||||
NInput: typeof import('naive-ui')['NInput']
|
||||
NInputNumber: typeof import('naive-ui')['NInputNumber']
|
||||
NModal: typeof import('naive-ui')['NModal']
|
||||
NPagination: typeof import('naive-ui')['NPagination']
|
||||
NSelect: typeof import('naive-ui')['NSelect']
|
||||
NSpace: typeof import('naive-ui')['NSpace']
|
||||
NTag: typeof import('naive-ui')['NTag']
|
||||
NTooltip: typeof import('naive-ui')['NTooltip']
|
||||
RouterLink: typeof import('vue-router')['RouterLink']
|
||||
RouterView: typeof import('vue-router')['RouterView']
|
||||
}
|
||||
}
|
||||
483
fe/dist/assets/DomainManager-2SUOMVR8.js
vendored
483
fe/dist/assets/DomainManager-2SUOMVR8.js
vendored
File diff suppressed because one or more lines are too long
1525
fe/dist/assets/FormItem-DHmVxm6n.js
vendored
1525
fe/dist/assets/FormItem-DHmVxm6n.js
vendored
File diff suppressed because one or more lines are too long
BIN
fe/dist/assets/IBMPlexMono-Regular-CAJ2AE84.ttf
vendored
BIN
fe/dist/assets/IBMPlexMono-Regular-CAJ2AE84.ttf
vendored
Binary file not shown.
BIN
fe/dist/assets/LatoLatin-Regular-Dmlz1U0B.woff2
vendored
BIN
fe/dist/assets/LatoLatin-Regular-Dmlz1U0B.woff2
vendored
Binary file not shown.
BIN
fe/dist/assets/LatoLatin-Semibold-Dbk81p2D.woff2
vendored
BIN
fe/dist/assets/LatoLatin-Semibold-Dbk81p2D.woff2
vendored
Binary file not shown.
1
fe/dist/assets/UrlManager-DG8i4_QJ.js
vendored
1
fe/dist/assets/UrlManager-DG8i4_QJ.js
vendored
File diff suppressed because one or more lines are too long
1
fe/dist/assets/index-Ca0u0JBQ.css
vendored
1
fe/dist/assets/index-Ca0u0JBQ.css
vendored
File diff suppressed because one or more lines are too long
1161
fe/dist/assets/index-DnpM0Ntg.js
vendored
1161
fe/dist/assets/index-DnpM0Ntg.js
vendored
File diff suppressed because one or more lines are too long
BIN
fe/dist/favicon.ico
vendored
BIN
fe/dist/favicon.ico
vendored
Binary file not shown.
|
Before Width: | Height: | Size: 4.2 KiB |
14
fe/dist/index.html
vendored
14
fe/dist/index.html
vendored
@ -1,14 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<link rel="icon" href="/favicon.ico">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Baidu Reporter</title>
|
||||
<script type="module" crossorigin src="/assets/index-DnpM0Ntg.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-Ca0u0JBQ.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
</body>
|
||||
</html>
|
||||
1
fe/env.d.ts
vendored
1
fe/env.d.ts
vendored
@ -1 +0,0 @@
|
||||
/// <reference types="vite/client" />
|
||||
@ -1,22 +0,0 @@
|
||||
import { globalIgnores } from 'eslint/config'
|
||||
import { defineConfigWithVueTs, vueTsConfigs } from '@vue/eslint-config-typescript'
|
||||
import pluginVue from 'eslint-plugin-vue'
|
||||
import skipFormatting from '@vue/eslint-config-prettier/skip-formatting'
|
||||
|
||||
// To allow more languages other than `ts` in `.vue` files, uncomment the following lines:
|
||||
// import { configureVueProject } from '@vue/eslint-config-typescript'
|
||||
// configureVueProject({ scriptLangs: ['ts', 'tsx'] })
|
||||
// More info at https://github.com/vuejs/eslint-config-typescript/#advanced-setup
|
||||
|
||||
export default defineConfigWithVueTs(
|
||||
{
|
||||
name: 'app/files-to-lint',
|
||||
files: ['**/*.{ts,mts,tsx,vue}'],
|
||||
},
|
||||
|
||||
globalIgnores(['**/dist/**', '**/dist-ssr/**', '**/coverage/**']),
|
||||
|
||||
pluginVue.configs['flat/essential'],
|
||||
vueTsConfigs.recommended,
|
||||
skipFormatting,
|
||||
)
|
||||
@ -1,13 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<link rel="icon" href="/favicon.ico">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Vite App</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
<script type="module" src="/src/main.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@ -1,46 +0,0 @@
|
||||
{
|
||||
"name": "fe",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "run-p type-check \"build-only {@}\" --",
|
||||
"preview": "vite preview",
|
||||
"build-only": "vite build",
|
||||
"type-check": "vue-tsc --build",
|
||||
"lint": "eslint . --fix",
|
||||
"format": "prettier --write src/"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tailwindcss/vite": "^4.1.1",
|
||||
"axios": "^1.8.4",
|
||||
"pinia": "^3.0.1",
|
||||
"tailwindcss": "^4.1.1",
|
||||
"vue": "^3.5.13",
|
||||
"vue-router": "^4.5.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tsconfig/node22": "^22.0.1",
|
||||
"@types/node": "^22.13.14",
|
||||
"@vicons/ionicons5": "^0.13.0",
|
||||
"@vitejs/plugin-vue": "^5.2.3",
|
||||
"@vitejs/plugin-vue-jsx": "^4.1.2",
|
||||
"@vue/eslint-config-prettier": "^10.2.0",
|
||||
"@vue/eslint-config-typescript": "^14.5.0",
|
||||
"@vue/tsconfig": "^0.7.0",
|
||||
"eslint": "^9.22.0",
|
||||
"eslint-plugin-vue": "~10.0.0",
|
||||
"jiti": "^2.4.2",
|
||||
"naive-ui": "^2.41.0",
|
||||
"npm-run-all2": "^7.0.2",
|
||||
"prettier": "3.5.3",
|
||||
"typescript": "~5.8.0",
|
||||
"unplugin-auto-import": "^19.1.2",
|
||||
"unplugin-vue-components": "^28.4.1",
|
||||
"vfonts": "^0.0.3",
|
||||
"vite": "^6.2.4",
|
||||
"vite-plugin-vue-devtools": "^7.7.2",
|
||||
"vue-tsc": "^2.2.8"
|
||||
}
|
||||
}
|
||||
4097
fe/pnpm-lock.yaml
generated
4097
fe/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
Before Width: | Height: | Size: 4.2 KiB |
@ -1,72 +0,0 @@
|
||||
<script setup lang="tsx">
|
||||
import { RouterLink, RouterView } from 'vue-router'
|
||||
import {
|
||||
NLayout,
|
||||
NMessageProvider,
|
||||
NConfigProvider,
|
||||
NLayoutHeader,
|
||||
NLayoutSider,
|
||||
NMenu,
|
||||
} from 'naive-ui'
|
||||
import { type MenuOption } from 'naive-ui'
|
||||
import { List, ColorWand } from '@vicons/ionicons5'
|
||||
|
||||
// 点击菜单时触发的操作，TODO 需要高亮对应的菜单项
|
||||
const handleUpdateValue = (v: string) => {
|
||||
console.log('handleUpdateValue: v')
|
||||
}
|
||||
|
||||
// 菜单定义
|
||||
const menuOpts: MenuOption[] = [
|
||||
{
|
||||
label: () => <RouterLink to={{ name: 'domain-manager' }}>域名管理</RouterLink>,
|
||||
key: 'rule-manager',
|
||||
icon: () => <List />,
|
||||
},
|
||||
{
|
||||
label: () => <RouterLink to={{ name: 'url-manager' }}>URL 管理</RouterLink>,
|
||||
key: 'rule-sniff',
|
||||
icon: () => <ColorWand />,
|
||||
},
|
||||
]
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<n-config-provider>
|
||||
<n-dialog-provider>
|
||||
<n-message-provider>
|
||||
<n-layout class="h-screen">
|
||||
<!-- header -->
|
||||
<n-layout-header
|
||||
class="h-16 p-5"
|
||||
style="background-color: oklch(62.3% 0.214 259.815); color: white"
|
||||
bordered
|
||||
>
|
||||
<span class="font-bold text-xl">BAIDU Reporter</span>
|
||||
</n-layout-header>
|
||||
|
||||
<n-layout position="absolute" has-sider style="top: 64px">
|
||||
<!-- sidebar -->
|
||||
<n-layout-sider
|
||||
width="8%"
|
||||
show-trigger
|
||||
show-collapsed-content
|
||||
:collapsed-width="64"
|
||||
content-style="padding: 8px; text-align:center;"
|
||||
:native-scrollbar="false"
|
||||
bordered
|
||||
collapse-mode="width"
|
||||
>
|
||||
<n-menu :indent="24" :options="menuOpts" @update:value="handleUpdateValue" />
|
||||
</n-layout-sider>
|
||||
|
||||
<!-- content -->
|
||||
<n-layout content-style="padding: 16px;" :native-scrollbar="false">
|
||||
<router-view />
|
||||
</n-layout>
|
||||
</n-layout>
|
||||
</n-layout>
|
||||
</n-message-provider>
|
||||
</n-dialog-provider>
|
||||
</n-config-provider>
|
||||
</template>
|
||||
@ -1,141 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, inject } from 'vue'
|
||||
import {
|
||||
NModal,
|
||||
NForm,
|
||||
NFormItem,
|
||||
NInputNumber,
|
||||
NCheckbox,
|
||||
NInput,
|
||||
NButton,
|
||||
NButtonGroup,
|
||||
useMessage,
|
||||
} from 'naive-ui'
|
||||
import type { FormRules } from 'naive-ui'
|
||||
import type { AxiosInstance } from 'axios'
|
||||
|
||||
const model = defineModel<boolean>('show', { required: true })
|
||||
const emit = defineEmits(['success'])
|
||||
|
||||
const axios = inject('axios') as AxiosInstance
|
||||
const message = useMessage()
|
||||
|
||||
const interval = ref<number>(1440) // 默认1天（1440分钟）
|
||||
const startImmediately = ref(true)
|
||||
const domains = ref('')
|
||||
const formRef = ref<InstanceType<typeof NForm> | null>(null)
|
||||
|
||||
const rules: FormRules = {
|
||||
interval: [
|
||||
{
|
||||
required: true,
|
||||
type: 'number',
|
||||
message: '请输入采集间隔',
|
||||
trigger: ['blur', 'change'],
|
||||
},
|
||||
{
|
||||
type: 'number',
|
||||
min: 1,
|
||||
message: '采集间隔必须大于0',
|
||||
trigger: ['blur', 'change'],
|
||||
},
|
||||
],
|
||||
domains: [
|
||||
{
|
||||
required: true,
|
||||
message: '请输入域名',
|
||||
trigger: ['blur', 'change'],
|
||||
},
|
||||
{
|
||||
validator: (_, value: string) => {
|
||||
if (!value.trim()) return true // 空值由 required 规则处理
|
||||
const domainList = value
|
||||
.split(/[\n,]/)
|
||||
.map((d) => d.trim())
|
||||
.filter((d) => d)
|
||||
if (domainList.length === 0) return false
|
||||
return true
|
||||
},
|
||||
message: '域名格式不正确',
|
||||
trigger: ['blur', 'change'],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
const handleConfirm = async () => {
|
||||
try {
|
||||
await formRef.value?.validate()
|
||||
} catch (errors) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// 处理域名输入，支持换行和逗号分隔
|
||||
const domainList = domains.value
|
||||
.split(/[\n,]/) // 同时支持换行和逗号分隔
|
||||
.map((domain) => domain.trim()) // 去除每个域名的前后空格
|
||||
.filter((domain) => domain) // 过滤空字符串
|
||||
|
||||
const response = await axios.post('/api/domain/v1/add', {
|
||||
domains: domainList,
|
||||
crawl_interval: interval.value,
|
||||
crawl_now: startImmediately.value,
|
||||
})
|
||||
|
||||
if (response.data.code === 20000) {
|
||||
message.success('添加成功')
|
||||
emit('success')
|
||||
handleClose()
|
||||
} else {
|
||||
message.error(`添加失败：${response.data.message}`)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('添加失败', error)
|
||||
message.error(`添加失败：${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
const handleClose = () => {
|
||||
// 清空表单
|
||||
interval.value = 1440
|
||||
startImmediately.value = true
|
||||
domains.value = ''
|
||||
formRef.value?.restoreValidation()
|
||||
model.value = false
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<n-modal
|
||||
v-model:show="model"
|
||||
preset="card"
|
||||
title="手动添加"
|
||||
:mask-closable="false"
|
||||
style="width: 600px"
|
||||
>
|
||||
<n-form size="small" ref="formRef" :model="{ interval, domains }" :rules="rules">
|
||||
<n-form-item path="interval" label="采集间隔（分钟）">
|
||||
<n-input-number v-model:value="interval" :min="1" />
|
||||
</n-form-item>
|
||||
|
||||
<n-form-item path="domains" label="域名列表">
|
||||
<n-input
|
||||
v-model:value="domains"
|
||||
type="textarea"
|
||||
:rows="10"
|
||||
placeholder="请输入域名，支持换行或英文逗号分隔"
|
||||
/>
|
||||
</n-form-item>
|
||||
|
||||
<n-form-item label="采集选项">
|
||||
<n-checkbox v-model:checked="startImmediately"> 立即开始采集 </n-checkbox>
|
||||
</n-form-item>
|
||||
</n-form>
|
||||
<template #action>
|
||||
<n-button-group size="small">
|
||||
<n-button type="primary" @click="handleConfirm">确认</n-button>
|
||||
<n-button @click="handleClose">关闭</n-button>
|
||||
</n-button-group>
|
||||
</template>
|
||||
</n-modal>
|
||||
</template>
|
||||
@ -1,115 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, defineProps, defineEmits, inject, defineModel, computed, watch } from 'vue'
|
||||
import { useMessage } from 'naive-ui'
|
||||
import type { AxiosInstance } from 'axios'
|
||||
import type { DataTableRowKey } from 'naive-ui'
|
||||
|
||||
const show = defineModel<boolean>('show')
|
||||
const props = defineProps<{
|
||||
// 要修改的域名 ID 列表
|
||||
domainIds: DataTableRowKey[] | null
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
(e: 'success'): void
|
||||
(e: 'close'): void
|
||||
}>()
|
||||
|
||||
const axios = inject('axios') as AxiosInstance
|
||||
const message = useMessage()
|
||||
|
||||
// 采集间隔需要用户输入
|
||||
const crawlInterval = ref<number | null>(null)
|
||||
const loading = ref(false)
|
||||
|
||||
// 动态计算弹窗标题
|
||||
const dialogTitle = computed(() => {
|
||||
const count = props.domainIds?.length || 0
|
||||
if (count > 1) {
|
||||
return `批量修改 ${count} 个域名的采集间隔`
|
||||
} else if (count === 1) {
|
||||
return '修改域名采集间隔'
|
||||
}
|
||||
return '修改采集间隔' // 应该不会出现，但作为备用
|
||||
})
|
||||
|
||||
// 弹窗显示时，重置 crawlInterval
|
||||
watch(show, (newShow) => {
|
||||
if (newShow) {
|
||||
crawlInterval.value = null // 每次打开都清空，强制用户输入
|
||||
}
|
||||
})
|
||||
|
||||
const handleSubmit = async () => {
|
||||
if (crawlInterval.value === null || crawlInterval.value < 1) {
|
||||
message.error('请输入有效的采集间隔（大于等于1的整数）')
|
||||
return
|
||||
}
|
||||
|
||||
if (!props.domainIds || props.domainIds.length === 0) {
|
||||
message.error('没有指定要修改的域名')
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
loading.value = true
|
||||
|
||||
const response = (
|
||||
await axios.post('/api/domain/v1/update', {
|
||||
domain_ids: props.domainIds, // 直接使用传入的 ID
|
||||
crawl_interval: crawlInterval.value,
|
||||
})
|
||||
).data
|
||||
|
||||
if (response.code !== 20000) {
|
||||
message.error(`更新失败：${response.message}`)
|
||||
return
|
||||
}
|
||||
|
||||
message.success('更新成功')
|
||||
emit('success')
|
||||
show.value = false // 关闭弹窗
|
||||
} catch (error) {
|
||||
console.error('更新失败', error)
|
||||
message.error(`更新失败：${error}`)
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
const handleClose = () => {
|
||||
show.value = false
|
||||
emit('close') // 触发 close 事件
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<n-modal
|
||||
v-model:show="show"
|
||||
preset="dialog"
|
||||
:title="dialogTitle"
|
||||
:loading="loading"
|
||||
@close="handleClose"
|
||||
>
|
||||
<!-- 批量编辑提示 -->
|
||||
<div v-if="(domainIds?.length || 0) > 1" class="mb-4 text-orange-500">
|
||||
你正在批量修改 {{ domainIds?.length }} 个域名的采集间隔。
|
||||
</div>
|
||||
|
||||
<n-form>
|
||||
<n-form-item label="采集间隔（分钟）" required>
|
||||
<n-input-number
|
||||
v-model:value="crawlInterval"
|
||||
:min="1"
|
||||
:step="1"
|
||||
style="width: 100%"
|
||||
placeholder="请输入采集间隔"
|
||||
/>
|
||||
</n-form-item>
|
||||
</n-form>
|
||||
<template #action>
|
||||
<n-button @click="handleClose">取消</n-button>
|
||||
<n-button type="primary" @click="handleSubmit" :loading="loading">确定</n-button>
|
||||
</template>
|
||||
</n-modal>
|
||||
</template>
|
||||
@ -1,144 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, inject } from 'vue'
|
||||
import {
|
||||
NModal,
|
||||
NForm,
|
||||
NFormItem,
|
||||
NInputNumber,
|
||||
NCheckbox,
|
||||
NUpload,
|
||||
NButton,
|
||||
NButtonGroup,
|
||||
useMessage,
|
||||
} from 'naive-ui'
|
||||
import type { FormRules } from 'naive-ui'
|
||||
import type { AxiosInstance } from 'axios'
|
||||
import type { UploadFileInfo } from 'naive-ui'
|
||||
|
||||
const model = defineModel<boolean>('show', { required: true })
|
||||
const emit = defineEmits(['success'])
|
||||
|
||||
const axios = inject('axios') as AxiosInstance
|
||||
const message = useMessage()
|
||||
|
||||
const interval = ref(1440) // 默认1天（1440分钟）
|
||||
const startImmediately = ref(true)
|
||||
const fileList = ref<UploadFileInfo[]>([])
|
||||
const formRef = ref<InstanceType<typeof NForm> | null>(null)
|
||||
|
||||
const rules: FormRules = {
|
||||
interval: [
|
||||
{
|
||||
type: 'number',
|
||||
required: true,
|
||||
message: '请输入采集间隔',
|
||||
trigger: ['blur', 'change'],
|
||||
},
|
||||
{
|
||||
type: 'number',
|
||||
min: 1,
|
||||
message: '采集间隔必须大于0',
|
||||
trigger: ['blur', 'change'],
|
||||
},
|
||||
],
|
||||
fileList: [
|
||||
{
|
||||
type: 'array',
|
||||
required: true,
|
||||
message: '请选择文件',
|
||||
trigger: ['change'],
|
||||
},
|
||||
{
|
||||
validator: (_, value: UploadFileInfo[]) => {
|
||||
if (!value || value.length === 0) return false
|
||||
return !!value[0].file
|
||||
},
|
||||
message: '文件无效',
|
||||
trigger: ['change'],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
const handleConfirm = async () => {
|
||||
try {
|
||||
await formRef.value?.validate()
|
||||
} catch (errors) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const formData = new FormData()
|
||||
const file = fileList.value[0].file
|
||||
if (!file) {
|
||||
message.error('文件无效')
|
||||
return
|
||||
}
|
||||
formData.append('file', file)
|
||||
formData.append('crawl_interval', interval.value.toString())
|
||||
formData.append('crawl_now', startImmediately.value.toString())
|
||||
|
||||
const response = await axios.post('/api/domain/v1/import', formData, {
|
||||
headers: {
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
})
|
||||
|
||||
if (response.data.code === 20000) {
|
||||
message.success('导入成功')
|
||||
emit('success')
|
||||
handleClose()
|
||||
} else {
|
||||
message.error(`导入失败：${response.data.message}`)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('导入失败', error)
|
||||
message.error(`导入失败：${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
const handleClose = () => {
|
||||
// 清空表单
|
||||
interval.value = 1440
|
||||
startImmediately.value = true
|
||||
fileList.value = []
|
||||
formRef.value?.restoreValidation()
|
||||
model.value = false
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<n-modal
|
||||
v-model:show="model"
|
||||
preset="card"
|
||||
title="通过文件导入"
|
||||
:mask-closable="false"
|
||||
style="width: 600px"
|
||||
>
|
||||
<n-form
|
||||
size="small"
|
||||
ref="formRef"
|
||||
:model="{ interval, fileList }"
|
||||
:rules="rules"
|
||||
label-placement="left"
|
||||
label-width="200"
|
||||
>
|
||||
<n-form-item path="interval" label="采集间隔（分钟）">
|
||||
<n-input-number v-model:value="interval" :min="1" />
|
||||
</n-form-item>
|
||||
<n-form-item path="fileList" label="选择文件">
|
||||
<n-upload v-model:file-list="fileList" :max="1" accept=".txt,.csv">
|
||||
<n-button>选择文件</n-button>
|
||||
</n-upload>
|
||||
</n-form-item>
|
||||
<n-form-item label="采集选项">
|
||||
<n-checkbox v-model:checked="startImmediately"> 立即开始采集 </n-checkbox>
|
||||
</n-form-item>
|
||||
</n-form>
|
||||
<template #action>
|
||||
<n-button-group size="small">
|
||||
<n-button type="primary" @click="handleConfirm">确认</n-button>
|
||||
<n-button @click="handleClose">关闭</n-button>
|
||||
</n-button-group>
|
||||
</template>
|
||||
</n-modal>
|
||||
</template>
|
||||
@ -1 +0,0 @@
|
||||
@import 'tailwindcss';
|
||||
@ -1,26 +0,0 @@
|
||||
import './main.css'
|
||||
|
||||
import { createApp } from 'vue'
|
||||
import { createPinia } from 'pinia'
|
||||
|
||||
import App from './App.vue'
|
||||
import router from './router'
|
||||
|
||||
import axios from 'axios'
|
||||
|
||||
import 'vfonts/Lato.css'
|
||||
import 'vfonts/IBMPlexMono.css'
|
||||
|
||||
const app = createApp(App)
|
||||
|
||||
app.use(createPinia())
|
||||
app.use(router)
|
||||
|
||||
const axiosInstance = axios.create({
|
||||
withCredentials: true,
|
||||
timeout: 9000,
|
||||
timeoutErrorMessage: 'E_NETWORK_TIMEOUT',
|
||||
})
|
||||
app.provide('axios', axiosInstance)
|
||||
|
||||
app.mount('#app')
|
||||
@ -1,23 +0,0 @@
|
||||
import { createRouter, createWebHistory } from 'vue-router'
|
||||
|
||||
const router = createRouter({
|
||||
history: createWebHistory(import.meta.env.BASE_URL),
|
||||
routes: [
|
||||
{
|
||||
path: '/',
|
||||
redirect: '/domain',
|
||||
},
|
||||
{
|
||||
path: '/domain',
|
||||
name: 'domain-manager',
|
||||
component: () => import('../views/DomainManager.vue'),
|
||||
},
|
||||
{
|
||||
path: '/url',
|
||||
name: 'url-manager',
|
||||
component: () => import('../views/UrlManager.vue'),
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
export default router
|
||||
@ -1,12 +0,0 @@
|
||||
import { ref, computed } from 'vue'
|
||||
import { defineStore } from 'pinia'
|
||||
|
||||
export const useCounterStore = defineStore('counter', () => {
|
||||
const count = ref(0)
|
||||
const doubleCount = computed(() => count.value * 2)
|
||||
function increment() {
|
||||
count.value++
|
||||
}
|
||||
|
||||
return { count, doubleCount, increment }
|
||||
})
|
||||
@ -1,5 +0,0 @@
|
||||
const convertTimestampToDate = (timestamp: number) => {
|
||||
return new Date(timestamp * 1000).toLocaleString()
|
||||
}
|
||||
|
||||
export { convertTimestampToDate }
|
||||
@ -1,430 +0,0 @@
|
||||
<script setup lang="tsx">
|
||||
import { onMounted, inject, ref, computed } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import type { AxiosInstance } from 'axios'
|
||||
import { type DataTableColumn, type DataTableRowKey, useMessage, useDialog } from 'naive-ui'
|
||||
import ImportDomainDialog from '../components/ImportDomainDialog.vue'
|
||||
import AddDomainDialog from '../components/AddDomainDialog.vue'
|
||||
import EditDomainDialog from '../components/EditDomainDialog.vue'
|
||||
import { convertTimestampToDate } from '@/utils/common'
|
||||
|
||||
const axios = inject('axios') as AxiosInstance
|
||||
const message = useMessage()
|
||||
const dialog = useDialog()
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
|
||||
// 筛选条件
|
||||
const filterForm = ref({
|
||||
domain: '',
|
||||
status: null as number | null,
|
||||
})
|
||||
|
||||
// 状态选项
|
||||
const statusOptions = [
|
||||
{ label: '全部', value: null },
|
||||
{ label: 'READY', value: 1 },
|
||||
{ label: 'QUEUE', value: 2 },
|
||||
{ label: 'CRAWLING', value: 3 },
|
||||
{ label: 'PAUSE', value: 999 },
|
||||
]
|
||||
|
||||
const showImportDialog = ref(false)
|
||||
const showAddDialog = ref(false)
|
||||
const showEditDialog = ref(false)
|
||||
|
||||
// 当前正在编辑的域名 ID 列表（单个或批量）
|
||||
const editingDomainIds = ref<DataTableRowKey[] | null>(null)
|
||||
|
||||
// 选中行的 Key
|
||||
const checkedRowKeys = ref<DataTableRowKey[]>([])
|
||||
|
||||
// 分页相关状态
|
||||
const pagination = ref({
|
||||
page: 1,
|
||||
pageSize: 50,
|
||||
itemCount: 0,
|
||||
showSizePicker: true,
|
||||
pageSizes: [10, 20, 50, 100, 200, 500, 1000],
|
||||
onChange: (page: number) => {
|
||||
pagination.value.page = page
|
||||
updateUrlParams()
|
||||
getDomainList()
|
||||
},
|
||||
onUpdatePageSize: (pageSize: number) => {
|
||||
pagination.value.pageSize = pageSize
|
||||
pagination.value.page = 1
|
||||
updateUrlParams()
|
||||
getDomainList()
|
||||
},
|
||||
})
|
||||
|
||||
// 更新URL参数
|
||||
const updateUrlParams = () => {
|
||||
router.push({
|
||||
query: {
|
||||
page: pagination.value.page,
|
||||
size: pagination.value.pageSize,
|
||||
domain: filterForm.value.domain || undefined,
|
||||
status: filterForm.value.status || undefined,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// 初始化分页参数
|
||||
const initPagination = () => {
|
||||
const page = Number(route.query.page) || 1
|
||||
const size = Number(route.query.size) || 50
|
||||
const domain = route.query.domain as string || ''
|
||||
const status = route.query.status ? Number(route.query.status) : null
|
||||
|
||||
pagination.value.page = page
|
||||
pagination.value.pageSize = size
|
||||
filterForm.value.domain = domain
|
||||
filterForm.value.status = status
|
||||
}
|
||||
|
||||
const columns: Array<DataTableColumn> = [
|
||||
{
|
||||
type: 'selection',
|
||||
},
|
||||
{
|
||||
title: '#',
|
||||
key: 'id',
|
||||
},
|
||||
{
|
||||
title: '域名',
|
||||
key: 'domain',
|
||||
},
|
||||
{
|
||||
title: '状态',
|
||||
key: 'status',
|
||||
render: (row) => {
|
||||
let statusText = ''
|
||||
let statusType = ''
|
||||
|
||||
switch (row.status) {
|
||||
case 1:
|
||||
statusText = 'READY'
|
||||
statusType = 'success'
|
||||
break
|
||||
case 2:
|
||||
statusText = 'QUENE'
|
||||
statusType = 'warning'
|
||||
break
|
||||
case 3:
|
||||
statusText = 'CRAWLING'
|
||||
statusType = 'info'
|
||||
break
|
||||
case 999:
|
||||
statusText = 'PAUSE'
|
||||
statusType = 'error'
|
||||
break
|
||||
default:
|
||||
statusText = 'UNKNOWN'
|
||||
statusType = 'error'
|
||||
}
|
||||
|
||||
return <n-tag type={statusType}>{statusText}</n-tag>
|
||||
},
|
||||
},
|
||||
{
|
||||
title: '采集间隔 (分钟)',
|
||||
key: 'crawl_interval',
|
||||
render: (row) => (
|
||||
<n-tooltip>
|
||||
{{
|
||||
trigger: () => <span>{row.crawl_interval}</span>,
|
||||
default: () => `约 ${(row.crawl_interval as number) / 60 / 24} 天`,
|
||||
}}
|
||||
</n-tooltip>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: '最近采集时间',
|
||||
key: 'latest_crawl_time',
|
||||
render: (row) => convertTimestampToDate(row.latest_crawl_time as number),
|
||||
},
|
||||
{
|
||||
title: '操作',
|
||||
key: 'action',
|
||||
render: (row) => (
|
||||
<div class="flex gap-2">
|
||||
<n-button size="small" type="primary" onClick={() => handleEdit(row)}>
|
||||
编辑
|
||||
</n-button>
|
||||
<n-button size="small" type="info" onClick={() => handleSingleCrawl(row)}>
|
||||
立即采集
|
||||
</n-button>
|
||||
<n-button size="small" type="error" onClick={() => handleDelete(row)}>
|
||||
删除
|
||||
</n-button>
|
||||
</div>
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
const domains = ref([])
|
||||
|
||||
// 是否有选中的行
|
||||
const hasSelectedRows = computed(() => checkedRowKeys.value.length > 0)
|
||||
|
||||
// 处理选中行变化
|
||||
const handleCheck = (rowKeys: DataTableRowKey[]) => {
|
||||
checkedRowKeys.value = rowKeys
|
||||
}
|
||||
|
||||
/** 批量删除域名 */
|
||||
const handleBatchDelete = () => {
|
||||
if (!hasSelectedRows.value) {
|
||||
message.warning('请至少选择一个域名')
|
||||
return
|
||||
}
|
||||
const removeSurl = ref(false)
|
||||
dialog.warning({
|
||||
title: '确认批量删除',
|
||||
content: () => (
|
||||
<div>
|
||||
<div class="mb-2">确定要删除选中的 {checkedRowKeys.value.length} 个域名吗？</div>
|
||||
<n-checkbox v-model:checked={removeSurl.value}>同时删除所有关联的 SURL</n-checkbox>
|
||||
</div>
|
||||
),
|
||||
positiveText: '确定',
|
||||
negativeText: '取消',
|
||||
onPositiveClick: async () => {
|
||||
try {
|
||||
const response = (
|
||||
await axios.post('/api/domain/v1/delete', {
|
||||
domain_ids: checkedRowKeys.value,
|
||||
remove_surl: removeSurl.value,
|
||||
})
|
||||
).data
|
||||
if (response.code !== 20000) {
|
||||
message.error(`批量删除域名失败，错误：${response.message}`)
|
||||
return
|
||||
}
|
||||
message.success('批量删除成功')
|
||||
checkedRowKeys.value = [] // 清空选择
|
||||
getDomainList()
|
||||
} catch (error) {
|
||||
console.error('批量删除域名失败', error)
|
||||
message.error(`批量删除域名失败，错误：${error}`)
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/** 批量修改采集间隔 */
|
||||
const handleBatchEdit = () => {
|
||||
if (!hasSelectedRows.value) {
|
||||
message.warning('请至少选择一个域名')
|
||||
return
|
||||
}
|
||||
editingDomainIds.value = [...checkedRowKeys.value] // 设置要编辑的 ID 为当前选中的 ID
|
||||
showEditDialog.value = true
|
||||
}
|
||||
|
||||
/** 批量立即采集 */
|
||||
const handleBatchCrawl = async () => {
|
||||
if (!hasSelectedRows.value) {
|
||||
message.warning('请至少选择一个域名')
|
||||
return
|
||||
}
|
||||
try {
|
||||
// TODO: 确认后端是否有 /api/domain/v1/crawl 接口
|
||||
const response = (
|
||||
await axios.post('/api/domain/v1/crawl', {
|
||||
domain_ids: checkedRowKeys.value,
|
||||
})
|
||||
).data
|
||||
|
||||
if (response.code !== 20000) {
|
||||
message.error(`批量触发采集失败：${response.message}`)
|
||||
return
|
||||
}
|
||||
|
||||
message.success('批量触发采集成功，已加入队列')
|
||||
checkedRowKeys.value = [] // 清空选择
|
||||
getDomainList() // 刷新列表查看状态变化
|
||||
} catch (error) {
|
||||
console.error('批量触发采集失败', error)
|
||||
message.error(`批量触发采集失败：${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** 单个立即采集 */
|
||||
const handleSingleCrawl = async (row: any) => {
|
||||
try {
|
||||
// 调用与批量采集相同的接口，但只传单个 ID
|
||||
const response = (
|
||||
await axios.post('/api/domain/v1/crawl', {
|
||||
domain_ids: [row.id],
|
||||
})
|
||||
).data
|
||||
|
||||
if (response.code !== 20000) {
|
||||
message.error(`触发采集失败：${response.message}`)
|
||||
return
|
||||
}
|
||||
|
||||
message.success(`域名 ${row.domain} 已加入采集队列`)
|
||||
getDomainList() // 刷新列表查看状态变化
|
||||
} catch (error) {
|
||||
console.error('触发采集失败', error)
|
||||
message.error(`触发采集失败：${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** 删除域名 */
|
||||
const handleDelete = async (row: any) => {
|
||||
const removeSurl = ref(false)
|
||||
|
||||
dialog.warning({
|
||||
title: '确认删除',
|
||||
content: () => (
|
||||
<div>
|
||||
<div class="mb-2">确定要删除域名 {row.domain} 吗？</div>
|
||||
<n-checkbox v-model:checked={removeSurl.value}>同时删除所有关联的 SURL</n-checkbox>
|
||||
</div>
|
||||
),
|
||||
positiveText: '确定',
|
||||
negativeText: '取消',
|
||||
onPositiveClick: async () => {
|
||||
try {
|
||||
const response = (
|
||||
await axios.post('/api/domain/v1/delete', {
|
||||
domain_ids: [row.id],
|
||||
remove_surl: removeSurl.value,
|
||||
})
|
||||
).data
|
||||
if (response.code !== 20000) {
|
||||
message.error(`删除域名失败，错误：${response.message}`)
|
||||
return
|
||||
}
|
||||
message.success('删除成功')
|
||||
// 如果删除的是选中的行，也从 checkedRowKeys 中移除
|
||||
const index = checkedRowKeys.value.findIndex((key) => key === row.id)
|
||||
if (index > -1) {
|
||||
checkedRowKeys.value.splice(index, 1)
|
||||
}
|
||||
getDomainList()
|
||||
} catch (error) {
|
||||
console.error('删除域名失败', error)
|
||||
message.error(`删除域名失败，错误：${error}`)
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/** 获取域名列表 */
|
||||
const getDomainList = async () => {
|
||||
try {
|
||||
const response = (
|
||||
await axios.get('/api/domain/v1/list', {
|
||||
params: {
|
||||
page: pagination.value.page,
|
||||
size: pagination.value.pageSize,
|
||||
domain: filterForm.value.domain || undefined,
|
||||
status: filterForm.value.status || undefined,
|
||||
},
|
||||
})
|
||||
).data
|
||||
if (response.code !== 20000) {
|
||||
message.error(`获取域名列表失败，错误：${response.message}`)
|
||||
return
|
||||
}
|
||||
|
||||
domains.value = response.data.rows
|
||||
pagination.value.itemCount = response.data.total
|
||||
} catch (error) {
|
||||
console.error('获取域名列表失败', error)
|
||||
message.error(`获取域名列表失败，错误：${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
const handleImportSuccess = () => {
|
||||
getDomainList()
|
||||
}
|
||||
|
||||
const handleAddSuccess = () => {
|
||||
getDomainList()
|
||||
}
|
||||
|
||||
/** 编辑域名 */
|
||||
const handleEdit = (row: any) => {
|
||||
editingDomainIds.value = [row.id] // 设置要编辑的 ID 为当前行的 ID
|
||||
showEditDialog.value = true
|
||||
}
|
||||
|
||||
const handleEditSuccess = () => {
|
||||
getDomainList()
|
||||
const editedCount = editingDomainIds.value?.length || 0
|
||||
editingDomainIds.value = null // 清空正在编辑的 ID
|
||||
// 如果是批量编辑，成功后清空表格的选择
|
||||
if (editedCount > 1) {
|
||||
checkedRowKeys.value = []
|
||||
}
|
||||
}
|
||||
|
||||
// 重置筛选
|
||||
const resetFilter = () => {
|
||||
filterForm.value = {
|
||||
domain: '',
|
||||
status: null,
|
||||
}
|
||||
pagination.value.page = 1
|
||||
updateUrlParams()
|
||||
getDomainList()
|
||||
}
|
||||
|
||||
// 应用筛选
|
||||
const applyFilter = () => {
|
||||
pagination.value.page = 1
|
||||
updateUrlParams()
|
||||
getDomainList()
|
||||
}
|
||||
|
||||
onMounted(async () => {
|
||||
initPagination()
|
||||
await getDomainList()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="text-2xl pb-4">域名管理</div>
|
||||
<div class="flex gap-2 mb-4">
|
||||
<n-button type="primary" @click="showImportDialog = true">通过文件导入</n-button>
|
||||
<n-button type="primary" @click="showAddDialog = true">手动添加</n-button>
|
||||
<n-button type="error" @click="handleBatchDelete" :disabled="!hasSelectedRows">批量删除</n-button>
|
||||
<n-button type="warning" @click="handleBatchEdit" :disabled="!hasSelectedRows">修改间隔</n-button>
|
||||
<n-button type="info" @click="handleBatchCrawl" :disabled="!hasSelectedRows">立即采集</n-button>
|
||||
</div>
|
||||
|
||||
<!-- 筛选表单 -->
|
||||
<n-form inline :model="filterForm" class="mb-4 p-4 bg-gray-50 rounded-lg">
|
||||
<n-form-item label="域名" path="domain">
|
||||
<n-input v-model:value="filterForm.domain" placeholder="请输入域名" clearable @keydown.enter="applyFilter" />
|
||||
</n-form-item>
|
||||
<n-form-item label="状态" path="status">
|
||||
<n-select v-model:value="filterForm.status" :options="statusOptions" placeholder="请选择状态" style="width: 200px" />
|
||||
</n-form-item>
|
||||
<n-form-item>
|
||||
<n-button type="primary" @click="applyFilter">筛选</n-button>
|
||||
<n-button class="ml-2" @click="resetFilter">重置</n-button>
|
||||
</n-form-item>
|
||||
</n-form>
|
||||
|
||||
<n-data-table :columns="columns" :data="domains" :row-key="(row: any) => row.id" :checked-row-keys="checkedRowKeys"
|
||||
@update:checked-row-keys="handleCheck" size="small" />
|
||||
|
||||
<div class="flex justify-center mt-4">
|
||||
<n-pagination v-model:page="pagination.page" :page-size="pagination.pageSize" :item-count="pagination.itemCount"
|
||||
:show-size-picker="pagination.showSizePicker" :page-sizes="pagination.pageSizes"
|
||||
@update:page-size="pagination.onUpdatePageSize" @update:page="pagination.onChange" />
|
||||
</div>
|
||||
|
||||
<ImportDomainDialog v-model:show="showImportDialog" @success="handleImportSuccess" />
|
||||
<AddDomainDialog v-model:show="showAddDialog" @success="handleAddSuccess" />
|
||||
<EditDomainDialog v-model:show="showEditDialog" :domain-ids="editingDomainIds" @success="handleEditSuccess"
|
||||
@close="editingDomainIds = null" />
|
||||
</template>
|
||||
@ -1,379 +0,0 @@
|
||||
<script setup lang="tsx">
|
||||
import { onMounted, inject, ref, computed } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import type { AxiosInstance } from 'axios'
|
||||
import { type DataTableColumn, type DataTableRowKey, useMessage, useDialog, NDropdown } from 'naive-ui'
|
||||
import { convertTimestampToDate } from '@/utils/common'
|
||||
|
||||
const axios = inject('axios') as AxiosInstance
|
||||
const message = useMessage()
|
||||
const dialog = useDialog()
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
|
||||
// 搜索条件
|
||||
const searchForm = ref({
|
||||
domain: '',
|
||||
surl: '',
|
||||
is_report_by_one: 2,
|
||||
is_report_by_site: 2,
|
||||
is_report_by_wap: 2,
|
||||
has_evidence: 2,
|
||||
})
|
||||
|
||||
// 选项数据
|
||||
const options = [
|
||||
{ label: '全部', value: 2 },
|
||||
{ label: '是', value: 1 },
|
||||
{ label: '否', value: 0 },
|
||||
]
|
||||
|
||||
// 分页相关状态
|
||||
const pagination = ref({
|
||||
page: 1,
|
||||
pageSize: 50,
|
||||
itemCount: 0,
|
||||
showSizePicker: true,
|
||||
pageSizes: [10, 20, 50, 100, 200, 500, 1000],
|
||||
onChange: (page: number) => {
|
||||
pagination.value.page = page
|
||||
updateUrlParams()
|
||||
getUrlList()
|
||||
},
|
||||
onUpdatePageSize: (pageSize: number) => {
|
||||
pagination.value.pageSize = pageSize
|
||||
pagination.value.page = 1
|
||||
updateUrlParams()
|
||||
getUrlList()
|
||||
},
|
||||
})
|
||||
|
||||
// 更新URL参数
|
||||
const updateUrlParams = () => {
|
||||
router.push({
|
||||
query: {
|
||||
page: pagination.value.page,
|
||||
size: pagination.value.pageSize,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// 初始化分页参数
|
||||
const initPagination = () => {
|
||||
const page = Number(route.query.page) || 1
|
||||
const size = Number(route.query.size) || 50
|
||||
pagination.value.page = page
|
||||
pagination.value.pageSize = size
|
||||
}
|
||||
|
||||
// 选中行的 Key
|
||||
const checkedRowKeys = ref<DataTableRowKey[]>([])
|
||||
|
||||
// 举报选项
|
||||
const reportOptions = [
|
||||
{
|
||||
label: '全部渠道',
|
||||
key: 'all',
|
||||
},
|
||||
{
|
||||
label: 'PC渠道',
|
||||
key: 'pc',
|
||||
},
|
||||
{
|
||||
label: 'SITE渠道',
|
||||
key: 'site',
|
||||
},
|
||||
{
|
||||
label: 'WAP渠道',
|
||||
key: 'wap',
|
||||
},
|
||||
]
|
||||
|
||||
const columns: Array<DataTableColumn> = [
|
||||
{
|
||||
type: 'selection',
|
||||
},
|
||||
{
|
||||
title: '#',
|
||||
key: 'id',
|
||||
minWidth: 60,
|
||||
},
|
||||
{
|
||||
title: '域名',
|
||||
key: 'domain',
|
||||
minWidth: 200,
|
||||
},
|
||||
{
|
||||
title: 'SURL',
|
||||
key: 'surl',
|
||||
minWidth: 100,
|
||||
},
|
||||
{
|
||||
title: 'Q',
|
||||
key: 'q',
|
||||
minWidth: 100,
|
||||
},
|
||||
{
|
||||
title: 'Token',
|
||||
key: 'token',
|
||||
minWidth: 100,
|
||||
},
|
||||
{
|
||||
title: '已通过PC举报',
|
||||
key: 'is_report_by_one',
|
||||
render: (row) => (
|
||||
<n-tag type={row.is_report_by_one ? 'success' : 'default'}>
|
||||
{row.is_report_by_one ? '是' : '否'}
|
||||
</n-tag>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: '已通过site举报',
|
||||
key: 'is_report_by_site',
|
||||
render: (row) => (
|
||||
<n-tag type={row.is_report_by_site ? 'success' : 'default'}>
|
||||
{row.is_report_by_site ? '是' : '否'}
|
||||
</n-tag>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: '已通过WAP举报',
|
||||
key: 'is_report_by_wap',
|
||||
render: (row) => (
|
||||
<n-tag type={row.is_report_by_wap ? 'success' : 'default'}>
|
||||
{row.is_report_by_wap ? '是' : '否'}
|
||||
</n-tag>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: '已收集证据',
|
||||
key: 'has_evidence',
|
||||
render: (row) => (
|
||||
<n-tag type={row.has_evidence ? 'success' : 'default'}>
|
||||
{row.has_evidence ? '是' : '否'}
|
||||
</n-tag>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: '操作',
|
||||
key: 'action',
|
||||
render: (row) => {
|
||||
return (
|
||||
<div class="flex gap-2">
|
||||
<n-dropdown
|
||||
trigger="click"
|
||||
options={reportOptions}
|
||||
onSelect={(key: string) => handleSingleReport(row, key)}
|
||||
>
|
||||
<n-button size="small" type="primary">举报</n-button>
|
||||
</n-dropdown>
|
||||
<n-button
|
||||
size="small"
|
||||
type="info"
|
||||
onClick={() => handleSingleCollectEvidence(row)}
|
||||
>
|
||||
收集证据
|
||||
</n-button>
|
||||
</div>
|
||||
)
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
const urls = ref([])
|
||||
|
||||
// 获取URL列表
|
||||
const getUrlList = async () => {
|
||||
try {
|
||||
const response = (
|
||||
await axios.get('/api/urls/v1/list', {
|
||||
params: {
|
||||
...searchForm.value,
|
||||
page: pagination.value.page,
|
||||
size: pagination.value.pageSize,
|
||||
},
|
||||
})
|
||||
).data
|
||||
|
||||
if (response.code !== 20000) {
|
||||
message.error(`获取URL列表失败：${response.message}`)
|
||||
return
|
||||
}
|
||||
|
||||
urls.value = response.data.data
|
||||
console.log('response.data.total:', response.data.total)
|
||||
pagination.value.itemCount = response.data.total
|
||||
} catch (error) {
|
||||
console.error('获取URL列表失败', error)
|
||||
message.error(`获取URL列表失败：${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
// 搜索
|
||||
const handleSearch = () => {
|
||||
pagination.value.page = 1
|
||||
getUrlList()
|
||||
}
|
||||
|
||||
// 重置搜索
|
||||
const handleReset = () => {
|
||||
searchForm.value = {
|
||||
domain: '',
|
||||
surl: '',
|
||||
is_report_by_one: 2,
|
||||
is_report_by_site: 2,
|
||||
is_report_by_wap: 2,
|
||||
has_evidence: 2,
|
||||
}
|
||||
handleSearch()
|
||||
}
|
||||
|
||||
// 处理举报
|
||||
const handleReport = async (ids: number[], option: string) => {
|
||||
// 检查是否有证据
|
||||
const selectedUrls = urls.value.filter((url: any) => ids.includes(url.id))
|
||||
const hasNoEvidence = selectedUrls.some((url: any) => !url.has_evidence)
|
||||
|
||||
if (hasNoEvidence) {
|
||||
message.warning('请先收集证据后再进行举报')
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const response = (await axios.post('/api/urls/v1/report', {
|
||||
ids,
|
||||
report_by_one: option === 'all' || option === 'pc',
|
||||
report_by_site: option === 'all' || option === 'site',
|
||||
report_by_wap: option === 'all' || option === 'wap',
|
||||
})).data
|
||||
|
||||
if (response.code !== 20000) {
|
||||
message.error(`举报失败：${response.message}`)
|
||||
return
|
||||
}
|
||||
|
||||
message.success('操作成功，已修改SURL状态，等待引擎调度')
|
||||
getUrlList()
|
||||
} catch (error) {
|
||||
console.error('举报失败', error)
|
||||
message.error(`举报失败：${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
// 处理收集证据
|
||||
const handleCollectEvidence = async (ids: number[]) => {
|
||||
try {
|
||||
const response = (await axios.post('/api/urls/v1/evidence', {
|
||||
ids,
|
||||
})).data
|
||||
|
||||
if (response.code !== 20000) {
|
||||
message.error(`收集证据失败：${response.message}`)
|
||||
return
|
||||
}
|
||||
|
||||
message.success('操作成功，已修改SURL状态，等待引擎调度')
|
||||
getUrlList()
|
||||
} catch (error) {
|
||||
console.error('收集证据失败', error)
|
||||
message.error(`收集证据失败：${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
// 处理单个URL的举报
|
||||
const handleSingleReport = (row: any, option: string) => {
|
||||
handleReport([row.id], option)
|
||||
}
|
||||
|
||||
// 处理单个URL的收集证据
|
||||
const handleSingleCollectEvidence = (row: any) => {
|
||||
handleCollectEvidence([row.id])
|
||||
}
|
||||
|
||||
// 处理批量举报
|
||||
const handleBatchReport = (option: string) => {
|
||||
if (checkedRowKeys.value.length === 0) {
|
||||
message.warning('请至少选择一个URL')
|
||||
return
|
||||
}
|
||||
handleReport(checkedRowKeys.value as number[], option)
|
||||
}
|
||||
|
||||
// 处理批量收集证据
|
||||
const handleBatchCollectEvidence = () => {
|
||||
if (checkedRowKeys.value.length === 0) {
|
||||
message.warning('请至少选择一个URL')
|
||||
return
|
||||
}
|
||||
handleCollectEvidence(checkedRowKeys.value as number[])
|
||||
}
|
||||
|
||||
// 是否有选中的行
|
||||
const hasSelectedRows = computed(() => checkedRowKeys.value.length > 0)
|
||||
|
||||
onMounted(() => {
|
||||
initPagination()
|
||||
getUrlList()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="p-4">
|
||||
<h1 class="text-2xl mb-4">URL管理</h1>
|
||||
|
||||
<!-- 搜索表单 -->
|
||||
<n-card class="mb-4">
|
||||
<n-form :model="searchForm" label-placement="left" label-width="auto" require-mark-placement="right-hanging">
|
||||
<n-grid :cols="24" :x-gap="24">
|
||||
<n-form-item-gi :span="8" label="域名">
|
||||
<n-input v-model:value="searchForm.domain" placeholder="请输入域名" @keyup.enter="handleSearch" />
|
||||
</n-form-item-gi>
|
||||
<n-form-item-gi :span="8" label="SURL">
|
||||
<n-input v-model:value="searchForm.surl" placeholder="请输入SURL" @keyup.enter="handleSearch" />
|
||||
</n-form-item-gi>
|
||||
<n-form-item-gi :span="8" label="是否已通过PC举报">
|
||||
<n-select v-model:value="searchForm.is_report_by_one" :options="options" placeholder="请选择" />
|
||||
</n-form-item-gi>
|
||||
<n-form-item-gi :span="8" label="是否已通过site举报">
|
||||
<n-select v-model:value="searchForm.is_report_by_site" :options="options" placeholder="请选择" />
|
||||
</n-form-item-gi>
|
||||
<n-form-item-gi :span="8" label="是否已通过WAP举报">
|
||||
<n-select v-model:value="searchForm.is_report_by_wap" :options="options" placeholder="请选择" />
|
||||
</n-form-item-gi>
|
||||
<n-form-item-gi :span="8" label="是否已收集证据">
|
||||
<n-select v-model:value="searchForm.has_evidence" :options="options" placeholder="请选择" />
|
||||
</n-form-item-gi>
|
||||
</n-grid>
|
||||
<div class="flex justify-end gap-2 mt-4">
|
||||
<n-button @click="handleReset">重置</n-button>
|
||||
<n-button type="primary" @click="handleSearch">搜索</n-button>
|
||||
</div>
|
||||
</n-form>
|
||||
</n-card>
|
||||
|
||||
<!-- 数据表格 -->
|
||||
<n-card>
|
||||
<div class="mb-4" v-if="hasSelectedRows">
|
||||
<n-space>
|
||||
<n-dropdown trigger="click" :options="reportOptions" @select="handleBatchReport">
|
||||
<n-button type="primary">批量举报 ({{ checkedRowKeys.length }})</n-button>
|
||||
</n-dropdown>
|
||||
<n-button type="info" @click="handleBatchCollectEvidence">
|
||||
批量收集证据 ({{ checkedRowKeys.length }})
|
||||
</n-button>
|
||||
</n-space>
|
||||
</div>
|
||||
|
||||
<n-data-table :columns="columns" :data="urls" :bordered="false" :row-key="(row: any) => row.id"
|
||||
:checked-row-keys="checkedRowKeys" @update:checked-row-keys="checkedRowKeys = $event" />
|
||||
|
||||
<div class="flex justify-center mt-4">
|
||||
<n-pagination v-model:page="pagination.page" :item-count="pagination.itemCount" :page-size="pagination.pageSize"
|
||||
:show-size-picker="pagination.showSizePicker" :page-sizes="pagination.pageSizes"
|
||||
:on-update:page-size="pagination.onUpdatePageSize" :on-change="pagination.onChange" />
|
||||
</div>
|
||||
</n-card>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped></style>
|
||||
@ -1,12 +0,0 @@
|
||||
{
|
||||
"extends": "@vue/tsconfig/tsconfig.dom.json",
|
||||
"include": ["env.d.ts", "src/**/*", "src/**/*.vue"],
|
||||
"exclude": ["src/**/__tests__/*"],
|
||||
"compilerOptions": {
|
||||
"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo",
|
||||
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
{
|
||||
"files": [],
|
||||
"references": [
|
||||
{
|
||||
"path": "./tsconfig.node.json"
|
||||
},
|
||||
{
|
||||
"path": "./tsconfig.app.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
{
|
||||
"extends": "@tsconfig/node22/tsconfig.json",
|
||||
"include": [
|
||||
"vite.config.*",
|
||||
"vitest.config.*",
|
||||
"cypress.config.*",
|
||||
"nightwatch.conf.*",
|
||||
"playwright.config.*",
|
||||
"eslint.config.*"
|
||||
],
|
||||
"compilerOptions": {
|
||||
"noEmit": true,
|
||||
"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo",
|
||||
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"types": ["node"]
|
||||
}
|
||||
}
|
||||
@ -1,50 +0,0 @@
|
||||
import {fileURLToPath, URL} from 'node:url'
|
||||
|
||||
import {defineConfig} from 'vite'
|
||||
import vue from '@vitejs/plugin-vue'
|
||||
import vueJsx from '@vitejs/plugin-vue-jsx'
|
||||
import vueDevTools from 'vite-plugin-vue-devtools'
|
||||
import tailwindcss from '@tailwindcss/vite'
|
||||
import AutoImport from 'unplugin-auto-import/vite'
|
||||
import Components from 'unplugin-vue-components/vite'
|
||||
import {NaiveUiResolver} from 'unplugin-vue-components/resolvers'
|
||||
|
||||
|
||||
// https://vite.dev/config/
|
||||
export default defineConfig({
|
||||
plugins: [
|
||||
tailwindcss(),
|
||||
vue(),
|
||||
vueJsx(),
|
||||
vueDevTools(),
|
||||
AutoImport({
|
||||
imports: [
|
||||
'vue',
|
||||
{
|
||||
'naive-ui': [
|
||||
'useDialog',
|
||||
'useMessage',
|
||||
'useNotification',
|
||||
'useLoadingBar'
|
||||
]
|
||||
}
|
||||
]
|
||||
}),
|
||||
Components({
|
||||
resolvers: [NaiveUiResolver()]
|
||||
})
|
||||
],
|
||||
resolve: {
|
||||
alias: {
|
||||
'@': fileURLToPath(new URL('./src', import.meta.url))
|
||||
},
|
||||
},
|
||||
server: {
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:3000',
|
||||
changeOrigin: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
@ -34,17 +34,13 @@ tab = browser.new_tab(f"https://www.baidu.com/s?wd={keyword}")
|
||||
# week_btn_el = tab.ele('t:li@@text()= 一月内 ')
|
||||
# week_btn_el.click(by_js=True)
|
||||
# tab.wait(2)
|
||||
print(f"{tab.url=}")
|
||||
|
||||
print("2222")
|
||||
tab.get("https://www.163.com/")
|
||||
print(f"{tab.url=}")
|
||||
# tab.ele(".content_none")
|
||||
# tab.wait.eles_loaded(["#container", ".content_none", "#content_left"], any_one=True)
|
||||
print("未找到相关结果" in tab.html)
|
||||
print("1111")
|
||||
|
||||
|
||||
|
||||
# if "未找到相关结果" in tab.html:
|
||||
# print("未找到相关结果")
|
||||
# else:
|
||||
|
||||
@ -1,272 +0,0 @@
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from enum import verify
|
||||
from pathlib import Path
|
||||
|
||||
import execjs
|
||||
import requests
|
||||
from DrissionPage import Chromium, ChromiumOptions
|
||||
from loguru import logger
|
||||
|
||||
from app.utils.common import get_all_cookies
|
||||
from app.utils.ydm_verify import YdmVerify
|
||||
|
||||
chrome_opts = ChromiumOptions()
|
||||
chrome_opts.mute(True) # 静音
|
||||
chrome_opts.no_imgs(False)
|
||||
chrome_opts.set_argument("--disable-gpu")
|
||||
chrome_opts.set_argument('--ignore-certificate-errors')
|
||||
chrome_opts.set_argument("--proxy-server=http://127.0.0.1:7890")
|
||||
# chrome_opts.incognito(True)
|
||||
chrome_opts.set_browser_path(r"C:\Program Files\Google\Chrome\Application\chrome.exe")
|
||||
chrome_opts.auto_port()
|
||||
browser = Chromium(addr_or_opts=chrome_opts)
|
||||
|
||||
# tab = browser.new_tab()
|
||||
# tab.listen.start([
|
||||
# "passport.baidu.com/cap/init",
|
||||
# "passport.baidu.com/cap/style",
|
||||
# ])
|
||||
# tab.get("https://wappass.baidu.com/static/captcha/tuxing_v2.html?&logid=10332554090053311096&ak=c27bbc89afca0463650ac9bde68ebe06&backurl=https%3A%2F%2Fwww.baidu.com%2Fs%3Fwd%3Dsite%253Altxbbs.com%26pn%3D50%26oq%3Dsite%253Altxbbs.com%26ct%3D2097152%26ie%3Dutf-8%26si%3Dltxbbs.com%26fenlei%3D256%26rsv_idx%3D1%26rsv_pq%3D99cae74f0003cd72%26rsv_t%3Dab2dk%252Fq4PohUCmoLbyMlEMrGJszk983ojkNLk%252FUiZGJ4ZLpwvZ46PtQUufk%26gpc%3Dstf%253D1741437499%252C1744115898%257Cstftype%253D1%26tfflag%3D1%26topic_pn%3D%26rsv_page%3D1&ext=x9G9QDmMXq%2FNo87gjGO0P1dyBXu4PagAZrreQL6%2Bticsr0rrDszYO2sAbAnT1vLIUgqUK9LXd1cIlztrhMwiv3XfcB99Y5gyF0c0ETsDFDls5CsGNJQRLPawcntn2ndVLHHLl46IaoOp8l%2FC1xtOHwMQi85PCzAojcSf2wQ76KRxVau99LtSYCIfwtv7By0w&signature=f2fbb1b81926e247835f69195661a06b×tamp=1744115910")
|
||||
# for pkg in tab.listen.steps():
|
||||
# print(f"{pkg.url=}")
|
||||
# print(f"{pkg.response.raw_body=}")
|
||||
# current_path = Path(__file__).resolve()
|
||||
# print(current_path)
|
||||
# current_dir = current_path.parent.parent
|
||||
# print(current_dir)
|
||||
# js_path = current_dir.joinpath("./js/mkd_v2_link_submit.js")
|
||||
# print(js_path.exists())
|
||||
|
||||
# with open("./js/mkd_v2_link_submit.js", "r", encoding="utf-8") as f:
|
||||
# ds_js = f.read()
|
||||
#
|
||||
|
||||
proxy_str = "http://127.0.0.1:7890"
|
||||
headers = {
|
||||
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-US;q=0.7,zh-TW;q=0.6',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
# 'Origin': 'https://jubao.baidu.com',
|
||||
'Pragma': 'no-cache',
|
||||
'Referer': "https://wappass.baidu.com/",
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0",
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
'sec-ch-ua_wap': '"Not(A:Brand";v="99", "Google Chrome";v="133", "Chromium";v="133"',
|
||||
'sec-ch-ua_wap-mobile': '?0',
|
||||
'sec-ch-ua_wap-platform': '"Windows"',
|
||||
"Cookie": "BDUSS=ldlSDMwdkg5VmlrbE5TZFdHUHVhWEFCTVNqcGtKZHhXeTNaTHFGZHY4Y3F5LVJiQVFBQUFBJCQAAAAAAAAAAAEAAADj3ycY1tC5zNXywO4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACo-vVsqPr1bS; PHPSESSID=f364o6o7tpsag92pd67630p870; lastIdentity=PassUserIdentity; BAIDUID=5C7396A6BE9E28B769E6E9815A1B8D5E:FG=1; BAIDUID_BFESS=5C7396A6BE9E28B769E6E9815A1B8D5E:FG=1; BDUSS_BFESS=ldlSDMwdkg5VmlrbE5TZFdHUHVhWEFCTVNqcGtKZHhXeTNaTHFGZHY4Y3F5LVJiQVFBQUFBJCQAAAAAAAAAAAEAAADj3ycY1tC5zNXywO4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACo-vVsqPr1bS",
|
||||
}
|
||||
|
||||
tab = browser.new_tab()
|
||||
tab.listen.start(r"/cap/(init|style|log)", is_regex=True)
|
||||
tab.get("https://www.baidu.com")
|
||||
|
||||
captcha_data = {}
|
||||
|
||||
|
||||
def listener():
|
||||
for pkg in tab.listen.steps():
|
||||
if "/cap/init" in pkg.url:
|
||||
captcha_data["init"] = pkg.response.body
|
||||
if "/cap/style" in pkg.url:
|
||||
captcha_data["style"] = pkg.response.body
|
||||
captcha_data["referer"] = pkg.request.headers.get("Referer")
|
||||
logger.debug(f"正确的 referer: {captcha_data["referer"]}")
|
||||
captcha_data["cookie"] = pkg.request.headers.get("Cookie")
|
||||
logger.debug(f"cookie: {captcha_data['cookie']}")
|
||||
if "/cap/log" in pkg.url:
|
||||
captcha_data["log"] = pkg.response.body
|
||||
|
||||
|
||||
thread = threading.Thread(target=listener, daemon=True)
|
||||
thread.start()
|
||||
|
||||
|
||||
def verify_captcha(current_url: str):
|
||||
headers["Referer"] = captcha_data["referer"]
|
||||
headers["Cookie"] = captcha_data["cookie"]
|
||||
|
||||
# 解出AS / TK
|
||||
as_value = captcha_data["init"]["data"]["as"]
|
||||
tk_value = captcha_data["init"]["data"]["tk"]
|
||||
# logger.debug(f"{as_value=}, {tk_value=}")
|
||||
# ts = time.time()
|
||||
# ts1 = int(ts)
|
||||
# ts2 = int(ts * 1000)
|
||||
# response = requests.post(
|
||||
# "https://passport.baidu.com/cap/init",
|
||||
# data={
|
||||
# "_": ts2,
|
||||
# "refer": re.sub(r'timestamp=\d+', f'timestamp={ts1}', captcha_data["referer"]),
|
||||
# "ak": "c27bbc89afca0463650ac9bde68ebe06",
|
||||
# "ver": "2",
|
||||
# "scene": "",
|
||||
# "ds": "",
|
||||
# "tk": "",
|
||||
# "as": "",
|
||||
# "reinit": 0
|
||||
# },
|
||||
# headers=headers,
|
||||
# proxies={
|
||||
# "http": proxy_str, "https": proxy_str
|
||||
# }
|
||||
# ).json()
|
||||
# as_value = response["data"]["as"]
|
||||
# tk_value = response["data"]["tk"]
|
||||
logger.debug(f"{as_value=}, {tk_value=}")
|
||||
|
||||
# 解出 style
|
||||
backstr = captcha_data["style"]["data"]["backstr"]
|
||||
captcha_link = captcha_data["style"]["data"]["captchalist"][0]["source"]["back"]["path"]
|
||||
# response = requests.post(
|
||||
# "https://passport.baidu.com/cap/style",
|
||||
# data={
|
||||
# "_": int(time.time() * 1000),
|
||||
# "refer": re.sub(r'timestamp=\d+', f'timestamp={ts1}', captcha_data["referer"]),
|
||||
# "ak": "c27bbc89afca0463650ac9bde68ebe06",
|
||||
# "tk": tk_value,
|
||||
# "scene": "",
|
||||
# "isios": "0",
|
||||
# "type": "spin",
|
||||
# "ver": "2"
|
||||
# },
|
||||
# headers=headers,
|
||||
# proxies={
|
||||
# "http": proxy_str, "https": proxy_str
|
||||
# }
|
||||
# )
|
||||
# logger.debug(f"{response.content=}")
|
||||
# response = response.json()
|
||||
# backstr = response["data"]["backstr"]
|
||||
# captcha_link = response["data"]["captchalist"][0]["source"]["back"]["path"]
|
||||
logger.debug(f"{backstr=}, {captcha_link=}")
|
||||
|
||||
# 下载验证码图片
|
||||
image_response = requests.get(captcha_link, headers=headers)
|
||||
with open("captcha.png", "wb") as f:
|
||||
f.write(image_response.content)
|
||||
logger.debug("download captcha.png")
|
||||
|
||||
# 识别验证码
|
||||
ydm = YdmVerify()
|
||||
with open("captcha.png", "rb") as fp:
|
||||
picture = fp.read()
|
||||
|
||||
slide_distance = ydm.rotate(picture)
|
||||
logger.debug(f"{slide_distance=}")
|
||||
if not slide_distance:
|
||||
logger.error("识别验证码失败")
|
||||
return None
|
||||
rotate_angle_rate = round(slide_distance / 360, 2)
|
||||
logger.debug(f"{rotate_angle_rate=}")
|
||||
|
||||
if not rotate_angle_rate:
|
||||
logger.debug("识别验证码失败")
|
||||
return None
|
||||
|
||||
# 发送验证码请求
|
||||
time_log = str(int(time.time() * 1000))
|
||||
with open("./js/mkd_v2_link_submit.js", 'r', encoding='utf-8') as f:
|
||||
ds_js = f.read()
|
||||
fs = execjs.compile(ds_js).call('getFs2', backstr, rotate_angle_rate, as_value)
|
||||
data = {
|
||||
"_": time_log,
|
||||
"refer": captcha_data["referer"],
|
||||
# "refer": "https://aigc.baidu.com/works",
|
||||
# "ak": self.get_ak(),
|
||||
"ak": "c27bbc89afca0463650ac9bde68ebe06", # c27bbc89afca0463650ac9bde68ebe06
|
||||
# "ak": "76AKmP4xDQjB3vAIPef3KxOlJZWCpw64", # c27bbc89afca0463650ac9bde68ebe06
|
||||
"as": as_value,
|
||||
"scene": "",
|
||||
"tk": tk_value,
|
||||
"ver": "2",
|
||||
"cv": "submit",
|
||||
"typeid": "spin-0",
|
||||
# fuid 短时间不会变, 指纹, 不同浏览器不一样
|
||||
# "Edge": "FOCoIC3q5fKa8fgJnwzbE67EJ49BGJeplOzf+4l4EOvDuu2RXBRv6R3A1AZMa49I27C0gDDLrJyxcIIeAeEhD8JYsoLTpBiaCXhLqvzbzmvy3SeAW17tKgNq/Xx+RgOdb8TWCFe62MVrDTY6lMf2GrfqL8c87KLF2qFER3obJGnCfjdYr2J6wEsox+bQtrTEGEimjy3MrXEpSuItnI4KDyOhCNLIvGcJ9TrqWJqhR97vnz96e18U/ntNdoDIMLzy/6P9rOWdIYWmTQAeeG69438PcpN++VzDmPtrURexo5YYWpVFkRs9k5n0AC3djzGRuXr1+yVZXtGGofFzxBmdr9HtaANtMMPysO2XXACLNUNkLLWJN9fLc3OAWce48tpeQQ2ufd7knx4Oo6OM0PpOVDwQcezbGX85VEmymh7f7M5kIyVl+w5yn2AY4BmLrEWEsyw9SzzW8eHQ5zYIUjiw9hXi7OMMoCL+ptOvZYbHZs0R5qLHmmDCW1M8MMX5yyJF0BV1dQvKslKnAJwZu4XCbsXKn3UXosU1U30/poiN2VeXkBPeo8+Xj/4BIoC2I7WZ6zkFa/Uwd5SvC91kvff2a/Z4OwyTQNM7ES9HmRhChdWg0SJ2xEs1aiXAit16RiTlf82esJH+X/j52G7R3ErwQeJT3QoDv64R2702+8NbGIjf1ZOfxhUCpmJqV4jeHSaHRmnKgJZsK91XhhrdJKXdsbt3phIOpxGLupULr2K+v1DNdId8/HuE0776+tTpUl7shVCeM/XWrdkhru42pifhiujnDhIblsLt8grnj5/GRqcD6ZPAXqJW3lLc0/ub9jXgvXK/EczRgKl+7/tTBkPTCrUVtajA0luHLQOrVsXuN1v0/PR3i09SuFzZJkJBKE3M6rYvPttK9NQiBxhxYWDhX82uQu2XK8+8oU3gxCIaJwsQmX/It0kaZ45PZHFqtD40uOX0sXuThvUin4N4RSI2G9d7jPkj5hbBFquQKM4S+tDJ34jmplOTrqqKT7PPVfrdgd4OkK13pEy86BsJ8M0gKXgtivUgM8Bjl1m/pkg0SuDyntWLdrmMxcZYvgySvSSwQ2Qtm8EkKHIMyR/XgfHnpX5vadGpRMro2qaE8u+x8w1gJHIRKib2u6Q1JtQiZE1Rde/vRx8xKfg6uYR37n0BvfgJE5+KbeuwCyAvJRGUA2fpt0VClIfV0m2PRG7bvH00OODKY6cFi7NgWAK6Jc1G4Ugkfp7W8I0ZYwNpTTxVoxIIBF37aBhyiPWPAOeYXBqA",
|
||||
# Chrome: "FOCoIC3q5fKa8fgJnwzbE67EJ49BGJeplOzf+4l4EOvDuu2RXBRv6R3A1AZMa49IZbsw3/U3NYEqD0LjhKzgMn8fIES5OyXlgwN5I+F8wHowpWWfXkQJw8/9AsO5Q2VOvnc2JlHGIlGS8Vq2z4OA80lVLon08EG3PPxkVZGm39fDi2exK9NDrZB+tNLX6ISxE5PzBgXpCOJ6oP9F1B0OBWaCMD/m01n8FhdDNCvP8EO5cetU79+pgL+ECRdtN6V4VElGJE0mxV4+4Zq4Jf/Xe/q8CkoTNf7Ti1glGYmN32UM9dg0uX+VzET/mmTRe4Dt+MuVHSzsI/bKCjPbpaOqfM8UsxDJUG9hyrGZ8QHa1kC04aTxkkTxI275dv3+ijS1zkWOdjFiy1eD/0R8HcRWYp2smk9EmXBkIAHL4H0gC9lQtdjey37/kyl4JA9Fp4zjuVO0arsD8MrGy1divU++B1KdawGqXpnbOcHZ3CctNGrpgmswaScc6DNWb34jFj0X3tdRE0uuHuqiYa5BClFS2V0TCorKi4CobgR419xWaX8IKLJiaNNLOShWdZdlQO2DXXVxcinzKHqUvWTYx45jsiUVlY78AHQGol6CJLQQ8Q797MShlazvdSwPXgJP5z0uMJp9L+3x/Y2GGhW5sit55sFuMXafALTYf69FCUw5+nVIRs150a4+KK+tA0Eu7Itiu3dM2pflKYWwPE6SDZznyejQ08vd+HpXRB/zhfSUcIYlT5gFEiMIA6SXZCo/XT7vC8D3gHdN+yr46XdVol/WkjFQof0JQH/Vhjj5C1xcAyNxq/VVBT01vdKk6zo6c08e84FEVMLd0m3XWtjFOYu7wRI7lldw2pSxyGnWvA4aiYWcWvvKNJtqB8wHqc5RPr9KRzhbxJnTM5K1vTx4xT/1ZUR3pU7nQKZo/4kP9XycIr/Jg3XMRSnqCBUJlagKAFPt2HF0LdsSk4WWcldb97Ar584nVGbSjPXEUVH0VgbUEm+dADzPoLP+NPMYOyhwgfADiqWaXyKT4UNESYXsPBkdGk6mLCaNSEQsDN1G2677Se3qjzDcyXBnEmHEFptRbmyJzKJ73veHPqfFYtsHO9jH0XnhYk8zKdRuqQ7dnuNIDwxm3UCPo22uFI0ZcgPvQm01s+8jYiMEFJDVra9jWyWTdMpMuhT3p2yYLf70CvUwIkw=",
|
||||
# fuid.length = 1280, length 235 变化
|
||||
# FOCoIC3q5fKa8fgJnwzbE67EJ49BGJeplOzf+4l4EOvDuu2RXBRv6R3A1AZMa49I27C0gDDLrJyxcIIeAeEhD8JYsoLTpBiaCXhLqvzbzmvy3SeAW17tKgNq/Xx+RgOdb8TWCFe62MVrDTY6lMf2GrfqL8c87KLF2qFER3obJGnCfjdYr2J6wEsox+bQtrTEGEimjy3MrXEpSuItnI4KDyOhCNLIvGcJ9TrqWJqhR97
|
||||
"fuid": "FOCoIC3q5fKa8fgJnwzbE67EJ49BGJeplOzf+4l4EOvDuu2RXBRv6R3A1AZMa49I27C0gDDLrJyxcIIeAeEhD8JYsoLTpBiaCXhLqvzbzmvy3SeAW17tKgNq/Xx+RgOdb8TWCFe62MVrDTY6lMf2GrfqL8c87KLF2qFER3obJGnfaJTn/Ne60I9LwR04t6XmGEimjy3MrXEpSuItnI4KD0FJKzTbw1AN69fBnzR2FuvMmmQZ+1zgJ72wdcVU+mcQxiE2ir0+TEYgjPJt1Qa3K1mLi+P4IWJeag2lvxB4yJ/GgLbz7OSojK1zRbqBESR5Pdk2R9IA3lxxOVzA+Iw1TWLSgWjlFVG9Xmh1+20oPSbrzvDjYtVPmZ+9/6evcXmhcO1Y58MgLozKnaQIaLfWRPAn9I0uOqAMff6fuUeWcH1mRYoTw2Nhr4J4agZi377iM/izL6cVCGRy2F8c0VpEvM5FjnYxYstXg/9EfB3EVmKAfzNRIeToJ5YV9twMcgdmlV1Uhhp5FAe6gNJIUptp7EMAaXYKm11G+JVPszQFdp9AJLcm4YSsYUXkaPI2Tl66J246cmjWQDTahAOINR5rXR5r/7VVI1RMZ8gb40q7az7vCK56XLooKT5a+rsFrf5Zu0yyCiiagElhrTEOtNdBJJq8eHwEHuFBni9ahSwpC7lbKkUwaKH69tf0DFV7hJROiLETSFloIVkHdy3+I2JUr1LsplAz0hMkWt/tE4tXVUV7QcTDTZWS/2mCoS/GV3N9awQ6iM6hs/BWjlgnEa1+5iP7WSc7RJ34FaE5PsyGXyoCWdXwNRGSZPSvVtB/Ea6w5FKazXcZ/j40FJv+iLGBn3nkkgHlne61I8I7KhtQgIkmBMJIjPMkS/L051MeqdGScsKYTJuSucgI5c3+79eVH+y2TvbOTuuHv1uGxwXFb2atIU1ZYPbmmXculmizKcKI/s44qf8uM8iBZLGkKeVyL74aPyLkg7Gk359g98BIGN/ZzJR/h+Y6AyFx+HlMoYJnS06dVmqFbvlCtSdGylKQ5f8eWtxPkJGqOFtWjIVteQYMsH/AaSJonqw+WLiZvGjYfm9p0alEyujapoTy77HzDcUoU1wUSXa5xS/Z6hXEr2OnLi0LdPVcGjz8lpLcdVeSfm9p0alEyujapoTy77HzDWf5PERRSTFqLd9BTUHLyY4Ji3EQLGQPaM1aeHxG1bJZH0s1Si/KwzTaTYzu6ziQiqwcr2kaYUiH+fMOxn69/BhNJVMhpQkhprc1KZuJRvXjppq0gKweencPxgS/jd0rjw==",
|
||||
"fs": fs
|
||||
}
|
||||
# logger.info(data)
|
||||
|
||||
response = requests.post(
|
||||
"https://passport.baidu.com/cap/log",
|
||||
headers=headers,
|
||||
data=data,
|
||||
proxies={"http": proxy_str, "https": proxy_str},
|
||||
).json()
|
||||
try:
|
||||
result = {
|
||||
"ds": response["data"]["ds"],
|
||||
"op": response["data"]["op"],
|
||||
"tk": response["data"]["tk"]
|
||||
}
|
||||
except KeyError:
|
||||
logger.error(f"验证码没转成功, response: {response=}")
|
||||
time.sleep(1)
|
||||
return None
|
||||
logger.debug(f"{result=}")
|
||||
|
||||
# 检查验证码是否正确
|
||||
if result["op"] != 1:
|
||||
logger.error(f"op != 1, 重试")
|
||||
return None
|
||||
|
||||
# 发送验证码请求 /cap/c 请求，获取待跳转的URL
|
||||
response = requests.post(
|
||||
"https://passport.baidu.com/cap/c?ak=c27bbc89afca0463650ac9bde68ebe06",
|
||||
headers=headers,
|
||||
json={
|
||||
"tk": result["tk"],
|
||||
"ds": result["ds"],
|
||||
"qrsign": "",
|
||||
"refer": captcha_data["referer"]
|
||||
},
|
||||
proxies={"http": proxy_str, "https": proxy_str},
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
if data["data"].get("f"):
|
||||
logger.error(f"验证码失败: {data['data'].get('f')}")
|
||||
return None
|
||||
if data["data"].get("s"):
|
||||
logger.debug("验证成功，URL：" + data["data"].get("s").get("url"))
|
||||
url = data["data"].get("s").get("url")
|
||||
url = url.encode("utf-8").decode("unicode-escape")
|
||||
logger.success("解码后的URL：" + url)
|
||||
return url
|
||||
|
||||
|
||||
current_page = 1
|
||||
while current_page < 15:
|
||||
tab.get(f"https://www.baidu.com/s?wd=site%3Abaidu.com&pn={(current_page - 1) * 10}")
|
||||
current_page += 1
|
||||
if "wappass.baidu.com/static/captcha/tuxing_v2.html" in tab.url:
|
||||
logger.debug("captcha!!!!")
|
||||
time.sleep(2)
|
||||
|
||||
idx = 0
|
||||
while idx < 3:
|
||||
idx += 1
|
||||
url = verify_captcha(tab.url)
|
||||
if not url:
|
||||
tab.refresh()
|
||||
time.sleep(3)
|
||||
else:
|
||||
tab.get(url)
|
||||
|
||||
time.sleep(30)
|
||||
|
||||
logger.debug(f"{captcha_data=}")
|
||||
# browser.quit()
|
||||
@ -1,8 +0,0 @@
|
||||
|
||||
import certifi
|
||||
print(certifi.where())
|
||||
|
||||
|
||||
url = r"https://www.baidu.com/s?wd=site%3Abaidu.com\u0026pn=10\u0026p_tk=30610C1sd8U0U%2BPJYAWv8nhtnx0emHFxWZ9edG%2BaRz9YAiXcODGGnlpuX%2FIMRoUmFESarFc5H8HQuG2nq8%2FVXRIsPZt%2BoxjJAmxxHNGCVs0oz%2FZSTZsdUlvw5a53dshtXQASLvZg71Bg4ZT6j%2B5a%2B%2FM3CHWuHs8cjlMBRCAX4l%2BZt8k%3D\u0026p_timestamp=1744202399\u0026p_sign=a1ee13c92f54d14d019cbdd8edcb4088\u0026p_signature=737f76b967318af4b309d30784d440c5\u0026__pc2ps_ab=30610C1sd8U0U%2BPJYAWv8nhtnx0emHFxWZ9edG%2BaRz9YAiXcODGGnlpuX%2FIMRoUmFESarFc5H8HQuG2nq8%2FVXRIsPZt%2BoxjJAmxxHNGCVs0oz%2FZSTZsdUlvw5a53dshtXQASLvZg71Bg4ZT6j%2B5a%2B%2FM3CHWuHs8cjlMBRCAX4l%2BZt8k%3D|1744202399|737f76b967318af4b309d30784d440c5|a1ee13c92f54d14d019cbdd8edcb4088"
|
||||
url = url.encode("utf-8").decode("unicode-escape")
|
||||
print(url)
|
||||
Loading…
x
Reference in New Issue
Block a user