42 lines
2.2 KiB
Python
42 lines
2.2 KiB
Python
import logging
|
|
from argparse import ArgumentParser
|
|
|
|
from config import GAME_VERSION, CrawlSettings
|
|
from fetcher import Fetcher
|
|
from spiders import SPIDERS
|
|
|
|
|
|
def main() -> None:
|
|
defaults = CrawlSettings()
|
|
parser = ArgumentParser(description="Scrape Wowhead in two stages: list -> CSV, then detail -> JSONL.")
|
|
parser.add_argument("stage", choices=["list", "detail", "all", "find-missing"])
|
|
parser.add_argument("target", choices=sorted(SPIDERS))
|
|
parser.add_argument("-v", "--version", default=GAME_VERSION, help=f"Wowhead path prefix. Default: {GAME_VERSION}")
|
|
parser.add_argument("--delay", type=float, default=defaults.delay, help="Min seconds between requests")
|
|
parser.add_argument("--jitter", type=float, default=defaults.jitter, help="Extra random delay, 0..N seconds")
|
|
parser.add_argument("--concurrency", type=int, default=defaults.concurrency, help="Parallel detail workers")
|
|
parser.add_argument("--max-retries", type=int, default=defaults.max_retries)
|
|
parser.add_argument("--backoff", type=float, default=defaults.backoff_base, help="First retry wait in seconds, doubles each retry")
|
|
parser.add_argument("--limit", type=int, help="Detail stage: only fetch this many pages")
|
|
parser.add_argument("--ids", type=int, nargs="+", help="Detail stage: only fetch these IDs")
|
|
parser.add_argument("--dry-run", action="store_true", help="find-missing: only list the skipped IDs")
|
|
parser.add_argument("--debug", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
settings = CrawlSettings(delay=args.delay, jitter=args.jitter, concurrency=args.concurrency,
|
|
max_retries=args.max_retries, backoff_base=args.backoff)
|
|
spider = SPIDERS[args.target](Fetcher(settings), version=args.version)
|
|
|
|
if args.stage in ("list", "all"):
|
|
spider.crawl_list()
|
|
if args.stage in ("detail", "all"):
|
|
spider.crawl_details(limit=args.limit, ids=args.ids)
|
|
if args.stage == "find-missing":
|
|
spider.find_missing(dry_run=args.dry_run)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|