initial commit

wowhead_scraper/runner.py

@@ -0,0 +1,41 @@
+import logging
+from argparse import ArgumentParser
+
+from config import GAME_VERSION, CrawlSettings
+from fetcher import Fetcher
+from spiders import SPIDERS
+
+
+def main() -> None:
+    defaults = CrawlSettings()
+    parser = ArgumentParser(description="Scrape Wowhead in two stages: list -> CSV, then detail -> JSONL.")
+    parser.add_argument("stage", choices=["list", "detail", "all", "find-missing"])
+    parser.add_argument("target", choices=sorted(SPIDERS))
+    parser.add_argument("-v", "--version", default=GAME_VERSION, help=f"Wowhead path prefix. Default: {GAME_VERSION}")
+    parser.add_argument("--delay", type=float, default=defaults.delay, help="Min seconds between requests")
+    parser.add_argument("--jitter", type=float, default=defaults.jitter, help="Extra random delay, 0..N seconds")
+    parser.add_argument("--concurrency", type=int, default=defaults.concurrency, help="Parallel detail workers")
+    parser.add_argument("--max-retries", type=int, default=defaults.max_retries)
+    parser.add_argument("--backoff", type=float, default=defaults.backoff_base, help="First retry wait in seconds, doubles each retry")
+    parser.add_argument("--limit", type=int, help="Detail stage: only fetch this many pages")
+    parser.add_argument("--ids", type=int, nargs="+", help="Detail stage: only fetch these IDs")
+    parser.add_argument("--dry-run", action="store_true", help="find-missing: only list the skipped IDs")
+    parser.add_argument("--debug", action="store_true")
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO,
+                        format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+    settings = CrawlSettings(delay=args.delay, jitter=args.jitter, concurrency=args.concurrency,
+                             max_retries=args.max_retries, backoff_base=args.backoff)
+    spider = SPIDERS[args.target](Fetcher(settings), version=args.version)
+
+    if args.stage in ("list", "all"):
+        spider.crawl_list()
+    if args.stage in ("detail", "all"):
+        spider.crawl_details(limit=args.limit, ids=args.ids)
+    if args.stage == "find-missing":
+        spider.find_missing(dry_run=args.dry_run)
+
+
+if __name__ == "__main__":
+    main()