import csv import json import logging from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Set from config import GAME_VERSION, OUTPUT_DIR, EntityConfig from fetcher import Fetcher, NotFound from parsers.common import filter_categories, list_page log = logging.getLogger(__name__) class Spider: """Two-stage crawler. Subclasses only say which entity it is and how to parse it. Stage 1 (crawl_list): walk the list pages and write one CSV row per entity. Stage 2 (crawl_details): visit each row's page and append one JSON line per entity. """ entity: EntityConfig listview_id: str csv_fields: List[str] def __init__(self, fetcher: Fetcher, version: str = GAME_VERSION) -> None: self.fetcher = fetcher self.version = version def list_row(self, raw: Dict[str, Any], categories: Dict[int, tuple]) -> Dict[str, Any]: raise NotImplementedError def parse_page(self, html: str, row: Dict[str, Any]) -> Dict[str, Any]: raise NotImplementedError def crawl_list(self) -> Path: """List pages cap out at 1000 rows, so split the ID range until no response is truncated. Wowhead silently drops a filter bound above the highest ID it knows (and keeps the other), so a response is a superset of the range asked for: - rows below `lo`: the lower bound was dropped, so nothing exists this high; skip it. - rows above `hi`: the upper bound was dropped; clipping to the range is still exact. """ rows: Dict[int, Dict[str, Any]] = {} expected_total = None ranges = [(1, self.entity.max_id)] while ranges: lo, hi = ranges.pop() html = self.fetcher.get(self.entity.list_url(lo, hi, self.version)) raw_rows, truncated, total = list_page(html, self.listview_id) expected_total = expected_total if expected_total is not None else total if any(r["id"] < lo for r in raw_rows): log.info("IDs %d-%d are past the highest ID Wowhead filters on, skipping", lo, hi) continue raw_rows = [r for r in raw_rows if r["id"] <= hi] if truncated and lo < hi: mid = (lo + hi) // 2 ranges += [(mid + 1, hi), (lo, mid)] log.info("IDs %d-%d truncated, splitting", lo, hi) continue categories = filter_categories(html) for raw in raw_rows: row = self.list_row(raw, categories) row["url"] = self.entity.detail_url(row["id"], self.version) rows[row["id"]] = row log.info("IDs %d-%d: %d rows (%d total)", lo, hi, len(raw_rows), len(rows)) path = self.entity.list_csv() OUTPUT_DIR.mkdir(exist_ok=True) with path.open("w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=self.csv_fields) writer.writeheader() writer.writerows(rows[i] for i in sorted(rows)) log.info("Wrote %d %ss to %s", len(rows), self.entity.name, path) if expected_total is not None and expected_total != len(rows): log.warning("Wowhead reports %d %ss in total but %d were listed", expected_total, self.entity.name, len(rows)) return path def crawl_details(self, limit: Optional[int] = None, ids: Optional[Iterable[int]] = None) -> Path: """Fetch detail pages for list rows not yet in the JSONL output, so reruns resume and retry failures.""" rows, done = self._list_rows(), self._done_ids() wanted = set(ids) if ids else None todo = [r for r in rows if int(r["id"]) not in done and (wanted is None or int(r["id"]) in wanted)] todo = todo[:limit] if limit else todo log.info("%d %ss to fetch (%d already done)", len(todo), self.entity.name, len(done)) return self._fetch_all(todo) def find_missing(self, dry_run: bool = False) -> List[int]: """Re-fetch rows skipped by an earlier detail run (e.g. while blocked). Rows are fetched in CSV order, so "skipped" means not in the output but below the highest ID that is. Rows past that point simply haven't been reached yet; `detail` picks those up. """ rows, done = self._list_rows(), self._done_ids() last_done = max(done, default=0) missing = [r for r in rows if int(r["id"]) not in done and int(r["id"]) < last_done] not_reached = sum(1 for r in rows if int(r["id"]) > last_done) missing_ids = [int(r["id"]) for r in missing] log.info("%d %ss skipped below ID %d: %s%s", len(missing), self.entity.name, last_done, missing_ids[:20], " ..." if len(missing) > 20 else "") log.info("%d %ss not reached yet (run `detail` for those)", not_reached, self.entity.name) if missing and not dry_run: self._fetch_all(missing) return missing_ids def _list_rows(self) -> List[Dict[str, Any]]: list_path = self.entity.list_csv() if not list_path.exists(): raise FileNotFoundError(f"{list_path} not found, run the list stage first") with list_path.open(newline="", encoding="utf-8") as f: return list(csv.DictReader(f)) def _done_ids(self) -> Set[int]: out_path = self.entity.detail_jsonl() if not out_path.exists(): return set() with out_path.open(encoding="utf-8") as f: return {json.loads(line)["id"] for line in f if line.strip()} def _fetch_all(self, todo: List[Dict[str, Any]]) -> Path: out_path = self.entity.detail_jsonl() ok = failed = 0 with out_path.open("a", encoding="utf-8") as out, \ ThreadPoolExecutor(self.fetcher.settings.concurrency) as pool: futures = {pool.submit(self._fetch_detail, row): row for row in todo} try: for n, future in enumerate(as_completed(futures), 1): row = futures[future] try: record = future.result() except Exception as e: failed += 1 log.error("%s %s failed: %s", self.entity.name, row["id"], e) continue out.write(json.dumps(record, ensure_ascii=False) + "\n") out.flush() ok += 1 if n % 50 == 0: log.info("%d/%d done", n, len(todo)) except KeyboardInterrupt: log.warning("Interrupted, finishing in-flight requests. Rerun to resume.") pool.shutdown(cancel_futures=True) raise log.info("Wrote %d records to %s, %d failed (rerun to retry)", ok, out_path, failed) return out_path def _fetch_detail(self, row: Dict[str, Any]) -> Dict[str, Any]: try: html = self.fetcher.get(row["url"]) except NotFound: # Recorded so reruns don't keep asking for it. return {"id": int(row["id"]), "url": row["url"], "not_found": True} return self.parse_page(html, row)