initial commit

wowhead_scraper/spiders/__init__.py

@@ -0,0 +1,7 @@
+from .object import ObjectSpider
+from .quest import QuestSpider
+
+SPIDERS = {
+    "quest": QuestSpider,
+    "object": ObjectSpider,
+}

wowhead_scraper/spiders/base.py

@@ -0,0 +1,153 @@
+import csv
+import json
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Set
+
+from config import GAME_VERSION, OUTPUT_DIR, EntityConfig
+from fetcher import Fetcher, NotFound
+from parsers.common import filter_categories, list_page
+
+log = logging.getLogger(__name__)
+
+
+class Spider:
+    """Two-stage crawler. Subclasses only say which entity it is and how to parse it.
+
+    Stage 1 (crawl_list): walk the list pages and write one CSV row per entity.
+    Stage 2 (crawl_details): visit each row's page and append one JSON line per entity.
+    """
+    entity: EntityConfig
+    listview_id: str
+    csv_fields: List[str]
+
+    def __init__(self, fetcher: Fetcher, version: str = GAME_VERSION) -> None:
+        self.fetcher = fetcher
+        self.version = version
+
+    def list_row(self, raw: Dict[str, Any], categories: Dict[int, tuple]) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    def parse_page(self, html: str, row: Dict[str, Any]) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    def crawl_list(self) -> Path:
+        """List pages cap out at 1000 rows, so split the ID range until no response is truncated.
+
+        Wowhead silently drops a filter bound above the highest ID it knows (and keeps the other),
+        so a response is a superset of the range asked for:
+          - rows below `lo`: the lower bound was dropped, so nothing exists this high; skip it.
+          - rows above `hi`: the upper bound was dropped; clipping to the range is still exact.
+        """
+        rows: Dict[int, Dict[str, Any]] = {}
+        expected_total = None
+        ranges = [(1, self.entity.max_id)]
+        while ranges:
+            lo, hi = ranges.pop()
+            html = self.fetcher.get(self.entity.list_url(lo, hi, self.version))
+            raw_rows, truncated, total = list_page(html, self.listview_id)
+            expected_total = expected_total if expected_total is not None else total
+            if any(r["id"] < lo for r in raw_rows):
+                log.info("IDs %d-%d are past the highest ID Wowhead filters on, skipping", lo, hi)
+                continue
+            raw_rows = [r for r in raw_rows if r["id"] <= hi]
+            if truncated and lo < hi:
+                mid = (lo + hi) // 2
+                ranges += [(mid + 1, hi), (lo, mid)]
+                log.info("IDs %d-%d truncated, splitting", lo, hi)
+                continue
+
+            categories = filter_categories(html)
+            for raw in raw_rows:
+                row = self.list_row(raw, categories)
+                row["url"] = self.entity.detail_url(row["id"], self.version)
+                rows[row["id"]] = row
+            log.info("IDs %d-%d: %d rows (%d total)", lo, hi, len(raw_rows), len(rows))
+
+        path = self.entity.list_csv()
+        OUTPUT_DIR.mkdir(exist_ok=True)
+        with path.open("w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=self.csv_fields)
+            writer.writeheader()
+            writer.writerows(rows[i] for i in sorted(rows))
+        log.info("Wrote %d %ss to %s", len(rows), self.entity.name, path)
+        if expected_total is not None and expected_total != len(rows):
+            log.warning("Wowhead reports %d %ss in total but %d were listed", expected_total, self.entity.name, len(rows))
+        return path
+
+    def crawl_details(self, limit: Optional[int] = None, ids: Optional[Iterable[int]] = None) -> Path:
+        """Fetch detail pages for list rows not yet in the JSONL output, so reruns resume and retry failures."""
+        rows, done = self._list_rows(), self._done_ids()
+        wanted = set(ids) if ids else None
+        todo = [r for r in rows if int(r["id"]) not in done and (wanted is None or int(r["id"]) in wanted)]
+        todo = todo[:limit] if limit else todo
+        log.info("%d %ss to fetch (%d already done)", len(todo), self.entity.name, len(done))
+        return self._fetch_all(todo)
+
+    def find_missing(self, dry_run: bool = False) -> List[int]:
+        """Re-fetch rows skipped by an earlier detail run (e.g. while blocked).
+
+        Rows are fetched in CSV order, so "skipped" means not in the output but below the highest ID
+        that is. Rows past that point simply haven't been reached yet; `detail` picks those up.
+        """
+        rows, done = self._list_rows(), self._done_ids()
+        last_done = max(done, default=0)
+        missing = [r for r in rows if int(r["id"]) not in done and int(r["id"]) < last_done]
+        not_reached = sum(1 for r in rows if int(r["id"]) > last_done)
+        missing_ids = [int(r["id"]) for r in missing]
+        log.info("%d %ss skipped below ID %d: %s%s", len(missing), self.entity.name, last_done,
+                 missing_ids[:20], " ..." if len(missing) > 20 else "")
+        log.info("%d %ss not reached yet (run `detail` for those)", not_reached, self.entity.name)
+        if missing and not dry_run:
+            self._fetch_all(missing)
+        return missing_ids
+
+    def _list_rows(self) -> List[Dict[str, Any]]:
+        list_path = self.entity.list_csv()
+        if not list_path.exists():
+            raise FileNotFoundError(f"{list_path} not found, run the list stage first")
+        with list_path.open(newline="", encoding="utf-8") as f:
+            return list(csv.DictReader(f))
+
+    def _done_ids(self) -> Set[int]:
+        out_path = self.entity.detail_jsonl()
+        if not out_path.exists():
+            return set()
+        with out_path.open(encoding="utf-8") as f:
+            return {json.loads(line)["id"] for line in f if line.strip()}
+
+    def _fetch_all(self, todo: List[Dict[str, Any]]) -> Path:
+        out_path = self.entity.detail_jsonl()
+        ok = failed = 0
+        with out_path.open("a", encoding="utf-8") as out, \
+                ThreadPoolExecutor(self.fetcher.settings.concurrency) as pool:
+            futures = {pool.submit(self._fetch_detail, row): row for row in todo}
+            try:
+                for n, future in enumerate(as_completed(futures), 1):
+                    row = futures[future]
+                    try:
+                        record = future.result()
+                    except Exception as e:
+                        failed += 1
+                        log.error("%s %s failed: %s", self.entity.name, row["id"], e)
+                        continue
+                    out.write(json.dumps(record, ensure_ascii=False) + "\n")
+                    out.flush()
+                    ok += 1
+                    if n % 50 == 0:
+                        log.info("%d/%d done", n, len(todo))
+            except KeyboardInterrupt:
+                log.warning("Interrupted, finishing in-flight requests. Rerun to resume.")
+                pool.shutdown(cancel_futures=True)
+                raise
+        log.info("Wrote %d records to %s, %d failed (rerun to retry)", ok, out_path, failed)
+        return out_path
+
+    def _fetch_detail(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        try:
+            html = self.fetcher.get(row["url"])
+        except NotFound:
+            # Recorded so reruns don't keep asking for it.
+            return {"id": int(row["id"]), "url": row["url"], "not_found": True}
+        return self.parse_page(html, row)

wowhead_scraper/spiders/object.py

@@ -0,0 +1,15 @@
+from config import ENTITIES
+from parsers import object as object_parser
+from spiders.base import Spider
+
+
+class ObjectSpider(Spider):
+    entity = ENTITIES["object"]
+    listview_id = object_parser.LISTVIEW_ID
+    csv_fields = object_parser.CSV_FIELDS
+
+    def list_row(self, raw, categories):
+        return object_parser.list_row(raw, categories)
+
+    def parse_page(self, html, row):
+        return object_parser.parse_page(html, row)

wowhead_scraper/spiders/quest.py

@@ -0,0 +1,15 @@
+from config import ENTITIES
+from parsers import quest
+from spiders.base import Spider
+
+
+class QuestSpider(Spider):
+    entity = ENTITIES["quest"]
+    listview_id = quest.LISTVIEW_ID
+    csv_fields = quest.CSV_FIELDS
+
+    def list_row(self, raw, categories):
+        return quest.list_row(raw, categories)
+
+    def parse_page(self, html, row):
+        return quest.parse_page(html, row)