initial commit

wowhead_scraper/config.py

@@ -0,0 +1,53 @@
+"""All URL and crawl configuration lives here."""
+from dataclasses import dataclass
+from pathlib import Path
+
+BASE_DIR = Path(__file__).parent
+OUTPUT_DIR = BASE_DIR / "output"
+
+BASE_URL = "https://www.wowhead.com"
+GAME_VERSION = "forever"
+
+# Wowhead list pages never return more than this many rows per request.
+LIST_ROW_CAP = 1000
+
+
+@dataclass(frozen=True)
+class EntityConfig:
+    name: str          # "quest"
+    list_path: str     # path of the list page, e.g. "quests"
+    id_filter: int     # Wowhead filter criterion id for "ID"
+    max_id: int        # upper bound for the ID-range bisection
+
+    def list_url(self, lo: int, hi: int, version: str = GAME_VERSION) -> str:
+        # filter=<criteria>;<operators>;<values>  operators: 2 is ">=", 4 is "<="
+        f = self.id_filter
+        return f"{BASE_URL}/{version}/{self.list_path}?filter={f}:{f};2:4;{lo}:{hi}"
+
+    def detail_url(self, entity_id: int, version: str = GAME_VERSION) -> str:
+        return f"{BASE_URL}/{version}/{self.name}={entity_id}"
+
+    def list_csv(self) -> Path:
+        return OUTPUT_DIR / f"{self.list_path}.csv"
+
+    def detail_jsonl(self) -> Path:
+        return OUTPUT_DIR / f"{self.list_path}.jsonl"
+
+
+ENTITIES = {
+    "quest": EntityConfig(name="quest", list_path="quests", id_filter=30, max_id=2_000_000),
+    "object": EntityConfig(name="object", list_path="objects", id_filter=15, max_id=2_000_000),
+    # To add items: look up the "ID" filter id in Filter.init on the /items list page.
+}
+
+
+@dataclass
+class CrawlSettings:
+    delay: float = 1.0           # minimum seconds between requests (shared across workers)
+    jitter: float = 2.5          # extra random delay, 0..jitter seconds
+    concurrency: int = 2         # parallel workers
+    timeout: float = 30.0
+    max_retries: int = 5
+    backoff_base: float = 2.0    # retry n waits backoff_base * 2**n seconds (capped)
+    backoff_max: float = 120.0
+    impersonate: str = "chrome"  # curl_cffi browser fingerprint