54 lines
1.9 KiB
Python
54 lines
1.9 KiB
Python
"""All URL and crawl configuration lives here."""
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
BASE_DIR = Path(__file__).parent
|
|
OUTPUT_DIR = BASE_DIR / "output"
|
|
|
|
BASE_URL = "https://www.wowhead.com"
|
|
GAME_VERSION = "forever"
|
|
|
|
# Wowhead list pages never return more than this many rows per request.
|
|
LIST_ROW_CAP = 1000
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EntityConfig:
|
|
name: str # "quest"
|
|
list_path: str # path of the list page, e.g. "quests"
|
|
id_filter: int # Wowhead filter criterion id for "ID"
|
|
max_id: int # upper bound for the ID-range bisection
|
|
|
|
def list_url(self, lo: int, hi: int, version: str = GAME_VERSION) -> str:
|
|
# filter=<criteria>;<operators>;<values> operators: 2 is ">=", 4 is "<="
|
|
f = self.id_filter
|
|
return f"{BASE_URL}/{version}/{self.list_path}?filter={f}:{f};2:4;{lo}:{hi}"
|
|
|
|
def detail_url(self, entity_id: int, version: str = GAME_VERSION) -> str:
|
|
return f"{BASE_URL}/{version}/{self.name}={entity_id}"
|
|
|
|
def list_csv(self) -> Path:
|
|
return OUTPUT_DIR / f"{self.list_path}.csv"
|
|
|
|
def detail_jsonl(self) -> Path:
|
|
return OUTPUT_DIR / f"{self.list_path}.jsonl"
|
|
|
|
|
|
ENTITIES = {
|
|
"quest": EntityConfig(name="quest", list_path="quests", id_filter=30, max_id=2_000_000),
|
|
"object": EntityConfig(name="object", list_path="objects", id_filter=15, max_id=2_000_000),
|
|
# To add items: look up the "ID" filter id in Filter.init on the /items list page.
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class CrawlSettings:
|
|
delay: float = 1.0 # minimum seconds between requests (shared across workers)
|
|
jitter: float = 2.5 # extra random delay, 0..jitter seconds
|
|
concurrency: int = 2 # parallel workers
|
|
timeout: float = 30.0
|
|
max_retries: int = 5
|
|
backoff_base: float = 2.0 # retry n waits backoff_base * 2**n seconds (capped)
|
|
backoff_max: float = 120.0
|
|
impersonate: str = "chrome" # curl_cffi browser fingerprint
|