initial commit

wowhead_scraper/.gitignore

@@ -0,0 +1,8 @@
+**/*.html
+**/*.json
+output/
+.idea/
+venv/
+**/__pycache__
+!tests/fixtures/*.html
+.pytest_cache/

wowhead_scraper/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Björn Böing
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

wowhead_scraper/README.md

@@ -0,0 +1,71 @@
+# Wowhead Scraper
+
+Scrapes quests and objects from [Wowhead](https://www.wowhead.com/forever) in two stages:
+
+1. **List**: crawls the list pages and writes one CSV row per entity (`output/quests.csv`, `output/objects.csv`).
+2. **Detail**: visits each row's page and appends one JSON object per line (`output/quests.jsonl`, `output/objects.jsonl`).
+
+## Setup
+
+```
+python -m venv venv
+venv\Scripts\activate
+pip install -r requirements.txt
+playwright install chromium
+```
+
+Wowhead is behind an AWS WAF JavaScript challenge. Requests go through `curl_cffi` with a Chrome
+fingerprint; when the challenge appears, headless Chromium solves it once and its cookie is reused.
+
+## Usage
+
+```
+python runner.py list quest            # -> output/quests.csv
+python runner.py detail quest          # -> output/quests.jsonl
+python runner.py all object            # both stages
+python runner.py detail quest --ids 166 98028
+python runner.py detail quest --limit 50
+python runner.py find-missing quest    # re-fetch quests skipped during a detail run
+python runner.py find-missing quest --dry-run   # only list them
+```
+
+`find-missing` only touches rows *below* the highest ID already in the JSONL, i.e. ones a detail run
+skipped (usually because a temporary block outlasted the retries). Rows past that point haven't been
+reached yet; `detail` continues with those.
+
+The detail stage skips IDs already in the JSONL file, so an interrupted run (or one with failures)
+resumes where it left off when rerun. Pages Wowhead doesn't have are written as `{"id": ..., "not_found": true}`.
+
+| Option          | Default   | Description                                             |
+|-----------------|-----------|---------------------------------------------------------|
+| `-v, --version` | `forever` | Wowhead path prefix                                     |
+| `--delay`       | `1.0`     | Minimum seconds between requests (across all workers)   |
+| `--jitter`      | `0.5`     | Extra random delay, 0..N seconds                        |
+| `--concurrency` | `2`       | Parallel detail workers                                 |
+| `--max-retries` | `5`       | Retries on 403/429/5xx/network errors                   |
+| `--backoff`     | `2.0`     | First retry wait in seconds; doubles each retry, max 120 |
+
+## How list retrieval works
+
+A list page holds at most 1000 rows (the `#200` style pagination is client side only), so the list
+stage requests ID ranges with Wowhead's ID filter and splits any range whose result was truncated.
+A full quest or object list takes about 30 requests. The final count is checked against Wowhead's
+own "N found" total.
+
+## Layout
+
+| Path            | Contents                                                   |
+|-----------------|------------------------------------------------------------|
+| `config.py`     | All URLs, filter ids and crawl settings                    |
+| `fetcher.py`    | HTTP client: throttle, back-off, WAF handling              |
+| `parsers/`      | Pure functions from page HTML to dicts, one module per type |
+| `spiders/`      | `Spider` base class with both stages; thin subclasses per type |
+| `tests/`        | Parser tests against `example_pages/` and `tests/fixtures/` |
+
+### Adding a type (e.g. items)
+
+1. Add an `EntityConfig` to `ENTITIES` in `config.py` (the "ID" filter id is in `Filter.init` on the list page).
+2. Write `parsers/item.py` with `LISTVIEW_ID`, `CSV_FIELDS`, `list_row()` and `parse_page()`.
+3. Add a `Spider` subclass in `spiders/` and register it in `SPIDERS`.
+
+Run the tests with `pytest`.

wowhead_scraper/config.py

@@ -0,0 +1,53 @@
+"""All URL and crawl configuration lives here."""
+from dataclasses import dataclass
+from pathlib import Path
+
+BASE_DIR = Path(__file__).parent
+OUTPUT_DIR = BASE_DIR / "output"
+
+BASE_URL = "https://www.wowhead.com"
+GAME_VERSION = "forever"
+
+# Wowhead list pages never return more than this many rows per request.
+LIST_ROW_CAP = 1000
+
+
+@dataclass(frozen=True)
+class EntityConfig:
+    name: str          # "quest"
+    list_path: str     # path of the list page, e.g. "quests"
+    id_filter: int     # Wowhead filter criterion id for "ID"
+    max_id: int        # upper bound for the ID-range bisection
+
+    def list_url(self, lo: int, hi: int, version: str = GAME_VERSION) -> str:
+        # filter=<criteria>;<operators>;<values>  operators: 2 is ">=", 4 is "<="
+        f = self.id_filter
+        return f"{BASE_URL}/{version}/{self.list_path}?filter={f}:{f};2:4;{lo}:{hi}"
+
+    def detail_url(self, entity_id: int, version: str = GAME_VERSION) -> str:
+        return f"{BASE_URL}/{version}/{self.name}={entity_id}"
+
+    def list_csv(self) -> Path:
+        return OUTPUT_DIR / f"{self.list_path}.csv"
+
+    def detail_jsonl(self) -> Path:
+        return OUTPUT_DIR / f"{self.list_path}.jsonl"
+
+
+ENTITIES = {
+    "quest": EntityConfig(name="quest", list_path="quests", id_filter=30, max_id=2_000_000),
+    "object": EntityConfig(name="object", list_path="objects", id_filter=15, max_id=2_000_000),
+    # To add items: look up the "ID" filter id in Filter.init on the /items list page.
+}
+
+
+@dataclass
+class CrawlSettings:
+    delay: float = 1.0           # minimum seconds between requests (shared across workers)
+    jitter: float = 2.5          # extra random delay, 0..jitter seconds
+    concurrency: int = 2         # parallel workers
+    timeout: float = 30.0
+    max_retries: int = 5
+    backoff_base: float = 2.0    # retry n waits backoff_base * 2**n seconds (capped)
+    backoff_max: float = 120.0
+    impersonate: str = "chrome"  # curl_cffi browser fingerprint

wowhead_scraper/fetcher.py

@@ -0,0 +1,121 @@
+"""HTTP client: browser-impersonating requests with a shared throttle, back-off and WAF handling.
+
+Wowhead sits behind an AWS WAF JavaScript challenge. curl_cffi alone can't solve it, so when a
+challenge page comes back we load the page once in headless Chromium (Playwright), take the
+`aws-waf-token` cookie it earns, and keep using the fast curl_cffi client with that cookie.
+"""
+import logging
+import random
+import threading
+import time
+
+from curl_cffi import requests
+from curl_cffi.requests.exceptions import RequestException
+
+from config import CrawlSettings
+
+log = logging.getLogger(__name__)
+
+# Playwright's default UA says "HeadlessChrome", which CloudFront rejects outright.
+BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+              "(KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36")
+RETRY_STATUSES = {403, 408, 429, 500, 502, 503, 504}
+
+
+class FetchError(Exception):
+    pass
+
+
+class NotFound(FetchError):
+    pass
+
+
+def is_challenge(html: str) -> bool:
+    # Real pages also set awsWafCookieDomainList, so look for the challenge loader itself.
+    return "AwsWafIntegration" in html and "challenge.js" in html
+
+
+class Fetcher:
+    def __init__(self, settings: CrawlSettings = None) -> None:
+        self.settings = settings or CrawlSettings()
+        self._local = threading.local()
+        self._throttle_lock = threading.Lock()
+        self._next_request_at = 0.0
+        self._waf_lock = threading.Lock()
+        self._cookies: dict = {}
+        self._cookie_generation = 0
+
+    def get(self, url: str) -> str:
+        s = self.settings
+        for attempt in range(s.max_retries + 1):
+            generation = self._cookie_generation
+            self._throttle()
+            try:
+                r = self._session().get(url, cookies=self._cookies, timeout=s.timeout)
+                status, text = r.status_code, r.text
+            except RequestException as e:
+                status, text, reason = None, "", f"{type(e).__name__}: {e}"
+            else:
+                if "notFound=" in r.url or status == 404:
+                    raise NotFound(url)
+                if is_challenge(text):
+                    self._solve_challenge(url, generation)
+                    continue
+                if status == 200:
+                    return text
+                if status not in RETRY_STATUSES:
+                    raise FetchError(f"HTTP {status} for {url}")
+                reason = f"HTTP {status}"
+
+            if attempt == s.max_retries:
+                break
+            wait = min(s.backoff_max, s.backoff_base * 2 ** attempt) + random.uniform(0, s.jitter)
+            log.warning("%s for %s, retry %d/%d in %.1fs", reason, url, attempt + 1, s.max_retries, wait)
+            time.sleep(wait)
+        raise FetchError(f"Giving up on {url} after {s.max_retries} retries")
+
+    def _session(self) -> requests.Session:
+        # curl_cffi sessions aren't thread-safe, so each worker thread gets its own.
+        if not hasattr(self._local, "session"):
+            self._local.session = requests.Session(impersonate=self.settings.impersonate)
+        return self._local.session
+
+    def _throttle(self) -> None:
+        with self._throttle_lock:
+            now = time.monotonic()
+            start_at = max(now, self._next_request_at)
+            self._next_request_at = start_at + self.settings.delay + random.uniform(0, self.settings.jitter)
+        time.sleep(start_at - now)
+
+    def _solve_challenge(self, url: str, seen_generation: int) -> None:
+        with self._waf_lock:
+            if self._cookie_generation != seen_generation:
+                return  # another worker refreshed the token while we waited
+            log.info("WAF challenge hit, solving it in headless Chromium")
+            self._cookies = solve_waf_challenge(url, self.settings.timeout)
+            self._cookie_generation += 1
+
+
+def solve_waf_challenge(url: str, timeout: float) -> dict:
+    from playwright.sync_api import Error as PlaywrightError, sync_playwright
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True, args=["--disable-blink-features=AutomationControlled"])
+        try:
+            context = browser.new_context(user_agent=BROWSER_UA, locale="en-US")
+            page = context.new_page()
+            page.goto(url, wait_until="domcontentloaded", timeout=timeout * 1000)
+            deadline = time.monotonic() + timeout
+            # The challenge script sets the cookie, then reloads into the real page.
+            while time.monotonic() < deadline:
+                cookies = {c["name"]: c["value"] for c in context.cookies()}
+                try:
+                    solved = "aws-waf-token" in cookies and not is_challenge(page.content())
+                except PlaywrightError:  # page is mid-reload
+                    solved = False
+                if solved:
+                    return cookies
+                page.wait_for_timeout(500)
+        finally:
+            browser.close()
+    raise FetchError(f"Could not solve WAF challenge for {url}")

wowhead_scraper/parsers/common.py

@@ -0,0 +1,142 @@
+"""Extractors for the data blobs Wowhead embeds in its pages.
+
+Almost everything worth having is in inline JSON / JS object literals rather than in the HTML,
+so these helpers find a blob by its marker and hand it to chompjs (which accepts unquoted keys
+and bare identifiers that plain json.loads would reject).
+"""
+import json
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+import chompjs
+from parsel import Selector
+
+from config import LIST_ROW_CAP
+
+SIDES = {1: "Alliance", 2: "Horde", 3: "Both"}
+
+_GATHERER_TYPES = {1: "npc", 2: "object", 3: "item", 5: "quest"}
+_ENTITY_LINK = re.compile(r"\[url=/\w+/(npc|object|item|quest|zone)=(\d+)[^\]]*\](.*?)\[/url\]")
+_MARKUP_TAG = re.compile(r"\[/?[a-z]+[^\]]*\]")
+_SMALL_WORDS = {"a", "an", "and", "at", "for", "in", "of", "on", "the", "to"}
+
+
+def js_object_after(html: str, marker: str) -> Optional[Any]:
+    """Parse the JS object/array literal that starts right after `marker`."""
+    i = html.find(marker)
+    if i < 0:
+        return None
+    return chompjs.parse_js_object(html[i + len(marker):])
+
+
+def json_script(html: str, script_id: str) -> Optional[Any]:
+    m = re.search(rf'<script type="application/json" id="{re.escape(script_id)}">(.*?)</script>', html, re.S)
+    return json.loads(m.group(1)) if m else None
+
+
+def listview(html: str, listview_id: str) -> Optional[Dict[str, Any]]:
+    """Return a `new Listview({...})` config by id. List pages may use either embedding style."""
+    for m in re.finditer(r"new Listview\(\{", html):
+        head = html[m.end():m.end() + 300]
+        if re.search(rf"\bid:\s*'{re.escape(listview_id)}'", head):
+            return chompjs.parse_js_object(html[m.end() - 1:])
+    for lv in json_script(html, "data.page.listPage.listviews") or []:
+        if lv.get("id") == listview_id:
+            return lv
+    return None
+
+
+def filter_categories(html: str) -> Dict[int, Tuple[str, Dict[int, str]]]:
+    """Category tree from the list page's Filter.init config: {id: (name, {sub_id: sub_name})}.
+
+    Top-level and sub-category ids overlap (Kalimdor and Dun Morogh are both 1), hence the nesting.
+    Names come from url slugs, so "elwynn-forest" becomes "Elwynn Forest".
+    """
+    config = js_object_after(html, "Filter.init(") or {}
+
+    def name(c):
+        words = c["url"].split("-")
+        return " ".join(w if i and w in _SMALL_WORDS else w.capitalize() for i, w in enumerate(words))
+
+    return {int(c["id"]): (name(c), {int(s["id"]): name(s) for s in c.get("categories") or []})
+            for c in config.get("categories") or []}
+
+
+def list_page(html: str, listview_id: str) -> Tuple[List[Dict[str, Any]], bool, Optional[int]]:
+    """Rows of a list page, whether Wowhead truncated them at its row cap, and its total match count."""
+    lv = listview(html, listview_id) or {}
+    rows = lv.get("data") or []
+    truncated = bool(lv.get("_truncated")) or len(rows) >= LIST_ROW_CAP
+    # "5,442 quests found (1,000 displayed)" only appears when truncated
+    m = re.match(r"([\d,]+) \w+ found", lv.get("note") or "")
+    total = lv.get("totalCount") or (int(m.group(1).replace(",", "")) if m else None)
+    return rows, truncated, total if truncated else len(rows)
+
+
+def entity_blob(html: str, kind: str, entity_id: int) -> Optional[Dict[str, Any]]:
+    """The `$.extend(g_quests[123], {...})` record a detail page carries for itself."""
+    return js_object_after(html, f"$.extend(g_{kind}s[{entity_id}], ")
+
+
+def gatherer(html: str) -> Dict[str, Dict[int, str]]:
+    """Names of every entity the page references, keyed by type then id."""
+    names: Dict[str, Dict[int, str]] = {}
+    for m in re.finditer(r"WH\.Gatherer\.addData\((\d+), \d+, ", html):
+        kind = _GATHERER_TYPES.get(int(m.group(1)))
+        if kind:
+            for eid, data in chompjs.parse_js_object(html[m.end():]).items():
+                names.setdefault(kind, {})[int(eid)] = data.get("name_enus")
+    return names
+
+
+def quick_facts(html: str) -> List[str]:
+    """The "Quick Facts" infobox as a list of raw markup lines ([li] items)."""
+    m = re.search(r'WH\.markup\.printHtml\(("(?:[^"\\]|\\.)*"), "infobox-contents-0"', html)
+    if not m:
+        return []
+    markup = json.loads(m.group(1))
+    return re.findall(r"\[li\](.*?)\[/li\]", markup)
+
+
+def markup_text(markup: str) -> str:
+    return " ".join(_MARKUP_TAG.sub("", markup).replace("&nbsp;", " ").split())
+
+
+def markup_links(markup: str) -> List[Dict[str, Any]]:
+    return [{"type": t, "id": int(i), "name": markup_text(n)} for t, i, n in _ENTITY_LINK.findall(markup)]
+
+
+def mapper(html: str) -> Optional[Dict[str, Any]]:
+    return js_object_after(html, "new Mapper(")
+
+
+def selector(html: str) -> Selector:
+    # Keep <br> line breaks in extracted text.
+    return Selector(text=re.sub(r"<br\s*/?>", "\n", html))
+
+
+def text_after(sel: Selector, xpath: str, stop_tags=("h2",)) -> Optional[str]:
+    """Text following the first node matching `xpath`, up to the first sibling in `stop_tags`."""
+    found = sel.xpath(xpath)
+    if not found:
+        return None
+    node = found[0].root
+    parts = [node.tail or ""]
+    for sibling in node.itersiblings():
+        if sibling.tag in stop_tags:
+            break
+        if sibling.tag not in ("script", "table"):
+            parts.append("".join(sibling.itertext()))
+        parts.append(sibling.tail or "")
+    return clean_text("".join(parts))
+
+
+def section_text(sel: Selector, heading: str) -> Optional[str]:
+    """Text of an <h2> section such as Description, Progress or Completion."""
+    return text_after(sel, f"//h2[normalize-space(.)='{heading}']")
+
+
+def clean_text(text: str) -> Optional[str]:
+    lines = [" ".join(line.split()) for line in text.splitlines()]
+    text = "\n".join(lines).strip()
+    return re.sub(r"\n{3,}", "\n\n", text) or None

wowhead_scraper/parsers/object.py

@@ -0,0 +1,68 @@
+from typing import Any, Dict, List
+
+from parsers.common import SIDES, entity_blob, js_object_after, listview, markup_text, quick_facts
+
+LISTVIEW_ID = "objects"
+CSV_FIELDS = ["id", "name", "type", "type_id", "skill", "zone_ids", "url"]
+
+
+def list_row(raw: Dict[str, Any], categories: Dict[int, tuple]) -> Dict[str, Any]:
+    type_name, _ = categories.get(raw.get("type"), (None, {}))
+    return {
+        "id": raw["id"],
+        "name": raw.get("name"),
+        "type": type_name,
+        "type_id": raw.get("type"),
+        "skill": raw.get("skill"),
+        # -1 entries are placeholders for "unknown location"
+        "zone_ids": ";".join(str(z) for z in raw.get("location", []) if z >= 0),
+    }
+
+
+def parse_page(html: str, row: Dict[str, Any]) -> Dict[str, Any]:
+    object_id = int(row["id"])
+    blob = entity_blob(html, "object", object_id) or {}
+    return {
+        "id": object_id,
+        "name": blob.get("name", row.get("name")),
+        "type": row.get("type"),
+        "type_id": blob.get("type"),
+        "skill": int(row["skill"]) if row.get("skill") not in (None, "") else None,
+        "locations": _locations(js_object_after(html, "var g_mapperData = ")),
+        "starts": _quests(html, "starts"),
+        "ends": _quests(html, "ends"),
+        "contains": _loot(html),
+        "quick_facts": [markup_text(f) for f in quick_facts(html)],
+        "url": row.get("url"),
+    }
+
+
+def _locations(mapper_data) -> List[Dict[str, Any]]:
+    locations = []
+    for zone_id, floors in (mapper_data or {}).items():
+        for floor in floors:
+            locations.append({
+                "zone_id": int(zone_id),
+                "zone": floor.get("uiMapName"),
+                "ui_map_id": floor.get("uiMapId"),
+                "count": floor.get("count"),
+                "coords": floor.get("coords"),
+            })
+    return locations
+
+
+def _quests(html: str, listview_id: str) -> List[Dict[str, Any]]:
+    lv = listview(html, listview_id) or {}
+    return [{"id": q["id"], "name": q.get("name"), "level": q.get("level"),
+             "reqlevel": q.get("reqlevel"), "side": SIDES.get(q.get("side"))}
+            for q in lv.get("data") or []]
+
+
+def _loot(html: str) -> List[Dict[str, Any]]:
+    lv = listview(html, "contains") or {}
+    opened = lv.get("_totalCount")
+    return [{"id": i["id"], "name": i.get("name"), "count": i.get("count"),
+             # same figure Wowhead shows in its "%" column
+             "percent": round(100 * i["count"] / opened, 2) if opened and i.get("count", -1) > 0 else None,
+             "stack": i.get("stack")}
+            for i in lv.get("data") or []]

wowhead_scraper/parsers/quest.py

@@ -0,0 +1,131 @@
+import re
+from typing import Any, Dict, List
+
+from parsers.common import (SIDES, entity_blob, gatherer, markup_links, markup_text, mapper, quick_facts,
+                            section_text, selector, text_after)
+
+LISTVIEW_ID = "quests"
+CSV_FIELDS = ["id", "name", "level", "reqlevel", "side", "category", "category_id",
+              "category2", "category2_id", "exp", "money", "url"]
+
+_MAPPER_TYPES = {1: "npc", 2: "object", 3: "item"}
+
+
+def list_row(raw: Dict[str, Any], categories: Dict[int, tuple]) -> Dict[str, Any]:
+    top_name, sub_names = categories.get(raw.get("category2"), (None, {}))
+    return {
+        "id": raw["id"],
+        "name": raw.get("name"),
+        "level": raw.get("level"),
+        "reqlevel": raw.get("reqlevel"),
+        "side": SIDES.get(raw.get("side")),
+        "category": sub_names.get(raw.get("category")),
+        "category_id": raw.get("category"),
+        "category2": top_name,
+        "category2_id": raw.get("category2"),
+        "exp": raw.get("xp"),
+        "money": raw.get("money"),
+    }
+
+
+def parse_page(html: str, row: Dict[str, Any]) -> Dict[str, Any]:
+    quest_id = int(row["id"])
+    sel = selector(html)
+    blob = entity_blob(html, "quest", quest_id) or {}
+    names = gatherer(html)
+    facts = quick_facts(html)
+
+    return {
+        "id": quest_id,
+        "name": blob.get("name", row.get("name")),
+        "level": blob.get("level"),
+        "reqlevel": blob.get("reqlevel"),
+        "side": SIDES.get(blob.get("side")),
+        "category": row.get("category"),
+        "category2": row.get("category2"),
+        "objectives": text_after(sel, "//h1[@class='heading-size-1']", stop_tags=("table", "div", "h2", "script")),
+        "objective_items": _objective_items(sel),
+        "description": section_text(sel, "Description"),
+        "progress": section_text(sel, "Progress"),
+        "completion": section_text(sel, "Completion"),
+        "start": _fact_links(facts, "Start:"),
+        "end": _fact_links(facts, "End:"),
+        "locations": _locations(mapper(html)),
+        "series": _series(sel, quest_id),
+        "rewards": {
+            "xp": blob.get("xp"),
+            # the page's record omits money when it is 0
+            "money": blob.get("money", 0),
+            "item_choices": _items(blob.get("itemchoices"), names),
+            "items": _items(blob.get("itemrewards"), names),
+            "reputation": _reputation(sel, blob.get("reprewards")),
+        },
+        "quick_facts": [markup_text(f) for f in facts],
+        "url": row.get("url"),
+    }
+
+
+def _fact_links(facts: List[str], label: str) -> List[Dict[str, Any]]:
+    for fact in facts:
+        if markup_text(fact).startswith(label):
+            return markup_links(fact)
+    return []
+
+
+def _locations(mapper_config) -> List[Dict[str, Any]]:
+    """Flatten mapper points: who/what is where, and its role (start, end, objective)."""
+    points = []
+    for zone_id, zone in ((mapper_config or {}).get("objectives") or {}).items():
+        for level in zone.get("levels", []):
+            for p in level:
+                points.append({
+                    "role": p.get("point"),
+                    "type": _MAPPER_TYPES.get(p.get("type"), p.get("type")),
+                    "id": p.get("id"),
+                    "name": p.get("name"),
+                    "zone_id": int(zone_id),
+                    "zone": zone.get("zone"),
+                    "coord": p.get("coord"),
+                    "coords": p.get("coords"),
+                })
+    return points
+
+
+def _objective_items(sel) -> List[Dict[str, Any]]:
+    result = []
+    for tr in sel.xpath("(//table[@class='icon-list'])[1]//tr"):
+        href = tr.xpath(".//a/@href").get() or ""
+        m = re.search(r"/(\w+)=(\d+)", href)
+        # A count for items/kills; a standing such as "Friendly" for faction objectives.
+        quantity = tr.attrib.get("data-icon-list-quantity", "1")
+        result.append({
+            "type": m.group(1) if m else None,
+            "id": int(m.group(2)) if m else None,
+            "name": " ".join(tr.xpath(".//a//text()").getall()).strip() or None,
+            "quantity": int(quantity) if quantity.isdigit() else None,
+            "standing": None if quantity.isdigit() else quantity,
+            "provided": "(Provided)" in "".join(tr.xpath(".//td//text()").getall()),
+        })
+    return result
+
+
+def _series(sel, quest_id: int) -> List[Dict[str, Any]]:
+    series = []
+    for td in sel.xpath("//table[@class='series']//tr/td"):
+        href = td.xpath(".//a/@href").get()
+        m = re.search(r"quest=(\d+)", href or "")
+        series.append({"id": int(m.group(1)) if m else quest_id,
+                       "name": " ".join(td.xpath(".//text()").getall()).strip()})
+    return series
+
+
+def _items(pairs, names) -> List[Dict[str, Any]]:
+    return [{"id": i, "name": names.get("item", {}).get(i), "count": c} for i, c in pairs or []]
+
+
+def _reputation(sel, pairs) -> List[Dict[str, Any]]:
+    factions = {}
+    for a in sel.xpath("//a[contains(@href, '/faction=')]"):
+        m = re.search(r"faction=(\d+)", a.attrib["href"])
+        factions[int(m.group(1))] = a.xpath("string()").get().strip()
+    return [{"faction_id": f, "faction": factions.get(f), "value": v} for f, v in pairs or []]

wowhead_scraper/pyproject.toml

@@ -0,0 +1,3 @@
+[tool.pytest.ini_options]
+pythonpath = ["."]
+testpaths = ["tests"]

wowhead_scraper/requirements.txt

@@ -0,0 +1,5 @@
+chompjs==1.4.1
+curl_cffi==0.17.0
+parsel==1.12.1
+playwright==1.64.1
+pytest==9.1.1

wowhead_scraper/runner.py

@@ -0,0 +1,41 @@
+import logging
+from argparse import ArgumentParser
+
+from config import GAME_VERSION, CrawlSettings
+from fetcher import Fetcher
+from spiders import SPIDERS
+
+
+def main() -> None:
+    defaults = CrawlSettings()
+    parser = ArgumentParser(description="Scrape Wowhead in two stages: list -> CSV, then detail -> JSONL.")
+    parser.add_argument("stage", choices=["list", "detail", "all", "find-missing"])
+    parser.add_argument("target", choices=sorted(SPIDERS))
+    parser.add_argument("-v", "--version", default=GAME_VERSION, help=f"Wowhead path prefix. Default: {GAME_VERSION}")
+    parser.add_argument("--delay", type=float, default=defaults.delay, help="Min seconds between requests")
+    parser.add_argument("--jitter", type=float, default=defaults.jitter, help="Extra random delay, 0..N seconds")
+    parser.add_argument("--concurrency", type=int, default=defaults.concurrency, help="Parallel detail workers")
+    parser.add_argument("--max-retries", type=int, default=defaults.max_retries)
+    parser.add_argument("--backoff", type=float, default=defaults.backoff_base, help="First retry wait in seconds, doubles each retry")
+    parser.add_argument("--limit", type=int, help="Detail stage: only fetch this many pages")
+    parser.add_argument("--ids", type=int, nargs="+", help="Detail stage: only fetch these IDs")
+    parser.add_argument("--dry-run", action="store_true", help="find-missing: only list the skipped IDs")
+    parser.add_argument("--debug", action="store_true")
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO,
+                        format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+    settings = CrawlSettings(delay=args.delay, jitter=args.jitter, concurrency=args.concurrency,
+                             max_retries=args.max_retries, backoff_base=args.backoff)
+    spider = SPIDERS[args.target](Fetcher(settings), version=args.version)
+
+    if args.stage in ("list", "all"):
+        spider.crawl_list()
+    if args.stage in ("detail", "all"):
+        spider.crawl_details(limit=args.limit, ids=args.ids)
+    if args.stage == "find-missing":
+        spider.find_missing(dry_run=args.dry_run)
+
+
+if __name__ == "__main__":
+    main()

wowhead_scraper/spiders/__init__.py

@@ -0,0 +1,7 @@
+from .object import ObjectSpider
+from .quest import QuestSpider
+
+SPIDERS = {
+    "quest": QuestSpider,
+    "object": ObjectSpider,
+}

wowhead_scraper/spiders/base.py

@@ -0,0 +1,153 @@
+import csv
+import json
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Set
+
+from config import GAME_VERSION, OUTPUT_DIR, EntityConfig
+from fetcher import Fetcher, NotFound
+from parsers.common import filter_categories, list_page
+
+log = logging.getLogger(__name__)
+
+
+class Spider:
+    """Two-stage crawler. Subclasses only say which entity it is and how to parse it.
+
+    Stage 1 (crawl_list): walk the list pages and write one CSV row per entity.
+    Stage 2 (crawl_details): visit each row's page and append one JSON line per entity.
+    """
+    entity: EntityConfig
+    listview_id: str
+    csv_fields: List[str]
+
+    def __init__(self, fetcher: Fetcher, version: str = GAME_VERSION) -> None:
+        self.fetcher = fetcher
+        self.version = version
+
+    def list_row(self, raw: Dict[str, Any], categories: Dict[int, tuple]) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    def parse_page(self, html: str, row: Dict[str, Any]) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    def crawl_list(self) -> Path:
+        """List pages cap out at 1000 rows, so split the ID range until no response is truncated.
+
+        Wowhead silently drops a filter bound above the highest ID it knows (and keeps the other),
+        so a response is a superset of the range asked for:
+          - rows below `lo`: the lower bound was dropped, so nothing exists this high; skip it.
+          - rows above `hi`: the upper bound was dropped; clipping to the range is still exact.
+        """
+        rows: Dict[int, Dict[str, Any]] = {}
+        expected_total = None
+        ranges = [(1, self.entity.max_id)]
+        while ranges:
+            lo, hi = ranges.pop()
+            html = self.fetcher.get(self.entity.list_url(lo, hi, self.version))
+            raw_rows, truncated, total = list_page(html, self.listview_id)
+            expected_total = expected_total if expected_total is not None else total
+            if any(r["id"] < lo for r in raw_rows):
+                log.info("IDs %d-%d are past the highest ID Wowhead filters on, skipping", lo, hi)
+                continue
+            raw_rows = [r for r in raw_rows if r["id"] <= hi]
+            if truncated and lo < hi:
+                mid = (lo + hi) // 2
+                ranges += [(mid + 1, hi), (lo, mid)]
+                log.info("IDs %d-%d truncated, splitting", lo, hi)
+                continue
+
+            categories = filter_categories(html)
+            for raw in raw_rows:
+                row = self.list_row(raw, categories)
+                row["url"] = self.entity.detail_url(row["id"], self.version)
+                rows[row["id"]] = row
+            log.info("IDs %d-%d: %d rows (%d total)", lo, hi, len(raw_rows), len(rows))
+
+        path = self.entity.list_csv()
+        OUTPUT_DIR.mkdir(exist_ok=True)
+        with path.open("w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=self.csv_fields)
+            writer.writeheader()
+            writer.writerows(rows[i] for i in sorted(rows))
+        log.info("Wrote %d %ss to %s", len(rows), self.entity.name, path)
+        if expected_total is not None and expected_total != len(rows):
+            log.warning("Wowhead reports %d %ss in total but %d were listed", expected_total, self.entity.name, len(rows))
+        return path
+
+    def crawl_details(self, limit: Optional[int] = None, ids: Optional[Iterable[int]] = None) -> Path:
+        """Fetch detail pages for list rows not yet in the JSONL output, so reruns resume and retry failures."""
+        rows, done = self._list_rows(), self._done_ids()
+        wanted = set(ids) if ids else None
+        todo = [r for r in rows if int(r["id"]) not in done and (wanted is None or int(r["id"]) in wanted)]
+        todo = todo[:limit] if limit else todo
+        log.info("%d %ss to fetch (%d already done)", len(todo), self.entity.name, len(done))
+        return self._fetch_all(todo)
+
+    def find_missing(self, dry_run: bool = False) -> List[int]:
+        """Re-fetch rows skipped by an earlier detail run (e.g. while blocked).
+
+        Rows are fetched in CSV order, so "skipped" means not in the output but below the highest ID
+        that is. Rows past that point simply haven't been reached yet; `detail` picks those up.
+        """
+        rows, done = self._list_rows(), self._done_ids()
+        last_done = max(done, default=0)
+        missing = [r for r in rows if int(r["id"]) not in done and int(r["id"]) < last_done]
+        not_reached = sum(1 for r in rows if int(r["id"]) > last_done)
+        missing_ids = [int(r["id"]) for r in missing]
+        log.info("%d %ss skipped below ID %d: %s%s", len(missing), self.entity.name, last_done,
+                 missing_ids[:20], " ..." if len(missing) > 20 else "")
+        log.info("%d %ss not reached yet (run `detail` for those)", not_reached, self.entity.name)
+        if missing and not dry_run:
+            self._fetch_all(missing)
+        return missing_ids
+
+    def _list_rows(self) -> List[Dict[str, Any]]:
+        list_path = self.entity.list_csv()
+        if not list_path.exists():
+            raise FileNotFoundError(f"{list_path} not found, run the list stage first")
+        with list_path.open(newline="", encoding="utf-8") as f:
+            return list(csv.DictReader(f))
+
+    def _done_ids(self) -> Set[int]:
+        out_path = self.entity.detail_jsonl()
+        if not out_path.exists():
+            return set()
+        with out_path.open(encoding="utf-8") as f:
+            return {json.loads(line)["id"] for line in f if line.strip()}
+
+    def _fetch_all(self, todo: List[Dict[str, Any]]) -> Path:
+        out_path = self.entity.detail_jsonl()
+        ok = failed = 0
+        with out_path.open("a", encoding="utf-8") as out, \
+                ThreadPoolExecutor(self.fetcher.settings.concurrency) as pool:
+            futures = {pool.submit(self._fetch_detail, row): row for row in todo}
+            try:
+                for n, future in enumerate(as_completed(futures), 1):
+                    row = futures[future]
+                    try:
+                        record = future.result()
+                    except Exception as e:
+                        failed += 1
+                        log.error("%s %s failed: %s", self.entity.name, row["id"], e)
+                        continue
+                    out.write(json.dumps(record, ensure_ascii=False) + "\n")
+                    out.flush()
+                    ok += 1
+                    if n % 50 == 0:
+                        log.info("%d/%d done", n, len(todo))
+            except KeyboardInterrupt:
+                log.warning("Interrupted, finishing in-flight requests. Rerun to resume.")
+                pool.shutdown(cancel_futures=True)
+                raise
+        log.info("Wrote %d records to %s, %d failed (rerun to retry)", ok, out_path, failed)
+        return out_path
+
+    def _fetch_detail(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        try:
+            html = self.fetcher.get(row["url"])
+        except NotFound:
+            # Recorded so reruns don't keep asking for it.
+            return {"id": int(row["id"]), "url": row["url"], "not_found": True}
+        return self.parse_page(html, row)

wowhead_scraper/spiders/object.py

@@ -0,0 +1,15 @@
+from config import ENTITIES
+from parsers import object as object_parser
+from spiders.base import Spider
+
+
+class ObjectSpider(Spider):
+    entity = ENTITIES["object"]
+    listview_id = object_parser.LISTVIEW_ID
+    csv_fields = object_parser.CSV_FIELDS
+
+    def list_row(self, raw, categories):
+        return object_parser.list_row(raw, categories)
+
+    def parse_page(self, html, row):
+        return object_parser.parse_page(html, row)

wowhead_scraper/spiders/quest.py

@@ -0,0 +1,15 @@
+from config import ENTITIES
+from parsers import quest
+from spiders.base import Spider
+
+
+class QuestSpider(Spider):
+    entity = ENTITIES["quest"]
+    listview_id = quest.LISTVIEW_ID
+    csv_fields = quest.CSV_FIELDS
+
+    def list_row(self, raw, categories):
+        return quest.list_row(raw, categories)
+
+    def parse_page(self, html, row):
+        return quest.parse_page(html, row)

wowhead_scraper/tests/fixtures/object_415106.html

@@ -0,0 +1,538 @@
+<!DOCTYPE html>
+<html lang="en-US">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="initial-scale=1">
+    <script>var _sf_startpt=(new Date()).getTime();</script>
+    <title>Burned-Out Remains - Object - Forever</title>
+    <link rel="preconnect" href="https://ex.co"><link rel="preconnect" href="https://bid.g.doubleclick.net"><link rel="preconnect" href="https://vast.doubleverify.com"><link rel="preconnect" href="https://ib.adnxs.com"><link rel="preconnect" href="https://fastlane.rubiconproject.com"><link rel="preconnect" href="https://hbopenbid.pubmatic.com"><link rel="preconnect" href="https://tlx.3lift.com"><link rel="preload" as="script" href="https://tagan.adlightning.com/zam/op.js" fetchpriority="high"><link rel="preload" as="script" href="https://securepubads.g.doubleclick.net/tag/js/gpt.js"><link rel="preload" as="script" href="https://cdn.dotaoutpost.com/stable/js/wowhead.js?type=ab"><link rel="preload" as="script" href="https://zaf.api.fanbyte.com/stable/js/wowhead.js"><link rel="preconnect" href="https://www.googletagmanager.com"><meta name="description" content="Burned-Out Remains is a World of Warcraft object that can be found in Westfall and The Barrens. In the Quest Objects category. Added in Classic World of Warcraft.">
+    <meta name="google-site-verification" content="SScjPPnjqJ0lGTlZeVs9x0D3_jmctKZ-6nE4cvfGL00">
+    <meta property="twitter:title" content="Burned-Out Remains">
+    <meta property="twitter:card" content="summary_large_image">
+    <meta property="twitter:image" content="https://wow.zamimg.com/uploads/screenshots/normal/1304352-burned-out-remains-this-is-the-turn-in-location-for-this-quest-in-the-barrons.jpg">
+    <meta property="twitter:imageUrl" content="https://wow.zamimg.com/uploads/screenshots/normal/1304352-burned-out-remains-this-is-the-turn-in-location-for-this-quest-in-the-barrons.jpg">
+    <meta property="twitter:site:id" content="17258481">
+    <meta property="twitter:site" content="@Wowhead">
+    <meta property="twitter:domain" content="wowhead.com">
+    <meta property="twitter:description" content="Burned-Out Remains is a World of Warcraft object that can be found in Westfall and The Barrens. In the Quest Objects category. Added in Classic World of Warcraft.">
+    <meta property="twitter:image:alt" content="Burned-Out Remains is a World of Warcraft object that can be found in Westfall and The Barrens. In the Quest Objects category. Added in Classic World of Warcraft.">
+    <meta property="og:title" content="Burned-Out Remains">
+    <meta property="og:type" content="article">
+    <meta property="og:image" content="https://wow.zamimg.com/uploads/screenshots/normal/1304352-burned-out-remains-this-is-the-turn-in-location-for-this-quest-in-the-barrons.jpg">
+    <meta property="og:site_name" content="Wowhead">
+    <meta property="og:url" content="https://www.wowhead.com/forever/object=415106/burned-out-remains">
+    <meta property="og:description" content="Burned-Out Remains is a World of Warcraft object that can be found in Westfall and The Barrens. In the Quest Objects category. Added in Classic World of Warcraft.">
+    <meta property="fb:pages" content="169121139638">
+    <link rel="canonical" href="https://www.wowhead.com/forever/object=415106/burned-out-remains">
+    <link rel="alternate" hreflang="en-US" href="https://www.wowhead.com/forever/object=415106/burned-out-remains">
+<link rel="alternate" hreflang="x-default" href="https://www.wowhead.com/forever/object=415106/burned-out-remains">
+<link rel="alternate" hreflang="fr-FR" href="https://www.wowhead.com/forever/fr/object=415106/restes-brûlés">
+<link rel="alternate" hreflang="de-DE" href="https://www.wowhead.com/forever/de/object=415106/ausgebrannte-überreste">
+<link rel="alternate" hreflang="es-ES" href="https://www.wowhead.com/forever/es/object=415106/restos-carbonizados">
+<link rel="alternate" hreflang="ru-RU" href="https://www.wowhead.com/forever/ru/object=415106/прогоревшие-останки">
+<link rel="alternate" hreflang="pt-BR" href="https://www.wowhead.com/forever/pt/object=415106/restos-queimados">
+<link rel="alternate" hreflang="zh-CN" href="https://www.wowhead.com/forever/cn/object=415106/焦焚残骸">
+<link rel="alternate" hreflang="ko-KR" href="https://www.wowhead.com/forever/ko/object=415106/불타버린-잔해">
+<link rel="alternate" hreflang="zh-TW" href="https://www.wowhead.com/forever/tw/object=415106/燒焦的遺骸">
+<link rel="alternate" hreflang="es-MX" href="https://www.wowhead.com/forever/mx/object=415106/restos-quemados">
+    <link rel="search" type="application/opensearchdescription+xml" href="https://www.wowhead.com/forever/opensearch/description?v=3"
+          title="Wowhead (Forever)">
+    <link rel="icon" type="image/png" href="https://wow.zamimg.com/images/logos/favicon-classicplus.png">
+    <link rel="apple-touch-icon-precomposed" href="https://wow.zamimg.com/apple-touch-icon.png">
+
+
+    <script type="application/ld+json">{"@context":"http:\/\/schema.org","@type":"Intangible","mainEntityOfPage":{"@type":"WebPage","@id":"https:\/\/www.wowhead.com\/forever\/object=415106\/burned-out-remains"},"name":"Burned-Out Remains","url":"https:\/\/www.wowhead.com\/forever\/object=415106\/burned-out-remains","logo":"https:\/\/wow.zamimg.com\/images\/logos\/wh-logo.png","publisher":{"@type":"Organization","name":"Wowhead Logo","logo":{"@type":"ImageObject","url":"https:\/\/wow.zamimg.com\/images\/logos\/amp-valid.png","width":395,"height":60}},"keywords":"Database, Forever Database, Objects","description":"Burned-Out Remains is a World of Warcraft object that can be found in Westfall and The Barrens. In the Quest Objects category. Added in Classic World of Warcraft.","image":{"@type":"ImageObject","url":"https:\/\/wow.zamimg.com\/uploads\/screenshots\/normal\/1304352-burned-out-remains-this-is-the-turn-in-location-for-this-quest-in-the-barrons.jpg","height":903,"width":1600}}</script>
+
+
+<script>
+let cdnErrorLeaving = false;
+function cdnErrorOnLeave() {
+    cdnErrorLeaving = true;
+    // A page that is still here a moment later had its unload cancelled (an unsaved-changes prompt). If the next
+    // page takes longer than this to start arriving, a cancel after the reset still redirects; that is rare enough
+    // to accept.
+    setTimeout(function () {
+        cdnErrorLeaving = false;
+    }, 1000);
+}
+addEventListener('beforeunload', cdnErrorOnLeave);
+// Firefox won't keep a page with a beforeunload listener in its back/forward cache, so drop the listener once every
+// stylesheet and script has loaded or failed, when there is nothing left to cancel.
+addEventListener('load', function () {
+    removeEventListener('beforeunload', cdnErrorOnLeave);
+});
+// Scripts get no retry (running one twice, or out of order, is worse than the error page), so a stop during a
+// script load still redirects.
+function cdnErrorHandler(type, el) {
+    if (cdnErrorLeaving) {
+        return;
+    }
+    if (el && el.tagName === 'LINK' && !el.hasAttribute('data-cdn-retry')) {
+        const retry = el.cloneNode();
+        retry.setAttribute('data-cdn-retry', '');
+        retry.addEventListener('load', function () {
+            el.remove();
+        });
+        el.after(retry);
+
+        return;
+    }
+    let url = "\/error-static-cdn?hash=303eb9adfb1&at=1793053487&redirect=1" + '&type=' + encodeURIComponent(type);
+    if (location.hash.length > 1) {
+        url += '&fragment=' + encodeURIComponent(location.hash.substr(1));
+    }
+    location.href = url;
+}
+</script><link rel="stylesheet" type="text/css" href="https://wow.zamimg.com/css/standard/universal.css?303eb9adfb1" onerror="cdnErrorHandler(&quot;css&quot;, this);">
+<link rel="stylesheet" type="text/css" href="https://wow.zamimg.com/css/standard/Components/universal/ZamUiPrimitives.css?303eb9adfb1">
+<link rel="stylesheet" type="text/css" href="https://wow.zamimg.com/css/standard/global.css?303eb9adfb1">
+<link rel="stylesheet" type="text/css" href="https://wow.zamimg.com/css/standard/Components/universal/ZamPrimitivesOverrides.css?303eb9adfb1">
+<link rel="stylesheet" type="text/css" href="https://wow.zamimg.com/css/standard/tools/book.css?303eb9adfb1">
+<script src="https://wow.zamimg.com/js/preamble.js?066f8967" onerror="cdnErrorHandler(&quot;js&quot;);"></script>
+    <script>
+        g_user = {"id":0,"name":"","roles":0,"ads":true,"canDeleteComments":false};
+    </script>
+<script src="https://cdn.cookielaw.org/scripttemplates/otSDKStub.js" type="text/javascript" charset="UTF-8" data-domain-script="11986010-b318-46fd-ab82-e05142e195e2" data-ignore-ga="true"></script><script>function OptanonWrapper() {}</script>
+
+
+<script>window.WH_WASBootstrap=new function(){this.initEarly=function(){let a=null;try{const e=document.getElementById("data.WAS.init");a=e?JSON.parse(e.textContent):null}catch(e){a=null}if(!a||!window.g_user||!g_user.ads){return}window.WH_WAS=window.WH_WAS||{};if(window.WH_WAS.initializedEarly){return}window.WH_WAS.initData=a;window.WH_WAS.deferredSlotRenderCallbacks=window.WH_WAS.deferredSlotRenderCallbacks||[];window.WH_WAS.deferredSkinRenderCallbacks=window.WH_WAS.deferredSkinRenderCallbacks||[];window.WH_WAS.deferredRejectedSlots=window.WH_WAS.deferredRejectedSlots||[];window.googletag=window.googletag||{};googletag.cmd=googletag.cmd||[];window.zamaf=window.zamaf||{};zamaf.cmd=zamaf.cmd||[];zamaf.setTargeting=function(){const e=a.targetData||{};googletag.pubads().setTargeting("Channel",e.channelName);googletag.pubads().setTargeting("ChannelGroup",e.channelGroup);googletag.pubads().setTargeting("DataEnvironment",e.dataEnv);googletag.pubads().setTargeting("DataTree",e.dataTree);if(e.entityType){googletag.pubads().setTargeting("EntityType",e.entityType)}googletag.pubads().setTargeting("Game",e.game);googletag.pubads().setTargeting("Universe",e.universe)};zamaf.slotRenderCallback=function(){if(window.WH&&WH.WAS&&typeof WH.WAS.slotRenderCallback==="function"){return WH.WAS.slotRenderCallback.apply(WH.WAS,arguments)}window.WH_WAS.deferredSlotRenderCallbacks.push(Array.from(arguments))};zamaf.skinRenderCallback=function(){if(window.WH&&WH.WAS&&typeof WH.WAS.displaySkin==="function"){return WH.WAS.displaySkin.apply(WH.WAS,arguments)}window.WH_WAS.deferredSkinRenderCallbacks.push(Array.from(arguments));return true};window.WH_WAS.rejectedHandler=function(){window.WH_WAS.deferredRejectedSlots.push(Array.from(arguments))};zamaf.cmd.push({event:"zaf:slot:rejected",handler:function(){return window.WH_WAS.rejectedHandler.apply(null,arguments)}});window.WH_WAS.zafRejectedHandlerRegistered=true;const n=new URLSearchParams(location.search);const d=n.get("zafAbId");const i=n.get("zafId");["https://tagan.adlightning.com/zam/op.js","https://securepubads.g.doubleclick.net/tag/js/gpt.js"].forEach(e);t((function(){["https://cdn.dotaoutpost.com/stable/js/wowhead.js?type=ab"+(d?"&id="+d:""),"https://zaf.api.fanbyte.com/stable/js/wowhead.js"+(i?"?id="+i:"")].forEach(e)}));window.WH_WAS.scriptsLoaded=true;if(zamaf.initialize){zamaf.initialize();window.WH_WAS.zafInitialized=true}window.WH_WAS.initializedEarly=true};function e(e){const t=document.createElement("script");t.async=true;t.src=e;if(e==="https://tagan.adlightning.com/zam/op.js"){t.setAttribute("fetchpriority","high")}document.head.appendChild(t)}function t(e){if(document.body){e();return}document.addEventListener("bodyAvailable",e,{once:true})}};</script>
+
+<script type="application/json" id="data.pageMeta">{"activeDbChanged":{"1":1791976965,"2":1792711558,"4":1761625051,"5":1793000437,"6":1733348219,"7":1734645791,"8":1711418938,"9":1785165687,"10":1790720440,"11":1785142632,"14":1785135856,"15":1786100712,"16":1792500090},"anonymousSubdomain":"nether","dataEnv":{"active":{"ptr":true,"ptr2":true,"beta":false,"d4ptr":false,"d4beta":false,"classicptr":true},"env":16,"tree":16,"versions":{"1":"12.1.5","2":"12.1.7","4":"1.15.8","5":"2.5.6","6":"2.6","7":"1.8.2","8":"3.4.3","9":"3.0.2","10":"12.1.5","11":"4.4.2","14":"1.15.8","15":"5.5.4","16":"1.60.1"},"phases":{"4":8,"8":null,"11":null,"14":7,"15":null,"16":null},"maxLevel":60},"dev":false,"envDomain":"wowhead.com","hasSidebar":true,"logo":"classicplus","modelViewerVersion":{"wow":{"wrath":"35ee72d","live":"35ee72d","ptr":"dd13b80","ptr2":"dd13b80","classic":"35ee72d","tbc":"35ee72d","mists":"35ee72d","staging":"e84c745","cata":"35ee72d","classicplus":"35ee72d"},"playerHousing":{"staging":"23759a2","live":"070f74e","ptr":"be2a862","ptr2":"be2a862"}},"page":"object","scriptVersionQueryString":"303eb9adfb1","serverTime":"2026-10-26T17:24:47-05:00","staticResizing":{"limit":3000,"throttle":{"start":3000,"percent":0}},"staticUrl":"https:\/\/wow.zamimg.com","theme":"standard","wowhead":true,"tracking":{"groups":{"premium":true,"house_ad":false,"ads":false,"notifications":false,"announcements":false,"model_viewer":false,"content_viewers":false,"navigation":false,"newsletter":false,"filters":false,"content":true,"performance":false,"errors":false,"diagnostics":false}},"availableDataEnvs":[1,2,4,5,6,7,8,9,10,11,14,15,16],"restrictedDataEnvs":[],"restrictedEntities":[],"disabledInTrees":[1,5,6,7,8,9,11,15]}</script>
+<script type="application/json" id="data.notifications.public-key">"BCv9I9cvqAYEi2ddxSOqAbkm8KTL-CNlIGmXk8R7ievkXR7QfBrNtkXQqk73vleylt5MlZQ8jFQ-tEQ_ObVxxMI"</script>
+<script type="application/json" id="data.newsletter.signup">{"newsletters":[{"value":"wow-retail","label":"Wowhead Retail News and Updates","checked":true,"subscribed":false,"popupEnvs":[1]},{"value":"wow-mists-classic","label":"Wowhead Mists of Pandaria Classic News and Updates","checked":false,"subscribed":false,"popupEnvs":[4,15]},{"value":"wow-forever","label":"Wowhead Forever News and Updates","checked":false,"subscribed":false,"popupEnvs":[16]}],"recaptchaSiteKey":"6LfPERwTAAAAAJJlVvIwSzYk_upePdoG-01tw0VX","tosUrl":"https:\/\/corp.fanbyte.com\/legal\/terms","privacyUrl":"https:\/\/corp.fanbyte.com\/legal\/privacy"}</script>
+<script type="application/json" id="data.page.info">{"commentsLocked":false,"entityId":415106,"entityType":2}</script>
+<script type="application/json" id="data.page.urlArticles">[]</script>
+
+<script type="application/json" id="data.WAS.init">{"targetData":{"channelGroup":"Database Detail","channelName":"Object","dataEnv":"Forever","dataTree":"Forever","entityType":"Object","game":"WoW","universe":"Warcraft"},"ads":["skin"],"email":null}</script><script>window.WH_WASBootstrap && window.WH_WASBootstrap.initEarly();</script>
+
+<script src="https://wow.zamimg.com/js/3rd-party/jquery.2.1.3.min.js?fa3be9bf"></script>
+<script src="https://wow.zamimg.com/js/tooltips.js?fdeadb5a"></script>
+<script src="https://wow.zamimg.com/global-strings/classicplus.enus.js?hash=PFLe03o84aagOxgJ9W-07w"></script>
+<script src="https://nether.wowhead.com/forever/menus/root?hash=7dbca72ff72b23dc23687fdc1ae3262e" onerror="cdnErrorHandler(&quot;anon-js&quot;);"></script>
+<script src="https://nether.wowhead.com/forever/menus?hash=7dbca72ff72b23dc23687fdc1ae3262e" async=""></script>
+<script src="https://wow.zamimg.com/js/locale/terms/enus.js?0173b0c1"></script>
+<script src="https://wow.zamimg.com/js/generated/game-env.js?e5c1729e"></script>
+<script src="https://nether.wowhead.com/forever/data/global?dv=86&amp;db=1792500090&amp;versionsSig=8bbc4bda5d5cb360c3dcb8590eb91625"></script>
+<script src="https://wow.zamimg.com/js/global-core.js?70b9b9fd"></script>
+<script src="https://wow.zamimg.com/js/global-tags.js?20399cbc"></script>
+<script src="https://wow.zamimg.com/js/react/runtime.b187eec5.js"></script>
+<script src="https://wow.zamimg.com/js/react/vendors-react.80ccf0e8.js"></script>
+<script src="https://wow.zamimg.com/js/react/vendors-fontawesome.7e553640.js"></script>
+<script src="https://wow.zamimg.com/js/react/main.d93c693f.js"></script>
+<script src="https://wow.zamimg.com/js/WH/Listview/Templates/comment.js?5a42e53b"></script>
+<script src="https://wow.zamimg.com/js/Book.js?070ad7e1"></script>
+
+<script>WH.WAS.init();</script>
+
+<script>
+WH.Track.gaInit();
+</script>
+<script>
+    (function(h,o,u,n,d) {
+        h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
+        d=o.createElement(u);d.async=1;d.src=n
+        n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
+    })(window,document,'script','https://www.datadoghq-browser-agent.com/us5/v6/datadog-rum.js','DD_RUM')
+    window.DD_RUM.onReady(function() {
+        window.DD_RUM.init({"clientToken":"pub983d2b36ec5246e30991ebd4b6dccaaa","applicationId":"f2771e64-a11a-4ddd-a6ab-08e7e43d6e6b","site":"us5.datadoghq.com","service":"wh-site","env":"prod","sessionSampleRate":0.03,"sessionReplaySampleRate":0,"trackUserInteractions":true,"trackResources":true,"trackLongTasks":true,"defaultPrivacyLevel":"mask-user-input","enableExperimentalFeatures":["feature_flags"],"version":"303eb9adfb1"});
+        window.DD_RUM?.addFeatureFlagEvaluation?.('ads', true);
+    });
+</script><script>window.awsWafCookieDomainList = ["wowhead.com"];</script>
+<script type="text/javascript" src="https://ad65c0e8b968.edge.sdk.awswaf.com/ad65c0e8b968/374c23fd6c5f/challenge.js" defer></script>
+
+    <script src="https://nether.wowhead.com/forever/data/item-bonuses?dv=19&db=1792500090"></script>
+    <script src="https://nether.wowhead.com/forever/data/zones?dv=23&db=1792500090"></script>
+    <script src="https://nether.wowhead.com/forever/data/difficulties?dv=2&db=1792500090"></script>
+
+
+</head>
+
+<body class="standard-layout locale-0 has-ads has-rail"><script>
+    // First statement after <body> on purpose: this stamps body[data-display] and body[data-rails]
+    // while the document is still only a <body> tag, so the first paint already has them. Anything
+    // later can be beaten by a paint mid-parse, and those two attributes drive the content column
+    // and the rail track. (The ZUL bar's mobile mode does NOT depend on this call site -- check()
+    // answers that from the viewport whether or not this ran.) Idempotent; Responsive.init() also
+    // calls it. Guarded like initPortalRooms() below: a bundle missing from a deploy must not kill
+    // the rest of inline page init with an uncaught TypeError.
+    if (window.WH && WH.Layout && WH.Layout.Responsive) {
+        WH.Layout.Responsive.initPrepaintState();
+    }
+</script>
+<a href="#main" class="wh-button zam-button skip-link">
+    <span class="wh-button-text body-text zam-button-text" data-text-style="body-small-bold">Skip to Main Content</span>
+</a>
+<script>
+    document.addEventListener('bodyAvailable', () => {
+        const skipLink = document.querySelector('.skip-link');
+
+        const updateSkipLink = () => {
+            const page = document.body.dataset.page;
+
+            if (page === 'guide') {
+                skipLink.setAttribute('href', '#interior-sidebar-content');
+            } else {
+                skipLink.setAttribute('href', '#main');
+            }
+        };
+
+        updateSkipLink();
+
+        const observer = new MutationObserver(updateSkipLink);
+        observer.observe(document.body, {
+            attributes: true,
+            attributeFilter: ['data-page']
+        });
+    });
+</script>
+
+<div id="react-root-main" style="display: none"></div>
+
+<script>
+    // Guard against a deploy that shipped without the React bundle: if WH.React is
+    // absent the page loses React features, but the rest of the inline page init
+    // (bodyAvailable, WH.WAS.init, page state) must still run rather than the whole
+    // page dying on an uncaught TypeError here.
+    if (window.WH && WH.React) {
+        WH.React.initPortalRooms();
+    } else if (window.WH && WH.error) {
+        WH.error('WH.React unavailable at initPortalRooms(); React bundle missing from this deploy.');
+    }
+    document.dispatchEvent(new CustomEvent('bodyAvailable'));
+</script>
+<noscript>
+    <div id="noscript-wrapper">
+        <div id="noscript-text">
+            <b>This site makes extensive use of JavaScript.</b><br>
+            Please enable JavaScript in your browser.        </div>
+    </div>
+</noscript>
+<script>
+    WH.Layout.initPageState(0, null);
+    Menu.processExtraInstructions();
+</script>
+
+
+<div class="header-wrapper" role="banner">
+<div class="header">
+    <div id="data-tree-switcher">
+        <script>WH.Layout.DataTreeSwitcher.init();</script>
+    </div>
+
+    <div class="header-left">
+        <a href="/forever" class="header-logo" aria-label="Home"></a>
+    </div>
+
+    <div class="header-right">
+        <div class="header-search">
+            <form action="/forever/search"></form>
+        </div>
+        <div class="data-env-links">
+    <div class="data-env-links-group"></div></div>
+<script>WH.Layout.DataTreeSwitcher.checkEnvDimensions();</script>
+    </div>
+
+    <script>WH.Layout.User.init();</script>
+</div>
+</div>
+
+<div id="newsletter-cta-wrapper"></div>
+
+<div class="layout-wrapper">
+<div class="layout">
+        <div class="skin-wrapper" data-position="left"></div>
+        <div id="page-content" class="page-content">
+    <div class="main" id="main">
+        <div class="blocks"><div class="blocks-video zaf-block" id="zaf-video-top" style="height: var(--ad-video-top-height, 250px)"></div><div class="zaf-block" id="zaf-billboard" data-zaf-slot="true"></div></div>        <div class="page-controls"></div>
+        <div class="main-contents" id="main-contents" role="main">
+            <div id="announcement-0"></div>
+        <div id="announcement-433"></div>
+<script>
+new WH.Announcement({"parent":"announcement-433","id":433,"text":"[div class=announcement-content-title style=\"padding-left: 130px\"][b]Help build out the Forever database[\/b][\/div]\r\n[div class=announcement-content-subtitle style=\"padding-left: 130px\"]Item stats stay hidden in Forever until someone actually discovers them, so the database can only be truly complete with your help. [url=https:\/\/www.wowhead.com\/client]Download the Wowhead Client[\/url] and help keep Wowhead's data up-to-date![\/div]","mobiletext":"","name":"Forever Client CTA - database pages","style":"background: url(https:\/\/wow.zamimg.com\/images\/logos\/horizontal\/classicplus-rocket-large.webp) -35px center \/ auto 100px no-repeat, linear-gradient(to right, #1d8dcd 0%, #303030 25%, #303030 100%)","class":"promo","status":1,"mode":1,"desktopOnly":true});
+</script>
+<script>
+WH.Layout.set({"pageName":"object","activeTab":0,"breadcrumb":[]});
+// WH.Layout.init() builds header navigation from async menu data.
+Menu.onLoad(WH.Layout.init);
+skipCaptcha = true;
+g_host = "https:\/\/www.wowhead.com";
+</script>
+
+<div id="infobox-original-position">
+<table class="infobox"><tr>
+            <td>
+            <div id="premium-house-block" class="house-cta-container"></div>    <script>
+        WH.React.initComponent(
+            WH.ge('premium-house-block'),
+            'ui/components/house-cta/PremiumCTA',
+            {"titleLines":["Drop the Ads.","Keep the Loot."],"buttonLabel":"Upgrade Now","trackingOverride":{"placement":"db_infobox_top","pageType":"database"}},
+        );
+    </script>
+        </td>
+        <td>
+    <table class="infobox-inner-table">
+    <tr class="infobox-heading">
+    <th>Quick Facts</th>    </tr>
+    <tr>
+        <td>
+        <div class="infobox-spacer"></div>
+                <div id="infobox-contents-0"></div>
+        <script>
+            WH.markup.printHtml("[ul][li]Added in patch [acronym=\"1.15.1.53247\"]1.15.1[\/acronym] \"Season of Discovery\"[\/li][\/ul]", "infobox-contents-0", {
+                allow: WH.markup.CLASS.STAFF,
+                dbPage: true,            });
+        </script>
+            </td></tr>
+    </table>
+    </td>
+    <td>
+    <table class="infobox-inner-table">
+    <tr class="infobox-heading">
+    <th id="infobox-screenshots">Screenshots</th>    </tr>
+    <tr>
+        <td>
+        <div class="infobox-spacer"></div>
+                <div id="infobox-sticky-ss"></div>            </td></tr>
+    </table>
+    </td>
+    <td>
+    <table class="infobox-inner-table">
+    <tr class="infobox-heading">
+    <th id="infobox-videos">Videos</th>    </tr>
+    <tr>
+        <td>
+        <div class="infobox-spacer"></div>
+                <div id="infobox-sticky-vi"></div>            </td></tr>
+    </table>
+    </td>
+</tr></table>
+</div>
+<script>WH.prepInfobox();</script>
+<script>
+        WH.onLoad(function () {
+            ss_appendSticky();
+            vi_appendSticky();
+        });
+    </script><script type="text/javascript">//<![CDATA[
+var _ = g_users;
+_["Onsokomaru"]={"border":0,"roles":0,"joined":"2020-11-24T09:23:04-06:00","posts":1,"gold":3,"silver":18,"reputation":10979,"copper":76,"tierClass":null,"tierTitle":null,"avatar":1,"avatarmore":"5332199"};
+WH.Gatherer.addData(2, 16, {"415106":{"name_enus":"Burned-Out Remains","displayName":""},"424010":{"name_enus":"Nailed Plank","displayName":""},"424005":{"name_enus":"Pocket Litter","displayName":""},"424012":{"name_enus":"Mound of Dirt","displayName":""},"424007":{"name_enus":"Carved Figurine","displayName":""},"406918":{"name_enus":"Messenger Bag","displayName":""},"424006":{"name_enus":"Hastily Rolled-Up Satchel","displayName":""}});
+WH.Gatherer.addData(3, 16, {"211527":{"name_enus":"Cozy Sleeping Bag","quality":3,"icon":"inv_misc_bag_bigbagofenchantments","screenshot":{"id":1306460,"imageType":2},"jsonequip":{"buyprice":171717,"cooldown":3600000,"itemSquishEraId":0,"maxcount":1,"reqlevel":1,"sellprice":1717},"attainable":0,"flags2":28672,"flags5":0,"displayName":"","qualityTier":0,"qualityTierTexture":""},"216619":{"name_enus":"Student Fodder","quality":2,"icon":"inv_misc_food_pinenut","screenshot":{},"jsonequip":{"cooldown":300000,"dmgrange":1,"health":500,"healthrgn":175,"itemSquishEraId":0},"attainable":0,"flags2":24580,"flags5":0,"displayName":"","qualityTier":0,"qualityTierTexture":""}});
+WH.Gatherer.addData(7, 16, {"17":{"name_enus":"The Barrens"},"40":{"name_enus":"Westfall"},"406":{"name_enus":"Stonetalon Mountains"},"38":{"name_enus":"Loch Modan"},"267":{"name_enus":"Hillsbrad Foothills"},"45":{"name_enus":"Arathi Highlands"}});
+var lv_comments0 = [{"commentv2":1,"number":0,"id":6433636,"nreplies":0,"sticky":0,"user":"Onsokomaru","body":"[b][u]Horde: follow below; Alliance: swap the order of first 2 steps[\/u][\/b]\n\nHow to obtain [item=211527] & 5 [item=216619]:\n\n1. 1. Travel to 46, 73 in [zone=17]\nHere, interact with [object=415106] and accept the quest.\n\n2. Travel to 37, 50 in [zone=40]\nInteract with [object=424010]\n\n3. Travel to 51, 51 in [zone=406]\nFrom the road, follow the opening into the mountains, and follow the path until you reach the camp\nAt the camp, interact with [object=424005]\nRun through the camp and jump down the mountains to reach the ledge with [object=424012] that you have to interact with\n\n4. Travel to the Dam at 49, 12 in [zone=38]\nJump down to the ledge and interact with [object=424007]\n\n5. Travel to 81, 56 in [zone=267]\nWhile facing the gate toward [zone=45], turn left and follow the wall until you reach the fallen cart\nFollow the little jumping puzzle toward the camp\nInteract with [object=406918] then interact with [object=424006]\n\n[b][u]Orc Hunter Journey to unlock it:[\/u][\/b] https:\/\/www.youtube.com\/watch?v=W_Yl_7lE_9Q\n\n[b][u]It was originally recorded for SoD, but the same quest carried over.[\/u][\/b]","date":"2026-09-22T16:31:56-05:00","rating":2,"indent":0,"roles":0,"deleted":0,"outofdate":0,"userRating":0,"replies":[],"dataTree":16,"markupDisabledEnvironments":{"ptr":false,"ptr2":false,"beta":false,"d4ptr":false,"d4beta":false,"classicptr":false},"lastEdit":["2026-09-22T16:39:19-05:00",1,"Onsokomaru"]}];
+</script>
+
+<noscript><div id="user-comments"><h2 class="heading-size-2">Comments</h2><div class="user-post-"><h3 class="heading-size-3">Comment by <b>Onsokomaru</b></h3><b><u>Horde: follow below; Alliance: swap the order of first 2 steps</u></b><br /><br />How to obtain <a href="/forever/item=211527/cozy-sleeping-bag">Cozy Sleeping Bag</a> &amp; 5 <a href="/forever/item=216619/student-fodder">Student Fodder</a>:<br /><br />1. 1. Travel to 46, 73 in <a href="/forever/zone=17/the-barrens">The Barrens</a><br />Here, interact with <a href="/forever/object=415106/burned-out-remains">Burned-Out Remains</a> and accept the quest.<br /><br />2. Travel to 37, 50 in <a href="/forever/zone=40/westfall">Westfall</a><br />Interact with <a href="/forever/object=424010/nailed-plank">Nailed Plank</a><br /><br />3. Travel to 51, 51 in <a href="/forever/zone=406/stonetalon-mountains">Stonetalon Mountains</a><br />From the road, follow the opening into the mountains, and follow the path until you reach the camp<br />At the camp, interact with <a href="/forever/object=424005/pocket-litter">Pocket Litter</a><br />Run through the camp and jump down the mountains to reach the ledge with <a href="/forever/object=424012/mound-of-dirt">Mound of Dirt</a> that you have to interact with<br /><br />4. Travel to the Dam at 49, 12 in <a href="/forever/zone=38/loch-modan">Loch Modan</a><br />Jump down to the ledge and interact with <a href="/forever/object=424007/carved-figurine">Carved Figurine</a><br /><br />5. Travel to 81, 56 in <a href="/forever/zone=267/hillsbrad-foothills">Hillsbrad Foothills</a><br />While facing the gate toward <a href="/forever/zone=45/arathi-highlands">Arathi Highlands</a>, turn left and follow the wall until you reach the fallen cart<br />Follow the little jumping puzzle toward the camp<br />Interact with <a href="/forever/object=406918/messenger-bag">Messenger Bag</a> then interact with <a href="/forever/object=424006/hastily-rolled-up-satchel">Hastily Rolled-Up Satchel</a><br /><br /><b><u>Orc Hunter Journey to unlock it:</u></b> <a href="https://www.youtube.com/watch?v=W_Yl_7lE_9Q" rel="nofollow ugc noopener">https://www.youtube.com/watch?v=W_Yl_7lE_9Q</a><br /><br /><b><u>It was originally recorded for SoD, but the same quest carried over.</u></b></div>
+
+</div>
+
+</noscript>
+
+<script type="text/javascript">
+var lv_screenshots = [{"id":1304352,"user":"YaketheSnake","date":"2026-09-30T00:06:12-05:00","width":1600,"height":903,"type":2,"typeId":415106,"imageType":2,"staffOnly":false,"caption":"This is the turn in location for this quest in the barrons.","sticky":1}];
+var lv_videos = [{"id":437914,"user":"Onsokomaru","date":"2026-09-22T16:31:56-05:00","videoType":1,"videoId":"W_Yl_7lE_9Q","type":2,"typeId":415106,"url":null,"sticky":1}];
+var g_pageInfo = {type: 2, typeId: 415106, name: "Burned-Out Remains"};
+WH.Layout.set({breadcrumb: [0, 5, -2]});
+//]]></script>
+
+<div class="text">
+
+<div class="db-action-buttons"><script>WH.onLoad(function () {
+    var linksButton = WH.ge('open-links-button');
+    if (linksButton) {
+        linksButton.dataset.displayId = 2610;
+        linksButton.dataset.modelEntityType = 2;
+        linksButton.dataset.modelEntityId = 415106;
+    }
+});</script><a href="javascript:" id="wh-mv-view-in-3d-button" class="btn btn-site btn-small" style="margin-left:5px;float:right" onclick="WH.Wow.ModelViewer.showLightbox({&quot;type&quot;:2,&quot;typeId&quot;:415106,&quot;displayId&quot;:2610})" data-mv-type="2" data-mv-type-id="415106" data-mv-display-id="2610">View in 3D</a><a href="javascript:" id="open-links-button" class="btn btn-site btn-small fa fa-fw fa-link" style="margin-left:5px;float:right" onclick="WH.Links.show(this, {&quot;type&quot;:2,&quot;typeId&quot;:415106}, event);">Links</a></div><h1 class="heading-size-1">Burned-Out Remains</h1><div>This object can be found in <span id="locations">                        <a href="javascript:" onclick="
+                            myMapper.update({
+                                zone: 40,
+                                level: 0,
+                            });
+                            WH.setSelectedLink(this, 'mapper');
+                            return false;
+                            " onmousedown="return false">Westfall</a>
+                        &nbsp;(2) and                         <a href="javascript:" onclick="
+                            myMapper.update({
+                                zone: 17,
+                                level: 0,
+                            });
+                            WH.setSelectedLink(this, 'mapper');
+                            return false;
+                            " onmousedown="return false">The Barrens</a>
+                        .</span>
+</div><div id="k6b43j6b"></div>
+<div style="clear:left"></div>
+
+<script>var g_mapperData = {"17":[{"count":1,"coords":[[46.4,73.9]],"uiMapId":1413,"uiMapName":"The Barrens"}],"40":[{"count":2,"coords":[[37.4,50.6],[37.5,50.7]],"uiMapId":1436,"uiMapName":"Westfall"}]};
+var myMapper = new Mapper({"parent":"k6b43j6b","name":"Burned-Out Remains"});
+WH.gE(WH.ge('locations'), 'a')[0].onclick();
+</script><div id="infobox-alternate-position"></div><script>WH.Responsive.placeInfobox();</script>
+    <div class="db-page-collection-tracker-cta"></div>
+    <script>
+        const cta = WH.Tools.createWowheadClientCta();
+        if (cta) {
+            $('.db-page-collection-tracker-cta').append(cta);
+        }
+    </script>
+
+<h2 class="heading-size-2 clear">Related</h2></div>
+
+<div id="jkbfksdbl4"></div>
+<div id="lkljbjkb574" class="listview">
+</div>
+<script type="text/javascript">//<![CDATA[
+WH.Gatherer.addData(2, 16, {"415106":{"name_enus":"Burned-Out Remains","displayName":""}});
+$.extend(g_objects[415106], {"displayName":"Burned-Out Remains","id":415106,"location":[40,17],"name":"Burned-Out Remains","type":-2,"envChange":{"status":"unchanged","labels":[],"lines":[]}});
+WH.Gatherer.addData(3, 16, {"2459":{"name_enus":"Swiftness Potion","quality":1,"icon":"inv_potion_95","screenshot":{},"jsonequip":{"avgbuyout":4800,"cooldown":120000,"itemSquishEraId":0,"reqlevel":5,"sellprice":25},"attainable":0,"flags2":24580,"flags5":0,"displayName":"","qualityTier":0,"qualityTierTexture":""},"3388":{"name_enus":"Lesser Troll's Blood Elixir","quality":1,"icon":"inv_potion_79","screenshot":{},"jsonequip":{"avgbuyout":1476,"cooldown":3000,"healthrgn":6,"itemSquishEraId":0,"reqlevel":15,"sellprice":40},"attainable":0,"flags2":24580,"flags5":0,"displayName":"","qualityTier":0,"qualityTierTexture":""},"1652":{"name_enus":"Sturdy Lunchbox","quality":1,"icon":"inv_box_01","screenshot":{},"jsonequip":{"avgbuyout":24549,"itemSquishEraId":0,"sellprice":5000,"slotbak":18},"attainable":0,"flags2":24580,"flags5":0,"displayName":"","qualityTier":0,"qualityTierTexture":""},"4470":{"name_enus":"Simple Wood","quality":1,"icon":"inv_tradeskillitem_01","screenshot":{},"jsonequip":{"avgbuyout":573,"buyprice":38,"itemSquishEraId":0,"sellprice":9},"attainable":0,"flags2":-2147459068,"flags5":0,"displayName":"","qualityTier":0,"qualityTierTexture":""},"4471":{"name_enus":"Flint and Tinder","quality":1,"icon":"inv_stone_11","screenshot":{},"jsonequip":{"avgbuyout":1198,"buyprice":135,"itemSquishEraId":0,"sellprice":33},"attainable":0,"flags2":24580,"flags5":0,"displayName":"","qualityTier":0,"qualityTierTexture":""},"221498":{"name_enus":"Old Toolbox","quality":1,"icon":"inv_box_02","screenshot":{},"jsonequip":{"dmgrange":1,"itemSquishEraId":0,"sellprice":5000,"slotbak":18},"attainable":0,"flags2":24580,"flags5":0,"displayName":"","qualityTier":0,"qualityTierTexture":""},"2901":{"name_enus":"Mining Pick","quality":1,"icon":"inv_pick_02","screenshot":{},"jsonequip":{"appearances":{"0":[1379,""]},"avgbuyout":393,"buyprice":10,"displayid":1379,"dmgmax1":4,"dmgmin1":2,"dmgrange":0.6,"dmgtype1":0,"dps":1.5,"itemSquishEraId":0,"mledmgmax":4,"mledmgmin":2,"mledps":1.5,"mlespeed":2,"reqlevel":1,"sellprice":2,"slotbak":21,"speed":2},"attainable":0,"flags2":24580,"flags5":0,"displayName":"","qualityTier":0,"qualityTierTexture":""},"3334":{"name_enus":"Farmer's Shovel","quality":2,"icon":"inv_misc_shovel_01","screenshot":{},"jsonequip":{"appearances":{"0":[7495,""]},"avgbuyout":5000,"displayid":7495,"dmgmax1":32,"dmgmin1":21,"dmgrange":0.4,"dmgtype1":0,"dps":7.16,"dura":35,"itemSquishEraId":0,"mledmgmax":32,"mledmgmin":21,"mledps":7.16,"mlespeed":3.7,"reqlevel":5,"sellprice":68,"sheathtype":1,"slotbak":17,"speed":3.7,"str":1},"attainable":0,"flags2":16388,"flags5":0,"displayName":"","qualityTier":0,"qualityTierTexture":""}});
+var tabsRelated = new Tabs({parent: WH.ge('jkbfksdbl4'), trackable: "WoW Object"});
+new Listview({template: 'quest', id: 'starts', name: WH.TERMS.starts, tabs: tabsRelated, parent: 'lkljbjkb574', data: [{"category":17,"category2":1,"id":79007,"itemrewards":[[2459,1],[3388,1]],"level":22,"name":"... and that note you found","reqlevel":14,"side":2,"xp":1750,"envChange":{"status":"updated","labels":["level_changed","rewards_changed","renamed","reworded"],"lines":["Renamed from ...and that note you found","Level 22 instead of 40","Description reworded","Experience 1750 instead of 3150","Rewards: adds Lesser Troll's Blood Elixir","Rewards: no longer Strong Troll's Blood Potion"],"notShown":["Description reworded","Experience 1750 instead of 3150","Rewards: adds Lesser Troll's Blood Elixir","Rewards: no longer Strong Troll's Blood Potion"]}},{"category":17,"category2":1,"id":79192,"itemchoices":[[2901,1],[3334,1]],"itemrewards":[[1652,1],[4470,1],[4471,1],[221498,1]],"level":24,"name":"Stepping Stones","reqlevel":14,"side":3,"xp":1950,"envChange":{"status":"updated","labels":["level_changed","rewards_changed"],"lines":["Level 24 instead of 40","Experience 1950 instead of 3150","Rewards: adds Old Toolbox"],"notShown":["Experience 1950 instead of 3150","Rewards: adds Old Toolbox"]}}]});
+new Listview({template: 'quest', id: 'ends', name: WH.TERMS.ends, tabs: tabsRelated, parent: 'lkljbjkb574', data: [{"category":40,"category2":0,"id":79008,"itemrewards":[[2459,1],[3388,1]],"level":22,"name":"... and that note you found","reqlevel":14,"side":1,"xp":1750,"envChange":{"status":"updated","labels":["level_changed","rewards_changed","renamed","reworded"],"lines":["Renamed from ...and that note you found","Level 22 instead of 40","Description reworded","Experience 1750 instead of 3150","Rewards: adds Lesser Troll's Blood Elixir","Rewards: no longer Strong Troll's Blood Potion"],"notShown":["Description reworded","Experience 1750 instead of 3150","Rewards: adds Lesser Troll's Blood Elixir","Rewards: no longer Strong Troll's Blood Potion"]}}]});
+new Listview({template: 'object', id: 'same-model-as', name: WH.TERMS.samemodelas_stc, tabs: tabsRelated, parent: 'lkljbjkb574', data: [{"displayName":"Dark Iron Dwarf Corpse","id":175608,"name":"Dark Iron Dwarf Corpse","type":-2,"envChange":{"status":"unchanged","labels":[],"lines":[]}},{"displayName":"Burned-Out Remains","id":415107,"location":[40],"name":"Burned-Out Remains","type":-2,"envChange":{"status":"unchanged","labels":[],"lines":[]}}]});
+</script><script>new Listview({
+    data: lv_comments0,
+    id: 'comments',
+    name: WH.TERMS.comments,
+    parent: lkljbjkb574,
+    
+    
+    sourceScopes: true,
+    tabOpts: {className: 'ugc-tab ugc-tab-comments'},
+    tabs: tabsRelated,
+    template: 'comment',
+});new Listview({
+    data: lv_screenshots,
+    id: 'screenshots',
+    name: WH.TERMS.screenshots,
+    parent: 'lkljbjkb574',
+    tabOpts: {className: 'ugc-tab ugc-tab-screenshots'},
+    tabs: tabsRelated,
+    template: 'screenshot',
+});if (lv_videos.length) {
+    new Listview({
+        data: lv_videos,
+        id: 'videos',
+        name: WH.TERMS.videos,
+        parent: 'lkljbjkb574',
+        tabs: tabsRelated,
+        tabOpts: {className: 'ugc-tab ugc-tab-videos'},
+        template: 'video',
+    });
+}
+if (window.lv_videos_english && lv_videos_english.length) {
+    new Listview({
+        data: lv_videos_english,
+        id: 'videos-english',
+        name: WH.TERMS.englishvideos,
+        parent: 'lkljbjkb574',
+        tabs: tabsRelated,
+        tabOpts: {className: 'ugc-tab ugc-tab-videos'},
+        template: 'video',
+    });
+}tabsRelated.flush();
+//]]></script>
+
+
+<h2 class="heading-size-2 wh-heading-responsive contribute-heading" data-text-style="heading-2-medium">Contribute</h2><div id="lkgljhkv456" class="comment-tabs"></div>
+
+<div class="text comment-tabs-body individual-tab">
+<div class="tabbed-contents" role="tabpanel" style="clear:none">
+<div id="tab-add-your-comment" style="display:none">
+Please keep the following in mind when posting a comment:<ul>
+    <li><div>Your comment must be in English or it will be removed.</div></li>
+    <li><div>Unsure how to post? Check out our <a href="/forever/help=commenting-and-you" target="_blank">handy guide</a>!</div></li>
+    <li><div>You might want to proof-read your comments before posting them.</div></li>
+    <li><div>Please post questions on our <a href="/forever/forums">forums</a> for quicker reply.</div></li>
+</ul>
+<div id="comment" class="add-comment-form">
+    <section class="comments-post-header">
+        <div class="imitation-heading heading-size-3 wh-heading-responsive contribute-heading-interior" data-text-style="heading-3">Post a Comment</div>    </section>
+<form>
+    <div class="comment-edit-body">
+        <textarea class="comment-editbox" rows="10" cols="40" name="commentbody" disabled="disabled"></textarea>
+    </div>
+    <small>You are not logged in. Please <a href="/forever/account=signin">log in</a> or <a href="/forever/auth/register">register an account</a> to add your comment.</small>
+</form>
+</div>
+</div>
+<div id="tab-submit-a-screenshot" style="display:none">
+    Simply browse for your screenshot using the form below.    <ul>
+        <li><div>Screenshots containing UI elements are generally declined on sight, the same goes for screenshots from the modelviewer or character selection screen.</div></li>
+        <li><div>The higher the quality the better!</div></li>
+        <li><div>Please review our <a href="/forever/help=screenshot-guidelines" target="_blank">Screenshot Guidelines</a> before submitting!</div></li>
+    </ul>
+    <form action="" method="post">
+        <input type="file" name="screenshotfile" disabled="disabled"><br>
+        <small>You are not logged in. Please <a href="/forever/account=signin">log in</a> to submit a screenshot.</small>
+    </form>
+</div>
+
+<div id="tab-suggest-a-video" style="display:none">
+    Simply type the URL of the video in the form below.    <div class="pad2"></div>
+    <form action="/forever/video=add&amp;2.415106" method="post" enctype="multipart/form-data"
+          onsubmit="return vi_validateForm(this)">
+
+        URL: <input type="text" name="videourl" style="width:35%">
+        <small>Supported: YouTube only</small>
+                    <div class="pad2"></div>
+            Title: <input type="text" name="videotitle" maxlength="200"> <small>Optional, up to 200 characters</small><br>
+                    <div class="pad"></div>
+        <input type="submit" value="Submit">
+
+        <div class="pad3"></div>
+        <small class="q0">Note: Your video will need to be approved before appearing on the site.</small>
+
+    </form>
+</div>
+<div id="tab-submit-your-data" style="display: none">
+    <p>The <b>Wowhead Client</b> is a little application we use to keep our database up to date, and to provide you with some nifty extra functionality on the website! &nbsp;</p>
+    <div class="pad2"></div>
+    <p>It serves 2 main purposes: &nbsp;</p>
+
+    <ol>
+        <li><div>It maintains a WoW addon called the <b>Wowhead Looter</b>, which collects data as you play the game! &nbsp;</div></li>
+        <li><div>It uploads the <b>collected data</b> to Wowhead in order to keep the database up-to-date! &nbsp;</div></li>
+    </ol>
+
+    <p>You can also use it to keep track of your completed quests, recipes, mounts, companion pets, and titles!&nbsp;</p>
+    <div class="pad2"></div>
+    <p>So, what are you waiting for? <a href="/forever/client">Download the client</a> and get started.&nbsp;</p>
+</div>
+</div>
+</div>
+
+<script>
+var tabsContribute = new Tabs({parent: WH.ge('lkgljhkv456')});
+        tabsContribute.add(WH.TERMS.addyourcomment_stc, {id: 'add-your-comment'});tabsContribute.add(WH.TERMS.submitascreenshot_stc, {id: 'submit-a-screenshot'});tabsContribute.add(WH.TERMS.suggestavideo_stc, {id: 'suggest-a-video'});
+tabsContribute.add(WH.TERMS.submityourdata_stc, {id: 'submit-your-data'});tabsContribute.flush();
+</script></div>    <div class="sidebar-wrapper" data-position="right">
+                    <div class="rail-track">
+                <div class="zaf-block" id="zaf-right-rail-1" data-zaf-slot="true"></div>
+            </div>
+            <div class="rail-track">
+                <div class="zaf-block" id="zaf-right-rail-2" data-zaf-slot="true"></div>
+            </div>
+                        </div>
+    </div></div>    <div class="skin-wrapper" data-position="right"></div>
+    </div><footer class="footer" data-cls-hack="true"></footer></div>    <script>
+        WH.Layout.Footer.init([[1,"All news posts","\/news\/rss\/all"],[2,"Classic series WoW news","\/news\/rss\/classic-series"],[3,"Retail WoW news","\/news\/rss\/retail"],[4,"In-development WoW news","\/news\/rss\/in-dev"],[5,"Other Blizzard games news","\/news\/rss\/other-blizzard-games"],[6,"Diablo news","\/news\/rss\/diablo"],[7,"Classic WoW news","\/news\/rss\/classic"],[26,"WoW Classic Era news","\/news\/rss\/classic-era"],[27,"WoW Classic Seasonal news","\/news\/rss\/classic-seasonal"],[8,"TBC WoW news","\/news\/rss\/burning-crusade-classic"],[22,"Wrath WoW news","\/news\/rss\/wotlk-classic"],[24,"Cataclysm WoW news","\/news\/rss\/cata-classic"],[25,"Mists WoW news","\/news\/rss\/mists-classic"],[9,"Diablo II: Resurrected news","\/news\/rss\/diablo-2"],[10,"Diablo Immortal news","\/news\/rss\/diablo-immortal"],[11,"Diablo IV news","\/news\/rss\/diablo-4"],[12,"All WoW news","\/news\/rss\/all-wow"],[13,"WoW PTR news","\/news\/rss\/wow-ptr"],[14,"WoW Beta news","\/news\/rss\/wow-beta"],[15,"WoW Live news","\/news\/rss\/wow-live"],[17,"Hearthstone news","\/news\/rss\/hearthstone"],[18,"Overwatch news","\/news\/rss\/overwatch"],[19,"Heroes of the Storm news","\/news\/rss\/heroes"],[20,"Warcraft III news","\/news\/rss\/wc3"],[21,"Starcraft news","\/news\/rss\/starcraft"],[23,"Warcraft Rumble news","\/news\/rss\/rumble"],[28,"Forever WoW news","\/news\/rss\/forever"]]);
+    </script>
+<script>
+    qcdata = {} || qcdata;
+    (function(){
+        var elem = document.createElement('script');
+        elem.src = (document.location.protocol === "https:" ? "https://secure" : "http://pixel") +
+            ".quantserve.com/aquant.js?a=p-80xBzMhbKolFI";
+        elem.async = true;
+        elem.type = "text/javascript";
+        var scpt = document.getElementsByTagName('script')[0];
+        scpt.parentNode.insertBefore(elem,scpt);
+    }());
+
+    var qcdata = {qacct: 'p-80xBzMhbKolFI', labels: ''};
+</script>
+<noscript>
+    <img src="//pixel.quantserve.com/pixel/p-80xBzMhbKolFI.gif?labels=_fp.event.Default" style="display: none;"
+         border="0" height="1" width="1" alt="Quantcast"/>
+</noscript>
+</body></html>

wowhead_scraper/tests/test_parsers.py

@@ -0,0 +1,110 @@
+from pathlib import Path
+
+import pytest
+
+from parsers import common, quest, object as object_parser
+
+ROOT = Path(__file__).parent.parent
+EXAMPLES = ROOT / "example_pages"
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def read(path: Path) -> str:
+    if not path.exists():
+        pytest.skip(f"{path.name} not available")
+    return path.read_text(encoding="utf-8")
+
+
+@pytest.fixture(scope="module")
+def quest_list():
+    return read(EXAMPLES / "https___www.wowhead.com_forever_quests#200.htm")
+
+
+@pytest.fixture(scope="module")
+def object_list():
+    return read(EXAMPLES / "https___www.wowhead.com_forever_objects.htm")
+
+
+def test_quest_list_is_truncated_listview(quest_list):
+    rows, truncated, _ = common.list_page(quest_list, quest.LISTVIEW_ID)
+    assert len(rows) == 1000 and truncated
+
+
+def test_quest_list_row(quest_list):
+    rows, _, _ = common.list_page(quest_list, quest.LISTVIEW_ID)
+    row = quest.list_row(next(r for r in rows if r["id"] == 96393), common.filter_categories(quest_list))
+    assert row == {
+        "id": 96393, "name": "Old Ironforge Incursion", "level": 16, "reqlevel": 9, "side": "Alliance",
+        "category": "The Hall of Thanes", "category_id": 16919, "category2": "Dungeons", "category2_id": 2,
+        "exp": 1450, "money": 800,
+    }
+
+
+def test_category_ids_overlap_between_levels(quest_list):
+    categories = common.filter_categories(quest_list)
+    assert categories[1][0] == "Kalimdor"
+    assert categories[0][1][1] == "Dun Morogh"  # zone 1 under Eastern Kingdoms
+
+
+def test_object_list_uses_json_script(object_list):
+    rows, truncated, _ = common.list_page(object_list, object_parser.LISTVIEW_ID)
+    assert len(rows) == 1000 and truncated
+    row = object_parser.list_row(next(r for r in rows if r["id"] == 3689), common.filter_categories(object_list))
+    assert row == {"id": 3689, "name": "Weapon Crate", "type": "Containers", "type_id": 3, "skill": 0,
+                   "zone_ids": "17;331"}
+
+
+def test_quest_page_baron_marinous():
+    html = read(EXAMPLES / "https___www.wowhead.com_forever_quest=98028_baron-marinous.htm")
+    q = quest.parse_page(html, {"id": "98028", "category": "Darkshore"})
+    assert q["name"] == "Baron Marinous"
+    assert (q["level"], q["reqlevel"], q["side"]) == (21, 17, "Alliance")
+    assert q["objectives"] == "Bring the Clouded Water Globe to Onu at the Grove of the Ancients in Darkshore."
+    assert q["objective_items"] == [{"type": "item", "id": 279275, "name": "Clouded Water Globe",
+                                     "quantity": 1, "standing": None, "provided": True}]
+    assert q["description"].startswith("All that remains of the powerful elemental")
+    assert q["progress"].startswith("I can sense something far older")
+    assert "\n\nStairs descending down in Mathystra?" in q["completion"]
+    assert q["start"] == []
+    assert q["end"] == [{"type": "npc", "id": 3616, "name": "Onu"}]
+    assert q["locations"][0] == {"role": "end", "type": "npc", "id": 3616, "name": "Onu", "zone_id": 148,
+                                 "zone": "Darkshore", "coord": [43.6, 76.4], "coords": [[43.4, 76.2], [43.6, 76.4]]}
+    assert q["rewards"]["xp"] == 1650
+    assert q["rewards"]["money"] == 0
+    assert [i["name"] for i in q["rewards"]["item_choices"]] == ["Mathystra Channeling Orb", "Onu's Patience"]
+    assert q["rewards"]["reputation"] == [{"faction_id": 69, "faction": "Darnassus", "value": 150}]
+    assert "Not sharable" in q["quick_facts"]
+
+
+def test_quest_page_with_giver_and_series():
+    q = quest.parse_page(read(FIXTURES / "quest_166.html"), {"id": "166"})
+    assert q["start"] == [{"type": "npc", "id": 234, "name": "Gryan Stoutmantle"}]
+    giver = next(loc for loc in q["locations"] if loc["role"] == "start")
+    assert (giver["name"], giver["zone"], giver["coord"]) == ("Gryan Stoutmantle", "Westfall", [56.2, 47.6])
+    assert q["series"] == [{"id": 155, "name": "The Defias Brotherhood"},
+                           {"id": 166, "name": "The Defias Brotherhood"}]
+
+
+def test_object_page_weapon_crate():
+    html = read(EXAMPLES / "https___www.wowhead.com_forever_object=3689_weapon-crate.htm")
+    o = object_parser.parse_page(html, {"id": "3689", "type": "Containers", "skill": "0"})
+    assert (o["name"], o["type"], o["type_id"], o["skill"]) == ("Weapon Crate", "Containers", 3, 0)
+    assert [(loc["zone"], loc["count"]) for loc in o["locations"]] == [("The Barrens", 16), ("Ashenvale", 7)]
+    assert len(o["locations"][0]["coords"]) == 16
+    bludgeon = next(i for i in o["contains"] if i["id"] == 1825)
+    assert bludgeon == {"id": 1825, "name": "Bulky Bludgeon", "count": 17, "percent": 0.4, "stack": [1, 1]}
+
+
+def test_object_page_starts_and_ends_quests():
+    o = object_parser.parse_page(read(FIXTURES / "object_415106.html"), {"id": "415106"})
+    assert [q["id"] for q in o["starts"]] == [79007, 79192]
+    assert [q["id"] for q in o["ends"]] == [79008]
+    assert o["contains"] == []
+
+
+def test_quest_page_with_reputation_objectives():
+    q = quest.parse_page(read(FIXTURES / "quest_1036.html"), {"id": "1036"})
+    assert [(o["type"], o["id"], o["name"], o["quantity"], o["standing"]) for o in q["objective_items"]] == [
+        ("faction", 87, "Bloodsail Buccaneers", None, "Friendly"),
+        ("faction", 21, "Booty Bay", None, "Hated"),
+    ]

wowhead_scraper/tests/test_spider.py

@@ -0,0 +1,112 @@
+import json
+import re
+
+import pytest
+
+import config
+from config import CrawlSettings, EntityConfig
+from fetcher import NotFound
+from spiders.base import Spider
+
+
+class FakeFetcher:
+    """Serves list pages for a fixed set of IDs the way Wowhead does: rows are capped per page,
+    and a filter bound above `filter_limit` is silently dropped."""
+
+    def __init__(self, ids, cap=4, filter_limit=1000):
+        self.ids, self.cap, self.filter_limit, self.urls = sorted(ids), cap, filter_limit, []
+        self.settings = CrawlSettings(concurrency=2)
+
+    def get(self, url):
+        self.urls.append(url)
+        m = re.search(r";(\d+):(\d+)$", url)
+        if m:
+            lo, hi = (b if b <= self.filter_limit else None for b in map(int, m.groups()))
+            rows = [{"id": i, "name": f"thing {i}"} for i in self.ids
+                    if (lo is None or i >= lo) and (hi is None or i <= hi)]
+            rows.sort(key=lambda r: r["id"] % 7)  # Wowhead sorts by popularity, not id
+            data = json.dumps(rows[:self.cap])
+            return f"new Listview({{template: 'thing', id: 'things', _truncated: {int(len(rows) > self.cap)}, data:{data}}});"
+        entity_id = int(url.rsplit("=", 1)[1])
+        if entity_id == 13:
+            raise NotFound(url)
+        return f"<h1>thing {entity_id}</h1>"
+
+
+class ThingSpider(Spider):
+    entity = EntityConfig(name="thing", list_path="things", id_filter=1, max_id=100_000)
+    listview_id = "things"
+    csv_fields = ["id", "name", "url"]
+
+    def list_row(self, raw, categories):
+        return {"id": raw["id"], "name": raw["name"]}
+
+    def parse_page(self, html, row):
+        return {"id": int(row["id"]), "title": html[4:-5]}
+
+
+@pytest.fixture(autouse=True)
+def tmp_output(tmp_path, monkeypatch):
+    monkeypatch.setattr(config, "OUTPUT_DIR", tmp_path)
+    monkeypatch.setattr("spiders.base.OUTPUT_DIR", tmp_path)
+    return tmp_path
+
+
+IDS = [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987]
+
+
+def test_list_bisects_until_nothing_is_truncated():
+    fetcher = FakeFetcher(IDS)
+    path = ThingSpider(fetcher).crawl_list()
+    lines = path.read_text(encoding="utf-8").splitlines()
+    assert lines[0] == "id,name,url"
+    assert [int(line.split(",")[0]) for line in lines[1:]] == IDS
+    assert lines[1].endswith("/forever/thing=1")
+
+
+def test_list_skips_ranges_past_the_filter_limit():
+    fetcher = FakeFetcher(IDS)
+    ThingSpider(fetcher).crawl_list()
+    # Ranges starting above 1000 are answered with "id <= hi" (everything) and skipped, not split.
+    assert len(fetcher.urls) < 40
+
+
+def test_list_page_total(tmp_output):
+    from parsers.common import list_page
+    html = "new Listview({id: 'x', note: \"5,442 quests found (1,000 displayed)\", _truncated: 1, data:[]});"
+    assert list_page(html, "x") == ([], True, 5442)
+    assert list_page("new Listview({id: 'x', data:[{\"id\": 1}]});", "x") == ([{"id": 1}], False, 1)
+
+
+def test_details_resume_and_record_missing():
+    fetcher = FakeFetcher(IDS)
+    spider = ThingSpider(fetcher)
+    spider.crawl_list()
+
+    path = spider.crawl_details(limit=5)
+    assert len(path.read_text(encoding="utf-8").splitlines()) == 5
+
+    fetcher.urls.clear()
+    spider.crawl_details()
+    records = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
+    assert sorted(r["id"] for r in records) == IDS
+    assert len(fetcher.urls) == len(IDS) - 5  # only the remainder was fetched
+    assert next(r for r in records if r["id"] == 13)["not_found"] is True
+    assert next(r for r in records if r["id"] == 8)["title"] == "thing 8"
+
+
+def test_find_missing_refetches_only_skipped_rows():
+    fetcher = FakeFetcher(IDS)
+    spider = ThingSpider(fetcher)
+    spider.crawl_list()
+    spider.crawl_details(limit=8)  # IDs 1..34
+    path = spider.entity.detail_jsonl()
+    kept = [line for line in path.read_text(encoding="utf-8").splitlines()
+            if json.loads(line)["id"] not in (3, 13)]  # pretend these were skipped while blocked
+    path.write_text("\n".join(kept) + "\n", encoding="utf-8")
+
+    assert spider.find_missing(dry_run=True) == [3, 13]
+    fetcher.urls.clear()
+    assert spider.find_missing() == [3, 13]
+    assert len(fetcher.urls) == 2  # IDs past 34 are left for `detail`
+    assert spider.find_missing() == []