initial commit

wowhead_scraper/parsers/quest.py

@@ -0,0 +1,131 @@
+import re
+from typing import Any, Dict, List
+
+from parsers.common import (SIDES, entity_blob, gatherer, markup_links, markup_text, mapper, quick_facts,
+                            section_text, selector, text_after)
+
+LISTVIEW_ID = "quests"
+CSV_FIELDS = ["id", "name", "level", "reqlevel", "side", "category", "category_id",
+              "category2", "category2_id", "exp", "money", "url"]
+
+_MAPPER_TYPES = {1: "npc", 2: "object", 3: "item"}
+
+
+def list_row(raw: Dict[str, Any], categories: Dict[int, tuple]) -> Dict[str, Any]:
+    top_name, sub_names = categories.get(raw.get("category2"), (None, {}))
+    return {
+        "id": raw["id"],
+        "name": raw.get("name"),
+        "level": raw.get("level"),
+        "reqlevel": raw.get("reqlevel"),
+        "side": SIDES.get(raw.get("side")),
+        "category": sub_names.get(raw.get("category")),
+        "category_id": raw.get("category"),
+        "category2": top_name,
+        "category2_id": raw.get("category2"),
+        "exp": raw.get("xp"),
+        "money": raw.get("money"),
+    }
+
+
+def parse_page(html: str, row: Dict[str, Any]) -> Dict[str, Any]:
+    quest_id = int(row["id"])
+    sel = selector(html)
+    blob = entity_blob(html, "quest", quest_id) or {}
+    names = gatherer(html)
+    facts = quick_facts(html)
+
+    return {
+        "id": quest_id,
+        "name": blob.get("name", row.get("name")),
+        "level": blob.get("level"),
+        "reqlevel": blob.get("reqlevel"),
+        "side": SIDES.get(blob.get("side")),
+        "category": row.get("category"),
+        "category2": row.get("category2"),
+        "objectives": text_after(sel, "//h1[@class='heading-size-1']", stop_tags=("table", "div", "h2", "script")),
+        "objective_items": _objective_items(sel),
+        "description": section_text(sel, "Description"),
+        "progress": section_text(sel, "Progress"),
+        "completion": section_text(sel, "Completion"),
+        "start": _fact_links(facts, "Start:"),
+        "end": _fact_links(facts, "End:"),
+        "locations": _locations(mapper(html)),
+        "series": _series(sel, quest_id),
+        "rewards": {
+            "xp": blob.get("xp"),
+            # the page's record omits money when it is 0
+            "money": blob.get("money", 0),
+            "item_choices": _items(blob.get("itemchoices"), names),
+            "items": _items(blob.get("itemrewards"), names),
+            "reputation": _reputation(sel, blob.get("reprewards")),
+        },
+        "quick_facts": [markup_text(f) for f in facts],
+        "url": row.get("url"),
+    }
+
+
+def _fact_links(facts: List[str], label: str) -> List[Dict[str, Any]]:
+    for fact in facts:
+        if markup_text(fact).startswith(label):
+            return markup_links(fact)
+    return []
+
+
+def _locations(mapper_config) -> List[Dict[str, Any]]:
+    """Flatten mapper points: who/what is where, and its role (start, end, objective)."""
+    points = []
+    for zone_id, zone in ((mapper_config or {}).get("objectives") or {}).items():
+        for level in zone.get("levels", []):
+            for p in level:
+                points.append({
+                    "role": p.get("point"),
+                    "type": _MAPPER_TYPES.get(p.get("type"), p.get("type")),
+                    "id": p.get("id"),
+                    "name": p.get("name"),
+                    "zone_id": int(zone_id),
+                    "zone": zone.get("zone"),
+                    "coord": p.get("coord"),
+                    "coords": p.get("coords"),
+                })
+    return points
+
+
+def _objective_items(sel) -> List[Dict[str, Any]]:
+    result = []
+    for tr in sel.xpath("(//table[@class='icon-list'])[1]//tr"):
+        href = tr.xpath(".//a/@href").get() or ""
+        m = re.search(r"/(\w+)=(\d+)", href)
+        # A count for items/kills; a standing such as "Friendly" for faction objectives.
+        quantity = tr.attrib.get("data-icon-list-quantity", "1")
+        result.append({
+            "type": m.group(1) if m else None,
+            "id": int(m.group(2)) if m else None,
+            "name": " ".join(tr.xpath(".//a//text()").getall()).strip() or None,
+            "quantity": int(quantity) if quantity.isdigit() else None,
+            "standing": None if quantity.isdigit() else quantity,
+            "provided": "(Provided)" in "".join(tr.xpath(".//td//text()").getall()),
+        })
+    return result
+
+
+def _series(sel, quest_id: int) -> List[Dict[str, Any]]:
+    series = []
+    for td in sel.xpath("//table[@class='series']//tr/td"):
+        href = td.xpath(".//a/@href").get()
+        m = re.search(r"quest=(\d+)", href or "")
+        series.append({"id": int(m.group(1)) if m else quest_id,
+                       "name": " ".join(td.xpath(".//text()").getall()).strip()})
+    return series
+
+
+def _items(pairs, names) -> List[Dict[str, Any]]:
+    return [{"id": i, "name": names.get("item", {}).get(i), "count": c} for i, c in pairs or []]
+
+
+def _reputation(sel, pairs) -> List[Dict[str, Any]]:
+    factions = {}
+    for a in sel.xpath("//a[contains(@href, '/faction=')]"):
+        m = re.search(r"faction=(\d+)", a.attrib["href"])
+        factions[int(m.group(1))] = a.xpath("string()").get().strip()
+    return [{"faction_id": f, "faction": factions.get(f), "value": v} for f, v in pairs or []]