initial commit

wowhead_scraper/parsers/object.py

@@ -0,0 +1,68 @@
+from typing import Any, Dict, List
+
+from parsers.common import SIDES, entity_blob, js_object_after, listview, markup_text, quick_facts
+
+LISTVIEW_ID = "objects"
+CSV_FIELDS = ["id", "name", "type", "type_id", "skill", "zone_ids", "url"]
+
+
+def list_row(raw: Dict[str, Any], categories: Dict[int, tuple]) -> Dict[str, Any]:
+    type_name, _ = categories.get(raw.get("type"), (None, {}))
+    return {
+        "id": raw["id"],
+        "name": raw.get("name"),
+        "type": type_name,
+        "type_id": raw.get("type"),
+        "skill": raw.get("skill"),
+        # -1 entries are placeholders for "unknown location"
+        "zone_ids": ";".join(str(z) for z in raw.get("location", []) if z >= 0),
+    }
+
+
+def parse_page(html: str, row: Dict[str, Any]) -> Dict[str, Any]:
+    object_id = int(row["id"])
+    blob = entity_blob(html, "object", object_id) or {}
+    return {
+        "id": object_id,
+        "name": blob.get("name", row.get("name")),
+        "type": row.get("type"),
+        "type_id": blob.get("type"),
+        "skill": int(row["skill"]) if row.get("skill") not in (None, "") else None,
+        "locations": _locations(js_object_after(html, "var g_mapperData = ")),
+        "starts": _quests(html, "starts"),
+        "ends": _quests(html, "ends"),
+        "contains": _loot(html),
+        "quick_facts": [markup_text(f) for f in quick_facts(html)],
+        "url": row.get("url"),
+    }
+
+
+def _locations(mapper_data) -> List[Dict[str, Any]]:
+    locations = []
+    for zone_id, floors in (mapper_data or {}).items():
+        for floor in floors:
+            locations.append({
+                "zone_id": int(zone_id),
+                "zone": floor.get("uiMapName"),
+                "ui_map_id": floor.get("uiMapId"),
+                "count": floor.get("count"),
+                "coords": floor.get("coords"),
+            })
+    return locations
+
+
+def _quests(html: str, listview_id: str) -> List[Dict[str, Any]]:
+    lv = listview(html, listview_id) or {}
+    return [{"id": q["id"], "name": q.get("name"), "level": q.get("level"),
+             "reqlevel": q.get("reqlevel"), "side": SIDES.get(q.get("side"))}
+            for q in lv.get("data") or []]
+
+
+def _loot(html: str) -> List[Dict[str, Any]]:
+    lv = listview(html, "contains") or {}
+    opened = lv.get("_totalCount")
+    return [{"id": i["id"], "name": i.get("name"), "count": i.get("count"),
+             # same figure Wowhead shows in its "%" column
+             "percent": round(100 * i["count"] / opened, 2) if opened and i.get("count", -1) > 0 else None,
+             "stack": i.get("stack")}
+            for i in lv.get("data") or []]