initial commit

wowhead_scraper/parsers/common.py

@@ -0,0 +1,142 @@
+"""Extractors for the data blobs Wowhead embeds in its pages.
+
+Almost everything worth having is in inline JSON / JS object literals rather than in the HTML,
+so these helpers find a blob by its marker and hand it to chompjs (which accepts unquoted keys
+and bare identifiers that plain json.loads would reject).
+"""
+import json
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+import chompjs
+from parsel import Selector
+
+from config import LIST_ROW_CAP
+
+SIDES = {1: "Alliance", 2: "Horde", 3: "Both"}
+
+_GATHERER_TYPES = {1: "npc", 2: "object", 3: "item", 5: "quest"}
+_ENTITY_LINK = re.compile(r"\[url=/\w+/(npc|object|item|quest|zone)=(\d+)[^\]]*\](.*?)\[/url\]")
+_MARKUP_TAG = re.compile(r"\[/?[a-z]+[^\]]*\]")
+_SMALL_WORDS = {"a", "an", "and", "at", "for", "in", "of", "on", "the", "to"}
+
+
+def js_object_after(html: str, marker: str) -> Optional[Any]:
+    """Parse the JS object/array literal that starts right after `marker`."""
+    i = html.find(marker)
+    if i < 0:
+        return None
+    return chompjs.parse_js_object(html[i + len(marker):])
+
+
+def json_script(html: str, script_id: str) -> Optional[Any]:
+    m = re.search(rf'<script type="application/json" id="{re.escape(script_id)}">(.*?)</script>', html, re.S)
+    return json.loads(m.group(1)) if m else None
+
+
+def listview(html: str, listview_id: str) -> Optional[Dict[str, Any]]:
+    """Return a `new Listview({...})` config by id. List pages may use either embedding style."""
+    for m in re.finditer(r"new Listview\(\{", html):
+        head = html[m.end():m.end() + 300]
+        if re.search(rf"\bid:\s*'{re.escape(listview_id)}'", head):
+            return chompjs.parse_js_object(html[m.end() - 1:])
+    for lv in json_script(html, "data.page.listPage.listviews") or []:
+        if lv.get("id") == listview_id:
+            return lv
+    return None
+
+
+def filter_categories(html: str) -> Dict[int, Tuple[str, Dict[int, str]]]:
+    """Category tree from the list page's Filter.init config: {id: (name, {sub_id: sub_name})}.
+
+    Top-level and sub-category ids overlap (Kalimdor and Dun Morogh are both 1), hence the nesting.
+    Names come from url slugs, so "elwynn-forest" becomes "Elwynn Forest".
+    """
+    config = js_object_after(html, "Filter.init(") or {}
+
+    def name(c):
+        words = c["url"].split("-")
+        return " ".join(w if i and w in _SMALL_WORDS else w.capitalize() for i, w in enumerate(words))
+
+    return {int(c["id"]): (name(c), {int(s["id"]): name(s) for s in c.get("categories") or []})
+            for c in config.get("categories") or []}
+
+
+def list_page(html: str, listview_id: str) -> Tuple[List[Dict[str, Any]], bool, Optional[int]]:
+    """Rows of a list page, whether Wowhead truncated them at its row cap, and its total match count."""
+    lv = listview(html, listview_id) or {}
+    rows = lv.get("data") or []
+    truncated = bool(lv.get("_truncated")) or len(rows) >= LIST_ROW_CAP
+    # "5,442 quests found (1,000 displayed)" only appears when truncated
+    m = re.match(r"([\d,]+) \w+ found", lv.get("note") or "")
+    total = lv.get("totalCount") or (int(m.group(1).replace(",", "")) if m else None)
+    return rows, truncated, total if truncated else len(rows)
+
+
+def entity_blob(html: str, kind: str, entity_id: int) -> Optional[Dict[str, Any]]:
+    """The `$.extend(g_quests[123], {...})` record a detail page carries for itself."""
+    return js_object_after(html, f"$.extend(g_{kind}s[{entity_id}], ")
+
+
+def gatherer(html: str) -> Dict[str, Dict[int, str]]:
+    """Names of every entity the page references, keyed by type then id."""
+    names: Dict[str, Dict[int, str]] = {}
+    for m in re.finditer(r"WH\.Gatherer\.addData\((\d+), \d+, ", html):
+        kind = _GATHERER_TYPES.get(int(m.group(1)))
+        if kind:
+            for eid, data in chompjs.parse_js_object(html[m.end():]).items():
+                names.setdefault(kind, {})[int(eid)] = data.get("name_enus")
+    return names
+
+
+def quick_facts(html: str) -> List[str]:
+    """The "Quick Facts" infobox as a list of raw markup lines ([li] items)."""
+    m = re.search(r'WH\.markup\.printHtml\(("(?:[^"\\]|\\.)*"), "infobox-contents-0"', html)
+    if not m:
+        return []
+    markup = json.loads(m.group(1))
+    return re.findall(r"\[li\](.*?)\[/li\]", markup)
+
+
+def markup_text(markup: str) -> str:
+    return " ".join(_MARKUP_TAG.sub("", markup).replace("&nbsp;", " ").split())
+
+
+def markup_links(markup: str) -> List[Dict[str, Any]]:
+    return [{"type": t, "id": int(i), "name": markup_text(n)} for t, i, n in _ENTITY_LINK.findall(markup)]
+
+
+def mapper(html: str) -> Optional[Dict[str, Any]]:
+    return js_object_after(html, "new Mapper(")
+
+
+def selector(html: str) -> Selector:
+    # Keep <br> line breaks in extracted text.
+    return Selector(text=re.sub(r"<br\s*/?>", "\n", html))
+
+
+def text_after(sel: Selector, xpath: str, stop_tags=("h2",)) -> Optional[str]:
+    """Text following the first node matching `xpath`, up to the first sibling in `stop_tags`."""
+    found = sel.xpath(xpath)
+    if not found:
+        return None
+    node = found[0].root
+    parts = [node.tail or ""]
+    for sibling in node.itersiblings():
+        if sibling.tag in stop_tags:
+            break
+        if sibling.tag not in ("script", "table"):
+            parts.append("".join(sibling.itertext()))
+        parts.append(sibling.tail or "")
+    return clean_text("".join(parts))
+
+
+def section_text(sel: Selector, heading: str) -> Optional[str]:
+    """Text of an <h2> section such as Description, Progress or Completion."""
+    return text_after(sel, f"//h2[normalize-space(.)='{heading}']")
+
+
+def clean_text(text: str) -> Optional[str]:
+    lines = [" ".join(line.split()) for line in text.splitlines()]
+    text = "\n".join(lines).strip()
+    return re.sub(r"\n{3,}", "\n\n", text) or None