143 lines
5.6 KiB
Python
143 lines
5.6 KiB
Python
"""Extractors for the data blobs Wowhead embeds in its pages.
|
|
|
|
Almost everything worth having is in inline JSON / JS object literals rather than in the HTML,
|
|
so these helpers find a blob by its marker and hand it to chompjs (which accepts unquoted keys
|
|
and bare identifiers that plain json.loads would reject).
|
|
"""
|
|
import json
|
|
import re
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
import chompjs
|
|
from parsel import Selector
|
|
|
|
from config import LIST_ROW_CAP
|
|
|
|
SIDES = {1: "Alliance", 2: "Horde", 3: "Both"}
|
|
|
|
_GATHERER_TYPES = {1: "npc", 2: "object", 3: "item", 5: "quest"}
|
|
_ENTITY_LINK = re.compile(r"\[url=/\w+/(npc|object|item|quest|zone)=(\d+)[^\]]*\](.*?)\[/url\]")
|
|
_MARKUP_TAG = re.compile(r"\[/?[a-z]+[^\]]*\]")
|
|
_SMALL_WORDS = {"a", "an", "and", "at", "for", "in", "of", "on", "the", "to"}
|
|
|
|
|
|
def js_object_after(html: str, marker: str) -> Optional[Any]:
|
|
"""Parse the JS object/array literal that starts right after `marker`."""
|
|
i = html.find(marker)
|
|
if i < 0:
|
|
return None
|
|
return chompjs.parse_js_object(html[i + len(marker):])
|
|
|
|
|
|
def json_script(html: str, script_id: str) -> Optional[Any]:
|
|
m = re.search(rf'<script type="application/json" id="{re.escape(script_id)}">(.*?)</script>', html, re.S)
|
|
return json.loads(m.group(1)) if m else None
|
|
|
|
|
|
def listview(html: str, listview_id: str) -> Optional[Dict[str, Any]]:
|
|
"""Return a `new Listview({...})` config by id. List pages may use either embedding style."""
|
|
for m in re.finditer(r"new Listview\(\{", html):
|
|
head = html[m.end():m.end() + 300]
|
|
if re.search(rf"\bid:\s*'{re.escape(listview_id)}'", head):
|
|
return chompjs.parse_js_object(html[m.end() - 1:])
|
|
for lv in json_script(html, "data.page.listPage.listviews") or []:
|
|
if lv.get("id") == listview_id:
|
|
return lv
|
|
return None
|
|
|
|
|
|
def filter_categories(html: str) -> Dict[int, Tuple[str, Dict[int, str]]]:
|
|
"""Category tree from the list page's Filter.init config: {id: (name, {sub_id: sub_name})}.
|
|
|
|
Top-level and sub-category ids overlap (Kalimdor and Dun Morogh are both 1), hence the nesting.
|
|
Names come from url slugs, so "elwynn-forest" becomes "Elwynn Forest".
|
|
"""
|
|
config = js_object_after(html, "Filter.init(") or {}
|
|
|
|
def name(c):
|
|
words = c["url"].split("-")
|
|
return " ".join(w if i and w in _SMALL_WORDS else w.capitalize() for i, w in enumerate(words))
|
|
|
|
return {int(c["id"]): (name(c), {int(s["id"]): name(s) for s in c.get("categories") or []})
|
|
for c in config.get("categories") or []}
|
|
|
|
|
|
def list_page(html: str, listview_id: str) -> Tuple[List[Dict[str, Any]], bool, Optional[int]]:
|
|
"""Rows of a list page, whether Wowhead truncated them at its row cap, and its total match count."""
|
|
lv = listview(html, listview_id) or {}
|
|
rows = lv.get("data") or []
|
|
truncated = bool(lv.get("_truncated")) or len(rows) >= LIST_ROW_CAP
|
|
# "5,442 quests found (1,000 displayed)" only appears when truncated
|
|
m = re.match(r"([\d,]+) \w+ found", lv.get("note") or "")
|
|
total = lv.get("totalCount") or (int(m.group(1).replace(",", "")) if m else None)
|
|
return rows, truncated, total if truncated else len(rows)
|
|
|
|
|
|
def entity_blob(html: str, kind: str, entity_id: int) -> Optional[Dict[str, Any]]:
|
|
"""The `$.extend(g_quests[123], {...})` record a detail page carries for itself."""
|
|
return js_object_after(html, f"$.extend(g_{kind}s[{entity_id}], ")
|
|
|
|
|
|
def gatherer(html: str) -> Dict[str, Dict[int, str]]:
|
|
"""Names of every entity the page references, keyed by type then id."""
|
|
names: Dict[str, Dict[int, str]] = {}
|
|
for m in re.finditer(r"WH\.Gatherer\.addData\((\d+), \d+, ", html):
|
|
kind = _GATHERER_TYPES.get(int(m.group(1)))
|
|
if kind:
|
|
for eid, data in chompjs.parse_js_object(html[m.end():]).items():
|
|
names.setdefault(kind, {})[int(eid)] = data.get("name_enus")
|
|
return names
|
|
|
|
|
|
def quick_facts(html: str) -> List[str]:
|
|
"""The "Quick Facts" infobox as a list of raw markup lines ([li] items)."""
|
|
m = re.search(r'WH\.markup\.printHtml\(("(?:[^"\\]|\\.)*"), "infobox-contents-0"', html)
|
|
if not m:
|
|
return []
|
|
markup = json.loads(m.group(1))
|
|
return re.findall(r"\[li\](.*?)\[/li\]", markup)
|
|
|
|
|
|
def markup_text(markup: str) -> str:
|
|
return " ".join(_MARKUP_TAG.sub("", markup).replace(" ", " ").split())
|
|
|
|
|
|
def markup_links(markup: str) -> List[Dict[str, Any]]:
|
|
return [{"type": t, "id": int(i), "name": markup_text(n)} for t, i, n in _ENTITY_LINK.findall(markup)]
|
|
|
|
|
|
def mapper(html: str) -> Optional[Dict[str, Any]]:
|
|
return js_object_after(html, "new Mapper(")
|
|
|
|
|
|
def selector(html: str) -> Selector:
|
|
# Keep <br> line breaks in extracted text.
|
|
return Selector(text=re.sub(r"<br\s*/?>", "\n", html))
|
|
|
|
|
|
def text_after(sel: Selector, xpath: str, stop_tags=("h2",)) -> Optional[str]:
|
|
"""Text following the first node matching `xpath`, up to the first sibling in `stop_tags`."""
|
|
found = sel.xpath(xpath)
|
|
if not found:
|
|
return None
|
|
node = found[0].root
|
|
parts = [node.tail or ""]
|
|
for sibling in node.itersiblings():
|
|
if sibling.tag in stop_tags:
|
|
break
|
|
if sibling.tag not in ("script", "table"):
|
|
parts.append("".join(sibling.itertext()))
|
|
parts.append(sibling.tail or "")
|
|
return clean_text("".join(parts))
|
|
|
|
|
|
def section_text(sel: Selector, heading: str) -> Optional[str]:
|
|
"""Text of an <h2> section such as Description, Progress or Completion."""
|
|
return text_after(sel, f"//h2[normalize-space(.)='{heading}']")
|
|
|
|
|
|
def clean_text(text: str) -> Optional[str]:
|
|
lines = [" ".join(line.split()) for line in text.splitlines()]
|
|
text = "\n".join(lines).strip()
|
|
return re.sub(r"\n{3,}", "\n\n", text) or None
|