hwilliams/wowhead-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
Hunter
2026-11-02 18:25:49 -05:00
commit 1367da24b0
36 changed files with 19448 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

328
quest_analyzer/experiments-1.ipynb Normal file
View File

@@ -0,0 +1,328 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "5c77481b",
"metadata": {},
"outputs": [],
"source": [
"# Requirements\n",
"import numpy as np\n",
"import pandas as pd\n",
"import io\n",
"import json\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d0b3427f",
"metadata": {},
"outputs": [],
"source": [
"path = rf\"D:\\Project\\Py\\WowheadScraper\\quest_analyzer\\resources\\quests.jsonl\"\n",
"df = pd.read_json(path, lines=True)\n",
"\n",
"if df is None:\n",
" raise Exception(\"DataFrame is None. Failed to read the CSV file.\")\n",
"\n",
"df.describe()\n",
"\n",
"horde_only_quests = df[(df[\"side\"] == \"Horde\") | (df[\"side\"] == \"Both\")]\n",
"all_zones = df[[\"category\", \"category2\"]].drop_duplicates().to_dict(orient='records')\n",
"\n",
"\n",
"def locations_contains_zone(row, zone_name):\n",
" locations = row.get(\"locations\")\n",
" if not isinstance(locations, list):\n",
" return False\n",
" for location in locations:\n",
" if location.get(\"role\") == \"start\" and zone_name in location[\"zone\"]:\n",
" return True\n",
" if location.get(\"role\") == \"end\" and zone_name in location[\"zone\"]:\n",
" return True\n",
" return False\n",
"\n",
"def get_quests_by_zone(_df: pd.DataFrame, zone_name: str) -> pd.DataFrame:\n",
" location_mask = _df.apply(lambda row: locations_contains_zone(row, zone_name), axis=1)\n",
" category_mask = (_df[\"category\"] == zone_name) | (_df[\"category2\"] == zone_name)\n",
" return _df[location_mask | category_mask]\n",
"\n",
"durotar_quests = get_quests_by_zone(horde_only_quests, \"Durotar\")\n",
"mulgore_quests = get_quests_by_zone(horde_only_quests, \"Mulgore\")\n",
"tirisfal_glades_quests = get_quests_by_zone(horde_only_quests, \"Tirisfal Glades\")\n",
"zephras_quests = get_quests_by_zone(horde_only_quests, \"Zephras Isle\")\n",
"\n",
"print(f\"Total quests for each starting zone:\")\n",
"print(f\"Durotar: {len(durotar_quests)}\")\n",
"print(f\"Mulgore: {len(mulgore_quests)}\")\n",
"print(f\"Tirisfal Glades: {len(tirisfal_glades_quests)}\")\n",
"print(f\"Zephras Isle: {len(zephras_quests)}\")\n",
"\n",
"# Now we limit the search to level brackets 1-10 and 11-20 for each zone\n",
"\n",
"durotar_quests_1_10 = durotar_quests[(durotar_quests[\"level\"] >= 1) & (durotar_quests[\"level\"] <= 10)]\n",
"durotar_quests_11_20 = durotar_quests[(durotar_quests[\"level\"] >= 11) & (durotar_quests[\"level\"] <= 20)]\n",
"\n",
"mulgore_quests_1_10 = mulgore_quests[(mulgore_quests[\"level\"] >= 1) & (mulgore_quests[\"level\"] <= 10)]\n",
"mulgore_quests_11_20 = mulgore_quests[(mulgore_quests[\"level\"] >= 11) & (mulgore_quests[\"level\"] <= 20)]\n",
"\n",
"tirisfal_glades_quests_1_10 = tirisfal_glades_quests[(tirisfal_glades_quests[\"level\"] >= 1) & (tirisfal_glades_quests[\"level\"] <= 10)]\n",
"tirisfal_glades_quests_11_20 = tirisfal_glades_quests[(tirisfal_glades_quests[\"level\"] >= 11) & (tirisfal_glades_quests[\"level\"] <= 20)]\n",
"\n",
"zephras_quests_1_10 = zephras_quests[(zephras_quests[\"level\"] >= 1) & (zephras_quests[\"level\"] <= 10)]\n",
"zephras_quests_11_20 = zephras_quests[(zephras_quests[\"level\"] >= 11) & (zephras_quests[\"level\"] <= 20)]\n",
"\n",
"print(f\"\\n\\nTotal quests for each starting zone and level bracket:\")\n",
"print(f\"Durotar 1-10: {len(durotar_quests_1_10)}\")\n",
"print(f\"Durotar 11-20: {len(durotar_quests_11_20)}\")\n",
"print(f\"Mulgore 1-10: {len(mulgore_quests_1_10)}\")\n",
"print(f\"Mulgore 11-20: {len(mulgore_quests_11_20)}\")\n",
"print(f\"Tirisfal Glades 1-10: {len(tirisfal_glades_quests_1_10)}\")\n",
"print(f\"Tirisfal Glades 11-20: {len(tirisfal_glades_quests_11_20)}\")\n",
"print(f\"Zephras Isle 1-10: {len(zephras_quests_1_10)}\")\n",
"print(f\"Zephras Isle 11-20: {len(zephras_quests_11_20)}\")\n",
"\n",
"\n",
"# Now we calculate the total experience available from quests for each zone and level bracket\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e826b3eb",
"metadata": {},
"outputs": [],
"source": [
"brackets = {\n",
" (\"Durotar\", \"1-10\"): durotar_quests_1_10,\n",
" (\"Durotar\", \"11-20\"): durotar_quests_11_20,\n",
" (\"Mulgore\", \"1-10\"): mulgore_quests_1_10,\n",
" (\"Mulgore\", \"11-20\"): mulgore_quests_11_20,\n",
" (\"Tirisfal Glades\", \"1-10\"): tirisfal_glades_quests_1_10,\n",
" (\"Tirisfal Glades\", \"11-20\"): tirisfal_glades_quests_11_20,\n",
" (\"Zephras Isle\", \"1-10\"): zephras_quests_1_10,\n",
" (\"Zephras Isle\", \"11-20\"): zephras_quests_11_20,\n",
"}\n",
"\n",
"def quest_xp(rewards) -> int:\n",
" if isinstance(rewards, dict) and rewards.get(\"xp\"):\n",
" return rewards[\"xp\"]\n",
" return 0\n",
"\n",
"xp_summary = pd.DataFrame([\n",
" {\"zone\": zone, \"bracket\": bracket, \"quests\": len(quests), \"total_xp\": quests[\"rewards\"].apply(quest_xp).sum()}\n",
" for (zone, bracket), quests in brackets.items()\n",
"])\n",
"xp_summary[\"avg_xp\"] = (xp_summary[\"total_xp\"] / xp_summary[\"quests\"]).round(0)\n",
"xp_summary"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ab76fa6f",
"metadata": {},
"outputs": [],
"source": [
"# Now we try to calculate travel requirements for each quest. We define travel as the distance from the quest giver to the objective area, and then from the objective area to the quest turn-in. For delivery/talk-to quests, we just measure straight from giver to turn-in.\n",
"# An additional challenge is that the coordinates given are percentage based based off of the map frame.\n",
"# Some digging online found some map size estimates, and they are the closest I have to a reliable source for now.\n",
"\n",
"OBJECTIVE_ROLES = (\"requirement\", \"sourcerequirement\")\n",
"\n",
"def zone_coords(locations, roles, zone_name):\n",
" return [loc[\"coord\"] for loc in locations\n",
" if loc.get(\"role\") in roles and loc.get(\"zone\") == zone_name and loc.get(\"coord\")]\n",
"\n",
"def distance(a, b):\n",
" if a is None or b is None:\n",
" return np.nan\n",
" return float(np.linalg.norm(np.subtract(a, b)))\n",
"\n",
"def quest_travel(row, zone_name):\n",
" locations = row[\"locations\"] if isinstance(row[\"locations\"], list) else []\n",
" starts = zone_coords(locations, (\"start\",), zone_name)\n",
" ends = zone_coords(locations, (\"end\",), zone_name)\n",
" objectives = zone_coords(locations, OBJECTIVE_ROLES, zone_name)\n",
"\n",
" start = starts[0] if starts else None\n",
" end = ends[0] if ends else None\n",
" # Objective area = mean of every objective npc/object's representative coord\n",
" objective = np.mean(objectives, axis=0) if objectives else None\n",
"\n",
" if objective is None:\n",
" # Delivery / talk-to quests: straight from giver to turn-in\n",
" to_objective, from_objective = np.nan, np.nan\n",
" total = distance(start, end)\n",
" else:\n",
" to_objective = distance(start, objective)\n",
" from_objective = distance(objective, end)\n",
" total = to_objective + from_objective\n",
"\n",
" return pd.Series({\n",
" \"start_x\": start[0] if start else np.nan,\n",
" \"start_y\": start[1] if start else np.nan,\n",
" \"objective_x\": objective[0] if objective is not None else np.nan,\n",
" \"objective_y\": objective[1] if objective is not None else np.nan,\n",
" \"end_x\": end[0] if end else np.nan,\n",
" \"end_y\": end[1] if end else np.nan,\n",
" \"to_objective\": to_objective,\n",
" \"from_objective\": from_objective,\n",
" \"total_travel\": total,\n",
" })\n",
"\n",
"def build_travel(quests: pd.DataFrame, zone_name: str, bracket: str) -> pd.DataFrame:\n",
" travel = quests.apply(lambda row: quest_travel(row, zone_name), axis=1)\n",
" travel.insert(0, \"zone\", zone_name)\n",
" travel.insert(1, \"bracket\", bracket)\n",
" travel.insert(2, \"id\", quests[\"id\"])\n",
" travel.insert(3, \"name\", quests[\"name\"])\n",
" travel.insert(4, \"level\", quests[\"level\"])\n",
" travel.insert(5, \"xp\", quests[\"rewards\"].apply(quest_xp))\n",
" return travel\n",
"\n",
"travel_df = pd.concat(\n",
" [build_travel(quests, zone, bracket) for (zone, bracket), quests in brackets.items()],\n",
" ignore_index=True,\n",
")\n",
"\n",
"travel_summary = travel_df.groupby([\"zone\", \"bracket\"]).agg(\n",
" quests=(\"id\", \"count\"),\n",
" with_objective=(\"objective_x\", \"count\"),\n",
" total_xp=(\"xp\", \"sum\"),\n",
" total_to_objective=(\"to_objective\", \"sum\"),\n",
" total_from_objective=(\"from_objective\", \"sum\"),\n",
" total_travel=(\"total_travel\", \"sum\"),\n",
" avg_travel=(\"total_travel\", \"mean\"),\n",
")\n",
"travel_summary[\"xp_per_travel\"] = (travel_summary[\"total_xp\"] / travel_summary[\"total_travel\"]).round(1)\n",
"travel_summary.round(1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7fe45672",
"metadata": {},
"outputs": [],
"source": [
"# Afterwards, we use X-means clustering to find hotspots for quest objective locations.\n",
"# X-means = k-means that keeps splitting clusters in two while the split improves the\n",
"# BIC score (Pelleg & Moore, 2000), so we don't have to pick k ourselves.\n",
"\n",
"def kmeans(points: np.ndarray, centers: np.ndarray, iterations: int = 100):\n",
" for _ in range(iterations):\n",
" labels = np.argmin(np.linalg.norm(points[:, None] - centers[None], axis=2), axis=1)\n",
" new_centers = np.array([\n",
" points[labels == k].mean(axis=0) if np.any(labels == k) else centers[k]\n",
" for k in range(len(centers))\n",
" ])\n",
" if np.allclose(new_centers, centers):\n",
" break\n",
" centers = new_centers\n",
" return labels, centers\n",
"\n",
"def bic(points: np.ndarray, labels: np.ndarray, centers: np.ndarray, min_variance: float = 1.25) -> float:\n",
" r, m = points.shape\n",
" k = len(centers)\n",
" if r <= k:\n",
" return -np.inf\n",
" # Many quests share the exact same objective coord, so without a floor a tiny cluster gets ~0 variance and an infinite likelihood. 1.0 = roughly 1% of the map.\n",
" variance = max(np.sum((points - centers[labels]) ** 2) / (m * (r - k)), min_variance)\n",
" log_likelihood = 0.0\n",
" for i in range(k):\n",
" r_i = np.sum(labels == i)\n",
" if r_i == 0:\n",
" continue\n",
" log_likelihood += (r_i * np.log(r_i) - r_i * np.log(r)\n",
" - r_i * m / 2 * np.log(2 * np.pi * variance)\n",
" - (r_i - 1) * m / 2)\n",
" free_params = k * (m + 1)\n",
" return log_likelihood - free_params / 2 * np.log(r)\n",
"\n",
"def best_split(members: np.ndarray, rng, attempts: int):\n",
" # k-means only finds a local optimum, so try a few random starts and keep the best split\n",
" best_score, best_children = -np.inf, None\n",
" for _ in range(attempts):\n",
" child_labels, children = kmeans(members, members[rng.choice(len(members), 2, replace=False)])\n",
" score = bic(members, child_labels, children)\n",
" if score > best_score:\n",
" best_score, best_children = score, children\n",
" return best_score, best_children\n",
"\n",
"def xmeans(points: np.ndarray, k_max: int = 15, split_attempts: int = 10, seed: int = 0):\n",
" rng = np.random.default_rng(seed)\n",
" labels, centers = kmeans(points, points.mean(axis=0, keepdims=True))\n",
" score = bic(points, labels, centers)\n",
" while True:\n",
" new_centers = []\n",
" for k, center in enumerate(centers):\n",
" members = points[labels == k]\n",
" if len(members) < 4:\n",
" new_centers.append(center)\n",
" continue\n",
" parent_score = bic(members, np.zeros(len(members), dtype=int), center[None])\n",
" child_score, children = best_split(members, rng, split_attempts)\n",
" new_centers.extend(children if child_score > parent_score else [center])\n",
" if len(new_centers) == len(centers) or len(new_centers) > k_max:\n",
" return labels, centers\n",
" # The per-cluster test is lenient on small clusters, so only keep the splits if they also improve the BIC of the whole zone\n",
" new_labels, new_centers = kmeans(points, np.array(new_centers))\n",
" new_score = bic(points, new_labels, new_centers)\n",
" if new_score <= score:\n",
" return labels, centers\n",
" labels, centers, score = new_labels, new_centers, new_score\n",
"\n",
"# Cluster each zone + bracket's objective areas separately, then classify every questin the travel data by the hotspot its objective falls in.\n",
"# Hotspot ids are only unique within a zone + bracket.\n",
"travel_df[\"objective_hotspot\"] = pd.NA\n",
"travel_df[\"hotspot_distance\"] = np.nan\n",
"hotspots = {}\n",
"\n",
"for (zone_name, bracket), bracket_travel in travel_df.groupby([\"zone\", \"bracket\"]):\n",
" has_objective = bracket_travel.dropna(subset=[\"objective_x\", \"objective_y\"])\n",
" points = has_objective[[\"objective_x\", \"objective_y\"]].to_numpy()\n",
" if len(points) < 2:\n",
" continue\n",
" labels, centers = xmeans(points)\n",
" hotspots[(zone_name, bracket)] = centers\n",
" travel_df.loc[has_objective.index, \"objective_hotspot\"] = labels\n",
" travel_df.loc[has_objective.index, \"hotspot_distance\"] = np.linalg.norm(points - centers[labels], axis=1)\n",
" print(f\"{zone_name} {bracket}: {len(centers)} hotspots from {len(points)} objectives\")\n",
"\n",
"hotspot_summary = travel_df.dropna(subset=[\"objective_hotspot\"]).groupby([\"zone\", \"bracket\", \"objective_hotspot\"]).agg(\n",
" center_x=(\"objective_x\", \"mean\"),\n",
" center_y=(\"objective_y\", \"mean\"),\n",
" quests=(\"id\", \"count\"),\n",
" total_xp=(\"xp\", \"sum\"),\n",
" avg_to_objective=(\"to_objective\", \"mean\"),\n",
" avg_from_objective=(\"from_objective\", \"mean\"),\n",
")\n",
"hotspot_summary.round(1)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv (3.14.6)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.14.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}