initial commit

wowhead_scraper/fetcher.py

@@ -0,0 +1,121 @@
+"""HTTP client: browser-impersonating requests with a shared throttle, back-off and WAF handling.
+
+Wowhead sits behind an AWS WAF JavaScript challenge. curl_cffi alone can't solve it, so when a
+challenge page comes back we load the page once in headless Chromium (Playwright), take the
+`aws-waf-token` cookie it earns, and keep using the fast curl_cffi client with that cookie.
+"""
+import logging
+import random
+import threading
+import time
+
+from curl_cffi import requests
+from curl_cffi.requests.exceptions import RequestException
+
+from config import CrawlSettings
+
+log = logging.getLogger(__name__)
+
+# Playwright's default UA says "HeadlessChrome", which CloudFront rejects outright.
+BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+              "(KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36")
+RETRY_STATUSES = {403, 408, 429, 500, 502, 503, 504}
+
+
+class FetchError(Exception):
+    pass
+
+
+class NotFound(FetchError):
+    pass
+
+
+def is_challenge(html: str) -> bool:
+    # Real pages also set awsWafCookieDomainList, so look for the challenge loader itself.
+    return "AwsWafIntegration" in html and "challenge.js" in html
+
+
+class Fetcher:
+    def __init__(self, settings: CrawlSettings = None) -> None:
+        self.settings = settings or CrawlSettings()
+        self._local = threading.local()
+        self._throttle_lock = threading.Lock()
+        self._next_request_at = 0.0
+        self._waf_lock = threading.Lock()
+        self._cookies: dict = {}
+        self._cookie_generation = 0
+
+    def get(self, url: str) -> str:
+        s = self.settings
+        for attempt in range(s.max_retries + 1):
+            generation = self._cookie_generation
+            self._throttle()
+            try:
+                r = self._session().get(url, cookies=self._cookies, timeout=s.timeout)
+                status, text = r.status_code, r.text
+            except RequestException as e:
+                status, text, reason = None, "", f"{type(e).__name__}: {e}"
+            else:
+                if "notFound=" in r.url or status == 404:
+                    raise NotFound(url)
+                if is_challenge(text):
+                    self._solve_challenge(url, generation)
+                    continue
+                if status == 200:
+                    return text
+                if status not in RETRY_STATUSES:
+                    raise FetchError(f"HTTP {status} for {url}")
+                reason = f"HTTP {status}"
+
+            if attempt == s.max_retries:
+                break
+            wait = min(s.backoff_max, s.backoff_base * 2 ** attempt) + random.uniform(0, s.jitter)
+            log.warning("%s for %s, retry %d/%d in %.1fs", reason, url, attempt + 1, s.max_retries, wait)
+            time.sleep(wait)
+        raise FetchError(f"Giving up on {url} after {s.max_retries} retries")
+
+    def _session(self) -> requests.Session:
+        # curl_cffi sessions aren't thread-safe, so each worker thread gets its own.
+        if not hasattr(self._local, "session"):
+            self._local.session = requests.Session(impersonate=self.settings.impersonate)
+        return self._local.session
+
+    def _throttle(self) -> None:
+        with self._throttle_lock:
+            now = time.monotonic()
+            start_at = max(now, self._next_request_at)
+            self._next_request_at = start_at + self.settings.delay + random.uniform(0, self.settings.jitter)
+        time.sleep(start_at - now)
+
+    def _solve_challenge(self, url: str, seen_generation: int) -> None:
+        with self._waf_lock:
+            if self._cookie_generation != seen_generation:
+                return  # another worker refreshed the token while we waited
+            log.info("WAF challenge hit, solving it in headless Chromium")
+            self._cookies = solve_waf_challenge(url, self.settings.timeout)
+            self._cookie_generation += 1
+
+
+def solve_waf_challenge(url: str, timeout: float) -> dict:
+    from playwright.sync_api import Error as PlaywrightError, sync_playwright
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True, args=["--disable-blink-features=AutomationControlled"])
+        try:
+            context = browser.new_context(user_agent=BROWSER_UA, locale="en-US")
+            page = context.new_page()
+            page.goto(url, wait_until="domcontentloaded", timeout=timeout * 1000)
+            deadline = time.monotonic() + timeout
+            # The challenge script sets the cookie, then reloads into the real page.
+            while time.monotonic() < deadline:
+                cookies = {c["name"]: c["value"] for c in context.cookies()}
+                try:
+                    solved = "aws-waf-token" in cookies and not is_challenge(page.content())
+                except PlaywrightError:  # page is mid-reload
+                    solved = False
+                if solved:
+                    return cookies
+                page.wait_for_timeout(500)
+        finally:
+            browser.close()
+    raise FetchError(f"Could not solve WAF challenge for {url}")