122 lines
4.9 KiB
Python
122 lines
4.9 KiB
Python
"""HTTP client: browser-impersonating requests with a shared throttle, back-off and WAF handling.
|
|
|
|
Wowhead sits behind an AWS WAF JavaScript challenge. curl_cffi alone can't solve it, so when a
|
|
challenge page comes back we load the page once in headless Chromium (Playwright), take the
|
|
`aws-waf-token` cookie it earns, and keep using the fast curl_cffi client with that cookie.
|
|
"""
|
|
import logging
|
|
import random
|
|
import threading
|
|
import time
|
|
|
|
from curl_cffi import requests
|
|
from curl_cffi.requests.exceptions import RequestException
|
|
|
|
from config import CrawlSettings
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
# Playwright's default UA says "HeadlessChrome", which CloudFront rejects outright.
|
|
BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36")
|
|
RETRY_STATUSES = {403, 408, 429, 500, 502, 503, 504}
|
|
|
|
|
|
class FetchError(Exception):
|
|
pass
|
|
|
|
|
|
class NotFound(FetchError):
|
|
pass
|
|
|
|
|
|
def is_challenge(html: str) -> bool:
|
|
# Real pages also set awsWafCookieDomainList, so look for the challenge loader itself.
|
|
return "AwsWafIntegration" in html and "challenge.js" in html
|
|
|
|
|
|
class Fetcher:
|
|
def __init__(self, settings: CrawlSettings = None) -> None:
|
|
self.settings = settings or CrawlSettings()
|
|
self._local = threading.local()
|
|
self._throttle_lock = threading.Lock()
|
|
self._next_request_at = 0.0
|
|
self._waf_lock = threading.Lock()
|
|
self._cookies: dict = {}
|
|
self._cookie_generation = 0
|
|
|
|
def get(self, url: str) -> str:
|
|
s = self.settings
|
|
for attempt in range(s.max_retries + 1):
|
|
generation = self._cookie_generation
|
|
self._throttle()
|
|
try:
|
|
r = self._session().get(url, cookies=self._cookies, timeout=s.timeout)
|
|
status, text = r.status_code, r.text
|
|
except RequestException as e:
|
|
status, text, reason = None, "", f"{type(e).__name__}: {e}"
|
|
else:
|
|
if "notFound=" in r.url or status == 404:
|
|
raise NotFound(url)
|
|
if is_challenge(text):
|
|
self._solve_challenge(url, generation)
|
|
continue
|
|
if status == 200:
|
|
return text
|
|
if status not in RETRY_STATUSES:
|
|
raise FetchError(f"HTTP {status} for {url}")
|
|
reason = f"HTTP {status}"
|
|
|
|
if attempt == s.max_retries:
|
|
break
|
|
wait = min(s.backoff_max, s.backoff_base * 2 ** attempt) + random.uniform(0, s.jitter)
|
|
log.warning("%s for %s, retry %d/%d in %.1fs", reason, url, attempt + 1, s.max_retries, wait)
|
|
time.sleep(wait)
|
|
raise FetchError(f"Giving up on {url} after {s.max_retries} retries")
|
|
|
|
def _session(self) -> requests.Session:
|
|
# curl_cffi sessions aren't thread-safe, so each worker thread gets its own.
|
|
if not hasattr(self._local, "session"):
|
|
self._local.session = requests.Session(impersonate=self.settings.impersonate)
|
|
return self._local.session
|
|
|
|
def _throttle(self) -> None:
|
|
with self._throttle_lock:
|
|
now = time.monotonic()
|
|
start_at = max(now, self._next_request_at)
|
|
self._next_request_at = start_at + self.settings.delay + random.uniform(0, self.settings.jitter)
|
|
time.sleep(start_at - now)
|
|
|
|
def _solve_challenge(self, url: str, seen_generation: int) -> None:
|
|
with self._waf_lock:
|
|
if self._cookie_generation != seen_generation:
|
|
return # another worker refreshed the token while we waited
|
|
log.info("WAF challenge hit, solving it in headless Chromium")
|
|
self._cookies = solve_waf_challenge(url, self.settings.timeout)
|
|
self._cookie_generation += 1
|
|
|
|
|
|
def solve_waf_challenge(url: str, timeout: float) -> dict:
|
|
from playwright.sync_api import Error as PlaywrightError, sync_playwright
|
|
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True, args=["--disable-blink-features=AutomationControlled"])
|
|
try:
|
|
context = browser.new_context(user_agent=BROWSER_UA, locale="en-US")
|
|
page = context.new_page()
|
|
page.goto(url, wait_until="domcontentloaded", timeout=timeout * 1000)
|
|
deadline = time.monotonic() + timeout
|
|
# The challenge script sets the cookie, then reloads into the real page.
|
|
while time.monotonic() < deadline:
|
|
cookies = {c["name"]: c["value"] for c in context.cookies()}
|
|
try:
|
|
solved = "aws-waf-token" in cookies and not is_challenge(page.content())
|
|
except PlaywrightError: # page is mid-reload
|
|
solved = False
|
|
if solved:
|
|
return cookies
|
|
page.wait_for_timeout(500)
|
|
finally:
|
|
browser.close()
|
|
raise FetchError(f"Could not solve WAF challenge for {url}")
|