diff --git a/backend/world_economy.py b/backend/world_economy.py index 0b75bca..95e6526 100644 --- a/backend/world_economy.py +++ b/backend/world_economy.py @@ -65,38 +65,74 @@ def _num(s): def _scrape_sync() -> list[dict]: - with sync_playwright() as p: - browser = p.chromium.launch(headless=True, **_proxy_arg()) - page = browser.new_page(user_agent=UA) - try: - page.goto(URL, wait_until="domcontentloaded", timeout=45000) - page.wait_for_selector("table tbody tr", timeout=20000) - rows = page.evaluate(r"""() => { - const t = document.querySelector("table"); - if (!t) return []; - return Array.from(t.querySelectorAll("tbody tr")).map(tr => - Array.from(tr.querySelectorAll("th,td")).map(c => (c.innerText || "").trim())); - }""") - finally: - browser.close() + # 1. Fast direct HTTP fetch + HTML table parsing (Table 1 has all 203 countries) + try: + import urllib.request + from bs4 import BeautifulSoup + req = urllib.request.Request(URL, headers={"User-Agent": UA}) + html = urllib.request.urlopen(req, timeout=12).read().decode('utf-8', errors='ignore') + soup = BeautifulSoup(html, 'html.parser') + tables = soup.find_all('table') + if len(tables) > 1: + out = [] + for tr in tables[1].find_all('tr'): + cells = [c.get_text(strip=True) for c in tr.find_all(['th', 'td'])] + if len(cells) >= 10 and cells[0] and cells[0] != 'Country': + out.append({ + "country": cells[0], + "gdp": _num(cells[1]), + "gdp_growth": _num(cells[2]), + "interest_rate": _num(cells[3]), + "inflation_rate": _num(cells[4]), + "jobless_rate": _num(cells[5]), + "gov_budget": _num(cells[6]), + "debt_gdp": _num(cells[7]), + "current_account": _num(cells[8]), + "population": _num(cells[9]), + }) + if out: + return out + except Exception as e: + print(f"[world] direct fetch error: {e}, falling back to playwright...") - out = [] - for r in rows: - if len(r) < 10 or not r[0]: - continue - out.append({ - "country": r[0], - "gdp": _num(r[1]), - "gdp_growth": _num(r[2]), - "interest_rate": _num(r[3]), - "inflation_rate": _num(r[4]), - "jobless_rate": _num(r[5]), - "gov_budget": _num(r[6]), - "debt_gdp": _num(r[7]), - "current_account": _num(r[8]), - "population": _num(r[9]), - }) - return out + # 2. Fallback to Playwright (targeting Table 1 instead of Table 0) + try: + with sync_playwright() as p: + browser = p.chromium.launch(headless=True, **_proxy_arg()) + page = browser.new_page(user_agent=UA) + try: + page.goto(URL, wait_until="domcontentloaded", timeout=45000) + page.wait_for_selector("table tbody tr", timeout=20000) + rows = page.evaluate(r"""() => { + const tables = document.querySelectorAll("table"); + const t = tables.length > 1 ? tables[1] : tables[0]; + if (!t) return []; + return Array.from(t.querySelectorAll("tbody tr")).map(tr => + Array.from(tr.querySelectorAll("th,td")).map(c => (c.innerText || "").trim())); + }""") + finally: + browser.close() + + out = [] + for r in rows: + if len(r) < 10 or not r[0] or r[0] == 'Country': + continue + out.append({ + "country": r[0], + "gdp": _num(r[1]), + "gdp_growth": _num(r[2]), + "interest_rate": _num(r[3]), + "inflation_rate": _num(r[4]), + "jobless_rate": _num(r[5]), + "gov_budget": _num(r[6]), + "debt_gdp": _num(r[7]), + "current_account": _num(r[8]), + "population": _num(r[9]), + }) + return out + except Exception as e: + print(f"[world] playwright fallback failed: {e}") + return [] def _save(rows: list[dict], now: str) -> None: