StatistaAmeri/backend/world_economy.py

"""
Scrapes the world-economy indicators matrix from Trading Economics
(tradingeconomics.com/matrix) — the same data behind TradingView's
"world-economy indicators heatmap". Macro data changes slowly, so this
refreshes every few hours. Caches the leading economies in memory
(served by /api/world-economy) and persists all countries.
"""
import asyncio
import os
from datetime import datetime, timezone
from playwright.sync_api import sync_playwright
from db import get_conn

# Trading Economics is geo-blocked from Iran; route the headless browser through
# an outbound proxy when one is configured (SCRAPER_PROXY / HTTPS_PROXY).
def _proxy_arg() -> dict:
    server = (
        os.environ.get("SCRAPER_PROXY")
        or os.environ.get("HTTPS_PROXY")
        or os.environ.get("HTTP_PROXY")
    )
    return {"proxy": {"server": server}} if server else {}

URL = "https://tradingeconomics.com/matrix"
UA = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
)
INTERVAL_SECONDS = 7 * 24 * 3600  # 1 week (macro data changes slowly)

# Leading economies to surface (must match Trading Economics' country names).
COUNTRIES = [
    "United States", "China", "Euro Area", "Germany", "Japan", "India",
    "United Kingdom", "France", "Italy", "Canada", "South Korea", "Russia",
    "Brazil", "Australia", "Spain", "Mexico", "Turkey", "Indonesia",
]

_latest: list[dict] = []


def create_table() -> None:
    with get_conn() as conn:
        conn.execute("""
            CREATE TABLE IF NOT EXISTS world_economy (
                country         TEXT PRIMARY KEY,
                gdp             REAL,
                gdp_growth      REAL,
                interest_rate   REAL,
                inflation_rate  REAL,
                jobless_rate    REAL,
                gov_budget      REAL,
                debt_gdp        REAL,
                current_account REAL,
                population      REAL,
                fetched_at      TEXT
            )
        """)


def _num(s):
    try:
        return float(str(s).replace(",", "").replace("%", "").strip())
    except (ValueError, AttributeError):
        return None


def _scrape_sync() -> list[dict]:
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True, **_proxy_arg())
        page = browser.new_page(user_agent=UA)
        try:
            page.goto(URL, wait_until="domcontentloaded", timeout=45000)
            page.wait_for_selector("table tbody tr", timeout=20000)
            rows = page.evaluate(r"""() => {
                const t = document.querySelector("table");
                if (!t) return [];
                return Array.from(t.querySelectorAll("tbody tr")).map(tr =>
                    Array.from(tr.querySelectorAll("th,td")).map(c => (c.innerText || "").trim()));
            }""")
        finally:
            browser.close()

    out = []
    for r in rows:
        if len(r) < 10 or not r[0]:
            continue
        out.append({
            "country": r[0],
            "gdp": _num(r[1]),
            "gdp_growth": _num(r[2]),
            "interest_rate": _num(r[3]),
            "inflation_rate": _num(r[4]),
            "jobless_rate": _num(r[5]),
            "gov_budget": _num(r[6]),
            "debt_gdp": _num(r[7]),
            "current_account": _num(r[8]),
            "population": _num(r[9]),
        })
    return out


def _save(rows: list[dict], now: str) -> None:
    with get_conn() as conn:
        for r in rows:
            conn.execute(
                """INSERT INTO world_economy
                   (country, gdp, gdp_growth, interest_rate, inflation_rate, jobless_rate,
                    gov_budget, debt_gdp, current_account, population, fetched_at)
                   VALUES(?,?,?,?,?,?,?,?,?,?,?)
                   ON CONFLICT(country) DO UPDATE SET
                     gdp=excluded.gdp, gdp_growth=excluded.gdp_growth,
                     interest_rate=excluded.interest_rate, inflation_rate=excluded.inflation_rate,
                     jobless_rate=excluded.jobless_rate, gov_budget=excluded.gov_budget,
                     debt_gdp=excluded.debt_gdp, current_account=excluded.current_account,
                     population=excluded.population, fetched_at=excluded.fetched_at""",
                (r["country"], r["gdp"], r["gdp_growth"], r["interest_rate"], r["inflation_rate"],
                 r["jobless_rate"], r["gov_budget"], r["debt_gdp"], r["current_account"],
                 r["population"], now),
            )


def _leading(rows: list[dict], now: str) -> list[dict]:
    order = {c: i for i, c in enumerate(COUNTRIES)}
    kept = [r for r in rows if r["country"] in order]
    kept.sort(key=lambda r: order[r["country"]])
    return [{**r, "fetched_at": now} for r in kept]


def _load_from_db() -> None:
    """Seed the cache from the DB on startup so the API has data before the
    first scrape completes."""
    global _latest
    try:
        with get_conn() as conn:
            rows = conn.execute("SELECT * FROM world_economy").fetchall()
        if rows:
            data = [dict(r) for r in rows]
            now = data[0].get("fetched_at") or ""
            _latest = _leading(data, now) or _latest
    except Exception as e:
        print(f"[world] could not seed from DB: {e}")


async def scrape_loop() -> None:
    global _latest
    create_table()
    _load_from_db()
    while True:
        try:
            rows = await asyncio.to_thread(_scrape_sync)
            if rows:
                now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
                await asyncio.to_thread(_save, rows, now)
                _latest = _leading(rows, now)
                print(f"[world] {len(_latest)}/{len(rows)} leading economies at {now}")
        except Exception as e:
            print(f"[world] error: {e}")
        await asyncio.sleep(INTERVAL_SECONDS)


def get_latest() -> list[dict]:
    return _latest