fix(world-economy): target data table and use fast direct parsing to populate matrix

backend/world_economy.py

@@ -65,38 +65,74 @@ def _num(s):
 
 
 def _scrape_sync() -> list[dict]:
-    with sync_playwright() as p:
-        browser = p.chromium.launch(headless=True, **_proxy_arg())
-        page = browser.new_page(user_agent=UA)
-        try:
-            page.goto(URL, wait_until="domcontentloaded", timeout=45000)
-            page.wait_for_selector("table tbody tr", timeout=20000)
-            rows = page.evaluate(r"""() => {
-                const t = document.querySelector("table");
-                if (!t) return [];
-                return Array.from(t.querySelectorAll("tbody tr")).map(tr =>
-                    Array.from(tr.querySelectorAll("th,td")).map(c => (c.innerText || "").trim()));
-            }""")
-        finally:
-            browser.close()
+    # 1. Fast direct HTTP fetch + HTML table parsing (Table 1 has all 203 countries)
+    try:
+        import urllib.request
+        from bs4 import BeautifulSoup
+        req = urllib.request.Request(URL, headers={"User-Agent": UA})
+        html = urllib.request.urlopen(req, timeout=12).read().decode('utf-8', errors='ignore')
+        soup = BeautifulSoup(html, 'html.parser')
+        tables = soup.find_all('table')
+        if len(tables) > 1:
+            out = []
+            for tr in tables[1].find_all('tr'):
+                cells = [c.get_text(strip=True) for c in tr.find_all(['th', 'td'])]
+                if len(cells) >= 10 and cells[0] and cells[0] != 'Country':
+                    out.append({
+                        "country": cells[0],
+                        "gdp": _num(cells[1]),
+                        "gdp_growth": _num(cells[2]),
+                        "interest_rate": _num(cells[3]),
+                        "inflation_rate": _num(cells[4]),
+                        "jobless_rate": _num(cells[5]),
+                        "gov_budget": _num(cells[6]),
+                        "debt_gdp": _num(cells[7]),
+                        "current_account": _num(cells[8]),
+                        "population": _num(cells[9]),
+                    })
+            if out:
+                return out
+    except Exception as e:
+        print(f"[world] direct fetch error: {e}, falling back to playwright...")
 
-    out = []
-    for r in rows:
-        if len(r) < 10 or not r[0]:
-            continue
-        out.append({
-            "country": r[0],
-            "gdp": _num(r[1]),
-            "gdp_growth": _num(r[2]),
-            "interest_rate": _num(r[3]),
-            "inflation_rate": _num(r[4]),
-            "jobless_rate": _num(r[5]),
-            "gov_budget": _num(r[6]),
-            "debt_gdp": _num(r[7]),
-            "current_account": _num(r[8]),
-            "population": _num(r[9]),
-        })
-    return out
+    # 2. Fallback to Playwright (targeting Table 1 instead of Table 0)
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True, **_proxy_arg())
+            page = browser.new_page(user_agent=UA)
+            try:
+                page.goto(URL, wait_until="domcontentloaded", timeout=45000)
+                page.wait_for_selector("table tbody tr", timeout=20000)
+                rows = page.evaluate(r"""() => {
+                    const tables = document.querySelectorAll("table");
+                    const t = tables.length > 1 ? tables[1] : tables[0];
+                    if (!t) return [];
+                    return Array.from(t.querySelectorAll("tbody tr")).map(tr =>
+                        Array.from(tr.querySelectorAll("th,td")).map(c => (c.innerText || "").trim()));
+                }""")
+            finally:
+                browser.close()
+
+        out = []
+        for r in rows:
+            if len(r) < 10 or not r[0] or r[0] == 'Country':
+                continue
+            out.append({
+                "country": r[0],
+                "gdp": _num(r[1]),
+                "gdp_growth": _num(r[2]),
+                "interest_rate": _num(r[3]),
+                "inflation_rate": _num(r[4]),
+                "jobless_rate": _num(r[5]),
+                "gov_budget": _num(r[6]),
+                "debt_gdp": _num(r[7]),
+                "current_account": _num(r[8]),
+                "population": _num(r[9]),
+            })
+        return out
+    except Exception as e:
+        print(f"[world] playwright fallback failed: {e}")
+        return []
 
 
 def _save(rows: list[dict], now: str) -> None: