fix(world-economy): target data table and use fast direct parsing to populate matrix

backend/world_economy.py

@@ -65,6 +65,38 @@ def _num(s):
 
 
 def _scrape_sync() -> list[dict]:
+    # 1. Fast direct HTTP fetch + HTML table parsing (Table 1 has all 203 countries)
+    try:
+        import urllib.request
+        from bs4 import BeautifulSoup
+        req = urllib.request.Request(URL, headers={"User-Agent": UA})
+        html = urllib.request.urlopen(req, timeout=12).read().decode('utf-8', errors='ignore')
+        soup = BeautifulSoup(html, 'html.parser')
+        tables = soup.find_all('table')
+        if len(tables) > 1:
+            out = []
+            for tr in tables[1].find_all('tr'):
+                cells = [c.get_text(strip=True) for c in tr.find_all(['th', 'td'])]
+                if len(cells) >= 10 and cells[0] and cells[0] != 'Country':
+                    out.append({
+                        "country": cells[0],
+                        "gdp": _num(cells[1]),
+                        "gdp_growth": _num(cells[2]),
+                        "interest_rate": _num(cells[3]),
+                        "inflation_rate": _num(cells[4]),
+                        "jobless_rate": _num(cells[5]),
+                        "gov_budget": _num(cells[6]),
+                        "debt_gdp": _num(cells[7]),
+                        "current_account": _num(cells[8]),
+                        "population": _num(cells[9]),
+                    })
+            if out:
+                return out
+    except Exception as e:
+        print(f"[world] direct fetch error: {e}, falling back to playwright...")
+
+    # 2. Fallback to Playwright (targeting Table 1 instead of Table 0)
+    try:
         with sync_playwright() as p:
             browser = p.chromium.launch(headless=True, **_proxy_arg())
             page = browser.new_page(user_agent=UA)
@@ -72,7 +104,8 @@ def _scrape_sync() -> list[dict]:
                 page.goto(URL, wait_until="domcontentloaded", timeout=45000)
                 page.wait_for_selector("table tbody tr", timeout=20000)
                 rows = page.evaluate(r"""() => {
-                const t = document.querySelector("table");
+                    const tables = document.querySelectorAll("table");
+                    const t = tables.length > 1 ? tables[1] : tables[0];
                     if (!t) return [];
                     return Array.from(t.querySelectorAll("tbody tr")).map(tr =>
                         Array.from(tr.querySelectorAll("th,td")).map(c => (c.innerText || "").trim()));
@@ -82,7 +115,7 @@ def _scrape_sync() -> list[dict]:
 
         out = []
         for r in rows:
-        if len(r) < 10 or not r[0]:
+            if len(r) < 10 or not r[0] or r[0] == 'Country':
                 continue
             out.append({
                 "country": r[0],
@@ -97,6 +130,9 @@ def _scrape_sync() -> list[dict]:
                 "population": _num(r[9]),
             })
         return out
+    except Exception as e:
+        print(f"[world] playwright fallback failed: {e}")
+        return []
 
 
 def _save(rows: list[dict], now: str) -> None: