Mr.Gay100
/
StatistaAmeri
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(world-economy): target data table and use fast direct parsing to populate matrix

This commit is contained in:
alireza 2026-08-26 14:25:47 +03:30
parent f96f39c962
commit b6d99c1270
1 changed files with 67 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
backend/world_economy.py
View File

@ -65,38 +65,74 @@ def _num(s):
def _scrape_sync() -> list[dict]: def _scrape_sync() -> list[dict]:
with sync_playwright() as p: # 1. Fast direct HTTP fetch + HTML table parsing (Table 1 has all 203 countries)
browser = p.chromium.launch(headless=True, **_proxy_arg()) try:
page = browser.new_page(user_agent=UA) import urllib.request
try: from bs4 import BeautifulSoup
page.goto(URL, wait_until="domcontentloaded", timeout=45000) req = urllib.request.Request(URL, headers={"User-Agent": UA})
page.wait_for_selector("table tbody tr", timeout=20000) html = urllib.request.urlopen(req, timeout=12).read().decode('utf-8', errors='ignore')
rows = page.evaluate(r"""() => { soup = BeautifulSoup(html, 'html.parser')
const t = document.querySelector("table"); tables = soup.find_all('table')
if (!t) return []; if len(tables) > 1:
return Array.from(t.querySelectorAll("tbody tr")).map(tr => out = []
Array.from(tr.querySelectorAll("th,td")).map(c => (c.innerText || "").trim())); for tr in tables[1].find_all('tr'):
}""") cells = [c.get_text(strip=True) for c in tr.find_all(['th', 'td'])]
finally: if len(cells) >= 10 and cells[0] and cells[0] != 'Country':
browser.close() out.append({
"country": cells[0],
"gdp": _num(cells[1]),
"gdp_growth": _num(cells[2]),
"interest_rate": _num(cells[3]),
"inflation_rate": _num(cells[4]),
"jobless_rate": _num(cells[5]),
"gov_budget": _num(cells[6]),
"debt_gdp": _num(cells[7]),
"current_account": _num(cells[8]),
"population": _num(cells[9]),
})
if out:
return out
except Exception as e:
print(f"[world] direct fetch error: {e}, falling back to playwright...")
out = [] # 2. Fallback to Playwright (targeting Table 1 instead of Table 0)
for r in rows: try:
if len(r) < 10 or not r[0]: with sync_playwright() as p:
continue browser = p.chromium.launch(headless=True, **_proxy_arg())
out.append({ page = browser.new_page(user_agent=UA)
"country": r[0], try:
"gdp": _num(r[1]), page.goto(URL, wait_until="domcontentloaded", timeout=45000)
"gdp_growth": _num(r[2]), page.wait_for_selector("table tbody tr", timeout=20000)
"interest_rate": _num(r[3]), rows = page.evaluate(r"""() => {
"inflation_rate": _num(r[4]), const tables = document.querySelectorAll("table");
"jobless_rate": _num(r[5]), const t = tables.length > 1 ? tables[1] : tables[0];
"gov_budget": _num(r[6]), if (!t) return [];
"debt_gdp": _num(r[7]), return Array.from(t.querySelectorAll("tbody tr")).map(tr =>
"current_account": _num(r[8]), Array.from(tr.querySelectorAll("th,td")).map(c => (c.innerText || "").trim()));
"population": _num(r[9]), }""")
}) finally:
return out browser.close()
out = []
for r in rows:
if len(r) < 10 or not r[0] or r[0] == 'Country':
continue
out.append({
"country": r[0],
"gdp": _num(r[1]),
"gdp_growth": _num(r[2]),
"interest_rate": _num(r[3]),
"inflation_rate": _num(r[4]),
"jobless_rate": _num(r[5]),
"gov_budget": _num(r[6]),
"debt_gdp": _num(r[7]),
"current_account": _num(r[8]),
"population": _num(r[9]),
})
return out
except Exception as e:
print(f"[world] playwright fallback failed: {e}")
return []
def _save(rows: list[dict], now: str) -> None: def _save(rows: list[dict], now: str) -> None: