""" Scrapes live Iranian market prices from tgju.org. Tables: gold, coins, currency, melted gold, silver, oil """ from datetime import datetime, timezone from playwright.sync_api import sync_playwright from db import get_conn, init_db URL = "https://www.tgju.org/gold-chart" def create_table(): with get_conn() as conn: conn.execute(""" CREATE TABLE IF NOT EXISTS tgju_prices ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, price TEXT, change_val TEXT, pct_change TEXT, low TEXT, high TEXT, fetched_at TEXT NOT NULL ) """) conn.execute("CREATE INDEX IF NOT EXISTS idx_tgju_name ON tgju_prices(name)") def scrape() -> list[dict]: with sync_playwright() as p: browser = p.chromium.launch(headless=True) page = browser.new_page( user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" ) page.goto(URL, wait_until="networkidle", timeout=30000) rows = page.evaluate("""() => { const results = []; document.querySelectorAll("table.data-table.market-table").forEach(table => { Array.from(table.querySelectorAll("tbody tr")).forEach(tr => { const cells = Array.from(tr.querySelectorAll("td")); if (cells.length < 2) return; const name = cells[0]?.innerText?.trim(); const price = cells[1]?.innerText?.trim(); const change = cells[2]?.innerText?.trim() || ""; const low = cells[3]?.innerText?.trim() || ""; const high = cells[4]?.innerText?.trim() || ""; if (name && price && price !== "-") { results.push({ name, price, change, low, high }); } }); }); return results; }""") browser.close() return rows def save(rows: list[dict]): if not rows: print("No data") return now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") with get_conn() as conn: conn.executemany( "INSERT INTO tgju_prices(name,price,change_val,pct_change,low,high,fetched_at) VALUES(?,?,?,?,?,?,?)", [(r["name"], r["price"], r["change"], "", r["low"], r["high"], now) for r in rows] ) print(f"Saved {len(rows)} rows at {now}") for r in rows[:8]: print(f" {r['name']}: {r['price']} ({r['change']})") if __name__ == "__main__": init_db() create_table() rows = scrape() print(f"Scraped {len(rows)} rows") save(rows)