73 lines
2.8 KiB
Python
73 lines
2.8 KiB
Python
"""
|
|
Scrapes live Iranian market prices from tgju.org.
|
|
Tables: gold, coins, currency, melted gold, silver, oil
|
|
"""
|
|
from datetime import datetime, timezone
|
|
from playwright.sync_api import sync_playwright
|
|
from db import get_conn, init_db
|
|
|
|
URL = "https://www.tgju.org/gold-chart"
|
|
|
|
def create_table():
|
|
with get_conn() as conn:
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS tgju_prices (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL, price TEXT, change_val TEXT,
|
|
pct_change TEXT, low TEXT, high TEXT,
|
|
fetched_at TEXT NOT NULL
|
|
)
|
|
""")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_tgju_name ON tgju_prices(name)")
|
|
|
|
def scrape() -> list[dict]:
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
page = browser.new_page(
|
|
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
|
)
|
|
page.goto(URL, wait_until="networkidle", timeout=30000)
|
|
|
|
rows = page.evaluate("""() => {
|
|
const results = [];
|
|
document.querySelectorAll("table.data-table.market-table").forEach(table => {
|
|
Array.from(table.querySelectorAll("tbody tr")).forEach(tr => {
|
|
const cells = Array.from(tr.querySelectorAll("td"));
|
|
if (cells.length < 2) return;
|
|
const name = cells[0]?.innerText?.trim();
|
|
const price = cells[1]?.innerText?.trim();
|
|
const change = cells[2]?.innerText?.trim() || "";
|
|
const low = cells[3]?.innerText?.trim() || "";
|
|
const high = cells[4]?.innerText?.trim() || "";
|
|
if (name && price && price !== "-") {
|
|
results.push({ name, price, change, low, high });
|
|
}
|
|
});
|
|
});
|
|
return results;
|
|
}""")
|
|
|
|
browser.close()
|
|
return rows
|
|
|
|
def save(rows: list[dict]):
|
|
if not rows:
|
|
print("No data")
|
|
return
|
|
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
with get_conn() as conn:
|
|
conn.executemany(
|
|
"INSERT INTO tgju_prices(name,price,change_val,pct_change,low,high,fetched_at) VALUES(?,?,?,?,?,?,?)",
|
|
[(r["name"], r["price"], r["change"], "", r["low"], r["high"], now) for r in rows]
|
|
)
|
|
print(f"Saved {len(rows)} rows at {now}")
|
|
for r in rows[:8]:
|
|
print(f" {r['name']}: {r['price']} ({r['change']})")
|
|
|
|
if __name__ == "__main__":
|
|
init_db()
|
|
create_table()
|
|
rows = scrape()
|
|
print(f"Scraped {len(rows)} rows")
|
|
save(rows)
|