77 lines
2.8 KiB
Python
77 lines
2.8 KiB
Python
"""
|
|
Scrapes commodity prices from tradingeconomics.com/commodities
|
|
using a headless browser. Saves to the SQLite DB.
|
|
Run every 10 minutes via cron / Task Scheduler.
|
|
"""
|
|
from datetime import datetime, timezone
|
|
from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout
|
|
from db import get_conn, init_db
|
|
|
|
URL = "https://tradingeconomics.com/commodities"
|
|
|
|
def scrape() -> list[dict]:
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
page = browser.new_page(
|
|
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
|
)
|
|
page.goto(URL, wait_until="domcontentloaded", timeout=30000)
|
|
try:
|
|
page.wait_for_selector("table", timeout=15000)
|
|
except PWTimeout:
|
|
browser.close()
|
|
return []
|
|
|
|
rows = page.evaluate("""() => {
|
|
const results = [];
|
|
document.querySelectorAll("table tr").forEach(tr => {
|
|
const cells = Array.from(tr.querySelectorAll("td"));
|
|
if (cells.length < 4) return;
|
|
const name = cells[0]?.innerText?.trim();
|
|
const price = cells[1]?.innerText?.trim();
|
|
const day = cells[2]?.innerText?.trim();
|
|
const pct = cells[3]?.innerText?.trim();
|
|
const unit = cells[4]?.innerText?.trim() || "";
|
|
if (name && price && !isNaN(parseFloat(price.replace(/,/g,"")))) {
|
|
results.push({ name, price, day, pct, unit });
|
|
}
|
|
});
|
|
return results;
|
|
}""")
|
|
|
|
browser.close()
|
|
return rows
|
|
|
|
def save(rows: list[dict]):
|
|
if not rows:
|
|
print("No data scraped")
|
|
return
|
|
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
with get_conn() as conn:
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS te_prices (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL,
|
|
price REAL,
|
|
day_change TEXT,
|
|
pct_change TEXT,
|
|
unit TEXT,
|
|
fetched_at TEXT NOT NULL
|
|
)
|
|
""")
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_te_name ON te_prices(name)")
|
|
conn.executemany(
|
|
"INSERT INTO te_prices(name, price, day_change, pct_change, unit, fetched_at) VALUES(?,?,?,?,?,?)",
|
|
[(r["name"], float(r["price"].replace(",","")), r["day"], r["pct"], r["unit"], now)
|
|
for r in rows]
|
|
)
|
|
print(f"Saved {len(rows)} prices at {now}")
|
|
|
|
if __name__ == "__main__":
|
|
init_db()
|
|
rows = scrape()
|
|
print(f"Scraped {len(rows)} rows")
|
|
for r in rows[:8]:
|
|
print(r)
|
|
save(rows)
|