steelforesight/panel/scraper.js

import * as cheerio from 'cheerio';
import { db, upsertPrice } from './db.js';

const SOURCES = [
  { url: 'https://www.tgju.org/basemetal',   kind: 'basemetal',  defaultUnit: 'دلار/تن' },
  { url: 'https://www.tgju.org/gold-global', kind: 'gold',       defaultUnit: 'دلار/اونس' },
];

const USER_AGENT =
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
  '(KHTML, like Gecko) Chrome/125.0 Safari/537.36';

async function fetchPage(url, timeoutMs = 20000) {
  const ctrl = new AbortController();
  const t = setTimeout(() => ctrl.abort(), timeoutMs);
  try {
    const res = await fetch(url, {
      signal: ctrl.signal,
      headers: {
        'User-Agent': USER_AGENT,
        Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'fa,en;q=0.8',
      },
    });
    if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
    return await res.text();
  } finally {
    clearTimeout(t);
  }
}

// Convert Persian/Arabic digits to ASCII, drop thousand separators, keep sign/decimal
function toNumber(raw) {
  if (raw == null) return null;
  const ascii = String(raw)
    .replace(/[۰-۹]/g, (d) => String('۰۱۲۳۴۵۶۷۸۹'.indexOf(d)))
    .replace(/[٠-٩]/g, (d) => String('٠١٢٣٤٥٦٧٨٩'.indexOf(d)))
    .replace(/[,،\s]/g, '')
    .replace(/[()٪%]/g, '')
    .replace(/[^\d.+\-]/g, '');
  if (!ascii || ascii === '+' || ascii === '-' || ascii === '.') return null;
  const n = parseFloat(ascii);
  return Number.isFinite(n) ? n : null;
}

// Parse the TGJU change cell — known formats:
//   "(0.26%) 12.38"      → pct=0.26,  value=12.38
//   "(-0.5%) -55"        → pct=-0.5,  value=-55
//   "0" or ""            → pct=0,     value=0
function parseChangeCell(raw) {
  if (!raw) return { changePct: null, changeValue: null };
  const text = raw.trim();
  if (!text || text === '0' || text === '۰' || text === '٠') {
    return { changePct: 0, changeValue: 0 };
  }
  let changePct = null;
  let changeValue = null;
  const pctMatch = text.match(/\(\s*(-?[\d.,۰-۹٠-٩]+)\s*%\s*\)/);
  if (pctMatch) changePct = toNumber(pctMatch[1]);
  // Remove the parenthesised pct, what's left should be the absolute change
  const withoutPct = text.replace(/\([^)]*\)/g, '').trim();
  if (withoutPct) changeValue = toNumber(withoutPct);
  return { changePct, changeValue };
}

function parsePrices(html, sourceUrl, defaultUnit) {
  const $ = cheerio.load(html);
  const byName = new Map();

  $('tr[data-market-row], tr[data-market-nameslug]').each((idx, tr) => {
    const $tr = $(tr);
    const symbolAttr = $tr.attr('data-market-row') || $tr.attr('data-market-nameslug') || `row-${idx}`;

    // Name lives in <th> for these tables (not the first <td>).
    let name = $tr.find('th').first().text().trim();
    if (!name) name = $tr.find('a').first().text().trim();
    name = name.replace(/\s+/g, ' ').trim();
    if (!name || /^(عنوان|بازار|نام)$/.test(name)) return;
    if (name.length > 60) return;

    // Price: prefer the row-level data-price attr (always clean numeric),
    // else first <td>'s text.
    const dataPrice = $tr.attr('data-price');
    let value = toNumber(dataPrice);
    if (value === null) {
      const firstTd = $tr.find('td').first().text().trim();
      value = toNumber(firstTd);
    }
    if (!Number.isFinite(value) || value === 0) return;

    // Change: TGJU puts the entire change expression in the SECOND <td>
    // ("(0.26%) 12.38" format). Earlier rows of the page may not have it,
    // so we tolerate missing data.
    const changeCellRaw = $tr.find('td').eq(1).text().trim();
    const { changePct, changeValue } = parseChangeCell(changeCellRaw);

    // Dedup by visible name; prefer non-generic symbols.
    const existing = byName.get(name);
    const isGeneric = /^general[_-]?\d+$/i.test(symbolAttr);
    if (existing) {
      const existingIsGeneric = /^general[_-]?\d+$/i.test(existing.symbol);
      if (!existingIsGeneric || isGeneric) return;
    }

    byName.set(name, {
      symbol: symbolAttr,
      name,
      value,
      unit: defaultUnit,
      changeValue,
      changePct,
      sourceUrl,
    });
  });

  return Array.from(byName.values());
}

export async function scrapeOnce() {
  // Collect results before mutating DB — if ALL sources fail, keep old data.
  const successful = [];
  for (const src of SOURCES) {
    try {
      const html = await fetchPage(src.url);
      const items = parsePrices(html, src.url, src.defaultUnit);
      successful.push({ src, items });
      console.log(`[scraper] ${src.kind}: ${items.length} prices parsed`);
    } catch (err) {
      console.error(`[scraper] ${src.kind} failed: ${err.message}`);
    }
  }
  if (successful.length === 0) {
    console.log('[scraper] all sources failed — keeping previous data');
    return 0;
  }
  // At least one source returned — replace the rows from those sources only.
  // This way a temporary failure of one page doesn't wipe the other's data.
  for (const { src, items } of successful) {
    db.prepare('DELETE FROM prices WHERE source_url = ?').run(src.url);
    for (const it of items) upsertPrice(it);
  }
  return successful.reduce((s, x) => s + x.items.length, 0);
}

let interval = null;
export function startScraperLoop(intervalMs = 2 * 60 * 1000) {
  if (interval) return;
  setTimeout(() => { scrapeOnce().catch((e) => console.error('[scraper] initial:', e)); }, 1500);
  interval = setInterval(() => {
    scrapeOnce().catch((e) => console.error('[scraper] tick:', e));
  }, intervalMs);
}

export function stopScraperLoop() {
  if (interval) { clearInterval(interval); interval = null; }
}