// ───────────────────────────────────────────────────────────────────────────── // Competitor news fetcher → Persian, with source + link // // For each steel company it pulls the latest headline from Google News RSS, // translates it to Persian, and writes public/competitor-news.json. The // dashboard ("آخرین خبر و اقدام" / رصد رقبا) reads that file at runtime. // // No API key required (uses Google News RSS + Google translate gtx endpoint). // // Usage: // node scripts/fetch-competitor-news.mjs # run once now // node scripts/fetch-competitor-news.mjs --daemon # run now, then daily 16:00 (Asia/Tehran) // // Schedule daily at 16:00 without the daemon: // • Linux/macOS cron: 30 12 * * * cd /path/to/repo && node scripts/fetch-competitor-news.mjs (12:30 UTC = 16:00 Tehran) // • Windows Task Scheduler: daily trigger 16:00 → program "node", args "scripts/fetch-competitor-news.mjs", start-in = repo root // // Requires Node 18+ (global fetch). // ───────────────────────────────────────────────────────────────────────────── import { readFile, writeFile, mkdir } from 'node:fs/promises' import { dirname, resolve } from 'node:path' import { fileURLToPath } from 'node:url' const __dirname = dirname(fileURLToPath(import.meta.url)) const OUT = resolve(__dirname, '..', 'public', 'competitor-news.json') const HISTORY = resolve(__dirname, 'news-history.json') // remembers topics already shown so they never repeat // keys MUST match competitor.name in src/pages/SteelDashboard/sections/IndustrySection.tsx const COMPANIES = [ { name: 'Foulad Mobarakeh', q: 'Mobarakeh Steel Company' }, { name: 'China Baowu', q: 'China Baowu steel' }, { name: 'ArcelorMittal', q: 'ArcelorMittal' }, { name: 'Ansteel Group', q: 'Ansteel Group steel' }, { name: 'Nippon Steel', q: 'Nippon Steel' }, { name: 'HBIS Group', q: 'HBIS Group steel' }, { name: 'POSCO Holdings', q: 'POSCO steel' }, { name: 'Tata Steel', q: 'Tata Steel' }, { name: 'JSW Steel', q: 'JSW Steel' }, { name: 'Nucor', q: 'Nucor steel' }, { name: 'Khouzestan Steel', q: 'Khouzestan Steel Company' }, { name: 'Esfahan Steel (Zob Ahan)', q: 'Esfahan Steel Zob Ahan' }, { name: 'Erdemir', q: 'Erdemir steel Turkey' }, { name: 'Tosyalı Holding', q: 'Tosyali steel' }, { name: 'EMSTEEL', q: 'EMSTEEL Emirates Steel' }, { name: 'Ezz Steel', q: 'Ezz Steel Egypt' }, ] const UA = { 'User-Agent': 'Mozilla/5.0 (compatible; SteelForesightNewsBot/1.0)' } async function fetchT(url, ms = 9000) { const ctrl = new AbortController() const id = setTimeout(() => ctrl.abort(), ms) try { return await fetch(url, { headers: UA, signal: ctrl.signal }) } finally { clearTimeout(id) } } function decode(s = '') { return s .replace(//g, '$1') .replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>') .replace(/"/g, '"').replace(/'/g, "'").replace(/'/g, "'") .trim() } async function fetchItems(query) { // last 30 days only; pull several so we can skip topics that are already taken const url = `https://news.google.com/rss/search?q=${encodeURIComponent(query + ' when:30d')}&hl=en-US&gl=US&ceid=US:en` const res = await fetchT(url) if (!res.ok) throw new Error(`news HTTP ${res.status}`) const xml = await res.text() const blocks = [...xml.matchAll(/([\s\S]*?)<\/item>/g)].map((m) => m[1]) return blocks .map((block) => { const rawTitle = decode(block.match(/([\s\S]*?)<\/title>/)?.[1] || '') const link = decode(block.match(/<link>([\s\S]*?)<\/link>/)?.[1] || '') const pubDate = decode(block.match(/<pubDate>([\s\S]*?)<\/pubDate>/)?.[1] || '') const source = decode(block.match(/<source[^>]*>([\s\S]*?)<\/source>/)?.[1] || '') // Google News titles end with " - Source" — strip it for a clean headline const title = source && rawTitle.endsWith(` - ${source}`) ? rawTitle.slice(0, -(source.length + 3)) : rawTitle return { title, link, pubDate, source } }) .filter((it) => it.title) } // normalised "topic" so near-identical headlines collapse to one key (for de-duping) function topicKey(title) { return title .toLowerCase() .replace(/[^a-z0-9؀-ۿ\s]/g, ' ') .replace(/\s+/g, ' ') .trim() .split(' ') .slice(0, 8) .join(' ') } async function loadHistory() { try { return new Set(JSON.parse(await readFile(HISTORY, 'utf8'))) } catch { return new Set() } } async function saveHistory(seen) { try { await writeFile(HISTORY, JSON.stringify([...seen].slice(-600)), 'utf8') } catch { /* ignore */ } } async function toPersian(text) { if (!text) return '' const url = `https://translate.googleapis.com/translate_a/single?client=gtx&sl=en&tl=fa&dt=t&q=${encodeURIComponent(text)}` const res = await fetchT(url) if (!res.ok) throw new Error(`translate HTTP ${res.status}`) const data = await res.json() return (data?.[0] || []).map((seg) => seg[0]).join('') } function faDate(iso) { try { return new Intl.DateTimeFormat('fa-IR', { year: 'numeric', month: 'long', day: 'numeric' }).format(new Date(iso)) } catch { return '' } } const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) async function runOnce() { const history = await loadHistory() // topics shown on previous days const usedThisRun = new Set() // topics already taken in this run const items = {} for (const c of COMPANIES) { try { const list = await fetchItems(c.q) // first headline whose topic is new — not taken this run AND not shown before let pick = list.find((it) => { const k = topicKey(it.title); return k && !usedThisRun.has(k) && !history.has(k) }) // if everything is a repeat, at least don't clash with another company in this same run if (!pick) pick = list.find((it) => !usedThisRun.has(topicKey(it.title))) if (!pick) { console.warn(`· no fresh news: ${c.name}`); continue } const key = topicKey(pick.title) usedThisRun.add(key); history.add(key) const newsFa = await toPersian(pick.title) items[c.name] = { newsFa, source: pick.source, link: pick.link, date: faDate(pick.pubDate) } console.log(`✓ ${c.name} — ${newsFa.slice(0, 64)}`) } catch (e) { console.warn(`✗ ${c.name}: ${e.message}`) } await sleep(1200) // stay polite to the free endpoints } await saveHistory(history) await mkdir(dirname(OUT), { recursive: true }) await writeFile(OUT, JSON.stringify({ updatedAt: new Date().toISOString(), items }, null, 2), 'utf8') console.log(`\n📰 wrote ${Object.keys(items).length}/${COMPANIES.length} items (no repeated topics) → ${OUT}`) } // 16:00 Asia/Tehran = 12:30 UTC (Iran dropped DST in 2022, fixed at UTC+3:30) function msUntilNextRun() { const now = new Date() const next = new Date(now) next.setUTCHours(12, 30, 0, 0) if (next <= now) next.setUTCDate(next.getUTCDate() + 1) return next.getTime() - now.getTime() } async function daemon() { console.log('🕓 news daemon up — runs now, then every day at 16:00 Asia/Tehran') await runOnce() const schedule = () => { const wait = msUntilNextRun() console.log(`next run in ${(wait / 3_600_000).toFixed(2)}h`) setTimeout(async () => { await runOnce(); schedule() }, wait) } schedule() } if (process.argv.includes('--daemon')) daemon() else runOnce()