174 lines
7.7 KiB
JavaScript
174 lines
7.7 KiB
JavaScript
// ─────────────────────────────────────────────────────────────────────────────
|
||
// Competitor news fetcher → Persian, with source + link
|
||
//
|
||
// For each steel company it pulls the latest headline from Google News RSS,
|
||
// translates it to Persian, and writes public/competitor-news.json. The
|
||
// dashboard ("آخرین خبر و اقدام" / رصد رقبا) reads that file at runtime.
|
||
//
|
||
// No API key required (uses Google News RSS + Google translate gtx endpoint).
|
||
//
|
||
// Usage:
|
||
// node scripts/fetch-competitor-news.mjs # run once now
|
||
// node scripts/fetch-competitor-news.mjs --daemon # run now, then daily 16:00 (Asia/Tehran)
|
||
//
|
||
// Schedule daily at 16:00 without the daemon:
|
||
// • Linux/macOS cron: 30 12 * * * cd /path/to/repo && node scripts/fetch-competitor-news.mjs (12:30 UTC = 16:00 Tehran)
|
||
// • Windows Task Scheduler: daily trigger 16:00 → program "node", args "scripts/fetch-competitor-news.mjs", start-in = repo root
|
||
//
|
||
// Requires Node 18+ (global fetch).
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
import { readFile, writeFile, mkdir } from 'node:fs/promises'
|
||
import { dirname, resolve } from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
|
||
const __dirname = dirname(fileURLToPath(import.meta.url))
|
||
const OUT = resolve(__dirname, '..', 'public', 'competitor-news.json')
|
||
const HISTORY = resolve(__dirname, 'news-history.json') // remembers topics already shown so they never repeat
|
||
|
||
// keys MUST match competitor.name in src/pages/SteelDashboard/sections/IndustrySection.tsx
|
||
const COMPANIES = [
|
||
{ name: 'Foulad Mobarakeh', q: 'Mobarakeh Steel Company' },
|
||
{ name: 'China Baowu', q: 'China Baowu steel' },
|
||
{ name: 'ArcelorMittal', q: 'ArcelorMittal' },
|
||
{ name: 'Ansteel Group', q: 'Ansteel Group steel' },
|
||
{ name: 'Nippon Steel', q: 'Nippon Steel' },
|
||
{ name: 'HBIS Group', q: 'HBIS Group steel' },
|
||
{ name: 'POSCO Holdings', q: 'POSCO steel' },
|
||
{ name: 'Tata Steel', q: 'Tata Steel' },
|
||
{ name: 'JSW Steel', q: 'JSW Steel' },
|
||
{ name: 'Nucor', q: 'Nucor steel' },
|
||
{ name: 'Khouzestan Steel', q: 'Khouzestan Steel Company' },
|
||
{ name: 'Esfahan Steel (Zob Ahan)', q: 'Esfahan Steel Zob Ahan' },
|
||
{ name: 'Erdemir', q: 'Erdemir steel Turkey' },
|
||
{ name: 'Tosyalı Holding', q: 'Tosyali steel' },
|
||
{ name: 'EMSTEEL', q: 'EMSTEEL Emirates Steel' },
|
||
{ name: 'Ezz Steel', q: 'Ezz Steel Egypt' },
|
||
]
|
||
|
||
const UA = { 'User-Agent': 'Mozilla/5.0 (compatible; SteelForesightNewsBot/1.0)' }
|
||
|
||
async function fetchT(url, ms = 9000) {
|
||
const ctrl = new AbortController()
|
||
const id = setTimeout(() => ctrl.abort(), ms)
|
||
try { return await fetch(url, { headers: UA, signal: ctrl.signal }) }
|
||
finally { clearTimeout(id) }
|
||
}
|
||
|
||
function decode(s = '') {
|
||
return s
|
||
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
|
||
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
||
.replace(/"/g, '"').replace(/'/g, "'").replace(/'/g, "'")
|
||
.trim()
|
||
}
|
||
|
||
async function fetchItems(query) {
|
||
// last 30 days only; pull several so we can skip topics that are already taken
|
||
const url = `https://news.google.com/rss/search?q=${encodeURIComponent(query + ' when:30d')}&hl=en-US&gl=US&ceid=US:en`
|
||
const res = await fetchT(url)
|
||
if (!res.ok) throw new Error(`news HTTP ${res.status}`)
|
||
const xml = await res.text()
|
||
const blocks = [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => m[1])
|
||
return blocks
|
||
.map((block) => {
|
||
const rawTitle = decode(block.match(/<title>([\s\S]*?)<\/title>/)?.[1] || '')
|
||
const link = decode(block.match(/<link>([\s\S]*?)<\/link>/)?.[1] || '')
|
||
const pubDate = decode(block.match(/<pubDate>([\s\S]*?)<\/pubDate>/)?.[1] || '')
|
||
const source = decode(block.match(/<source[^>]*>([\s\S]*?)<\/source>/)?.[1] || '')
|
||
// Google News titles end with " - Source" — strip it for a clean headline
|
||
const title = source && rawTitle.endsWith(` - ${source}`) ? rawTitle.slice(0, -(source.length + 3)) : rawTitle
|
||
return { title, link, pubDate, source }
|
||
})
|
||
.filter((it) => it.title)
|
||
}
|
||
|
||
// normalised "topic" so near-identical headlines collapse to one key (for de-duping)
|
||
function topicKey(title) {
|
||
return title
|
||
.toLowerCase()
|
||
.replace(/[^a-z0-9-ۿ\s]/g, ' ')
|
||
.replace(/\s+/g, ' ')
|
||
.trim()
|
||
.split(' ')
|
||
.slice(0, 8)
|
||
.join(' ')
|
||
}
|
||
|
||
async function loadHistory() {
|
||
try { return new Set(JSON.parse(await readFile(HISTORY, 'utf8'))) } catch { return new Set() }
|
||
}
|
||
async function saveHistory(seen) {
|
||
try { await writeFile(HISTORY, JSON.stringify([...seen].slice(-600)), 'utf8') } catch { /* ignore */ }
|
||
}
|
||
|
||
async function toPersian(text) {
|
||
if (!text) return ''
|
||
const url = `https://translate.googleapis.com/translate_a/single?client=gtx&sl=en&tl=fa&dt=t&q=${encodeURIComponent(text)}`
|
||
const res = await fetchT(url)
|
||
if (!res.ok) throw new Error(`translate HTTP ${res.status}`)
|
||
const data = await res.json()
|
||
return (data?.[0] || []).map((seg) => seg[0]).join('')
|
||
}
|
||
|
||
function faDate(iso) {
|
||
try {
|
||
return new Intl.DateTimeFormat('fa-IR', { year: 'numeric', month: 'long', day: 'numeric' }).format(new Date(iso))
|
||
} catch {
|
||
return ''
|
||
}
|
||
}
|
||
|
||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
||
|
||
async function runOnce() {
|
||
const history = await loadHistory() // topics shown on previous days
|
||
const usedThisRun = new Set() // topics already taken in this run
|
||
const items = {}
|
||
for (const c of COMPANIES) {
|
||
try {
|
||
const list = await fetchItems(c.q)
|
||
// first headline whose topic is new — not taken this run AND not shown before
|
||
let pick = list.find((it) => { const k = topicKey(it.title); return k && !usedThisRun.has(k) && !history.has(k) })
|
||
// if everything is a repeat, at least don't clash with another company in this same run
|
||
if (!pick) pick = list.find((it) => !usedThisRun.has(topicKey(it.title)))
|
||
if (!pick) { console.warn(`· no fresh news: ${c.name}`); continue }
|
||
const key = topicKey(pick.title)
|
||
usedThisRun.add(key); history.add(key)
|
||
const newsFa = await toPersian(pick.title)
|
||
items[c.name] = { newsFa, source: pick.source, link: pick.link, date: faDate(pick.pubDate) }
|
||
console.log(`✓ ${c.name} — ${newsFa.slice(0, 64)}`)
|
||
} catch (e) {
|
||
console.warn(`✗ ${c.name}: ${e.message}`)
|
||
}
|
||
await sleep(1200) // stay polite to the free endpoints
|
||
}
|
||
await saveHistory(history)
|
||
await mkdir(dirname(OUT), { recursive: true })
|
||
await writeFile(OUT, JSON.stringify({ updatedAt: new Date().toISOString(), items }, null, 2), 'utf8')
|
||
console.log(`\n📰 wrote ${Object.keys(items).length}/${COMPANIES.length} items (no repeated topics) → ${OUT}`)
|
||
}
|
||
|
||
// 16:00 Asia/Tehran = 12:30 UTC (Iran dropped DST in 2022, fixed at UTC+3:30)
|
||
function msUntilNextRun() {
|
||
const now = new Date()
|
||
const next = new Date(now)
|
||
next.setUTCHours(12, 30, 0, 0)
|
||
if (next <= now) next.setUTCDate(next.getUTCDate() + 1)
|
||
return next.getTime() - now.getTime()
|
||
}
|
||
|
||
async function daemon() {
|
||
console.log('🕓 news daemon up — runs now, then every day at 16:00 Asia/Tehran')
|
||
await runOnce()
|
||
const schedule = () => {
|
||
const wait = msUntilNextRun()
|
||
console.log(`next run in ${(wait / 3_600_000).toFixed(2)}h`)
|
||
setTimeout(async () => { await runOnce(); schedule() }, wait)
|
||
}
|
||
schedule()
|
||
}
|
||
|
||
if (process.argv.includes('--daemon')) daemon()
|
||
else runOnce()
|