118 lines
4.9 KiB
JavaScript
118 lines
4.9 KiB
JavaScript
// Refreshable news fetcher for the /observe map.
|
||
// Scrapes Iranian steel/mining news sites (all WordPress → search-RSS at
|
||
// /?s=<query>&feed=rss2) per company and writes src/content/iranSitesNews.json.
|
||
//
|
||
// node scripts/fetch-news.mjs
|
||
//
|
||
// Each item: real title, article link (clickable Iranian source), date, and
|
||
// the post excerpt as a 1-paragraph summary. Re-run anytime to refresh.
|
||
|
||
import fs from 'node:fs/promises'
|
||
import path from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
|
||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||
const ROOT = path.join(__dirname, '..')
|
||
const PER_SITE = 5
|
||
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36'
|
||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
||
|
||
// WordPress sources (base + Persian display name)
|
||
const SRC = {
|
||
chilan: { base: 'https://www.chilanonline.com', name: 'چیلان' },
|
||
felezat: { base: 'https://felezatonline.ir', name: 'فلزات آنلاین' },
|
||
madanmedia: { base: 'https://madanmedia.ir', name: 'رسانه معدن' },
|
||
madannews: { base: 'https://www.madannews.ir', name: 'معدننیوز' },
|
||
madanpress: { base: 'https://madanpress.com', name: 'معدنپرس' },
|
||
madanbidar: { base: 'https://madanbidar.ir', name: 'معدن بیدار' },
|
||
madanotoseeh: { base: 'https://madanotoseeh.ir', name: 'معدن و توسعه' },
|
||
}
|
||
const STEEL_SRC = ['chilan', 'felezat', 'madanmedia']
|
||
const MINE_SRC = ['madannews', 'madanpress', 'madanbidar', 'madanmedia', 'madanotoseeh', 'felezat']
|
||
|
||
// generic words to drop when picking a company's distinctive tokens
|
||
const STOP = new Set(['فولاد', 'سنگآهن', 'سنگ', 'آهن', 'مس', 'و', 'معدن', 'سرب', 'روی', 'طلای', 'طلا', 'بوکسیت', 'کرومیت', 'زغالسنگ', 'زغال', 'گروه', 'ملی', 'صنعتی', 'ایران', 'شرکت', 'مجتمع'])
|
||
|
||
const norm = (s) => s.replace(//g, '').replace(/ي/g, 'ی').replace(/ك/g, 'ک').replace(/\(.*?\)/g, '').trim()
|
||
const decode = (s) =>
|
||
s
|
||
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
|
||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(+n))
|
||
.replace(/ | /g, ' ')
|
||
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
||
.replace(/"/g, '"').replace(/'/g, "'").replace(/ /g, ' ')
|
||
.trim()
|
||
const stripTags = (s) => decode(s.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' '))
|
||
|
||
async function get(url, timeout = 18000) {
|
||
const ac = new AbortController()
|
||
const t = setTimeout(() => ac.abort(), timeout)
|
||
try {
|
||
const r = await fetch(url, { headers: { 'User-Agent': UA }, redirect: 'follow', signal: ac.signal })
|
||
return r.ok ? await r.text() : ''
|
||
} catch {
|
||
return ''
|
||
} finally {
|
||
clearTimeout(t)
|
||
}
|
||
}
|
||
|
||
function tag(b, name) {
|
||
const m = b.match(new RegExp(`<${name}[^>]*>([\\s\\S]*?)<\\/${name}>`, 'i'))
|
||
return m ? m[1] : ''
|
||
}
|
||
|
||
function parseRss(xml, sourceName) {
|
||
return [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => {
|
||
const b = m[1]
|
||
let summary = stripTags(tag(b, 'description'))
|
||
summary = summary.replace(/\[…\]|\[…\]|…\s*$/g, '').trim()
|
||
return {
|
||
title: decode(tag(b, 'title')),
|
||
url: decode(tag(b, 'link')).trim(),
|
||
source: sourceName,
|
||
date: tag(b, 'pubDate').trim(),
|
||
summary,
|
||
}
|
||
})
|
||
}
|
||
|
||
async function main() {
|
||
const src = await fs.readFile(path.join(ROOT, 'src/content/iranSites.ts'), 'utf8')
|
||
const sites = [...src.matchAll(/id:\s*'([^']+)',\s*name:\s*\{\s*fa:\s*'([^']+)'[\s\S]*?type:\s*'([^']+)'/g)].map((m) => ({ id: m[1], fa: m[2], type: m[3] }))
|
||
console.log(`Fetching news for ${sites.length} sites…`)
|
||
|
||
const out = {}
|
||
for (const s of sites) {
|
||
const tokens = norm(s.fa).split(/\s+/).filter((w) => w.length > 1 && !STOP.has(w)).map(norm)
|
||
const srcKeys = s.type === 'steel' ? STEEL_SRC : MINE_SRC
|
||
const collected = []
|
||
for (const key of srcKeys) {
|
||
const { base, name } = SRC[key]
|
||
const xml = await get(`${base}/?s=${encodeURIComponent(s.fa)}&feed=rss2`)
|
||
if (xml) collected.push(...parseRss(xml, name))
|
||
await sleep(400)
|
||
}
|
||
// relevance: title or summary must contain a distinctive token of the company
|
||
const hay = (it) => norm(it.title + ' ' + it.summary)
|
||
const relevant = tokens.length ? collected.filter((it) => tokens.some((t) => hay(it).includes(t))) : collected
|
||
// dedup by url, then by title
|
||
const seen = new Set()
|
||
const unique = relevant.filter((it) => {
|
||
const k = it.url || norm(it.title).slice(0, 50)
|
||
if (seen.has(k)) return false
|
||
seen.add(k)
|
||
return true
|
||
})
|
||
unique.sort((a, b) => (Date.parse(b.date) || 0) - (Date.parse(a.date) || 0))
|
||
out[s.id] = unique.slice(0, PER_SITE)
|
||
console.log(` ${s.id} (${s.type}): ${out[s.id].length} items [${collected.length} raw]`)
|
||
}
|
||
|
||
const dest = path.join(ROOT, 'src/content/iranSitesNews.json')
|
||
await fs.writeFile(dest, JSON.stringify(out, null, 2), 'utf8')
|
||
console.log(`Wrote ${dest}`)
|
||
}
|
||
|
||
main()
|