// Refreshable news fetcher for the /observe map.
// Scrapes Iranian steel/mining news sites (all WordPress → search-RSS at
// /?s=<query>&feed=rss2) per company and writes src/content/iranSitesNews.json.
//
//   node scripts/fetch-news.mjs
//
// Each item: real title, article link (clickable Iranian source), date, and
// the post excerpt as a 1-paragraph summary. Re-run anytime to refresh.

import fs from 'node:fs/promises'
import path from 'node:path'
import { fileURLToPath } from 'node:url'

const __dirname = path.dirname(fileURLToPath(import.meta.url))
const ROOT = path.join(__dirname, '..')
const PER_SITE = 5
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36'
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))

// WordPress sources (base + Persian display name)
const SRC = {
  chilan: { base: 'https://www.chilanonline.com', name: 'چیلان' },
  felezat: { base: 'https://felezatonline.ir', name: 'فلزات آنلاین' },
  madanmedia: { base: 'https://madanmedia.ir', name: 'رسانه معدن' },
  madannews: { base: 'https://www.madannews.ir', name: 'معدن‌نیوز' },
  madanpress: { base: 'https://madanpress.com', name: 'معدن‌پرس' },
  madanbidar: { base: 'https://madanbidar.ir', name: 'معدن بیدار' },
  madanotoseeh: { base: 'https://madanotoseeh.ir', name: 'معدن و توسعه' },
}
const STEEL_SRC = ['chilan', 'felezat', 'madanmedia']
const MINE_SRC = ['madannews', 'madanpress', 'madanbidar', 'madanmedia', 'madanotoseeh', 'felezat']

// generic words to drop when picking a company's distinctive tokens
const STOP = new Set(['فولاد', 'سنگ‌آهن', 'سنگ', 'آهن', 'مس', 'و', 'معدن', 'سرب', 'روی', 'طلای', 'طلا', 'بوکسیت', 'کرومیت', 'زغال‌سنگ', 'زغال', 'گروه', 'ملی', 'صنعتی', 'ایران', 'شرکت', 'مجتمع'])

const norm = (s) => s.replace(/‌/g, '').replace(/ي/g, 'ی').replace(/ك/g, 'ک').replace(/\(.*?\)/g, '').trim()
const decode = (s) =>
  s
    .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(+n))
    .replace(/&#xA0;|&#160;/g, ' ')
    .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, ' ')
    .trim()
const stripTags = (s) => decode(s.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' '))

async function get(url, timeout = 18000) {
  const ac = new AbortController()
  const t = setTimeout(() => ac.abort(), timeout)
  try {
    const r = await fetch(url, { headers: { 'User-Agent': UA }, redirect: 'follow', signal: ac.signal })
    return r.ok ? await r.text() : ''
  } catch {
    return ''
  } finally {
    clearTimeout(t)
  }
}

function tag(b, name) {
  const m = b.match(new RegExp(`<${name}[^>]*>([\\s\\S]*?)<\\/${name}>`, 'i'))
  return m ? m[1] : ''
}

function parseRss(xml, sourceName) {
  return [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => {
    const b = m[1]
    let summary = stripTags(tag(b, 'description'))
    summary = summary.replace(/\[…\]|\[&hellip;\]|…\s*$/g, '').trim()
    return {
      title: decode(tag(b, 'title')),
      url: decode(tag(b, 'link')).trim(),
      source: sourceName,
      date: tag(b, 'pubDate').trim(),
      summary,
    }
  })
}

async function main() {
  const src = await fs.readFile(path.join(ROOT, 'src/content/iranSites.ts'), 'utf8')
  const sites = [...src.matchAll(/id:\s*'([^']+)',\s*name:\s*\{\s*fa:\s*'([^']+)'[\s\S]*?type:\s*'([^']+)'/g)].map((m) => ({ id: m[1], fa: m[2], type: m[3] }))
  console.log(`Fetching news for ${sites.length} sites…`)

  const out = {}
  for (const s of sites) {
    const tokens = norm(s.fa).split(/\s+/).filter((w) => w.length > 1 && !STOP.has(w)).map(norm)
    const srcKeys = s.type === 'steel' ? STEEL_SRC : MINE_SRC
    const collected = []
    for (const key of srcKeys) {
      const { base, name } = SRC[key]
      const xml = await get(`${base}/?s=${encodeURIComponent(s.fa)}&feed=rss2`)
      if (xml) collected.push(...parseRss(xml, name))
      await sleep(400)
    }
    // relevance: title or summary must contain a distinctive token of the company
    const hay = (it) => norm(it.title + ' ' + it.summary)
    const relevant = tokens.length ? collected.filter((it) => tokens.some((t) => hay(it).includes(t))) : collected
    // dedup by url, then by title
    const seen = new Set()
    const unique = relevant.filter((it) => {
      const k = it.url || norm(it.title).slice(0, 50)
      if (seen.has(k)) return false
      seen.add(k)
      return true
    })
    unique.sort((a, b) => (Date.parse(b.date) || 0) - (Date.parse(a.date) || 0))
    out[s.id] = unique.slice(0, PER_SITE)
    console.log(`  ${s.id} (${s.type}): ${out[s.id].length} items  [${collected.length} raw]`)
  }

  const dest = path.join(ROOT, 'src/content/iranSitesNews.json')
  await fs.writeFile(dest, JSON.stringify(out, null, 2), 'utf8')
  console.log(`Wrote ${dest}`)
}

main()