steelforesight/panel/content-generator.js

import * as cheerio from 'cheerio';

const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125 Safari/537.36';
const STATUS_BY_JOB = new Map();
let seq = 0;

const CFG = {
  ai: {
    baseUrl: process.env.AI_BASE_URL || 'https://api.openai.com/v1',
    apiKey: process.env.AI_API_KEY || '',
    model: process.env.AI_MODEL || 'gpt-4.1-mini',
  },
  panel: {
    baseUrl: process.env.PANEL_BASE_URL || process.env.PUBLIC_ORIGIN || 'http://localhost:3000',
    username: process.env.ADMIN_USERNAME || 'admin',
    password: process.env.ADMIN_PASSWORD || '',
  },
  systemPrompt: `You are a Persian steel-sector editorial generator.

Return JSON only.

For "At a Glance":
- short, compact, 2-3 sentences
- one takeaway
- no long analysis

For "Radar Future":
- long-form analytical article
- 4-5 paragraphs
- multiple sources and in-text citations
- intro, build-up, conclusion
- neutral, formal Persian

Do not mix the two formats.
Do not invent facts. Use only the provided news.`
};

const AT_A_GLANCE_CATEGORIES = [
  { code: 'market', label: 'بازارهای جهانی و اقتصاد کلان', query: 'فولاد بازار قیمت' },
  { code: 'tech', label: 'نوآوری و فناوری‌های تحول‌آفرین', query: 'فولاد فناوری نوآوری' },
  { code: 'commodity', label: 'مواد معدنی حیاتی و زنجیره تامین', query: 'سنگ آهن قراضه فولاد' },
  { code: 'geo', label: 'ژئوپلیتیک و امنیت اقتصادی', query: 'فولاد تعرفه تحریم ژئوپلیتیک' },
  { code: 'energy', label: 'تجارت و کسب‌وکار بین‌الملل', query: 'فولاد انرژی صادرات تجارت' },
];

const RADAR_FUTURE_CATEGORIES = [
  { label: 'بازار و زنجیره فولاد', query: 'steel market supply chain pricing tariffs' },
  { label: 'پایداری و فولاد سبز', query: 'green steel hydrogen decarbonization' },
  { label: 'ژئوپلیتیک و اقتصاد جهانی', query: 'steel geopolitics tariffs sanctions trade' },
  { label: 'فناوری و نوآوری', query: 'steel automation ai digital twin robotics' },
];

const RADAR_FUTURE_PROFILES = {
  'بازار و زنجیره فولاد': 'Focus on prices, supply-demand balance, iron ore, scrap, coking coal, capacity, trade flows, logistics, inventory, margins, and steel value-chain risk.',
  'پایداری و فولاد سبز': 'Focus on green steel, hydrogen DRI, EAF, emissions, CBAM, ETS, renewable power, regulation, capital expenditure, and industrial decarbonization.',
  'ژئوپلیتیک و اقتصاد جهانی': 'Focus on tariffs, sanctions, industrial policy, China, US, EU, India, supply security, trade conflict, currency pressure, and geopolitical risk.',
  'فناوری و نوآوری': 'Focus on AI, automation, robotics, sensors, digital twins, process control, predictive maintenance, advanced materials, and steel plant productivity.',
};

function buildSystemPrompt(kind, categoryLabel) {
  if (kind === 'radarFuture') {
    return `You are the senior Persian editor for the "Radar Future" section of a steel-industry think tank.

Section category: ${categoryLabel}
Category specialization: ${RADAR_FUTURE_PROFILES[categoryLabel] || 'Steel-industry foresight and market analysis.'}

Output JSON only.

Radar Future requirements:
- Write in formal Persian.
- Target length: about 900 Persian words, acceptable range 800-1000 words.
- Analytical article, not a short news brief.
- Build an introduction, evidence-led body, strategic implications, and conclusion.
- Use multiple sources. Prefer at least 3 distinct source links when provided.
- Use numbered in-text citations exactly like [1], [2], [3].
- Citation [1] must correspond to sources[0], citation [2] to sources[1], and so on.
- Every source in sources[] should be cited at least once in the text.
- Do not invent facts. Use only the provided source material.
- Do not add image blocks, chart blocks, image URLs, or captions. The editor will add images manually.
- blocks[] must contain only {"type":"text","value":"..."} objects.`;
  }
  return `You are the senior Persian editor for the "At a Glance" section of a steel-industry think tank.

Output JSON only.

At a Glance requirements:
- Write in formal Persian.
- Short, compact, and decision-useful.
- 2-3 sentences per item.
- One clear takeaway per item.
- No long essay.
- Do not invent facts. Use only the provided source material.`;
}

function publicJob(job) {
  return {
    id: job.id,
    kind: job.kind,
    input: job.input,
    status: job.status,
    step: job.step,
    progress: job.progress,
    message: job.message,
    createdAt: job.createdAt,
    startedAt: job.startedAt,
    finishedAt: job.finishedAt,
    error: job.error,
    result: job.result,
  };
}

export function getJob(id) {
  return STATUS_BY_JOB.get(id) || null;
}

export function listJobs() {
  return [...STATUS_BY_JOB.values()].map(publicJob);
}

function createJob(kind, input) {
  const id = `gen_${Date.now()}_${++seq}`;
  const job = {
    id,
    kind,
    input,
    status: 'queued',
    step: 'waiting',
    progress: 0,
    message: 'queued',
    createdAt: new Date().toISOString(),
    startedAt: null,
    finishedAt: null,
    error: null,
    result: null,
  };
  STATUS_BY_JOB.set(id, job);
  return job;
}

function updateJob(job, patch) {
  Object.assign(job, patch);
  return publicJob(job);
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

function decode(s = '') {
  return s
    .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
    .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&apos;/g, "'")
    .trim();
}

async function fetchWithTimeout(url, init = {}, ms = 15000) {
  const ctrl = new AbortController();
  const t = setTimeout(() => ctrl.abort(), ms);
  try {
    return await fetch(url, { ...init, signal: ctrl.signal });
  } finally {
    clearTimeout(t);
  }
}

async function fetchNews(query) {
  const url = `https://news.google.com/rss/search?q=${encodeURIComponent(query + ' when:30d')}&hl=en-US&gl=US&ceid=US:en`;
  const sources = [
    { name: 'google-rss', url, init: { headers: { 'User-Agent': UA } } },
    { name: 'jina-rss', url: `https://r.jina.ai/http://${url.replace(/^https?:\/\//, '')}`, init: {} },
  ];
  let xml = '';
  let lastErr = null;
  for (const src of sources) {
    try {
      const res = await fetchWithTimeout(src.url, src.init, 15000);
      if (!res.ok) throw new Error(`news HTTP ${res.status}`);
      xml = await res.text();
      if (xml.trim()) break;
    } catch (err) {
      lastErr = new Error(`${src.name}: ${err.message || err}`);
    }
  }
  if (!xml.trim()) throw new Error(`news fetch failed: ${lastErr?.message || 'no source returned content'}`);
  const items = [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => {
    const block = m[1];
    const t = (tag) => decode(block.match(new RegExp(`<${tag}>([\\s\\S]*?)<\\/${tag}>`))?.[1] || '');
    return { title: t('title'), link: t('link'), pubDate: t('pubDate'), source: t('source') };
  });
  return items.filter((x) => x.title);
}

function sourceHost(url) {
  try { return new URL(url).hostname.replace(/^www\./, ''); }
  catch { return ''; }
}

function trimSourceText(text, max = 3500) {
  return String(text || '').replace(/\s+/g, ' ').trim().slice(0, max);
}

function extractReadableText(raw, contentType = '', url = '') {
  if (!raw) return null;
  if (/html/i.test(contentType) || /<html|<body|<article|<!doctype/i.test(raw.slice(0, 500))) {
    const $ = cheerio.load(raw);
    $('script,style,noscript,svg,iframe,nav,footer,header').remove();
    const title = trimSourceText($('meta[property="og:title"]').attr('content') || $('title').first().text() || $('h1').first().text() || url, 220);
    const text = trimSourceText($('article').text() || $('main').text() || $('body').text());
    return text ? { title, text } : null;
  }
  const titleMatch = raw.match(/^Title:\s*(.+)$/im);
  const title = trimSourceText(titleMatch?.[1] || url, 220);
  const text = trimSourceText(raw);
  return text ? { title, text } : null;
}

function jinaReaderUrl(url) {
  const u = String(url || '').replace(/^https?:\/\//, '');
  return `https://r.jina.ai/http://${u}`;
}

async function fetchReadableUrl(url) {
  const attempts = [
    { name: 'direct', url },
    { name: 'reader', url: jinaReaderUrl(url) },
  ];
  let lastErr = null;
  for (const attempt of attempts) {
    try {
      const res = await fetchWithTimeout(attempt.url, {
        headers: {
          'User-Agent': UA,
          Accept: 'text/html,text/plain,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        },
      });
      if (!res.ok) throw new Error(`${res.status}`);
      const raw = await res.text();
      const parsed = extractReadableText(raw, res.headers.get('content-type') || '', url);
      if (parsed?.text) return { ...parsed, url };
    } catch (err) {
      lastErr = new Error(`${attempt.name}: ${err.message || err}`);
    }
  }
  throw lastErr || new Error('read failed');
}

async function collectSourceMaterials(query, sourceRows = [], maxItems = 12) {
  const rows = Array.isArray(sourceRows) ? sourceRows.filter((s) => s?.url).slice(0, 80) : [];
  if (!rows.length) return [];
  const materials = [];
  for (const src of rows) {
    if (materials.length >= maxItems) break;
    const host = sourceHost(src.url);
    const type = src.sourceType || src.source_type || 'auto';
    const label = src.label || host || src.url;
    try {
      if (type === 'link' || (new URL(src.url).pathname || '/') !== '/') {
        const page = await fetchReadableUrl(src.url);
        materials.push({ ...page, sourceLabel: label });
      } else {
        const hits = (await fetchNews(`${query} site:${host}`)).slice(0, 2);
        for (const hit of hits) {
          if (materials.length >= maxItems) break;
          try {
            const page = await fetchReadableUrl(hit.link);
            materials.push({ ...page, title: page.title || hit.title, sourceLabel: label });
          } catch {
            materials.push({ title: hit.title, url: hit.link, sourceLabel: label, text: hit.title });
          }
        }
      }
    } catch {
      // A large source library will always have a few blocked sites. Keep the job moving.
    }
  }
  return materials;
}

function sourceMaterialsText(materials) {
  return materials.map((m, i) => `${i + 1}. ${m.title || m.sourceLabel || 'Source'}
source: ${m.sourceLabel || ''}
url: ${m.url || ''}
excerpt: ${trimSourceText(m.text, 2200)}`).join('\n\n');
}

function uniqueSourceList(materials, max = 6) {
  const seen = new Set();
  const out = [];
  for (const m of materials || []) {
    if (!m?.url || seen.has(m.url)) continue;
    seen.add(m.url);
    out.push({ title: m.title || m.sourceLabel || sourceHost(m.url) || m.url, url: m.url });
    if (out.length >= max) break;
  }
  return out;
}

function newsToMaterials(news) {
  return (news || []).map((n) => ({
    title: n.title,
    url: n.link,
    sourceLabel: n.source || sourceHost(n.link),
    text: n.title,
  }));
}

function textOnlyBlocks(blocks, fallbackText = '') {
  const out = (Array.isArray(blocks) ? blocks : [])
    .filter((b) => b?.type === 'text' && String(b.value || '').trim())
    .map((b) => ({ type: 'text', value: String(b.value || '').trim() }));
  if (!out.length && fallbackText) out.push({ type: 'text', value: String(fallbackText).trim() });
  return out;
}

function alignCitations(blocks, sources) {
  if (!blocks.length || !sources.length) return blocks;
  const maxSource = sources.length;
  for (const b of blocks) {
    b.value = String(b.value || '').replace(/\[(\d+)\]/g, (m, n) => {
      const idx = Number(n);
      return idx >= 1 && idx <= maxSource ? m : '';
    }).replace(/\s{2,}/g, ' ').trim();
  }
  const joined = blocks.map((b) => b.value).join('\n');
  const cited = new Set([...joined.matchAll(/\[(\d+)\]/g)].map((m) => Number(m[1])));
  const minimum = Math.min(3, maxSource);
  for (let i = 1; i <= minimum; i++) {
    if (cited.has(i)) continue;
    const target = blocks[Math.min(i - 1, blocks.length - 1)];
    target.value = `${target.value.replace(/[.。؟?]*$/, '')} [${i}].`;
  }
  return blocks;
}

function extractStringField(text, key) {
  const rx = new RegExp(`"${key}"\\s*:\\s*"((?:[^"\\\\]|\\\\.)*)"`, 's');
  const m = text.match(rx);
  return m ? m[1]
    .replace(/\\"/g, '"')
    .replace(/\\\\/g, '\\')
    .replace(/\\n/g, '\n')
    .replace(/\\r/g, '\r')
    .replace(/\\t/g, '\t')
    .trim() : '';
}

function extractArrayField(text, key) {
  const rx = new RegExp(`"${key}"\\s*:\\s*(\\[[\\s\\S]*?\\])`, 's');
  const m = text.match(rx);
  if (!m) return [];
  try {
    return JSON.parse(m[1].replace(/,(\s*[}\]])/g, '$1'));
  } catch {
    return [];
  }
}

function salvageJson(text) {
  const cleaned = String(text || '')
    .replace(/^```(?:json)?\s*/i, '')
    .replace(/```$/i, '')
    .trim();
  const obj = {
    titleFa: extractStringField(cleaned, 'titleFa'),
    titleEn: extractStringField(cleaned, 'titleEn'),
    publishDateFa: extractStringField(cleaned, 'publishDateFa'),
    publishDateEn: extractStringField(cleaned, 'publishDateEn'),
    summaryFa: extractStringField(cleaned, 'summaryFa'),
    summaryEn: extractStringField(cleaned, 'summaryEn'),
    authorName: extractStringField(cleaned, 'authorName'),
    authorAvatar: extractStringField(cleaned, 'authorAvatar'),
    tags: extractArrayField(cleaned, 'tags'),
    sources: extractArrayField(cleaned, 'sources'),
    blocks: extractArrayField(cleaned, 'blocks'),
  };
  if (!obj.blocks.length && cleaned) obj.blocks = [{ type: 'text', value: cleaned.slice(0, 900) }];
  return obj;
}

async function aiJson(messages, maxTokens = 6000) {
  if (!CFG.ai.apiKey) throw new Error('AI_API_KEY is missing');
  let res;
  try {
    res = await fetchWithTimeout(`${CFG.ai.baseUrl}/chat/completions`, {
      method: 'POST',
      headers: { Authorization: `Bearer ${CFG.ai.apiKey}`, 'Content-Type': 'application/json' },
      body: JSON.stringify({ model: CFG.ai.model, messages, max_tokens: maxTokens, temperature: 0.7 }),
    }, 45000);
  } catch (err) {
    throw new Error(`ai fetch failed: ${err.message || err}`);
  }
  if (!res.ok) throw new Error(`ai ${res.status}: ${(await res.text()).slice(0, 200)}`);
  const j = await res.json();
  const text = String(j.choices?.[0]?.message?.content || '').trim();
  const candidates = [
    text,
    text.replace(/^```(?:json)?\s*/i, '').replace(/```$/i, '').trim(),
  ];
  for (const candidate of candidates) {
    const s = candidate.indexOf('{');
    const e = candidate.lastIndexOf('}');
    if (s === -1 || e === -1) continue;
    const body = candidate.slice(s, e + 1)
      .replace(/,(\s*[}\]])/g, '$1')
      .replace(/^\uFEFF/, '');
    try {
      return JSON.parse(body);
    } catch {}
  }
  const salvage = salvageJson(text);
  if (salvage.titleFa || salvage.summaryFa || salvage.blocks.length || salvage.sources.length) return salvage;
  throw new Error(`ai json parse failed: ${text.slice(0, 240)}`);
}

async function login(panelBaseUrl, username, password) {
  const res = await fetch(`${panelBaseUrl}/api/auth/login`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ username, password }),
  });
  if (!res.ok) throw new Error(`panel login failed (${res.status})`);
  const cookies = res.headers.getSetCookie?.() || [];
  const session = cookies.map((c) => c.split(';')[0]).find((c) => c.startsWith('session='));
  if (!session) throw new Error('no session cookie returned');
  return session;
}

async function createRadarDraft(panelBaseUrl, session, payload) {
  const res = await fetch(`${panelBaseUrl}/api/radar`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json', Cookie: session },
    body: JSON.stringify(payload),
  });
  if (!res.ok) throw new Error(`radar POST ${res.status}: ${(await res.text()).slice(0, 140)}`);
  return res.json();
}

async function createArticleDraft(panelBaseUrl, session, payload) {
  const res = await fetch(`${panelBaseUrl}/api/articles`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json', Cookie: session },
    body: JSON.stringify(payload),
  });
  if (!res.ok) throw new Error(`article POST ${res.status}: ${(await res.text()).slice(0, 140)}`);
  return res.json();
}

async function generateAtAGlance(job, panelBaseUrl, session) {
  updateJob(job, { status: 'running', startedAt: new Date().toISOString(), step: 'fetching', progress: 10, message: 'fetching news' });
  const cat = AT_A_GLANCE_CATEGORIES.find((c) => c.code === job.input.category) || AT_A_GLANCE_CATEGORIES[0];
  const trusted = await collectSourceMaterials(cat.query, job.input.sources, 10);
  const news = trusted.length ? [] : (await fetchNews(cat.query)).slice(0, 8);
  updateJob(job, { step: 'generating', progress: 35, message: trusted.length ? `using ${trusted.length} saved source items` : `news ${news.length} items` });
  const newsText = trusted.length
    ? sourceMaterialsText(trusted)
    : news.map((n, i) => `${i + 1}. ${n.title}\nsource: ${n.link}`).join('\n\n');
  const ai = await aiJson([
    { role: 'system', content: buildSystemPrompt('glance', cat.label) },
    { role: 'user', content: `Write short "At a Glance" items for category: ${cat.label}. Return JSON only: {"items":[{"titleFa":"","excerptFa":"","bodyFa":"","analysisFa":"","tags":[],"sourceUrl":""}]}. Use the saved source material/news below. Prefer exact source URLs from the material.\n\n${newsText}` },
  ], 7000);
  const items = Array.isArray(ai?.items) ? ai.items : [];
  updateJob(job, { step: 'saving', progress: 70, message: `drafting ${items.length} items` });
  const dateFa = new Intl.DateTimeFormat('fa-IR-u-ca-persian', { year: 'numeric', month: 'long', day: 'numeric' }).format(new Date());
  const saved = [];
  for (const item of items.slice(0, 3)) {
    const row = await createRadarDraft(panelBaseUrl, session, {
      category: cat.code,
      titleFa: String(item.titleFa || '').trim(),
      excerptFa: item.excerptFa || null,
      bodyFa: item.bodyFa || null,
      analysisFa: item.analysisFa || null,
      tags: Array.isArray(item.tags) ? item.tags : [],
      source: item.sourceUrl || null,
      dateFa,
      status: 'draft',
    });
    saved.push(row.id);
  }
  updateJob(job, { status: 'done', step: 'done', progress: 100, message: 'finished', finishedAt: new Date().toISOString(), result: { ids: saved } });
}

async function generateRadarFuture(job, panelBaseUrl, session) {
  updateJob(job, { status: 'running', startedAt: new Date().toISOString(), step: 'fetching', progress: 10, message: 'fetching news' });
  const selected = job.input.category && job.input.category !== 'all'
    ? RADAR_FUTURE_CATEGORIES.filter((c) => c.label === job.input.category)
    : RADAR_FUTURE_CATEGORIES;
  const saved = [];
  for (let idx = 0; idx < selected.length; idx++) {
    const cat = selected[idx];
    const pct = 10 + Math.round((idx / Math.max(selected.length, 1)) * 70);
    updateJob(job, { step: 'fetching', progress: pct, message: `fetching sources for ${cat.label}` });
    const trusted = await collectSourceMaterials(cat.query, job.input.sources, 14);
    const news = trusted.length ? [] : (await fetchNews(cat.query)).slice(0, 12);
    updateJob(job, { step: 'generating', progress: pct + 10, message: trusted.length ? `writing ${cat.label} from ${trusted.length} saved source items` : `writing ${cat.label} from ${news.length} news items` });
    const materials = (trusted.length ? trusted : newsToMaterials(news)).slice(0, 6);
    const citationSources = uniqueSourceList(materials, 6);
    const newsText = sourceMaterialsText(materials);
    const ai = await aiJson([
      { role: 'system', content: buildSystemPrompt('radarFuture', cat.label) },
      { role: 'user', content: `Write one long-form Radar Future article for category "${cat.label}".

Return JSON only in this exact shape:
{"titleFa":"","titleEn":"","publishDateFa":"","publishDateEn":"","summaryFa":"","summaryEn":"","authorName":"","authorAvatar":"","tags":[],"sources":[{"title":"","url":""}],"blocks":[{"type":"text","value":""}]}

Rules:
- Target about 900 Persian words, acceptable range 800-1000.
- Use 5-7 paragraph-style text blocks.
- Do not include images, image blocks, chart blocks, img fields, image URLs, or captions.
- Use multiple source mentions in the body.
- Cite facts with numbered in-text citations like [1], [2], [3].
- Citation [1] must map to sources[0], [2] to sources[1], etc.
- Keep sources[] in the same order as the citation numbers.
- Use at least 3 source links if at least 3 are provided below.
- Intro, build-up, strategic implications, conclusion.
- Keep it analytical.
- Use only facts supported by the saved source material/news.
- Keep sources array to the strongest 3-5 exact links.
- Return a single valid JSON object only. No markdown, no code fences, no explanation text.

Available source material, already numbered for citation use:
${newsText}` },
    ], 12000);
    updateJob(job, { step: 'saving', progress: pct + 18, message: `saving ${cat.label}` });
    const blocks = alignCitations(textOnlyBlocks(ai?.blocks, ai?.body || ''), citationSources);
    const article = await createArticleDraft(panelBaseUrl, session, {
      title: ai?.titleFa || cat.label,
      category: cat.label,
      type: 'radarfuture',
      author: ai?.authorName || 'تحریریه',
      authorRole: 'تحریریه',
      authorInitial: 'ت',
      publishDate: ai?.publishDateFa || new Intl.DateTimeFormat('fa-IR-u-ca-persian', { year: 'numeric', month: 'long', day: 'numeric' }).format(new Date()),
      pages: 0,
      price: 0,
      isFree: true,
      summary: ai?.summaryFa || '',
      body: blocks.map((b) => b.value).join('\n\n'),
      coverImage: '',
      blocks,
      sources: citationSources.slice(0, Math.max(3, Math.min(citationSources.length, 6))),
      tags: Array.isArray(ai?.tags) ? ai.tags : [],
      featured: false,
      status: 'draft',
      keyData: null,
      type: 'radarfuture',
    });
    saved.push({ id: article.id, category: cat.label });
    await sleep(50);
  }
  updateJob(job, { status: 'done', step: 'done', progress: 100, message: 'finished', finishedAt: new Date().toISOString(), result: { articles: saved } });
}

export async function runGenerator(input) {
  const kind = input.kind === 'radarFuture' ? 'radarFuture' : 'glance';
  const job = createJob(kind, input);
  const panelBaseUrl = input.panelBaseUrl || CFG.panel.baseUrl;
  const session = input.sessionCookie || await login(panelBaseUrl, input.username || CFG.panel.username, input.password || CFG.panel.password);
  Promise.resolve().then(async () => {
    try {
      if (kind === 'radarFuture') await generateRadarFuture(job, panelBaseUrl, session);
      else await generateAtAGlance(job, panelBaseUrl, session);
    } catch (err) {
      updateJob(job, { status: 'error', step: 'error', progress: job.progress || 0, message: String(err.message || err), finishedAt: new Date().toISOString(), error: String(err.message || err) });
    }
  });
  return publicJob(job);
}