seedproject-web/content-pipeline/scripts/research.mjs

#!/usr/bin/env node
// Editorial research: top up the EVERGREEN backlog in calendar.json (additive, deduped).
// Timely news is handled separately by the news radar. Usage:
//   node content-pipeline/scripts/research.mjs [count]
import { readdirSync, existsSync } from "node:fs";
import { dirname, resolve, join } from "node:path";
import { fileURLToPath } from "node:url";
import { runClaude } from "./lib/claude.mjs";
import { loadJson, todayInTz, addDays } from "./lib/util.mjs";
import { readCalendar, mergeEntries } from "./lib/calendar.mjs";

const HERE = dirname(fileURLToPath(import.meta.url));
const PIPELINE = resolve(HERE, "..");
const cfg = loadJson(join(PIPELINE, "config.json"));
const root = cfg.paths.projectRoot;

const publishedSlugs = () => {
  const d = join(root, cfg.paths.blogContentDir);
  return existsSync(d)
    ? readdirSync(d, { withFileTypes: true }).filter((x) => x.isDirectory()).map((x) => x.name)
    : [];
};

/** Scan for evergreen topics and additively merge them into calendar.json. Returns count added. */
export async function runResearch({ count } = {}) {
  const n = count || cfg.editorial.calendarDays;
  const today = todayInTz(cfg.editorial.timezone);
  const calendarPath = join(root, cfg.paths.calendar);
  const scanPath = join(root, "content-pipeline", "research-scan.json");

  // Per-type target counts from the configured mix.
  const types = Object.keys(cfg.editorial.contentMix);
  const counts = {};
  let assigned = 0;
  for (const t of types) {
    counts[t] = Math.round(cfg.editorial.contentMix[t] * n);
    assigned += counts[t];
  }
  const biggest = types.reduce((a, b) => (counts[a] >= counts[b] ? a : b));
  counts[biggest] += n - assigned;

  const exclude = [...new Set([...readCalendar(calendarPath).map((e) => e.slug), ...publishedSlugs()])];

  const prompt = `Propose ${n} evergreen Comiida topics.

- Write a JSON array of ${n} topic proposals to: ${scanPath}
- Do NOT assign dates (the pipeline schedules them).
- Allowed content types: ${types.join(", ")}
- Rough target counts per type: ${JSON.stringify(counts)}
- DO NOT reuse any of these slugs: ${exclude.join(", ") || "(none yet)"}
- Read existing posts if useful: ${join(root, cfg.paths.blogContentDir)}

Follow the output contract in your system prompt exactly.`;

  console.log(`[research] proposing ${n} evergreen topics, mix ${JSON.stringify(counts)}`);

  await runClaude({
    prompt,
    systemPromptFile: join(PIPELINE, "prompts/research.system.md"),
    model: cfg.models.research,
    allowedTools: "Read Glob Grep WebSearch Write",
    addDirs: [root],
    cwd: root,
    logFile: join(root, cfg.paths.logsDir, `research-${today}.log`),
  });

  const proposals = existsSync(scanPath) ? loadJson(scanPath) : [];
  if (!proposals.length) {
    console.log("[research] no proposals produced.");
    return 0;
  }

  // Schedule new topics one per day, starting the day after the latest existing date.
  const dates = readCalendar(calendarPath).map((e) => (e.date || "").slice(0, 10)).filter(Boolean);
  const maxDate = dates.sort().pop();
  let next = maxDate && maxDate >= today ? addDays(maxDate, 1) : today;

  const entries = proposals.map((p) => {
    const entry = { ...p, date: next, status: "planned" };
    next = addDays(next, 1);
    return entry;
  });

  const added = await mergeEntries(calendarPath, entries);
  const planned = readCalendar(calendarPath).filter((e) => e.status === "planned").length;
  console.log(`[research] added ${added} evergreen topics. Planned backlog now: ${planned}.`);
  return added;
}

// CLI
if (import.meta.url === `file://${process.argv[1]}`) {
  const count = parseInt(process.argv[2], 10) || undefined;
  await runResearch({ count });
}