website/scripts/cache-images.mjs

import { createHash } from "node:crypto";
import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";

const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const contentRoot = path.join(root, "src", "data");
const outputRoot = path.join(root, "public", "media", "news");
const manifestPath = path.join(root, "src", "lib", "image-map.generated.ts");
const remoteUrlPattern = /https?:\/\/[^\s)>'"]+/g;
const imageExtensionPattern = /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#].*)?$/i;

async function collectMarkdownFiles(directory) {
  const entries = await readdir(directory, { withFileTypes: true });
  const files = [];

  for (const entry of entries) {
    const entryPath = path.join(directory, entry.name);
    if (entry.isDirectory()) files.push(...await collectMarkdownFiles(entryPath));
    else if (entry.isFile() && entry.name.endsWith(".md")) files.push(entryPath);
  }

  return files;
}

function cleanUrl(value) {
  return value.replace(/[.,;:!?]+$/, "");
}

function getExtension(url, contentType) {
  const pathname = new URL(url).pathname;
  const extension = path.extname(pathname).toLowerCase();
  if (extension && imageExtensionPattern.test(pathname)) return extension;
  const typeExtension = contentType.split("/")[1]?.split(";")[0]?.toLowerCase();
  return typeExtension === "jpeg" ? ".jpg" : `.${typeExtension || "bin"}`;
}

async function downloadImage(url) {
  const response = await fetch(url, {
    redirect: "follow",
    signal: AbortSignal.timeout(15000),
  });
  if (!response.ok) throw new Error(`${response.status} ${response.statusText}`);

  const contentType = response.headers.get("content-type") || "application/octet-stream";
  if (!contentType.startsWith("image/")) throw new Error(`unexpected content type ${contentType}`);

  const bytes = Buffer.from(await response.arrayBuffer());
  const hash = createHash("sha256").update(bytes).digest("hex").slice(0, 16);
  const fileName = `${hash}${getExtension(url, contentType)}`;
  await writeFile(path.join(outputRoot, fileName), bytes);
  return `/media/news/${fileName}`;
}

const markdownFiles = await collectMarkdownFiles(contentRoot);
const remoteUrls = new Set();
for (const filePath of markdownFiles) {
  const content = await readFile(filePath, "utf8");
  for (const match of content.matchAll(remoteUrlPattern)) {
    const url = cleanUrl(match[0]);
    if (imageExtensionPattern.test(url)) remoteUrls.add(url);
  }
}

await mkdir(outputRoot, { recursive: true });
const imageMap = {};
let failed = 0;
const urls = [...remoteUrls];
const concurrency = 8;
for (let index = 0; index < urls.length; index += concurrency) {
  const batch = urls.slice(index, index + concurrency);
  await Promise.all(batch.map(async (url) => {
    try {
      imageMap[url] = await downloadImage(url);
    } catch (error) {
      failed += 1;
      console.warn(`[cache-images] skipped ${url}: ${error.message}`);
    }
  }));
}

const generated = `// Generated by scripts/cache-images.mjs. Do not edit.\nexport const imageMap: Record<string, string> = ${JSON.stringify(imageMap, null, 2)};\n`;
await writeFile(manifestPath, generated, "utf8");
console.log(`[cache-images] cached ${Object.keys(imageMap).length}/${remoteUrls.size} images${failed ? `; ${failed} failed` : ""}`);