website/scripts/cache-images.mjs

import { createHash } from "node:crypto";
import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";

const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const contentRoot = path.join(root, "src", "data");
const outputRoot = path.join(root, "public", "media", "news");
const manifestPath = path.join(root, "src", "lib", "image-map.generated.ts");
const remoteUrlPattern = /https?:\/\/[^\s)>'"]+/g;
const imageExtensionPattern = /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#].*)?$/i;
const browserHeaders = {
  "accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
  "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
  "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
};
const requestTimeoutMs = 30000;
const maxAttempts = 3;

function wait(milliseconds) {
  return new Promise((resolve) => setTimeout(resolve, milliseconds));
}

async function readCachedImageMap() {
  try {
    const source = await readFile(manifestPath, "utf8");
    const match = source.match(/imageMap: Record<string, string> = (\{[\s\S]*\});\s*$/);
    return match ? JSON.parse(match[1]) : {};
  } catch {
    return {};
  }
}

async function cachedFileExists(imagePath) {
  try {
    await access(path.join(root, "public", imagePath.replace(/^\//, "")));
    return true;
  } catch {
    return false;
  }
}

async function collectMarkdownFiles(directory) {
  const entries = await readdir(directory, { withFileTypes: true });
  const files = [];

  for (const entry of entries) {
    const entryPath = path.join(directory, entry.name);
    if (entry.isDirectory()) files.push(...await collectMarkdownFiles(entryPath));
    else if (entry.isFile() && entry.name.endsWith(".md")) files.push(entryPath);
  }

  return files;
}

function cleanUrl(value) {
  return value.replace(/[.,;:!?]+$/, "");
}

function getExtension(url, contentType) {
  const pathname = new URL(url).pathname;
  const extension = path.extname(pathname).toLowerCase();
  if (extension && imageExtensionPattern.test(pathname)) return extension;
  const typeExtension = contentType.split("/")[1]?.split(";")[0]?.toLowerCase();
  return typeExtension === "jpeg" ? ".jpg" : `.${typeExtension || "bin"}`;
}

async function downloadImage(url) {
  let lastError;

  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
    try {
      const response = await fetch(url, {
        redirect: "follow",
        headers: browserHeaders,
        signal: AbortSignal.timeout(requestTimeoutMs),
      });
      if (!response.ok) throw new Error(`${response.status} ${response.statusText}`);

      const contentType = response.headers.get("content-type") || "application/octet-stream";
      if (!contentType.startsWith("image/")) throw new Error(`unexpected content type ${contentType}`);

      const bytes = Buffer.from(await response.arrayBuffer());
      const hash = createHash("sha256").update(bytes).digest("hex").slice(0, 16);
      const fileName = `${hash}${getExtension(url, contentType)}`;
      await writeFile(path.join(outputRoot, fileName), bytes);
      return `/media/news/${fileName}`;
    } catch (error) {
      lastError = error;
      if (attempt < maxAttempts) {
        const retryDelayMs = attempt * 2000;
        console.warn(`[cache-images] retrying ${url} in ${retryDelayMs / 1000}s (attempt ${attempt + 1}/${maxAttempts})`);
        await wait(retryDelayMs);
      }
    }
  }

  throw lastError;
}

const markdownFiles = await collectMarkdownFiles(contentRoot);
const remoteUrls = new Set();
for (const filePath of markdownFiles) {
  const content = await readFile(filePath, "utf8");
  for (const match of content.matchAll(remoteUrlPattern)) {
    const url = cleanUrl(match[0]);
    if (imageExtensionPattern.test(url)) remoteUrls.add(url);
  }
}

await mkdir(outputRoot, { recursive: true });
const previousImageMap = await readCachedImageMap();
const imageMap = {};
let failed = 0;
let reused = 0;
const urls = [...remoteUrls];
const concurrency = Math.max(1, Number.parseInt(process.env.CACHE_IMAGES_CONCURRENCY || "3", 10) || 3);
for (let index = 0; index < urls.length; index += concurrency) {
  const batch = urls.slice(index, index + concurrency);
  await Promise.all(batch.map(async (url) => {
    try {
      const cachedPath = previousImageMap[url];
      if (cachedPath && await cachedFileExists(cachedPath)) {
        imageMap[url] = cachedPath;
        reused += 1;
        return;
      }
      imageMap[url] = await downloadImage(url);
    } catch (error) {
      failed += 1;
      console.warn(`[cache-images] skipped ${url}: ${error.message}`);
    }
  }));
}

const generated = `// Generated by scripts/cache-images.mjs. Do not edit.\nexport const imageMap: Record<string, string> = ${JSON.stringify(imageMap, null, 2)};\n`;
await writeFile(manifestPath, generated, "utf8");
console.log(`[cache-images] cached ${Object.keys(imageMap).length}/${remoteUrls.size} images; reused ${reused}${failed ? `; ${failed} failed` : ""}`);