import { createHash } from "node:crypto"; import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const contentRoot = path.join(root, "src", "data"); const outputRoot = path.join(root, "public", "media", "news"); const manifestPath = path.join(root, "src", "lib", "image-map.generated.ts"); const remoteUrlPattern = /https?:\/\/[^\s)>'"]+/g; const imageExtensionPattern = /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#].*)?$/i; async function collectMarkdownFiles(directory) { const entries = await readdir(directory, { withFileTypes: true }); const files = []; for (const entry of entries) { const entryPath = path.join(directory, entry.name); if (entry.isDirectory()) files.push(...await collectMarkdownFiles(entryPath)); else if (entry.isFile() && entry.name.endsWith(".md")) files.push(entryPath); } return files; } function cleanUrl(value) { return value.replace(/[.,;:!?]+$/, ""); } function getExtension(url, contentType) { const pathname = new URL(url).pathname; const extension = path.extname(pathname).toLowerCase(); if (extension && imageExtensionPattern.test(pathname)) return extension; const typeExtension = contentType.split("/")[1]?.split(";")[0]?.toLowerCase(); return typeExtension === "jpeg" ? ".jpg" : `.${typeExtension || "bin"}`; } async function downloadImage(url) { const response = await fetch(url, { redirect: "follow", signal: AbortSignal.timeout(15000), }); if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); const contentType = response.headers.get("content-type") || "application/octet-stream"; if (!contentType.startsWith("image/")) throw new Error(`unexpected content type ${contentType}`); const bytes = Buffer.from(await response.arrayBuffer()); const hash = createHash("sha256").update(bytes).digest("hex").slice(0, 16); const fileName = `${hash}${getExtension(url, contentType)}`; await writeFile(path.join(outputRoot, fileName), bytes); return `/media/news/${fileName}`; } const markdownFiles = await collectMarkdownFiles(contentRoot); const remoteUrls = new Set(); for (const filePath of markdownFiles) { const content = await readFile(filePath, "utf8"); for (const match of content.matchAll(remoteUrlPattern)) { const url = cleanUrl(match[0]); if (imageExtensionPattern.test(url)) remoteUrls.add(url); } } await mkdir(outputRoot, { recursive: true }); const imageMap = {}; let failed = 0; const urls = [...remoteUrls]; const concurrency = 8; for (let index = 0; index < urls.length; index += concurrency) { const batch = urls.slice(index, index + concurrency); await Promise.all(batch.map(async (url) => { try { imageMap[url] = await downloadImage(url); } catch (error) { failed += 1; console.warn(`[cache-images] skipped ${url}: ${error.message}`); } })); } const generated = `// Generated by scripts/cache-images.mjs. Do not edit.\nexport const imageMap: Record = ${JSON.stringify(imageMap, null, 2)};\n`; await writeFile(manifestPath, generated, "utf8"); console.log(`[cache-images] cached ${Object.keys(imageMap).length}/${remoteUrls.size} images${failed ? `; ${failed} failed` : ""}`);