import { createHash } from "node:crypto"; import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const contentRoot = path.join(root, "src", "data"); const outputRoot = path.join(root, "public", "media", "news"); const manifestPath = path.join(root, "src", "lib", "image-map.generated.ts"); const remoteUrlPattern = /https?:\/\/[^\s)>'"]+/g; const imageExtensionPattern = /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#].*)?$/i; const browserHeaders = { "accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", "accept-language": "zh-CN,zh;q=0.9,en;q=0.8", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36", }; const requestTimeoutMs = 30000; const maxAttempts = 3; function wait(milliseconds) { return new Promise((resolve) => setTimeout(resolve, milliseconds)); } async function readCachedImageMap() { try { const source = await readFile(manifestPath, "utf8"); const match = source.match(/imageMap: Record = (\{[\s\S]*\});\s*$/); return match ? JSON.parse(match[1]) : {}; } catch { return {}; } } async function cachedFileExists(imagePath) { try { await access(path.join(root, "public", imagePath.replace(/^\//, ""))); return true; } catch { return false; } } async function collectMarkdownFiles(directory) { const entries = await readdir(directory, { withFileTypes: true }); const files = []; for (const entry of entries) { const entryPath = path.join(directory, entry.name); if (entry.isDirectory()) files.push(...await collectMarkdownFiles(entryPath)); else if (entry.isFile() && entry.name.endsWith(".md")) files.push(entryPath); } return files; } function cleanUrl(value) { return value.replace(/[.,;:!?]+$/, ""); } function getExtension(url, contentType) { const pathname = new URL(url).pathname; const extension = path.extname(pathname).toLowerCase(); if (extension && imageExtensionPattern.test(pathname)) return extension; const typeExtension = contentType.split("/")[1]?.split(";")[0]?.toLowerCase(); return typeExtension === "jpeg" ? ".jpg" : `.${typeExtension || "bin"}`; } async function downloadImage(url) { let lastError; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { try { const response = await fetch(url, { redirect: "follow", headers: browserHeaders, signal: AbortSignal.timeout(requestTimeoutMs), }); if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); const contentType = response.headers.get("content-type") || "application/octet-stream"; if (!contentType.startsWith("image/")) throw new Error(`unexpected content type ${contentType}`); const bytes = Buffer.from(await response.arrayBuffer()); const hash = createHash("sha256").update(bytes).digest("hex").slice(0, 16); const fileName = `${hash}${getExtension(url, contentType)}`; await writeFile(path.join(outputRoot, fileName), bytes); return `/media/news/${fileName}`; } catch (error) { lastError = error; if (attempt < maxAttempts) { const retryDelayMs = attempt * 2000; console.warn(`[cache-images] retrying ${url} in ${retryDelayMs / 1000}s (attempt ${attempt + 1}/${maxAttempts})`); await wait(retryDelayMs); } } } throw lastError; } const markdownFiles = await collectMarkdownFiles(contentRoot); const remoteUrls = new Set(); for (const filePath of markdownFiles) { const content = await readFile(filePath, "utf8"); for (const match of content.matchAll(remoteUrlPattern)) { const url = cleanUrl(match[0]); if (imageExtensionPattern.test(url)) remoteUrls.add(url); } } await mkdir(outputRoot, { recursive: true }); const previousImageMap = await readCachedImageMap(); const imageMap = {}; let failed = 0; let reused = 0; const urls = [...remoteUrls]; const concurrency = Math.max(1, Number.parseInt(process.env.CACHE_IMAGES_CONCURRENCY || "3", 10) || 3); for (let index = 0; index < urls.length; index += concurrency) { const batch = urls.slice(index, index + concurrency); await Promise.all(batch.map(async (url) => { try { const cachedPath = previousImageMap[url]; if (cachedPath && await cachedFileExists(cachedPath)) { imageMap[url] = cachedPath; reused += 1; return; } imageMap[url] = await downloadImage(url); } catch (error) { failed += 1; console.warn(`[cache-images] skipped ${url}: ${error.message}`); } })); } const generated = `// Generated by scripts/cache-images.mjs. Do not edit.\nexport const imageMap: Record = ${JSON.stringify(imageMap, null, 2)};\n`; await writeFile(manifestPath, generated, "utf8"); console.log(`[cache-images] cached ${Object.keys(imageMap).length}/${remoteUrls.size} images; reused ${reused}${failed ? `; ${failed} failed` : ""}`);