修改图片下载脚本

2026-11-03 11:06:32 +09:00 · 2026-09-16 12:28:06 +08:00
parent 36b6012809
commit 706463d12f
3 changed files with 413 additions and 32 deletions
@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
+import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath } from "node:url";

@@ -9,6 +9,36 @@ const outputRoot = path.join(root, "public", "media", "news");
 const manifestPath = path.join(root, "src", "lib", "image-map.generated.ts");
 const remoteUrlPattern = /https?:\/\/[^\s)>'"]+/g;
 const imageExtensionPattern = /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#].*)?$/i;
+const browserHeaders = {
+  "accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
+  "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+  "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+};
+const requestTimeoutMs = 30000;
+const maxAttempts = 3;
+
+function wait(milliseconds) {
+  return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+
+async function readCachedImageMap() {
+  try {
+    const source = await readFile(manifestPath, "utf8");
+    const match = source.match(/imageMap: Record<string, string> = (\{[\s\S]*\});\s*$/);
+    return match ? JSON.parse(match[1]) : {};
+  } catch {
+    return {};
+  }
+}
+
+async function cachedFileExists(imagePath) {
+  try {
+    await access(path.join(root, "public", imagePath.replace(/^\//, "")));
+    return true;
+  } catch {
+    return false;
+  }
+}

 async function collectMarkdownFiles(directory) {
  const entries = await readdir(directory, { withFileTypes: true });
@@ -36,20 +66,36 @@ function getExtension(url, contentType) {
 }

 async function downloadImage(url) {
-  const response = await fetch(url, {
-    redirect: "follow",
-    signal: AbortSignal.timeout(15000),
-  });
-  if (!response.ok) throw new Error(`${response.status} ${response.statusText}`);
+  let lastError;

-  const contentType = response.headers.get("content-type") || "application/octet-stream";
-  if (!contentType.startsWith("image/")) throw new Error(`unexpected content type ${contentType}`);
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      const response = await fetch(url, {
+        redirect: "follow",
+        headers: browserHeaders,
+        signal: AbortSignal.timeout(requestTimeoutMs),
+      });
+      if (!response.ok) throw new Error(`${response.status} ${response.statusText}`);

-  const bytes = Buffer.from(await response.arrayBuffer());
-  const hash = createHash("sha256").update(bytes).digest("hex").slice(0, 16);
-  const fileName = `${hash}${getExtension(url, contentType)}`;
-  await writeFile(path.join(outputRoot, fileName), bytes);
-  return `/media/news/${fileName}`;
+      const contentType = response.headers.get("content-type") || "application/octet-stream";
+      if (!contentType.startsWith("image/")) throw new Error(`unexpected content type ${contentType}`);
+
+      const bytes = Buffer.from(await response.arrayBuffer());
+      const hash = createHash("sha256").update(bytes).digest("hex").slice(0, 16);
+      const fileName = `${hash}${getExtension(url, contentType)}`;
+      await writeFile(path.join(outputRoot, fileName), bytes);
+      return `/media/news/${fileName}`;
+    } catch (error) {
+      lastError = error;
+      if (attempt < maxAttempts) {
+        const retryDelayMs = attempt * 2000;
+        console.warn(`[cache-images] retrying ${url} in ${retryDelayMs / 1000}s (attempt ${attempt + 1}/${maxAttempts})`);
+        await wait(retryDelayMs);
+      }
+    }
+  }
+
+  throw lastError;
 }

 const markdownFiles = await collectMarkdownFiles(contentRoot);
@@ -63,14 +109,22 @@ for (const filePath of markdownFiles) {
 }

 await mkdir(outputRoot, { recursive: true });
+const previousImageMap = await readCachedImageMap();
 const imageMap = {};
 let failed = 0;
+let reused = 0;
 const urls = [...remoteUrls];
-const concurrency = 8;
+const concurrency = Math.max(1, Number.parseInt(process.env.CACHE_IMAGES_CONCURRENCY || "3", 10) || 3);
 for (let index = 0; index < urls.length; index += concurrency) {
  const batch = urls.slice(index, index + concurrency);
  await Promise.all(batch.map(async (url) => {
    try {
+      const cachedPath = previousImageMap[url];
+      if (cachedPath && await cachedFileExists(cachedPath)) {
+        imageMap[url] = cachedPath;
+        reused += 1;
+        return;
+      }
      imageMap[url] = await downloadImage(url);
    } catch (error) {
      failed += 1;
@@ -81,4 +135,4 @@ for (let index = 0; index < urls.length; index += concurrency) {

 const generated = `// Generated by scripts/cache-images.mjs. Do not edit.\nexport const imageMap: Record<string, string> = ${JSON.stringify(imageMap, null, 2)};\n`;
 await writeFile(manifestPath, generated, "utf8");
-console.log(`[cache-images] cached ${Object.keys(imageMap).length}/${remoteUrls.size} images${failed ? `; ${failed} failed` : ""}`);
+console.log(`[cache-images] cached ${Object.keys(imageMap).length}/${remoteUrls.size} images; reused ${reused}${failed ? `; ${failed} failed` : ""}`);