Upload prerendered pages with revalidate-based expiry

This makes cloudfront request them again periodically.
Not yet useful on its own, but required for rerendering.

Author: jvarho

packages/libs/s3-static-assets/src/index.ts

@@ -7,19 +7,25 @@ import {
   SERVER_NO_CACHE_CACHE_CONTROL_HEADER,
   SERVER_CACHE_CONTROL_HEADER
 } from "./lib/constants";
+import getPageName from "./lib/getPageName";
 import S3ClientFactory, { Credentials } from "./lib/s3";
 import pathToPosix from "./lib/pathToPosix";
-import { PrerenderManifest } from "next/dist/build/index";
+import { PrerenderManifest, SsgRoute } from "next/dist/build/index";
 import getPublicAssetCacheControl, {
   PublicDirectoryCache
 } from "./lib/getPublicAssetCacheControl";
 
+type PrerenderRoutes = {
+  [path: string]: SsgRoute;
+};
+
 type UploadStaticAssetsOptions = {
   bucketName: string;
   basePath: string;
   nextConfigDir: string;
   nextStaticDir?: string;
   credentials: Credentials;
+  prerenderRoutes?: PrerenderRoutes;
   publicDirectoryCache?: PublicDirectoryCache;
 };
 
@@ -29,6 +35,7 @@ type AssetDirectoryFileCachePoliciesOptions = {
   // .i.e. by default .serverless_nextjs
   serverlessBuildOutDir: string;
   nextStaticDir?: string;
+  prerenderRoutes: PrerenderRoutes;
   publicDirectoryCache?: PublicDirectoryCache;
 };
 
@@ -38,13 +45,19 @@ type AssetDirectoryFileCachePoliciesOptions = {
 const getAssetDirectoryFileCachePolicies = (
   options: AssetDirectoryFileCachePoliciesOptions
 ): Array<{
-  cacheControl: string | undefined;
+  cacheControl?: string;
+  expires?: Date;
   path: {
     relative: string;
     absolute: string;
   };
 }> => {
-  const { basePath, publicDirectoryCache, serverlessBuildOutDir } = options;
+  const {
+    basePath,
+    prerenderRoutes,
+    publicDirectoryCache,
+    serverlessBuildOutDir
+  } = options;
 
   const normalizedBasePath = basePath ? basePath.slice(1) : "";
 
@@ -75,20 +88,39 @@ const getAssetDirectoryFileCachePolicies = (
 
   // Upload Next.js data files
 
-  const nextDataFiles = readDirectoryFiles(
-    path.join(assetsOutputDirectory, normalizedBasePath, "_next", "data")
+  const nextDataDir = path.join(
+    assetsOutputDirectory,
+    normalizedBasePath,
+    "_next",
+    "data"
   );
+  const nextDataFiles = readDirectoryFiles(nextDataDir);
 
-  const nextDataFilesUploads = nextDataFiles.map((fileItem) => ({
-    path: fileItem.path,
-    cacheControl: SERVER_CACHE_CONTROL_HEADER
-  }));
+  const nextDataFilesUploads = nextDataFiles.map((fileItem) => {
+    const route = prerenderRoutes[getPageName(fileItem.path, nextDataDir)];
+    if (route && route.initialRevalidateSeconds) {
+      const expires = new Date(
+        new Date().getTime() + 1000 * route.initialRevalidateSeconds
+      );
+      return {
+        path: fileItem.path,
+        expires
+      };
+    }
+    return {
+      path: fileItem.path,
+      cacheControl: SERVER_CACHE_CONTROL_HEADER
+    };
+  });
 
   // Upload Next.js HTML pages
 
-  const htmlPages = readDirectoryFiles(
-    path.join(assetsOutputDirectory, normalizedBasePath, "static-pages")
+  const htmlDir = path.join(
+    assetsOutputDirectory,
+    normalizedBasePath,
+    "static-pages"
   );
+  const htmlPages = readDirectoryFiles(htmlDir);
 
   const htmlPagesUploads = htmlPages.map((fileItem) => {
     // Dynamic fallback HTML pages should never be cached as it will override actual pages once generated and stored in S3.
@@ -98,12 +130,23 @@ const getAssetDirectoryFileCachePolicies = (
         path: fileItem.path,
         cacheControl: SERVER_NO_CACHE_CACHE_CONTROL_HEADER
       };
-    } else {
+    }
+
+    const route = prerenderRoutes[getPageName(fileItem.path, htmlDir)];
+    if (route && route.initialRevalidateSeconds) {
+      const expires = new Date(
+        new Date().getTime() + 1000 * route.initialRevalidateSeconds
+      );
       return {
         path: fileItem.path,
-        cacheControl: SERVER_CACHE_CONTROL_HEADER
+        expires
       };
     }
+
+    return {
+      path: fileItem.path,
+      cacheControl: SERVER_CACHE_CONTROL_HEADER
+    };
   });
 
   // Upload user static and public files
@@ -132,14 +175,14 @@ const getAssetDirectoryFileCachePolicies = (
     ...htmlPagesUploads,
     ...publicAndStaticUploads,
     buildIdUpload
-  ].map(({ cacheControl, path: absolutePath }) => ({
-    cacheControl,
+  ].map(({ path: absolutePath, ...rest }) => ({
     path: {
       // Path relative to the assets folder, used for the S3 upload key
       relative: path.relative(assetsOutputDirectory, absolutePath),
       // Absolute path of local asset
       absolute: absolutePath
-    }
+    },
+    ...rest
   }));
 };
 
@@ -155,11 +198,14 @@ const uploadStaticAssetsFromBuild = async (
     bucketName,
     credentials,
     basePath,
+    prerenderRoutes,
     publicDirectoryCache,
     nextConfigDir
   } = options;
+
   const files = getAssetDirectoryFileCachePolicies({
     basePath,
+    prerenderRoutes: prerenderRoutes ?? {},
     publicDirectoryCache,
     serverlessBuildOutDir: path.join(nextConfigDir, ".serverless_nextjs")
   });
@@ -173,7 +219,8 @@ const uploadStaticAssetsFromBuild = async (
       s3.uploadFile({
         s3Key: pathToPosix(file.path.relative),
         filePath: file.path.absolute,
-        cacheControl: file.cacheControl
+        cacheControl: file.cacheControl,
+        expires: file.expires
       })
     )
   );

packages/libs/s3-static-assets/src/lib/getPageName.ts

@@ -0,0 +1,8 @@
+const getPageName = (file: string, base: string): string => {
+  const relative = file.slice(base.length + 1);
+  const withoutBuildId = relative.split("/", 2)[1];
+  const withoutExtension = withoutBuildId.replace(/\.(html|json)$/, "");
+  return `/${withoutExtension}`;
+};
+
+export default getPageName;

packages/libs/s3-static-assets/src/lib/s3.ts

@@ -12,6 +12,7 @@ type S3ClientFactoryOptions = {
 type UploadFileOptions = {
   filePath: string;
   cacheControl?: string;
+  expires?: Date;
   s3Key?: string;
 };
 
@@ -73,7 +74,7 @@ export default async ({
     uploadFile: async (
       options: UploadFileOptions
     ): Promise<AWS.S3.ManagedUpload.SendData> => {
-      const { filePath, cacheControl, s3Key } = options;
+      const { filePath, cacheControl, expires, s3Key } = options;
 
       const fileBody = await fse.readFile(filePath);
 
@@ -83,7 +84,8 @@ export default async ({
           Key: s3Key || filePath,
           Body: fileBody,
           ContentType: getMimeType(filePath),
-          CacheControl: cacheControl || undefined
+          CacheControl: cacheControl || undefined,
+          Expires: expires
         })
         .promise();
     },

packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/index.html renamed to packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build-basepath/.serverless_nextjs/assets/basepath/static-pages/zsWqBqLjpgRmswfQomanp/index.html

@@ -43,6 +43,13 @@ const upload = (
       secretAccessKey: "fake-secret-key",
       sessionToken: "fake-session-token"
     },
+    prerenderRoutes: {
+      "/revalidate": {
+        initialRevalidateSeconds: 60,
+        srcRoute: "/revalidate",
+        dataRoute: "/_next/data/zsWqBqLjpgRmswfQomanp/revalidate.json"
+      }
+    },
     publicDirectoryCache: publicAssetCache
   });
 };
@@ -163,6 +170,27 @@ describe.each`
       );
     });
 
+    it("uploads revalidate HTML pages with expires instead of cache-control", async () => {
+      expect(mockUpload).toBeCalledWith(
+        expect.objectContaining({
+          Key: "static-pages/zsWqBqLjpgRmswfQomanp/revalidate.html",
+          ContentType: "text/html"
+        })
+      );
+      const call = mockUpload.mock.calls.find(
+        (call) =>
+          call[0].Key === "static-pages/zsWqBqLjpgRmswfQomanp/revalidate.html"
+      );
+      expect(call[0]).toHaveProperty("Expires");
+      expect(call[0].CacheControl).toEqual(undefined);
+      expect(new Date(call[0].Expires).getTime()).toBeGreaterThan(
+        new Date().getTime()
+      );
+      expect(new Date(call[0].Expires).getTime()).toBeLessThan(
+        new Date().getTime() + 60000
+      );
+    });
+
     it("uploads staticProps JSON files in _next/data", async () => {
       expect(mockUpload).toBeCalledWith(
         expect.objectContaining({
@@ -197,6 +225,27 @@ describe.each`
       );
     });
 
+    it("uploads revalidate _next/data JSON with expires instead of cache-control", async () => {
+      expect(mockUpload).toBeCalledWith(
+        expect.objectContaining({
+          Key: "_next/data/zsWqBqLjpgRmswfQomanp/revalidate.json",
+          ContentType: "application/json"
+        })
+      );
+      const call = mockUpload.mock.calls.find(
+        (call) =>
+          call[0].Key === "_next/data/zsWqBqLjpgRmswfQomanp/revalidate.json"
+      );
+      expect(call[0]).toHaveProperty("Expires");
+      expect(call[0].CacheControl).toEqual(undefined);
+      expect(new Date(call[0].Expires).getTime()).toBeGreaterThan(
+        new Date().getTime()
+      );
+      expect(new Date(call[0].Expires).getTime()).toBeLessThan(
+        new Date().getTime() + 60000
+      );
+    });
+
     it("uploads files in the public folder", async () => {
       expect(mockUpload).toBeCalledWith(
         expect.objectContaining({

packages/libs/s3-static-assets/tests/fixtures/app-basic-upload-from-build/.serverless_nextjs/assets/_next/data/zsWqBqLjpgRmswfQomanp/revalidate.json

@@ -347,6 +347,7 @@ class NextjsComponent extends Component {
         nextConfigDir: nextConfigPath,
         nextStaticDir: nextStaticPath,
         credentials: this.context.credentials.aws,
+        prerenderRoutes: defaultBuildManifest.pages.ssg.nonDynamic,
         publicDirectoryCache: inputs.publicDirectoryCache
       });
     } else {