Incremental static regeneration

packages/libs/lambda-at-edge/package.json

@@ -62,6 +62,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "1.0.0-rc.3",
+    "@aws-sdk/client-sqs": "1.0.0-rc.3",
     "@hapi/accept": "5.0.1",
     "@vercel/nft": "^0.9.3",
     "cookie": "^0.4.1",

packages/libs/lambda-at-edge/rollup.config.js

@@ -47,5 +47,7 @@ export default [
   { filename: "api-handler", minify: false },
   { filename: "api-handler", minify: true },
   { filename: "image-handler", minify: false },
-  { filename: "image-handler", minify: true }
+  { filename: "image-handler", minify: true },
+  { filename: "regeneration-handler", minify: false },
+  { filename: "regeneration-handler", minify: true }
 ].map(generateConfig);

packages/libs/lambda-at-edge/src/build.ts

@@ -31,6 +31,7 @@ import { Job } from "@vercel/nft/out/node-file-trace";
 export const DEFAULT_LAMBDA_CODE_DIR = "default-lambda";
 export const API_LAMBDA_CODE_DIR = "api-lambda";
 export const IMAGE_LAMBDA_CODE_DIR = "image-lambda";
+export const REGENERATION_LAMBDA_CODE_DIR = "regeneration-lambda";
 export const ASSETS_DIR = "assets";
 
 type BuildOptions = {
@@ -230,7 +231,11 @@ class Builder {
    * @param shouldMinify
    */
   async processAndCopyHandler(
-    handlerType: "api-handler" | "default-handler" | "image-handler",
+    handlerType:
+      | "api-handler"
+      | "default-handler"
+      | "image-handler"
+      | "regeneration-handler",
     destination: string,
     shouldMinify: boolean
   ) {
@@ -243,9 +248,9 @@ class Builder {
     await fse.copy(source, destination);
   }
 
-  async buildDefaultLambda(
+  async copyTraces(
     buildManifest: OriginRequestDefaultHandlerManifest
-  ): Promise<void[]> {
+  ): Promise<void> {
     let copyTraces: Promise<void>[] = [];
 
     if (this.buildOptions.useServerlessTraceTarget) {
@@ -282,7 +287,13 @@ class Builder {
       );
     }
 
-    let prerenderManifest = require(join(
+    await Promise.all(copyTraces);
+  }
+
+  async buildDefaultLambda(
+    buildManifest: OriginRequestDefaultHandlerManifest
+  ): Promise<void[]> {
+    const prerenderManifest = require(join(
       this.dotNextDir,
       "prerender-manifest.json"
     ));
@@ -292,7 +303,7 @@ class Builder {
     );
 
     return Promise.all([
-      ...copyTraces,
+      this.copyTraces(buildManifest),
       this.processAndCopyHandler(
         "default-handler",
         join(this.outputDir, DEFAULT_LAMBDA_CODE_DIR, "index.js"),
@@ -412,6 +423,48 @@ class Builder {
     ]);
   }
 
+  async buildRegenerationHandler(
+    buildManifest: OriginRequestDefaultHandlerManifest
+  ): Promise<void> {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const prerenderManifest = require(join(
+      this.dotNextDir,
+      "prerender-manifest.json"
+    ));
+    await Promise.all([
+      this.copyTraces(buildManifest),
+      this.processAndCopyHandler(
+        "regeneration-handler",
+        join(this.outputDir, REGENERATION_LAMBDA_CODE_DIR, "index.js"),
+        !!this.buildOptions.minifyHandlers
+      ),
+      fse.copy(
+        join(this.serverlessDir, "pages"),
+        join(this.outputDir, REGENERATION_LAMBDA_CODE_DIR, "pages"),
+        {
+          recursive: true,
+          filter: (file: string) => {
+            const isNotPrerenderedHTMLPage = path.extname(file) !== ".html";
+            const isNotStaticPropsJSONFile = path.extname(file) !== ".json";
+            const isNotApiPage = pathToPosix(file).indexOf("pages/api") === -1;
+            const isDirectory = fse.statSync(file).isDirectory();
+            const isPrerenderedJSFile = this.isPrerenderedJSFile(
+              prerenderManifest,
+              path.relative(join(this.serverlessDir, "pages"), file)
+            );
+
+            return (
+              isNotApiPage &&
+              isNotPrerenderedHTMLPage &&
+              isNotStaticPropsJSONFile &&
+              (isPrerenderedJSFile || isDirectory)
+            );
+          }
+        }
+      )
+    ]);
+  }
+
   /**
    * Build image optimization lambda (supported by Next.js 10)
    * @param buildManifest
@@ -941,9 +994,9 @@ class Builder {
       path.join(dotNextDirectory, "prerender-manifest.json")
     );
 
-    let prerenderManifestJSONPropFileAssets: Promise<void>[] = [];
-    let prerenderManifestHTMLPageAssets: Promise<void>[] = [];
-    let fallbackHTMLPageAssets: Promise<void>[] = [];
+    const prerenderManifestJSONPropFileAssets: Promise<void>[] = [];
+    const prerenderManifestHTMLPageAssets: Promise<void>[] = [];
+    const fallbackHTMLPageAssets: Promise<void>[] = [];
 
     // Copy locale-specific prerendered files if defined, otherwise use empty locale
     // which would copy to root only
@@ -1119,6 +1172,7 @@ class Builder {
     await fse.emptyDir(join(this.outputDir, DEFAULT_LAMBDA_CODE_DIR));
     await fse.emptyDir(join(this.outputDir, API_LAMBDA_CODE_DIR));
     await fse.emptyDir(join(this.outputDir, IMAGE_LAMBDA_CODE_DIR));
+    await fse.emptyDir(join(this.outputDir, REGENERATION_LAMBDA_CODE_DIR));
     await fse.emptyDir(join(this.outputDir, ASSETS_DIR));
 
     const { restoreUserConfig } = await createServerlessConfig(
@@ -1160,6 +1214,7 @@ class Builder {
     } = await this.prepareBuildManifests(routesManifest, prerenderManifest);
 
     await this.buildDefaultLambda(defaultBuildManifest);
+    await this.buildRegenerationHandler(defaultBuildManifest);
 
     const hasAPIPages =
       Object.keys(apiBuildManifest.apis.nonDynamic).length > 0 ||

packages/libs/lambda-at-edge/src/default-handler.ts

@@ -325,8 +325,8 @@ const handleOriginRequest = async ({
   const decodedUri = decodeURI(uri);
   const { pages, publicFiles } = manifest;
 
-  let isPublicFile = publicFiles[decodedUri];
-  let isDataReq = isDataRequest(uri);
+  const isPublicFile = publicFiles[decodedUri];
+  const isDataReq = isDataRequest(uri);
 
   // Handle redirects
   // TODO: refactor redirect logic to another file since this is getting quite large
@@ -602,12 +602,95 @@ const handleOriginResponse = async ({
   const request = event.Records[0].cf.request;
   const { uri } = request;
   const { status } = response;
+  const { region, domainName } = request.origin?.s3 || {};
+  const bucketName = domainName?.replace(`.s3.${region}.amazonaws.com`, "");
+
   if (status !== "403") {
     // Set 404 status code for 404.html page. We do not need normalised URI as it will always be "/404.html"
     if (uri.endsWith("/404.html")) {
       response.status = "404";
       response.statusDescription = "Not Found";
+      return response;
+    }
+
+    const initialRevalidateSeconds =
+      manifest.pages.ssg.nonDynamic?.[uri.replace(".html", "")]
+        ?.initialRevalidateSeconds;
+    const lastModifiedHeaderString =
+      response.headers?.["last-modified"]?.[0]?.value;
+    const lastModifiedAt = lastModifiedHeaderString
+      ? new Date(lastModifiedHeaderString)
+      : null;
+    if (typeof initialRevalidateSeconds === "number" && lastModifiedAt) {
+      const createdAgo =
+        // LastModified should always be defined
+        (Date.now() - (lastModifiedAt.getTime() || Date.now())) / 1000;
+
+      const timeToRevalidate = Math.floor(
+        initialRevalidateSeconds - createdAgo
+      );
+
+      response.headers["cache-control"] = [
+        {
+          key: "Cache-Control",
+          value:
+            timeToRevalidate < 0
+              ? "public, max-age=0, s-maxage=0, must-revalidate"
+              : `public, max-age=0, s-maxage=${timeToRevalidate}, must-revalidate`
+        }
+      ];
+
+      if (timeToRevalidate < 0) {
+        const { SQSClient, SendMessageCommand } = await import(
+          "@aws-sdk/client-sqs"
+        );
+        const sqs = new SQSClient({
+          region,
+          maxAttempts: 3,
+          retryStrategy: await buildS3RetryStrategy()
+        });
+        await sqs.send(
+          new SendMessageCommand({
+            QueueUrl: `https://sqs.${region}.amazonaws.com/${bucketName}.fifo`,
+            MessageBody: uri,
+            MessageAttributes: {
+              BucketRegion: {
+                DataType: "String",
+                StringValue: region
+              },
+              BucketName: {
+                DataType: "String",
+                StringValue: bucketName
+              },
+              CloudFrontEventRequest: {
+                DataType: "String",
+                StringValue: JSON.stringify(request)
+              },
+              Manifest: {
+                DataType: "String",
+                StringValue: JSON.stringify(manifest)
+              },
+              ...(basePath
+                ? {
+                    BasePath: {
+                      DataType: "String",
+                      StringValue: basePath
+                    }
+                  }
+                : {})
+            },
+            // We only want to trigger the regeneration once for every previous
+            // update. This will prevent the case where this page is being
+            // requested again whilst its already started to regenerate.
+            MessageDeduplicationId: lastModifiedAt.getTime().toString(),
+            // Only deduplicate based on the object, i.e. we can generate
+            // different pages in parallel, just not the same one
+            MessageGroupId: uri
+          })
+        );
+      }
     }
+
     return response;
   }
 
@@ -616,9 +699,6 @@ const handleOriginResponse = async ({
     return response;
   }
 
-  const { domainName, region } = request.origin!.s3!;
-  const bucketName = domainName.replace(`.s3.${region}.amazonaws.com`, "");
-
   // Lazily import only S3Client to reduce init times until actually needed
   const { S3Client } = await import("@aws-sdk/client-s3/S3Client");
 

packages/libs/lambda-at-edge/src/regeneration-handler.ts

@@ -0,0 +1,86 @@
+import lambdaAtEdgeCompat from "@sls-next/next-aws-cloudfront";
+import { OriginRequestDefaultHandlerManifest } from "./types";
+import { S3Client } from "@aws-sdk/client-s3";
+import { buildS3RetryStrategy } from "./s3/s3RetryStrategy";
+
+export const handler: AWSLambda.SQSHandler = async (event) => {
+  await Promise.all(
+    event.Records.map(async (record) => {
+      const bucketName = record.messageAttributes.BucketName?.stringValue;
+      const bucketRegion = record.messageAttributes.BucketRegion?.stringValue;
+      const manifestString = record.messageAttributes.Manifest?.stringValue;
+      const basePath = record.messageAttributes.BasePath?.stringValue;
+      const cloudFrontEventRequestString =
+        record.messageAttributes.CloudFrontEventRequest?.stringValue;
+      if (
+        !bucketName ||
+        !bucketRegion ||
+        !cloudFrontEventRequestString ||
+        !manifestString
+      ) {
+        throw new Error(
+          "Expected BucketName, BucketRegion, CloudFrontEventRequest & EnableHTTPCompression message attributes"
+        );
+      }
+      const cloudFrontEventRequest: AWSLambda.CloudFrontRequest = JSON.parse(
+        cloudFrontEventRequestString
+      );
+      const manifest: OriginRequestDefaultHandlerManifest = JSON.parse(
+        manifestString
+      );
+
+      const s3 = new S3Client({
+        region: cloudFrontEventRequest.origin?.s3?.region,
+        maxAttempts: 3,
+        retryStrategy: await buildS3RetryStrategy()
+      });
+
+      const { req, res } = lambdaAtEdgeCompat(
+        { request: cloudFrontEventRequest },
+        { enableHTTPCompression: manifest.enableHTTPCompression }
+      );
+
+      const baseKey = cloudFrontEventRequest.uri
+        .replace(/^\//, "")
+        .replace(/\.(json|html)$/, "")
+        .replace(/^_next\/data\/[^\/]*\//, "");
+
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const page = require(`./pages/${baseKey}`);
+
+      const jsonKey = `_next/data/${manifest.buildId}/${baseKey}.json`;
+      const htmlKey = `static-pages/${manifest.buildId}/${baseKey}.html`;
+
+      const { renderOpts, html } = await page.renderReqToHTML(
+        req,
+        res,
+        "passthrough"
+      );
+
+      const s3BasePath = basePath ? `${basePath.replace(/^\//, "")}/` : "";
+      const s3JsonParams = {
+        Bucket: bucketName,
+        Key: `${s3BasePath}${jsonKey}`,
+        Body: JSON.stringify(renderOpts.pageData),
+        ContentType: "application/json",
+        CacheControl: "public, max-age=0, s-maxage=2678400, must-revalidate"
+      };
+
+      const s3HtmlParams = {
+        Bucket: bucketName,
+        Key: `${s3BasePath}${htmlKey}`,
+        Body: html,
+        ContentType: "text/html",
+        CacheControl: "public, max-age=0, s-maxage=2678400, must-revalidate"
+      };
+
+      const { PutObjectCommand } = await import(
+        "@aws-sdk/client-s3/commands/PutObjectCommand"
+      );
+      await Promise.all([
+        s3.send(new PutObjectCommand(s3JsonParams)),
+        s3.send(new PutObjectCommand(s3HtmlParams))
+      ]);
+    })
+  );
+};

packages/libs/lambda-at-edge/tsconfig.build.json

@@ -9,6 +9,7 @@
     "node_modules",
     "./src/api-handler.ts",
     "./src/default-handler.ts",
-    "./src/image-handler.ts"
+    "./src/image-handler.ts",
+    "./src/regeneration-handler.ts",
   ]
 }

packages/libs/lambda-at-edge/tsconfig.bundle.json

@@ -11,5 +11,10 @@
     "allowJs": true,
     "resolveJsonModule": true
   },
-  "include": ["./src/default-handler.ts", "./src/api-handler.ts", "./src/image-handler.ts"]
-}
+  "include": [
+    "./src/default-handler.ts",
+    "./src/api-handler.ts",
+    "./src/image-handler.ts",
+    "./src/regeneration-handler.ts",
+  ]
+}