refactor: nextjs packaging, use npm packages to bundle instead of shell script, parameters added (#22)

Co-authored-by: Jan Soukup <[email protected]>

Author: sladg

Diff for: lib/cli.ts

@@ -1,19 +1,17 @@
 #!/usr/bin/env node
-import { exec as child_exec } from 'child_process'
-import util from 'util'
+
+import { Command } from 'commander'
+import { mkdirSync, rmSync, symlinkSync, writeFileSync } from 'fs'
+import { tmpdir } from 'os'
 import path from 'path'
-import packageJson from '../package.json'
 import { simpleGit } from 'simple-git'
-import { Command } from 'commander'
-import { bumpCalculator, bumpMapping, BumpType, isValidTag, replaceVersionInCommonFiles } from './utils'
-
-const exec = util.promisify(child_exec)
+import packageJson from '../package.json'
+import { bumpCalculator, bumpMapping, BumpType, findInFile, isValidTag, replaceVersionInCommonFiles, zipFolder, zipMultipleFoldersOrFiles } from './utils'
 
 const skipCiFlag = '[skip ci]'
-
+const commandCwd = process.cwd()
+const nextServerConfigRegex = /(?<=conf: )(.*)(?=,)/
 const scriptDir = path.dirname(__filename)
-const scriptPath = path.resolve(`${scriptDir}/../../scripts/pack-nextjs.sh`)
-const handlerPath = path.resolve(`${scriptDir}/../server-handler/index.js`)
 
 const program = new Command()
 
@@ -26,21 +24,107 @@ program
 program
 	.command('pack')
 	.description('Package standalone Next12 build into Lambda compatible ZIPs.')
-	.option('--output', 'folder where to save output', 'next.out')
-	.option('--publicFolder', 'folder where public assets are located', 'public')
-	.option('--handler', 'custom handler to deal with ApiGw events', handlerPath)
-	.option('--grepBy', 'keyword to identify configuration inside server.js', 'webpack')
-	.action(async (str, options) => {
-		// @TODO: Ensure path exists.
-		// @TODO: Ensure.next folder exists with standalone folder inside.
-
-		// @TODO: Transform into code, move away from script.
-		// Also, pass parameters and options.
-		console.log('Starting packaging of your NextJS project!')
-
-		await exec(`chmod +x ${scriptPath} && ${scriptPath}`)
-			.then(({ stdout }) => console.log(stdout))
-			.catch(console.error)
+	.option(
+		'--standaloneFolder',
+		'Folder including NextJS standalone build. Parental folder should include more folders as well.',
+		path.resolve(commandCwd, '.next/standalone'),
+	)
+	.option(
+		'--publicFolder',
+		'Folder where public assets are located, typically this folder is located in root of the project.',
+		path.resolve(commandCwd, './public'),
+	)
+	.option(
+		'--handlerPath',
+		'Path to custom handler to be used to handle ApiGw events. By default this is provided for you.',
+		path.resolve(scriptDir, './../server-handler/index.js'),
+	)
+	.option(
+		'--outputFolder',
+		'Path to folder which should be used for outputting bundled ZIP files for your Lambda. It will be cleared before every script run.',
+		path.resolve(commandCwd, './next.out'),
+	)
+	.action(async (options) => {
+		const { standaloneFolder, publicFolder, handlerPath, outputFolder } = options
+
+		// Dependencies layer configuration
+		const nodeModulesFolderPath = path.resolve(standaloneFolder, './node_modules')
+		const depsLambdaFolder = 'nodejs/node_modules'
+		const lambdaNodeModulesPath = path.resolve('/opt', depsLambdaFolder)
+		const dependenciesOutputPath = path.resolve(outputFolder, 'dependenciesLayer.zip')
+
+		// Code layer configuration
+		const generatedNextServerPath = path.resolve(standaloneFolder, './server.js')
+		const codeOutputPath = path.resolve(outputFolder, 'code.zip')
+
+		// Assets bundle configuration
+		const generatedStaticContentPath = path.resolve(commandCwd, '.next/static')
+		const generatedStaticRemapping = '_next/static'
+		const assetsOutputPath = path.resolve(outputFolder, 'assetsLayer.zip')
+
+		// Clean output directory before continuing
+		rmSync(outputFolder, { force: true, recursive: true })
+		mkdirSync(outputFolder)
+
+		// Zip dependencies from standalone output in a layer-compatible format.
+		await zipFolder({
+			outputName: dependenciesOutputPath,
+			folderPath: nodeModulesFolderPath,
+			dir: depsLambdaFolder,
+		})
+
+		// Zip staticly generated assets and public folder.
+		await zipMultipleFoldersOrFiles({
+			outputName: assetsOutputPath,
+			inputDefinition: [
+				{
+					path: publicFolder,
+				},
+				{
+					path: generatedStaticContentPath,
+					dir: generatedStaticRemapping,
+				},
+			],
+		})
+
+		// Create a symlink for node_modules so they point to the separately packaged layer.
+		// We need to create symlink because we are not using NodejsFunction in CDK as bundling is custom.
+		const tmpFolder = tmpdir()
+
+		const symlinkPath = path.resolve(tmpFolder, `./node_modules_${Math.random()}`)
+		symlinkSync(lambdaNodeModulesPath, symlinkPath)
+
+		const nextConfig = findInFile(generatedNextServerPath, nextServerConfigRegex)
+		const configPath = path.resolve(tmpFolder, `./config.json_${Math.random()}`)
+		writeFileSync(configPath, nextConfig, 'utf-8')
+
+		// Zip codebase including symlinked node_modules and handler.
+		await zipMultipleFoldersOrFiles({
+			outputName: codeOutputPath,
+			inputDefinition: [
+				{
+					isGlob: true,
+					cwd: standaloneFolder,
+					path: '**/*',
+					ignore: ['**/node_modules/**', '*.zip'],
+				},
+				{
+					isFile: true,
+					path: handlerPath,
+					name: 'handler.js',
+				},
+				{
+					isFile: true,
+					path: symlinkPath,
+					name: 'node_modules',
+				},
+				{
+					isFile: true,
+					path: configPath,
+					name: 'config.json',
+				},
+			],
+		})
 
 		console.log('Your NextJS project was succefully prepared for Lambda.')
 	})

Diff for: lib/image-handler.ts

@@ -1,7 +1,6 @@
-process.env.NODE_ENV = 'production'
-// Set NEXT_SHARP_PATH environment variable
 // ! Make sure this comes before the fist import
 process.env.NEXT_SHARP_PATH = require.resolve('sharp')
+process.env.NODE_ENV = 'production'
 
 import type { APIGatewayProxyEventV2, APIGatewayProxyStructuredResultV2 } from 'aws-lambda'
 import { defaultConfig, NextConfigComplete } from 'next/dist/server/config-shared'

Diff for: lib/server-handler.ts

@@ -1,5 +1,6 @@
-process.env.NODE_ENV = 'production'
+// ! This is needed for nextjs to correctly resolve.
 process.chdir(__dirname)
+process.env.NODE_ENV = 'production'
 
 import NextServer from 'next/dist/server/next-server'
 import slsHttp from 'serverless-http'

Diff for: lib/utils.ts

@@ -1,7 +1,10 @@
 import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'
+import archiver from 'archiver'
+import { createWriteStream, readFileSync, symlinkSync } from 'fs'
+import { IOptions as GlobOptions } from 'glob'
 import { IncomingMessage, ServerResponse } from 'http'
-import { replaceInFileSync } from 'replace-in-file'
 import { NextUrlWithParsedQuery } from 'next/dist/server/request-meta'
+import { replaceInFileSync } from 'replace-in-file'
 import { Readable } from 'stream'
 
 // Make header keys lowercase to ensure integrity.
@@ -145,6 +148,7 @@ export const replaceVersionInCommonFiles = (oldVersion: string, newVersion: stri
 		],
 		files: [
 			'package.json',
+			'**/package.json', // Useful for workspaces with nested package.jsons also including versions.
 			'package-lock.json',
 			'package-lock.json', // Duplicate because lock file contains two occurences.
 			// 'yarn.lock', Yarn3 lock file does not contain version from package.json
@@ -170,3 +174,83 @@ export const replaceVersionInCommonFiles = (oldVersion: string, newVersion: stri
 
 	return results
 }
+
+export const findInFile = (filePath: string, regex: RegExp): string => {
+	const content = readFileSync(filePath, 'utf-8')
+	const data = content.match(regex)
+
+	if (!data?.[0]) {
+		throw new Error('Unable to match Next server configuration.')
+	}
+
+	return data[0]
+}
+
+interface ZipFolderProps {
+	outputName: string
+	folderPath: string
+	dir?: string
+}
+
+export const zipFolder = async ({ folderPath, outputName, dir }: ZipFolderProps) =>
+	zipMultipleFoldersOrFiles({
+		outputName,
+		inputDefinition: [{ path: folderPath, dir }],
+	})
+
+interface FolderInput {
+	path: string
+	dir?: string
+}
+
+interface FileInput {
+	path: string
+	name: string
+	isFile: true
+}
+
+interface SymlinkInput {
+	source: string
+	target: string
+	isSymlink: true
+}
+
+interface GlobInput extends GlobOptions {
+	path: string
+	isGlob: true
+}
+
+interface ZipProps {
+	outputName: string
+	inputDefinition: (FolderInput | FileInput | SymlinkInput | GlobInput)[]
+}
+
+export const zipMultipleFoldersOrFiles = async ({ outputName, inputDefinition }: ZipProps) => {
+	const archive = archiver('zip', { zlib: { level: 5 } })
+	const stream = createWriteStream(outputName)
+
+	return new Promise((resolve, reject) => {
+		inputDefinition.forEach((props) => {
+			if ('isFile' in props) {
+				archive.file(props.path, { name: props.name })
+			} else if ('isSymlink' in props) {
+				archive.symlink(props.source, props.target)
+			} else if ('isGlob' in props) {
+				archive.glob(props.path, props)
+			} else {
+				archive.directory(props.path, props.dir ?? false)
+			}
+		})
+
+		archive.on('error', (err) => reject(err)).pipe(stream)
+		stream.on('close', resolve)
+		archive.finalize()
+	})
+}
+
+interface SymlinkProps {
+	sourcePath: string
+	linkLocation: string
+}
+
+export const createSymlink = ({ linkLocation, sourcePath }: SymlinkProps) => symlinkSync(sourcePath, linkLocation)