From 6b64c74d49c3087b3ea1274b93ff8742f1185fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Wilczyn=CC=81ski?= Date: Sat, 20 Nov 2021 19:20:17 +0100 Subject: [PATCH 1/6] Implement requirements layer caching --- lib/layer.js | 31 ++++++++++++++++++++++++++----- lib/shared.js | 23 +++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/lib/layer.js b/lib/layer.js index 12d338ec..fd5efeae 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -3,6 +3,7 @@ const fse = require('fs-extra'); const path = require('path'); const JSZip = require('jszip'); const { writeZip, addTree } = require('./zipTree'); +const {sha256Path, getRequirementsLayerPath} = require("./shared"); BbPromise.promisifyAll(fse); @@ -11,13 +12,33 @@ BbPromise.promisifyAll(fse); * @return {Promise} the JSZip object constructed. */ function zipRequirements() { - const rootZip = new JSZip(); const src = path.join('.serverless', 'requirements'); - const runtimepath = 'python'; - - return addTree(rootZip.folder(runtimepath), src).then(() => - writeZip(rootZip, path.join('.serverless', 'pythonRequirements.zip')) + const reqChecksum = sha256Path(path.join('.serverless', 'requirements.txt')); + const targetZipPath = path.join('.serverless', 'pythonRequirements.zip'); + const zipCachePath = getRequirementsLayerPath( + reqChecksum, + targetZipPath, + this.options ); + + const promises = []; + if (fse.existsSync(zipCachePath)) { + this.serverless.cli.log("Found cached Python Requirements Lambda Layer file"); + } else { + const rootZip = new JSZip(); + const runtimepath = 'python'; + + promises.push( + addTree(rootZip.folder(runtimepath), src).then(() => + writeZip(rootZip, zipCachePath) + ) + ); + } + return BbPromise.all(promises).then(() => { + if (zipCachePath !== targetZipPath) { + fse.symlink(zipCachePath, targetZipPath, 'file') + } + }); } /** diff --git a/lib/shared.js b/lib/shared.js index 79b60cef..b877acd2 100644 --- a/lib/shared.js +++ b/lib/shared.js @@ -76,6 +76,28 @@ function getRequirementsWorkingPath( return path.join(requirementsTxtDirectory, 'requirements'); } +/** + * Path of a cached requirements layer archive file + * @param {string} subfolder + * @param {string} fallback + * @param {Object} options + * @return {string} + */ +function getRequirementsLayerPath( + hash, + fallback, + options +) { + // If we want to use the static cache + if (hash && options && options.useStaticCache) { + hash = hash + '_slspyc.zip'; + return path.join(getUserCachePath(options), hash); + } + + // If we don't want to use the static cache, then fallback to requirements file in .serverless directory + return fallback; +} + /** * The static cache path that will be used for this system + options, used if static cache is enabled * @param {Object} options @@ -107,6 +129,7 @@ function sha256Path(fullpath) { module.exports = { checkForAndDeleteMaxCacheVersions, getRequirementsWorkingPath, + getRequirementsLayerPath, getUserCachePath, sha256Path, }; From 40c81d62273d3597b7433b64f02f1ef860b4f23a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Wilczyn=CC=81ski?= Date: Sat, 20 Nov 2021 19:32:34 +0100 Subject: [PATCH 2/6] Copy instead of linking on Windows --- lib/layer.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/layer.js b/lib/layer.js index fd5efeae..3657b5fd 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -36,7 +36,11 @@ function zipRequirements() { } return BbPromise.all(promises).then(() => { if (zipCachePath !== targetZipPath) { - fse.symlink(zipCachePath, targetZipPath, 'file') + if (process.platform === 'win32') { + fse.copySync(zipCachePath, targetZipPath); + } else { + fse.symlink(zipCachePath, targetZipPath, 'file') + } } }); } From b0426fdfac37b62381e272d9371603f83041672d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Wilczyn=CC=81ski?= Date: Wed, 1 Dec 2021 12:33:06 +0100 Subject: [PATCH 3/6] Linting --- lib/layer.js | 10 ++++++---- lib/shared.js | 6 +----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/layer.js b/lib/layer.js index 5d0837b8..12482fa2 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -3,7 +3,7 @@ const fse = require('fs-extra'); const path = require('path'); const JSZip = require('jszip'); const { writeZip, addTree } = require('./zipTree'); -const {sha256Path, getRequirementsLayerPath} = require("./shared"); +const { sha256Path, getRequirementsLayerPath } = require('./shared'); BbPromise.promisifyAll(fse); @@ -23,14 +23,16 @@ function zipRequirements() { const promises = []; if (fse.existsSync(zipCachePath)) { - this.serverless.cli.log("Found cached Python Requirements Lambda Layer file"); + this.serverless.cli.log( + 'Found cached Python Requirements Lambda Layer file' + ); } else { const rootZip = new JSZip(); const runtimepath = 'python'; promises.push( addTree(rootZip.folder(runtimepath), src).then(() => - writeZip(rootZip, zipCachePath) + writeZip(rootZip, zipCachePath) ) ); } @@ -39,7 +41,7 @@ function zipRequirements() { if (process.platform === 'win32') { fse.copySync(zipCachePath, targetZipPath); } else { - fse.symlink(zipCachePath, targetZipPath, 'file') + fse.symlink(zipCachePath, targetZipPath, 'file'); } } }); diff --git a/lib/shared.js b/lib/shared.js index dad6ed4d..77a11538 100644 --- a/lib/shared.js +++ b/lib/shared.js @@ -90,11 +90,7 @@ function getRequirementsWorkingPath( * @param {Object} options * @return {string} */ -function getRequirementsLayerPath( - hash, - fallback, - options -) { +function getRequirementsLayerPath(hash, fallback, options) { // If we want to use the static cache if (hash && options && options.useStaticCache) { hash = hash + '_slspyc.zip'; From b19c3305129e5eda547292bdbe8cfbad0c0fb84f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Wilczyn=CC=81ski?= Date: Thu, 2 Dec 2021 09:57:55 +0100 Subject: [PATCH 4/6] Add multi-arch support to cached ZIP archives --- lib/layer.js | 3 ++- lib/shared.js | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/layer.js b/lib/layer.js index 12482fa2..eee55cb8 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -18,7 +18,8 @@ function zipRequirements() { const zipCachePath = getRequirementsLayerPath( reqChecksum, targetZipPath, - this.options + this.options, + this.serverless ); const promises = []; diff --git a/lib/shared.js b/lib/shared.js index fb05d35d..bebb3f09 100644 --- a/lib/shared.js +++ b/lib/shared.js @@ -91,12 +91,14 @@ function getRequirementsWorkingPath( * @param {string} subfolder * @param {string} fallback * @param {Object} options + * @param {Object} serverless * @return {string} */ -function getRequirementsLayerPath(hash, fallback, options) { +function getRequirementsLayerPath(hash, fallback, options, serverless) { // If we want to use the static cache if (hash && options && options.useStaticCache) { - hash = hash + '_slspyc.zip'; + const architecture = serverless.service.provider.architecture || 'x86_64'; + hash = `${hash}_${architecture}_slspyc.zip`; return path.join(getUserCachePath(options), hash); } From 33721243261481cb0328cefbf7b25428db08a8e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Wilczyn=CC=81ski?= Date: Thu, 2 Dec 2021 11:20:14 +0100 Subject: [PATCH 5/6] Add v3 logging support --- lib/layer.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/layer.js b/lib/layer.js index eee55cb8..d020bd33 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -24,9 +24,17 @@ function zipRequirements() { const promises = []; if (fse.existsSync(zipCachePath)) { - this.serverless.cli.log( - 'Found cached Python Requirements Lambda Layer file' - ); + let layerProgress; + if (this.progress && this.log) { + layerProgress = this.progress.get('python-layer-requirements'); + layerProgress.update('Using cached Python Requirements Lambda Layer file'); + this.log.info('Found cached Python Requirements Lambda Layer file'); + } else { + this.serverless.cli.log( + 'Found cached Python Requirements Lambda Layer file' + ); + } + } else { const rootZip = new JSZip(); const runtimepath = 'python'; From 4970e5adf7deed4719db210ccfdcc2b5bde050b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Wilczyn=CC=81ski?= Date: Thu, 2 Dec 2021 11:54:58 +0100 Subject: [PATCH 6/6] Linting --- lib/layer.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/layer.js b/lib/layer.js index d020bd33..6fe9ca4c 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -27,14 +27,15 @@ function zipRequirements() { let layerProgress; if (this.progress && this.log) { layerProgress = this.progress.get('python-layer-requirements'); - layerProgress.update('Using cached Python Requirements Lambda Layer file'); + layerProgress.update( + 'Using cached Python Requirements Lambda Layer file' + ); this.log.info('Found cached Python Requirements Lambda Layer file'); } else { this.serverless.cli.log( 'Found cached Python Requirements Lambda Layer file' ); } - } else { const rootZip = new JSZip(); const runtimepath = 'python';