diff --git a/.github/docker/Dockerfile.glibc b/.github/docker/Dockerfile.glibc
index b22a180..cfd02fc 100644
--- a/.github/docker/Dockerfile.glibc
+++ b/.github/docker/Dockerfile.glibc
@@ -11,7 +11,7 @@ ENV PATH=$PATH:/nodejs/bin
WORKDIR /mongodb-client-encryption
COPY . .
-RUN apt-get -qq update && apt-get -qq install -y python3 build-essential && ldd --version
+RUN apt-get -qq update && apt-get -qq install -y python3 build-essential git && ldd --version
RUN npm run install:libmongocrypt
diff --git a/.github/scripts/get-commit-from-ref.sh b/.github/scripts/get-commit-from-ref.sh
new file mode 100644
index 0000000..823b4aa
--- /dev/null
+++ b/.github/scripts/get-commit-from-ref.sh
@@ -0,0 +1,14 @@
+#! /usr/bin/env bash
+set -o errexit
+
+git clone https://github.com/mongodb/libmongocrypt.git _libmongocrypt
+cd _libmongocrypt
+
+git checkout --detach $REF
+
+COMMIT_HASH=$(git rev-parse HEAD)
+
+echo "COMMIT_HASH=$COMMIT_HASH"
+
+cd -
+rm -rf _libmongocrypt
diff --git a/.github/scripts/libmongocrypt.mjs b/.github/scripts/libmongocrypt.mjs
index 0f83095..d61a7bd 100644
--- a/.github/scripts/libmongocrypt.mjs
+++ b/.github/scripts/libmongocrypt.mjs
@@ -8,14 +8,7 @@ import events from 'node:events';
import path from 'node:path';
import https from 'node:https';
import stream from 'node:stream/promises';
-import url from 'node:url';
-
-const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
-
-/** Resolves to the root of this repository */
-function resolveRoot(...paths) {
- return path.resolve(__dirname, '..', '..', ...paths);
-}
+import { buildLibmongocryptDownloadUrl, getLibmongocryptPrebuildName, resolveRoot, run } from './utils.mjs';
async function parseArguments() {
const pkg = JSON.parse(await fs.readFile(resolveRoot('package.json'), 'utf8'));
@@ -26,7 +19,6 @@ async function parseArguments() {
clean: { short: 'c', type: 'boolean', default: false },
build: { short: 'b', type: 'boolean', default: false },
dynamic: { type: 'boolean', default: false },
- fastDownload: { type: 'boolean', default: false }, // Potentially incorrect download, only for the brave and impatient
'skip-bindings': { type: 'boolean', default: false },
help: { short: 'h', type: 'boolean', default: false }
};
@@ -46,7 +38,6 @@ async function parseArguments() {
return {
url: args.values.gitURL,
ref: args.values.libVersion,
- fastDownload: args.values.fastDownload,
clean: args.values.clean,
build: args.values.build,
dynamic: args.values.dynamic,
@@ -55,26 +46,6 @@ async function parseArguments() {
};
}
-/** `xtrace` style command runner, uses spawn so that stdio is inherited */
-async function run(command, args = [], options = {}) {
- const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`;
- console.error(commandDetails);
- const proc = child_process.spawn(command, args, {
- shell: process.platform === 'win32',
- stdio: 'inherit',
- cwd: resolveRoot('.'),
- ...options
- });
- await events.once(proc, 'exit');
-
- if (proc.exitCode != 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`);
-}
-
-/** CLI flag maker: `toFlags({a: 1, b: 2})` yields `['-a=1', '-b=2']` */
-function toFlags(object) {
- return Array.from(Object.entries(object)).map(([k, v]) => `-${k}=${v}`);
-}
-
export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) {
console.error('fetching libmongocrypt...', { url, ref });
await fs.rm(libmongocryptRoot, { recursive: true, force: true });
@@ -87,6 +58,11 @@ export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) {
}
export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, options) {
+ /** CLI flag maker: `toFlags({a: 1, b: 2})` yields `['-a=1', '-b=2']` */
+ function toCLIFlags(object) {
+ return Array.from(Object.entries(object)).map(([k, v]) => `-${k}=${v}`);
+ }
+
console.error('building libmongocrypt...');
const nodeBuildRoot = resolveRoot(nodeDepsRoot, 'tmp', 'libmongocrypt-build');
@@ -94,7 +70,7 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, option
await fs.rm(nodeBuildRoot, { recursive: true, force: true });
await fs.mkdir(nodeBuildRoot, { recursive: true });
- const CMAKE_FLAGS = toFlags({
+ const CMAKE_FLAGS = toCLIFlags({
/**
* We provide crypto hooks from Node.js binding to openssl (so disable system crypto)
* TODO: NODE-5455
@@ -127,12 +103,12 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, option
const WINDOWS_CMAKE_FLAGS =
process.platform === 'win32' // Windows is still called "win32" when it is 64-bit
- ? toFlags({ Thost: 'x64', A: 'x64', DENABLE_WINDOWS_STATIC_RUNTIME: 'ON' })
+ ? toCLIFlags({ Thost: 'x64', A: 'x64', DENABLE_WINDOWS_STATIC_RUNTIME: 'ON' })
: [];
const DARWIN_CMAKE_FLAGS =
process.platform === 'darwin' // The minimum darwin target version we want for
- ? toFlags({ DCMAKE_OSX_DEPLOYMENT_TARGET: '10.12' })
+ ? toCLIFlags({ DCMAKE_OSX_DEPLOYMENT_TARGET: '10.12' })
: [];
const cmakeProgram = process.platform === 'win32' ? 'cmake.exe' : 'cmake';
@@ -149,11 +125,10 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, option
});
}
-export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload }) {
- const downloadURL =
- ref === 'latest'
- ? 'https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz'
- : `https://mciuploads.s3.amazonaws.com/libmongocrypt/all/${ref}/libmongocrypt-all.tar.gz`;
+export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) {
+ const prebuild = getLibmongocryptPrebuildName();
+
+ const downloadURL = buildLibmongocryptDownloadUrl(ref, prebuild);
console.error('downloading libmongocrypt...', downloadURL);
const destination = resolveRoot(`_libmongocrypt-${ref}`);
@@ -161,23 +136,7 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload })
await fs.rm(destination, { recursive: true, force: true });
await fs.mkdir(destination);
- const platformMatrix = {
- ['darwin-arm64']: 'macos',
- ['darwin-x64']: 'macos',
- ['linux-ppc64']: 'rhel-71-ppc64el',
- ['linux-s390x']: 'rhel72-zseries-test',
- ['linux-arm64']: 'ubuntu1804-arm64',
- ['linux-x64']: 'rhel-70-64-bit',
- ['win32-x64']: 'windows-test'
- };
-
- const detectedPlatform = `${process.platform}-${process.arch}`;
- const prebuild = platformMatrix[detectedPlatform];
- if (prebuild == null) throw new Error(`Unsupported: ${detectedPlatform}`);
-
- console.error(`Platform: ${detectedPlatform} Prebuild: ${prebuild}`);
-
- const downloadDestination = `${prebuild}/nocrypto`;
+ const downloadDestination = `nocrypto`;
const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, downloadDestination];
console.error(`+ tar ${unzipArgs.join(' ')}`);
const unzip = child_process.spawn('tar', unzipArgs, {
@@ -190,35 +149,8 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload })
const start = performance.now();
- let signal;
- if (fastDownload) {
- /**
- * Tar will print out each file it finds inside MEMBER (ex. macos/nocrypto)
- * For each file it prints, we give it a deadline of 3 seconds to print the next one.
- * If nothing prints after 3 seconds we exit early.
- * This depends on the tar file being in order and un-tar-able in under 3sec.
- */
- const controller = new AbortController();
- signal = controller.signal;
- let firstMemberSeen = true;
- let timeout;
- unzip.stderr.on('data', chunk => {
- process.stderr.write(chunk, () => {
- if (firstMemberSeen) {
- firstMemberSeen = false;
- timeout = setTimeout(() => {
- clearTimeout(timeout);
- unzip.stderr.removeAllListeners('data');
- controller.abort();
- }, 3_000);
- }
- timeout?.refresh();
- });
- });
- }
-
try {
- await stream.pipeline(response, unzip.stdin, { signal });
+ await stream.pipeline(response, unzip.stdin);
} catch {
await fs.access(path.join(`_libmongocrypt-${ref}`, downloadDestination));
}
@@ -228,7 +160,7 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload })
console.error(`downloaded libmongocrypt in ${(end - start) / 1000} secs...`);
await fs.rm(nodeDepsRoot, { recursive: true, force: true });
- await fs.cp(resolveRoot(destination, prebuild, 'nocrypto'), nodeDepsRoot, { recursive: true });
+ await fs.cp(resolveRoot(destination, 'nocrypto'), nodeDepsRoot, { recursive: true });
const potentialLib64Path = path.join(nodeDepsRoot, 'lib64');
try {
await fs.rename(potentialLib64Path, path.join(nodeDepsRoot, 'lib'));
diff --git a/.github/scripts/utils.mjs b/.github/scripts/utils.mjs
new file mode 100644
index 0000000..1a54385
--- /dev/null
+++ b/.github/scripts/utils.mjs
@@ -0,0 +1,89 @@
+// @ts-check
+
+import { execSync } from "child_process";
+import path from "path";
+import url from 'node:url';
+import { spawn } from "node:child_process";
+import { once } from "node:events";
+
+const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
+
+/** Resolves to the root of this repository */
+export function resolveRoot(...paths) {
+ return path.resolve(__dirname, '..', '..', ...paths);
+}
+
+export function getCommitFromRef(ref) {
+ console.error(`resolving ref: ${ref}`);
+ const script = resolveRoot('.github', 'scripts', 'get-commit-from-ref.sh');
+ const output = execSync(`bash ${script}`, { env: { REF: ref }, encoding: 'utf-8' })
+
+ const regex = /COMMIT_HASH=(?<hash>[a-zA-Z0-9]+)/
+ const result = regex.exec(output);
+
+ if (!result?.groups) throw new Error('unable to parse ref.')
+
+ const { hash } = result.groups;
+
+ console.error(`resolved to: ${hash}`);
+ return hash;
+}
+
+export function buildLibmongocryptDownloadUrl(ref, platform) {
+ const hash = getCommitFromRef(ref);
+
+ // sort of a hack - if we have an official release version, it'll be in the form `major.minor`. otherwise,
+ // we'd expect a commit hash or `master`.
+ if (ref.includes('.')) {
+ const [major, minor, _patch] = ref.split('.');
+
+ // Just a note: it may appear that this logic _doesn't_ support patch releases but it actually does.
+ // libmongocrypt's creates release branches for minor releases in the form `r<major>.<minor>`.
+ // Any patches made to this branch are committed as tags in the form <major>.<minor>.<patch>.
+ // So, the branch that is used for the AWS s3 upload is `r<major>.<minor>` and the commit hash
+ // is the commit hash we parse from the `getCommitFromRef()` (which handles switching to git tags and
+ // getting the commit hash at that tag just fine).
+ const branch = `r${major}.${minor}`
+
+ return `https://mciuploads.s3.amazonaws.com/libmongocrypt-release/${platform}/${branch}/${hash}/libmongocrypt.tar.gz`;
+ }
+
+ // just a note here - `master` refers to the branch, the hash is the commit on that branch.
+ // if we ever need to download binaries from a non-master branch (or non-release branch),
+ // this will need to be modified somehow.
+ return `https://mciuploads.s3.amazonaws.com/libmongocrypt/${platform}/master/${hash}/libmongocrypt.tar.gz`;
+}
+
+export function getLibmongocryptPrebuildName() {
+ const platformMatrix = {
+ ['darwin-arm64']: 'macos',
+ ['darwin-x64']: 'macos',
+ ['linux-ppc64']: 'rhel-71-ppc64el',
+ ['linux-s390x']: 'rhel72-zseries-test',
+ ['linux-arm64']: 'ubuntu1804-arm64',
+ ['linux-x64']: 'rhel-70-64-bit',
+ ['win32-x64']: 'windows-test'
+ };
+
+ const detectedPlatform = `${process.platform}-${process.arch}`;
+ const prebuild = platformMatrix[detectedPlatform];
+
+ if (prebuild == null) throw new Error(`Unsupported: ${detectedPlatform}`);
+
+ return prebuild;
+}
+
+/** `xtrace` style command runner, uses spawn so that stdio is inherited */
+export async function run(command, args = [], options = {}) {
+ const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`;
+ console.error(commandDetails);
+ const proc = spawn(command, args, {
+ shell: process.platform === 'win32',
+ stdio: 'inherit',
+ cwd: resolveRoot('.'),
+ ...options
+ });
+ await once(proc, 'exit');
+
+ if (proc.exitCode != 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`);
+}
\ No newline at end of file