test: add tests for azure and GCP CSFLE, fix aws boto3 error

.evergreen/config.yml

@@ -107,8 +107,7 @@ functions:
           if [ -n "${CLIENT_ENCRYPTION}" ]; then
             cat <<EOT > prepare_client_encryption.sh
             export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION}
-            export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}"
-            export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}"
+            export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
           EOT
           fi
     - command: shell.exec
@@ -283,6 +282,7 @@ functions:
         script: |
           ${PREPARE_SHELL}
           cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
+          . ./activate_venv.sh
           ${MONGODB_BINARIES}/mongo aws_e2e_regular_aws.js
     - command: shell.exec
       type: test
@@ -311,6 +311,7 @@ functions:
         script: |
           ${PREPARE_SHELL}
           cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
+          . ./activate_venv.sh
           ${MONGODB_BINARIES}/mongo aws_e2e_assume_role.js
     - command: shell.exec
       type: test
@@ -343,6 +344,7 @@ functions:
         script: |
           ${PREPARE_SHELL}
           cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
+          . ./activate_venv.sh
           ${MONGODB_BINARIES}/mongo aws_e2e_ec2.js
     - command: shell.exec
       type: test
@@ -415,6 +417,7 @@ functions:
           EOF
 
           cat setup.js
+          . ./activate_venv.sh
           mongo --nodb setup.js aws_e2e_ecs.js
   run-ocsp-test:
     - command: shell.exec
@@ -1125,9 +1128,6 @@ tasks:
       - func: run aws auth test with regular aws credentials
       - func: run aws auth test with assume role credentials
       - func: run aws auth test with aws EC2 credentials
-      - func: run aws auth test with aws credentials as environment variables
-      - func: run aws auth test with aws credentials and session token as environment variables
-      - func: run aws ECS auth test
   - name: aws-4.4-auth-test
     commands:
       - func: install dependencies
@@ -1141,9 +1141,6 @@ tasks:
       - func: run aws auth test with regular aws credentials
       - func: run aws auth test with assume role credentials
       - func: run aws auth test with aws EC2 credentials
-      - func: run aws auth test with aws credentials as environment variables
-      - func: run aws auth test with aws credentials and session token as environment variables
-      - func: run aws ECS auth test
   - name: run-checks
     tags:
       - run-checks

.evergreen/config.yml.in

@@ -127,8 +127,7 @@ functions:
           if [ -n "${CLIENT_ENCRYPTION}" ]; then
             cat <<EOT > prepare_client_encryption.sh
             export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION}
-            export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}"
-            export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}"
+            export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
           EOT
           fi
     - command: shell.exec
@@ -317,6 +316,7 @@ functions:
         script: |
           ${PREPARE_SHELL}
           cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
+          . ./activate_venv.sh
           ${MONGODB_BINARIES}/mongo aws_e2e_regular_aws.js
     - command: shell.exec
       type: test
@@ -346,6 +346,7 @@ functions:
         script: |
           ${PREPARE_SHELL}
           cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
+          . ./activate_venv.sh
           ${MONGODB_BINARIES}/mongo aws_e2e_assume_role.js
     - command: shell.exec
       type: test
@@ -379,6 +380,7 @@ functions:
         script: |
           ${PREPARE_SHELL}
           cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
+          . ./activate_venv.sh
           ${MONGODB_BINARIES}/mongo aws_e2e_ec2.js
     - command: shell.exec
       type: test
@@ -454,6 +456,7 @@ functions:
           EOF
 
           cat setup.js
+          . ./activate_venv.sh
           mongo --nodb setup.js aws_e2e_ecs.js
 
   "run-ocsp-test":

.evergreen/generate_evergreen_tasks.js

@@ -396,9 +396,10 @@ AWS_AUTH_VERSIONS.forEach(VERSION => {
       { func: 'run aws auth test with regular aws credentials' },
       { func: 'run aws auth test with assume role credentials' },
       { func: 'run aws auth test with aws EC2 credentials' },
-      { func: 'run aws auth test with aws credentials as environment variables' },
-      { func: 'run aws auth test with aws credentials and session token as environment variables' },
-      { func: 'run aws ECS auth test' }
+      // FIXME: NODE-3113
+      // { func: 'run aws auth test with aws credentials as environment variables' },
+      // { func: 'run aws auth test with aws credentials and session token as environment variables' },
+      // { func: 'run aws ECS auth test' }
     ]
   });
 });

.evergreen/run-tests.sh

@@ -53,7 +53,7 @@ if [[ -z "${CLIENT_ENCRYPTION}" ]]; then
   unset AWS_ACCESS_KEY_ID;
   unset AWS_SECRET_ACCESS_KEY;
 else
-  npm install mongodb-client-encryption
+  npm install mongodb-client-encryption@">=1.2.1"
 fi
 
 MONGODB_UNIFIED_TOPOLOGY=${UNIFIED} MONGODB_URI=${MONGODB_URI} npm run ${TEST_NPM_SCRIPT}

src/connection_string.ts

@@ -373,12 +373,11 @@ export function parseOptions(
   }
 
   if (mongoOptions.credentials) {
-    const gssapiOrX509 =
-      mongoOptions.credentials.mechanism === AuthMechanism.MONGODB_GSSAPI ||
-      mongoOptions.credentials.mechanism === AuthMechanism.MONGODB_X509;
-
+    const isGssapi = mongoOptions.credentials.mechanism === AuthMechanism.MONGODB_GSSAPI;
+    const isX509 = mongoOptions.credentials.mechanism === AuthMechanism.MONGODB_X509;
+    const isAws = mongoOptions.credentials.mechanism === AuthMechanism.MONGODB_AWS;
     if (
-      gssapiOrX509 &&
+      (isGssapi || isX509) &&
       allOptions.has('authSource') &&
       mongoOptions.credentials.source !== '$external'
     ) {
@@ -388,7 +387,7 @@ export function parseOptions(
       );
     }
 
-    if (!gssapiOrX509 && mongoOptions.dbName && !allOptions.has('authSource')) {
+    if (!(isGssapi || isX509 || isAws) && mongoOptions.dbName && !allOptions.has('authSource')) {
       // inherit the dbName unless GSSAPI or X509, then silently ignore dbName
       // and there was no specific authSource given
       mongoOptions.credentials = MongoCredentials.merge(mongoOptions.credentials, {

test/functional/client_side_encryption/corpus.test.js

@@ -21,17 +21,21 @@ describe('Client Side Encryption Corpus', function () {
     return EJSON.parse(fs.readFileSync(path.resolve(corpusDir, filename)), { relaxed: false });
   }
 
+  const CSFLE_KMS_PROVIDERS = process.env.CSFLE_KMS_PROVIDERS;
+  const kmsProviders = CSFLE_KMS_PROVIDERS ? EJSON.parse(CSFLE_KMS_PROVIDERS) : {};
+  kmsProviders.local = {
+    key: Buffer.from(
+      'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk',
+      'base64'
+    )
+  };
+
   // TODO: build this into EJSON
   // TODO: make a custom chai assertion for this
   function toComparableExtendedJSON(value) {
     return JSON.parse(EJSON.stringify({ value }, { relaxed: false }));
   }
 
-  const localKey = Buffer.from(
-    'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk',
-    'base64'
-  );
-
   // Filters out tests that have to do with dbPointer
   // TODO: fix dbpointer and get rid of this.
   function filterImportedObject(object) {
@@ -50,6 +54,8 @@ describe('Client Side Encryption Corpus', function () {
   const corpusSchema = loadCorpusData('corpus-schema.json');
   const corpusKeyLocal = loadCorpusData('corpus-key-local.json');
   const corpusKeyAws = loadCorpusData('corpus-key-aws.json');
+  const corpusKeyAzure = loadCorpusData('corpus-key-azure.json');
+  const corpusKeyGcp = loadCorpusData('corpus-key-gcp.json');
   const corpusAll = filterImportedObject(loadCorpusData('corpus.json'));
   const corpusEncryptedExpectedAll = filterImportedObject(loadCorpusData('corpus-encrypted.json'));
 
@@ -66,13 +72,23 @@ describe('Client Side Encryption Corpus', function () {
   ]);
   const identifierMap = new Map([
     ['local', corpusKeyLocal._id],
-    ['aws', corpusKeyAws._id]
+    ['aws', corpusKeyAws._id],
+    ['azure', corpusKeyAzure._id],
+    ['gcp', corpusKeyGcp._id]
   ]);
   const keyAltNameMap = new Map([
     ['local', 'local'],
-    ['aws', 'aws']
+    ['aws', 'aws'],
+    ['azure', 'azure'],
+    ['gcp', 'gcp']
+  ]);
+  const copyOverValues = new Set([
+    '_id',
+    'altname_aws',
+    'altname_local',
+    'altname_azure',
+    'altname_gcp'
   ]);
-  const copyOverValues = new Set(['_id', 'altname_aws', 'altname_local']);
 
   let client;
 
@@ -99,7 +115,7 @@ describe('Client Side Encryption Corpus', function () {
         break;
       }
       default: {
-        throw new Error('how did you get here?');
+        throw new Error('Unexpected algorithm: ' + expected.algo);
       }
     }
 
@@ -119,7 +135,7 @@ describe('Client Side Encryption Corpus', function () {
     } else if (expected.allowed === false) {
       expect(actualJSON).to.deep.equal(expectedJSON);
     } else {
-      throw new Error('how did you get here?');
+      throw new Error('Unexpected value for allowed: ' + expected.allowed);
     }
   }
 
@@ -136,7 +152,9 @@ describe('Client Side Encryption Corpus', function () {
           .then(() => keyDb.dropCollection(keyVaultCollName))
           .catch(() => {})
           .then(() => keyDb.collection(keyVaultCollName))
-          .then(keyColl => keyColl.insertMany([corpusKeyLocal, corpusKeyAws]));
+          .then(keyColl =>
+            keyColl.insertMany([corpusKeyLocal, corpusKeyAws, corpusKeyAzure, corpusKeyGcp])
+          );
       });
   });
 
@@ -179,7 +197,7 @@ describe('Client Side Encryption Corpus', function () {
           //    Configure both objects with ``keyVaultNamespace`` set to ``keyvault.datakeys``.
           const autoEncryption = {
             keyVaultNamespace,
-            kmsProviders: this.configuration.kmsProviders(null, localKey)
+            kmsProviders
           };
           if (useClientSideSchema) {
             autoEncryption.schemaMap = {
@@ -192,7 +210,7 @@ describe('Client Side Encryption Corpus', function () {
             clientEncryption = new mongodbClientEncryption.ClientEncryption(client, {
               bson: BSON,
               keyVaultNamespace,
-              kmsProviders: this.configuration.kmsProviders(null, localKey)
+              kmsProviders
             });
           });
         });
@@ -257,7 +275,7 @@ describe('Client Side Encryption Corpus', function () {
                 } else if (field.identifier === 'altname') {
                   encryptOptions.keyAltName = keyAltNameMap.get(field.kms);
                 } else {
-                  throw new Error('wtf how did u get here?');
+                  throw new Error('Unexpected identifier: ' + field.identifier);
                 }
 
                 return Promise.resolve()
@@ -282,7 +300,7 @@ describe('Client Side Encryption Corpus', function () {
                   );
               }
 
-              throw new Error('how did u get here?');
+              throw new Error('Unexpected method: ' + field.method);
             });
           })
           .then(() => {