feat(NODE-6507): generate encryption configuration on mongoose connect

docs/field-level-encryption.md

@@ -151,3 +151,68 @@ To declare a field as encrypted, you must:
 2. Choose an encryption type for the schema and configure the schema for the encryption type
 
 Not all schematypes are supported for CSFLE and QE.  For an overview of valid schema types, refer to MongoDB's documentation.
+
+### Registering Models
+
+Encrypted schemas must be registered on a connection, not the Mongoose global:
+
+```javascript
+
+const connection = mongoose.createConnection();
+const UserModel = connection.model('User', encryptedUserSchema);
+```
+
+### Connecting and configuring encryption options
+
+CSFLE/QE in Mongoose work by generating the encryption schema that the MongoDB driver expects for each encrypted model on the connection.  This happens automatically the model's connection is established.
+
+Queryable encryption and CSFLE requires all the same configuration as outlined in <>, except for the schemaMap or encryptedFieldsMap options.
+
+```javascript
+const keyVaultNamespace = 'client.encryption';
+const kmsProviders = { local: { key } };
+await connection.openUri(`mongodb://localhost:27017`, {
+  // Configure auto encryption
+  autoEncryption: {
+    keyVaultNamespace: 'datakeys.datakeys',
+    kmsProviders
+  }
+});
+```
+
+Once the connection is established, Mongoose's operations will work as usual.  Writes are encrypted automatically by the MongoDB driver prior to sending them to the server and reads are decrypted by the driver after fetching documents from the server.
+
+### Discriminators
+
+Discriminators are supported for encrypted models as well:
+
+```javascript
+const connection = createConnection();
+
+const schema = new Schema({
+  name: {
+    type: String, encrypt: { keyId }
+  }
+}, {
+  encryptionType: 'queryableEncryption'
+});
+
+const Model = connection.model('BaseUserModel', schema);
+const ModelWithAge = model.discriminator('ModelWithAge', new Schema({
+  age: {
+    type: Int32, encrypt: { keyId: keyId2 }
+  }
+}, {
+  encryptionType: 'queryableEncryption'
+}));
+
+const ModelWithBirthday = model.discriminator('ModelWithBirthday', new Schema({
+  dob: {
+    type: Int32, encrypt: { keyId: keyId3 }
+  }
+}, {
+  encryptionType: 'queryableEncryption'
+}));
+```
+
+When generating encryption schemas, Mongoose merges all discriminators together for the all discriminators declared on the same namespace.  As a result, discriminators that declare the same key with different types are not supported.  Furthermore, all discriminators must share the same encryption type - it is not possible to configure discriminators on the same model for both CSFLE and QE.

lib/drivers/node-mongodb-native/connection.js

@@ -12,6 +12,7 @@ const pkg = require('../../../package.json');
 const processConnectionOptions = require('../../helpers/processConnectionOptions');
 const setTimeout = require('../../helpers/timers').setTimeout;
 const utils = require('../../utils');
+const Schema = require('../../schema');
 
 /**
  * A [node-mongodb-native](https://github.com/mongodb/node-mongodb-native) connection implementation.
@@ -320,6 +321,20 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio
     };
   }
 
+  const { schemaMap, encryptedFieldsMap } = this._buildEncryptionSchemas();
+
+  if ((Object.keys(schemaMap).length > 0 || Object.keys(encryptedFieldsMap).length) && !options.autoEncryption) {
+    throw new Error('Must provide `autoEncryption` when connecting with encrypted schemas.');
+  }
+
+  if (Object.keys(schemaMap).length > 0) {
+    options.autoEncryption.schemaMap = schemaMap;
+  }
+
+  if (Object.keys(encryptedFieldsMap).length > 0) {
+    options.autoEncryption.encryptedFieldsMap = encryptedFieldsMap;
+  }
+
   this.readyState = STATES.connecting;
   this._connectionString = uri;
 
@@ -343,6 +358,56 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio
   return this;
 };
 
+/**
+ * Given a connection, which may or may not have encrypted models, build
+ * a schemaMap and/or an encryptedFieldsMap for the connection, combining all models
+ * into a single schemaMap and encryptedFields map.
+ *
+ * @returns the generated schemaMap and encryptedFieldsMap
+  */
+NativeConnection.prototype._buildEncryptionSchemas = function() {
+  const qeMappings = {};
+  const csfleMappings = {};
+
+  const encryptedModels = Object.values(this.models).filter(model => model.schema._hasEncryptedFields());
+
+  // If discriminators are configured for the collection, there might be multiple models
+  // pointing to the same namespace.  For this scenario, we merge all the schemas for each namespace
+  // into a single schema and then generate a schemaMap/encryptedFieldsMap for the combined schema.
+  for (const model of encryptedModels) {
+    const { schema, collection: { collectionName } } = model;
+    const namespace = `${this.$dbName}.${collectionName}`;
+    const mappings = schema.encryptionType() === 'csfle' ? csfleMappings : qeMappings;
+
+    mappings[namespace] ??= new Schema({}, { encryptionType: schema.encryptionType() });
+
+    const isNonRootDiscriminator = schema.discriminatorMapping && !schema.discriminatorMapping.isRoot;
+    if (isNonRootDiscriminator) {
+      const rootSchema = schema._baseSchema;
+      schema.eachPath((pathname) => {
+        if (rootSchema.path(pathname)) return;
+        if (!mappings[namespace]._hasEncryptedField(pathname)) return;
+
+        throw new Error(`Cannot have duplicate keys in discriminators with encryption. key=${pathname}`);
+      });
+    }
+
+    mappings[namespace].add(schema);
+  }
+
+  const schemaMap = Object.fromEntries(Object.entries(csfleMappings).map(
+    ([namespace, schema]) => ([namespace, schema._buildSchemaMap()])
+  ));
+
+  const encryptedFieldsMap = Object.fromEntries(Object.entries(qeMappings).map(
+    ([namespace, schema]) => ([namespace, schema._buildEncryptedFields()])
+  ));
+
+  return {
+    schemaMap, encryptedFieldsMap
+  };
+};
+
 /*!
  * ignore
  */

lib/helpers/model/discriminator.js

@@ -95,6 +95,13 @@ module.exports = function discriminator(model, name, schema, tiedValue, applyPlu
     const baseSchemaPaths = Object.keys(baseSchema.paths);
     const conflictingPaths = [];
 
+
+    baseSchema.eachPath((pathname) => {
+      if (schema._hasEncryptedField(pathname)) {
+        throw new Error(`cannot declare an encrypted field on child schema overriding base schema. key=${pathname}`);
+      }
+    });
+
     for (const path of baseSchemaPaths) {
       if (schema.nested[path]) {
         conflictingPaths.push(path);

lib/schema.js

@@ -721,7 +721,6 @@ Schema.prototype.encryptionType = function encryptionType(encryptionType) {
 Schema.prototype.add = function add(obj, prefix) {
   if (obj instanceof Schema || (obj != null && obj.instanceOfSchema)) {
     merge(this, obj);
-
     return this;
   }
 
@@ -914,6 +913,77 @@ Schema.prototype._hasEncryptedFields = function _hasEncryptedFields() {
   return Object.keys(this.encryptedFields).length > 0;
 };
 
+/**
+ * @api private
+ */
+Schema.prototype._hasEncryptedField = function _hasEncryptedField(path) {
+  return path in this.encryptedFields;
+};
+
+
+/**
+ * Builds an encryptedFieldsMap for the schema.
+ */
+Schema.prototype._buildEncryptedFields = function() {
+  const fields = Object.entries(this.encryptedFields).map(
+    ([path, config]) => {
+      const bsonType = this.path(path).autoEncryptionType();
+      // { path, bsonType, keyId, queries? }
+      return { path, bsonType, ...config };
+    });
+
+  return { fields };
+};
+
+/**
+ * Builds a schemaMap for the schema, if the schema is configured for client-side field level encryption.
+ */
+Schema.prototype._buildSchemaMap = function() {
+  /**
+   * `schemaMap`s are JSON schemas, which use the following structure to represent objects:
+   *    { field: { bsonType: 'object', properties: { ... } } }
+   *
+   * for example, a schema that looks like this `{ a: { b: int32 } }` would be encoded as
+   * `{ a: { bsonType: 'object', properties: { b: < encryption configuration > } } }`
+   *
+   * This function takes an array of path segments, an output object (that gets mutated) and
+   * a value to associated with the full path, and constructs a valid CSFLE JSON schema path for
+   * the object.  This works for deeply nested properties as well.
+   *
+   * @param {string[]} path array of path components
+   * @param {object} object the object in which to build a JSON schema of `path`'s properties
+   * @param {object} value the value to associate with the path in object
+   */
+  function buildNestedPath(path, object, value) {
+    let i = 0, component = path[i];
+    for (; i < path.length - 1; ++i, component = path[i]) {
+      object[component] = object[component] == null ? {
+        bsonType: 'object',
+        properties: {}
+      } : object[component];
+      object = object[component].properties;
+    }
+    object[component] = value;
+  }
+
+  const schemaMapPropertyReducer = (accum, [path, propertyConfig]) => {
+    const bsonType = this.path(path).autoEncryptionType();
+    const pathComponents = path.split('.');
+    const configuration = { encrypt: { ...propertyConfig, bsonType } };
+    buildNestedPath(pathComponents, accum, configuration);
+    return accum;
+  };
+
+  const properties = Object.entries(this.encryptedFields).reduce(
+    schemaMapPropertyReducer,
+    {});
+
+  return {
+    bsonType: 'object',
+    properties
+  };
+};
+
 /**
  * Add an alias for `path`. This means getting or setting the `alias`
  * is equivalent to getting or setting the `path`.

lib/schema/bigint.js

@@ -255,7 +255,7 @@ SchemaBigInt.prototype.toJSONSchema = function toJSONSchema(options) {
 };
 
 SchemaBigInt.prototype.autoEncryptionType = function autoEncryptionType() {
-  return 'int64';
+  return 'long';
 };
 
 /*!

lib/schema/boolean.js

@@ -305,7 +305,7 @@ SchemaBoolean.prototype.toJSONSchema = function toJSONSchema(options) {
 };
 
 SchemaBoolean.prototype.autoEncryptionType = function autoEncryptionType() {
-  return 'boolean';
+  return 'bool';
 };
 
 /*!

lib/schema/buffer.js

@@ -315,7 +315,7 @@ SchemaBuffer.prototype.toJSONSchema = function toJSONSchema(options) {
 };
 
 SchemaBuffer.prototype.autoEncryptionType = function autoEncryptionType() {
-  return 'binary';
+  return 'binData';
 };
 
 /*!

lib/schema/decimal128.js

@@ -236,7 +236,7 @@ SchemaDecimal128.prototype.toJSONSchema = function toJSONSchema(options) {
 };
 
 SchemaDecimal128.prototype.autoEncryptionType = function autoEncryptionType() {
-  return 'decimal128';
+  return 'decimal';
 };
 
 /*!

lib/schema/int32.js

@@ -261,7 +261,7 @@ SchemaInt32.prototype.toJSONSchema = function toJSONSchema(options) {
 };
 
 SchemaInt32.prototype.autoEncryptionType = function autoEncryptionType() {
-  return 'int32';
+  return 'int';
 };
 
 

lib/schema/map.js

@@ -95,6 +95,10 @@ class SchemaMap extends SchemaType {
 
     return result;
   }
+
+  autoEncryptionType() {
+    return 'object';
+  }
 }
 
 /**

lib/schema/objectId.js

@@ -305,7 +305,7 @@ SchemaObjectId.prototype.toJSONSchema = function toJSONSchema(options) {
 };
 
 SchemaObjectId.prototype.autoEncryptionType = function autoEncryptionType() {
-  return 'objectid';
+  return 'objectId';
 };
 
 /*!

scripts/configure-cluster-with-encryption.sh

@@ -4,11 +4,11 @@
 
 # this script downloads all tools required to use FLE with mongodb, then starts a cluster of the provided configuration (sharded on 8.0 server)
 
-export CWD=$(pwd);
-export DRIVERS_TOOLS_PINNED_COMMIT=35d0592c76f4f3d25a5607895eb21b491dd52543;
+export CWD=$(pwd)
+export DRIVERS_TOOLS_PINNED_COMMIT=4e18803c074231ec9fc3ace8f966e2c49d9874bb
 
 # install extra dependency
-npm install mongodb-client-encryption
+npm install --no-save mongodb-client-encryption
 
 # set up mongodb cluster and encryption configuration if the data/ folder does not exist
 if [ ! -d "data" ]; then
@@ -33,11 +33,9 @@ if [ ! -d "data" ]; then
 
   # configure cluster settings
   export DRIVERS_TOOLS=$CWD/data/drivers-evergreen-tools
-  export MONGODB_VERSION=8.0
-  export AUTH=true
+  export AUTH=auth
   export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin
-  export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo
-  export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration
+  export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration
   export TOPOLOGY=sharded_cluster
   export SSL=nossl
 
@@ -46,12 +44,12 @@ if [ ! -d "data" ]; then
   mkdir mo
   cd -
 
-  rm expansions.sh 2> /dev/null
+  rm expansions.sh 2>/dev/null
 
   echo 'Configuring Cluster...'
 
   # start cluster
-  (bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh) 1> /dev/null 2> /dev/null
+  (bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh) 1>/dev/null 2>/dev/null
 
   echo 'Cluster Configuration Finished!'