From 63d453f67baabb70ecd90fffbcc1cd668ea9ca04 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 17 Feb 2025 00:02:00 +0000 Subject: [PATCH 01/15] add test files --- .../etc/data/lookup/key-doc.json | 30 +++++++++++++++++++ .../etc/data/lookup/schema-csfle.json | 19 ++++++++++++ .../etc/data/lookup/schema-csfle2.json | 19 ++++++++++++ .../etc/data/lookup/schema-qe.json | 20 +++++++++++++ .../etc/data/lookup/schema-qe2.json | 20 +++++++++++++ 5 files changed, 108 insertions(+) create mode 100644 source/client-side-encryption/etc/data/lookup/key-doc.json create mode 100644 source/client-side-encryption/etc/data/lookup/schema-csfle.json create mode 100644 source/client-side-encryption/etc/data/lookup/schema-csfle2.json create mode 100644 source/client-side-encryption/etc/data/lookup/schema-qe.json create mode 100644 source/client-side-encryption/etc/data/lookup/schema-qe2.json diff --git a/source/client-side-encryption/etc/data/lookup/key-doc.json b/source/client-side-encryption/etc/data/lookup/key-doc.json new file mode 100644 index 0000000000..566b56c354 --- /dev/null +++ b/source/client-side-encryption/etc/data/lookup/key-doc.json @@ -0,0 +1,30 @@ +{ + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } +} diff --git a/source/client-side-encryption/etc/data/lookup/schema-csfle.json b/source/client-side-encryption/etc/data/lookup/schema-csfle.json new file mode 100644 index 0000000000..29ac9ad5da --- /dev/null +++ b/source/client-side-encryption/etc/data/lookup/schema-csfle.json @@ -0,0 +1,19 @@ +{ + "properties": { + "csfle": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" +} diff --git a/source/client-side-encryption/etc/data/lookup/schema-csfle2.json b/source/client-side-encryption/etc/data/lookup/schema-csfle2.json new file mode 100644 index 0000000000..3f1c02781c --- /dev/null +++ b/source/client-side-encryption/etc/data/lookup/schema-csfle2.json @@ -0,0 +1,19 @@ +{ + "properties": { + "csfle2": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" +} diff --git a/source/client-side-encryption/etc/data/lookup/schema-qe.json b/source/client-side-encryption/etc/data/lookup/schema-qe.json new file mode 100644 index 0000000000..9428ea1b45 --- /dev/null +++ b/source/client-side-encryption/etc/data/lookup/schema-qe.json @@ -0,0 +1,20 @@ +{ + "escCollection": "enxcol_.qe.esc", + "ecocCollection": "enxcol_.qe.ecoc", + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "qe", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": 0 + } + } + ] +} diff --git a/source/client-side-encryption/etc/data/lookup/schema-qe2.json b/source/client-side-encryption/etc/data/lookup/schema-qe2.json new file mode 100644 index 0000000000..77d5bd37cb --- /dev/null +++ b/source/client-side-encryption/etc/data/lookup/schema-qe2.json @@ -0,0 +1,20 @@ +{ + "escCollection": "enxcol_.qe2.esc", + "ecocCollection": "enxcol_.qe2.ecoc", + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "qe2", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": 0 + } + } + ] +} From bf4971ddb254392ef94d964ad942cfaef492c6fa Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 17 Feb 2025 00:02:25 +0000 Subject: [PATCH 02/15] add prose test 25: Test $lookup --- source/client-side-encryption/tests/README.md | 255 ++++++++++++++++++ 1 file changed, 255 insertions(+) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 304ff52ee6..5f1367d5c4 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3413,3 +3413,258 @@ Repeat this test with the `azure` and `gcp` masterKeys. 2. Call `client_encryption.createDataKey()` with "aws" as the provider. Expect this to fail. Repeat this test with the `azure` and `gcp` masterKeys. + +### 25. Test $lookup + +This test requires libmongocrypt 1.13.0. Unless otherwise noted, tests require mongocryptd/crypt_shared 8.1+. + +The syntax `` is used to refer to files in +[source/client-side-encryption/etc/data/lookup](/source/client-side-encryption/etc/data/lookup/). + +#### Setup + +Create an unencrypted MongoClient. Drop database `db`. + +Insert `` into `db.keyvault`. + +Create the following collections: + +- `db.csfle` with options: `{ "validator": { "$jsonSchema": ""}}`. +- `db.csfle2` with options: `{ "validator": { "$jsonSchema": ""}}`. +- `db.qe` with options: `{ "encryptedFields": ""}`. +- `db.qe2` with options: `{ "encryptedFields": ""}`. +- `db.no_schema` with no options. +- `db.no_schema2` with no options. + +Create an encrypted MongoClient configured with: + +```python +AutoEncryptionOpts( + keyVaultNamespace="db.keyvault", + kmsProviders={"local": { "key": "" }} +) +``` + +Insert documents with the encrypted MongoClient: + +- `{"csfle": "csfle"}` into `db.csfle` + - Use the unencrypted client to retrieve it. Assert the `csfle` field is BSON binary. +- `{"csfle2": "csfle2"}` into `db.csfle2` + - Use the unencrypted client to retrieve it. Assert the `csfle2` field is BSON binary. +- `{"qe": "qe"}` into `db.qe` + - Use the unencrypted client to retrieve it. Assert the `qe` field is BSON binary. +- `{"qe2": "qe2"}` into `db.qe2` + - Use the unencrypted client to retrieve it. Assert the `qe2` field is BSON binary. +- `{"no_schema": "no_schema"}` into `db.no_schema` +- `{"no_schema2": "no_schema2"}` into `db.no_schema2` + +#### Case 1: `db.csfle` joins `db.no_schema` + +Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents +schema caching from impacting the test). + +Run an aggregate operation on `db.csfle` with the following pipeline: + +```json +[ + {"$match" : {"csfle" : "csfle"}}, + { + "$lookup" : { + "from" : "no_schema", + "as" : "matched", + "pipeline" : [ {"$match" : {"no_schema" : "no_schema"}}, {"$project" : {"_id" : 0}} ] + } + }, + {"$project" : {"_id" : 0}} +] +``` + +Expect one document to be returned matching: `{"csfle" : "csfle", "matched" : [ {"no_schema" : "no_schema"} ]}`. + +#### Case 2: `db.qe` joins `db.no_schema` + +Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents +schema caching from impacting the test). + +Run an aggregate operation on `db.qe` with the following pipeline: + +```json +[ + {"$match" : {"qe" : "qe"}}, + { + "$lookup" : { + "from" : "no_schema", + "as" : "matched", + "pipeline" : + [ {"$match" : {"no_schema" : "no_schema"}}, {"$project" : {"_id" : 0, "__safeContent__" : 0}} ] + } + }, + {"$project" : {"_id" : 0, "__safeContent__" : 0}} +] +``` + +Expect one document to be returned matching: `{"qe" : "qe", "matched" : [ {"no_schema" : "no_schema"} ]}`. + +#### Case 3: `db.no_schema` joins `db.csfle` + +Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents +schema caching from impacting the test). + +Run an aggregate operation on `db.no_schema` with the following pipeline: + +```json +[ + {"$match" : {"no_schema" : "no_schema"}}, + { + "$lookup" : { + "from" : "csfle", + "as" : "matched", + "pipeline" : [ {"$match" : {"csfle" : "csfle"}}, {"$project" : {"_id" : 0}} ] + } + }, + {"$project" : {"_id" : 0}} +] +``` + +Expect one document to be returned matching: `{"no_schema" : "no_schema", "matched" : [ {"csfle" : "csfle"} ]}`. + +#### Case 4: `db.no_schema` joins `db.qe` + +Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents +schema caching from impacting the test). + +Run an aggregate operation on `db.no_schema` with the following pipeline: + +```json +[ + {"$match" : {"no_schema" : "no_schema"}}, + { + "$lookup" : { + "from" : "qe", + "as" : "matched", + "pipeline" : [ {"$match" : {"qe" : "qe"}}, {"$project" : {"_id" : 0, "__safeContent__" : 0}} ] + } + }, + {"$project" : {"_id" : 0}} +] +``` + +Expect one document to be returned matching: `{"no_schema" : "no_schema", "matched" : [ {"qe" : "qe"} ]}`. + +#### Case 5: `db.csfle` joins `db.csfle2` + +Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents +schema caching from impacting the test). + +Run an aggregate operation on `db.csfle` with the following pipeline: + +```json +[ + {"$match" : {"csfle" : "csfle"}}, + { + "$lookup" : { + "from" : "csfle2", + "as" : "matched", + "pipeline" : [ {"$match" : {"csfle2" : "csfle2"}}, {"$project" : {"_id" : 0}} ] + } + }, + {"$project" : {"_id" : 0}} +] +``` + +Expect one document to be returned matching: `{"csfle" : "csfle", "matched" : [ {"csfle2" : "csfle2"} ]}`. + +#### Case 6: `db.qe` joins `db.qe2` + +Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents +schema caching from impacting the test). + +Run an aggregate operation on `db.qe` with the following pipeline: + +```json +[ + {"$match" : {"qe" : "qe"}}, + { + "$lookup" : { + "from" : "qe2", + "as" : "matched", + "pipeline" : [ {"$match" : {"qe2" : "qe2"}}, {"$project" : {"_id" : 0, "__safeContent__" : 0}} ] + } + }, + {"$project" : {"_id" : 0, "__safeContent__" : 0}} +] +``` + +Expect one document to be returned matching: `{"qe" : "qe", "matched" : [ {"qe2" : "qe2"} ]}`. + +#### Case 7: `db.no_schema` joins `db.no_schema2` + +Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents +schema caching from impacting the test). + +Run an aggregate operation on `db.no_schema` with the following pipeline: + +```json +[ + {"$match" : {"no_schema" : "no_schema"}}, + { + "$lookup" : { + "from" : "no_schema2", + "as" : "matched", + "pipeline" : [ {"$match" : {"no_schema2" : "no_schema2"}}, {"$project" : {"_id" : 0}} ] + } + }, + {"$project" : {"_id" : 0}} +] +``` + +Expect one document to be returned matching: +`{"no_schema" : "no_schema", "matched" : [ {"no_schema2" : "no_schema2"} ]}`. + +#### Case 8: `db.csfle` joins `db.qe` + +Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents +schema caching from impacting the test). + +Run an aggregate operation on `db.csfle` with the following pipeline: + +```json +[ + {"$match" : {"csfle" : "qe"}}, + { + "$lookup" : { + "from" : "qe", + "as" : "matched", + "pipeline" : [ {"$match" : {"qe" : "qe"}}, {"$project" : {"_id" : 0}} ] + } + }, + {"$project" : {"_id" : 0}} +] +``` + +Expect an exception to be thrown with a message containing the substring `not supported`. + +#### Case 9: test error with \<8.1 + +This case requires mongocryptd/crypt_shared \<8.1. + +Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents +schema caching from impacting the test). + +Run an aggregate operation on `db.csfle` with the following pipeline: + +```json +[ + {"$match" : {"csfle" : "csfle"}}, + { + "$lookup" : { + "from" : "no_schema", + "as" : "matched", + "pipeline" : [ {"$match" : {"no_schema" : "no_schema"}}, {"$project" : {"_id" : 0}} ] + } + }, + {"$project" : {"_id" : 0}} +] +``` + +Expect an exception to be thrown with a message containing the substring `Upgrade`. From b805cd891680013b1d530e7d13512402ad188caf Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Thu, 20 Feb 2025 13:44:33 +0000 Subject: [PATCH 03/15] use relative link --- source/client-side-encryption/tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 5f1367d5c4..6fe66a9a8b 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3419,7 +3419,7 @@ Repeat this test with the `azure` and `gcp` masterKeys. This test requires libmongocrypt 1.13.0. Unless otherwise noted, tests require mongocryptd/crypt_shared 8.1+. The syntax `` is used to refer to files in -[source/client-side-encryption/etc/data/lookup](/source/client-side-encryption/etc/data/lookup/). +[source/client-side-encryption/etc/data/lookup](../etc/data/lookup/). #### Setup From 3558b582e7994fe800f1a10e5c1fa32d699bcc30 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Thu, 20 Feb 2025 13:47:36 +0000 Subject: [PATCH 04/15] note server version requirement --- source/client-side-encryption/tests/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 6fe66a9a8b..db4ea29e52 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3418,6 +3418,8 @@ Repeat this test with the `azure` and `gcp` masterKeys. This test requires libmongocrypt 1.13.0. Unless otherwise noted, tests require mongocryptd/crypt_shared 8.1+. +Tests require server support of QE: Require MongoDB server 7.0+. Skip on standalone. + The syntax `` is used to refer to files in [source/client-side-encryption/etc/data/lookup](../etc/data/lookup/). From 95a910c2b91ddedf3a554a9228b5669910c111ee Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Thu, 20 Feb 2025 14:01:33 +0000 Subject: [PATCH 05/15] remove relative link Appears to error in `mkdocs`. Link is not very useful regardless. --- source/client-side-encryption/tests/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index db4ea29e52..f24877c5a7 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3420,8 +3420,7 @@ This test requires libmongocrypt 1.13.0. Unless otherwise noted, tests require m Tests require server support of QE: Require MongoDB server 7.0+. Skip on standalone. -The syntax `` is used to refer to files in -[source/client-side-encryption/etc/data/lookup](../etc/data/lookup/). +The syntax `` is used to refer to files in `../etc/data/lookup`. #### Setup From c68a5bc2fd467f5b64981f3b4176c9df129e0e56 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 24 Feb 2025 14:48:57 -0500 Subject: [PATCH 06/15] use plural "these tests require" Co-authored-by: Maxim Katcharov --- source/client-side-encryption/tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index f24877c5a7..a9fa5ffd58 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3416,7 +3416,7 @@ Repeat this test with the `azure` and `gcp` masterKeys. ### 25. Test $lookup -This test requires libmongocrypt 1.13.0. Unless otherwise noted, tests require mongocryptd/crypt_shared 8.1+. +Unless otherwise noted, these tests require: libmongocrypt 1.13.0, mongocryptd/crypt_shared 8.1+. Tests require server support of QE: Require MongoDB server 7.0+. Skip on standalone. From c127c75fe2a633b9d29ab154a955b829925556b0 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 24 Feb 2025 14:49:23 -0500 Subject: [PATCH 07/15] clarify `` refers to file contents Co-authored-by: Maxim Katcharov --- source/client-side-encryption/tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index a9fa5ffd58..2dd7ff9ea5 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3420,7 +3420,7 @@ Unless otherwise noted, these tests require: libmongocrypt 1.13.0, mongocryptd/c Tests require server support of QE: Require MongoDB server 7.0+. Skip on standalone. -The syntax `` is used to refer to files in `../etc/data/lookup`. +The syntax `` is used to refer to the content of the corresponding file in `../etc/data/lookup`. #### Setup From 861cf7a0864f9ac6936195fc989608bb5161b7bf Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 24 Feb 2025 19:53:15 +0000 Subject: [PATCH 08/15] use majority write concern to insert key document --- source/client-side-encryption/tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 2dd7ff9ea5..d9c128279b 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3426,7 +3426,7 @@ The syntax `` is used to refer to the content of the correspondin Create an unencrypted MongoClient. Drop database `db`. -Insert `` into `db.keyvault`. +Insert `` into `db.keyvault` with majority write concern. Create the following collections: From a6b8c7d8262bbdcc86b89afcaf72060653b947a6 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 24 Feb 2025 20:00:57 +0000 Subject: [PATCH 09/15] name clients --- source/client-side-encryption/tests/README.md | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index d9c128279b..3b52094401 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3424,7 +3424,7 @@ The syntax `` is used to refer to the content of the correspondin #### Setup -Create an unencrypted MongoClient. Drop database `db`. +Create an unencrypted MongoClient named `unencryptedClient`. Drop database `db`. Insert `` into `db.keyvault` with majority write concern. @@ -3437,7 +3437,7 @@ Create the following collections: - `db.no_schema` with no options. - `db.no_schema2` with no options. -Create an encrypted MongoClient configured with: +Create an encrypted MongoClient named `encryptedClient` configured with: ```python AutoEncryptionOpts( @@ -3446,23 +3446,23 @@ AutoEncryptionOpts( ) ``` -Insert documents with the encrypted MongoClient: +Insert documents with `encryptedClient`: - `{"csfle": "csfle"}` into `db.csfle` - - Use the unencrypted client to retrieve it. Assert the `csfle` field is BSON binary. + - Use `unencryptedClient` to retrieve it. Assert the `csfle` field is BSON binary. - `{"csfle2": "csfle2"}` into `db.csfle2` - - Use the unencrypted client to retrieve it. Assert the `csfle2` field is BSON binary. + - Use `unencryptedClient` to retrieve it. Assert the `csfle2` field is BSON binary. - `{"qe": "qe"}` into `db.qe` - - Use the unencrypted client to retrieve it. Assert the `qe` field is BSON binary. + - Use `unencryptedClient` to retrieve it. Assert the `qe` field is BSON binary. - `{"qe2": "qe2"}` into `db.qe2` - - Use the unencrypted client to retrieve it. Assert the `qe2` field is BSON binary. + - Use `unencryptedClient` to retrieve it. Assert the `qe2` field is BSON binary. - `{"no_schema": "no_schema"}` into `db.no_schema` - `{"no_schema2": "no_schema2"}` into `db.no_schema2` #### Case 1: `db.csfle` joins `db.no_schema` -Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents -schema caching from impacting the test). +Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from +impacting the test). Run an aggregate operation on `db.csfle` with the following pipeline: @@ -3484,8 +3484,8 @@ Expect one document to be returned matching: `{"csfle" : "csfle", "matched" : [ #### Case 2: `db.qe` joins `db.no_schema` -Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents -schema caching from impacting the test). +Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from +impacting the test). Run an aggregate operation on `db.qe` with the following pipeline: @@ -3508,8 +3508,8 @@ Expect one document to be returned matching: `{"qe" : "qe", "matched" : [ {"no_s #### Case 3: `db.no_schema` joins `db.csfle` -Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents -schema caching from impacting the test). +Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from +impacting the test). Run an aggregate operation on `db.no_schema` with the following pipeline: @@ -3531,8 +3531,8 @@ Expect one document to be returned matching: `{"no_schema" : "no_schema", "match #### Case 4: `db.no_schema` joins `db.qe` -Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents -schema caching from impacting the test). +Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from +impacting the test). Run an aggregate operation on `db.no_schema` with the following pipeline: @@ -3554,8 +3554,8 @@ Expect one document to be returned matching: `{"no_schema" : "no_schema", "match #### Case 5: `db.csfle` joins `db.csfle2` -Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents -schema caching from impacting the test). +Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from +impacting the test). Run an aggregate operation on `db.csfle` with the following pipeline: @@ -3577,8 +3577,8 @@ Expect one document to be returned matching: `{"csfle" : "csfle", "matched" : [ #### Case 6: `db.qe` joins `db.qe2` -Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents -schema caching from impacting the test). +Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from +impacting the test). Run an aggregate operation on `db.qe` with the following pipeline: @@ -3600,8 +3600,8 @@ Expect one document to be returned matching: `{"qe" : "qe", "matched" : [ {"qe2" #### Case 7: `db.no_schema` joins `db.no_schema2` -Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents -schema caching from impacting the test). +Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from +impacting the test). Run an aggregate operation on `db.no_schema` with the following pipeline: @@ -3624,8 +3624,8 @@ Expect one document to be returned matching: #### Case 8: `db.csfle` joins `db.qe` -Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents -schema caching from impacting the test). +Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from +impacting the test). Run an aggregate operation on `db.csfle` with the following pipeline: @@ -3649,8 +3649,8 @@ Expect an exception to be thrown with a message containing the substring `not su This case requires mongocryptd/crypt_shared \<8.1. -Create a new encrypted MongoClient with the same `AutoEncryptionOpts` as the setup. (Creating a new client prevents -schema caching from impacting the test). +Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from +impacting the test). Run an aggregate operation on `db.csfle` with the following pipeline: From dc9f3151fb0787ffa73e8dc222cd928fe1fe8c6f Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Mon, 24 Feb 2025 20:13:04 +0000 Subject: [PATCH 10/15] drop and create collections with `encryptedClient` Use `encryptedClient` to also drop state collections created for QE. --- source/client-side-encryption/tests/README.md | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 3b52094401..4ce5ac41e3 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3424,11 +3424,18 @@ The syntax `` is used to refer to the content of the correspondin #### Setup -Create an unencrypted MongoClient named `unencryptedClient`. Drop database `db`. +Create an encrypted MongoClient named `encryptedClient` configured with: + +```python +AutoEncryptionOpts( + keyVaultNamespace="db.keyvault", + kmsProviders={"local": { "key": "" }} +) +``` -Insert `` into `db.keyvault` with majority write concern. +Use `encryptedClient` to drop `db.keyvault`. Insert `` into `db.keyvault` with majority write concern. -Create the following collections: +Use `encryptedClient` to drop and create the following collections: - `db.csfle` with options: `{ "validator": { "$jsonSchema": ""}}`. - `db.csfle2` with options: `{ "validator": { "$jsonSchema": ""}}`. @@ -3437,14 +3444,7 @@ Create the following collections: - `db.no_schema` with no options. - `db.no_schema2` with no options. -Create an encrypted MongoClient named `encryptedClient` configured with: - -```python -AutoEncryptionOpts( - keyVaultNamespace="db.keyvault", - kmsProviders={"local": { "key": "" }} -) -``` +Create an unencrypted MongoClient named `unencryptedClient`. Insert documents with `encryptedClient`: From 240ce7232f95725aa82f120d2afe674013c3d561 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Tue, 25 Feb 2025 14:48:42 +0000 Subject: [PATCH 11/15] also require server 8.1+ Testing with crypt_shared/mongocryptd 8.1+ and pre-8.1 server results in not matching QE fields in sub-pipelines. This is a known server limitation. --- source/client-side-encryption/tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 4ce5ac41e3..03a5e8aa8c 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3416,7 +3416,7 @@ Repeat this test with the `azure` and `gcp` masterKeys. ### 25. Test $lookup -Unless otherwise noted, these tests require: libmongocrypt 1.13.0, mongocryptd/crypt_shared 8.1+. +Unless otherwise noted, these tests require: libmongocrypt 1.13.0, mongocryptd/crypt_shared 8.1+, and server 8.1+. Tests require server support of QE: Require MongoDB server 7.0+. Skip on standalone. From 5879061762a919fcf09d8c9ba6bef015be4df10f Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Tue, 25 Feb 2025 12:49:01 -0500 Subject: [PATCH 12/15] Remove redundant condition Co-authored-by: Maxim Katcharov --- source/client-side-encryption/tests/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 03a5e8aa8c..32703f9960 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3416,9 +3416,7 @@ Repeat this test with the `azure` and `gcp` masterKeys. ### 25. Test $lookup -Unless otherwise noted, these tests require: libmongocrypt 1.13.0, mongocryptd/crypt_shared 8.1+, and server 8.1+. - -Tests require server support of QE: Require MongoDB server 7.0+. Skip on standalone. +Unless otherwise noted, these tests require: libmongocrypt 1.13.0, mongocryptd/crypt_shared 8.1+, and server 8.1+. Skip on standalone. The syntax `` is used to refer to the content of the corresponding file in `../etc/data/lookup`. From d212c24b87ef508c7059f9977d29e0f058c83dfc Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Tue, 25 Feb 2025 17:52:57 +0000 Subject: [PATCH 13/15] fix lint --- source/client-side-encryption/tests/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 32703f9960..04627388e6 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3416,7 +3416,8 @@ Repeat this test with the `azure` and `gcp` masterKeys. ### 25. Test $lookup -Unless otherwise noted, these tests require: libmongocrypt 1.13.0, mongocryptd/crypt_shared 8.1+, and server 8.1+. Skip on standalone. +Unless otherwise noted, these tests require: libmongocrypt 1.13.0, mongocryptd/crypt_shared 8.1+, and server 8.1+. Skip +on standalone. The syntax `` is used to refer to the content of the corresponding file in `../etc/data/lookup`. From fe9431060d0d7dfb5d3f1de00ddc8894f91850c8 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Wed, 26 Feb 2025 18:48:13 +0000 Subject: [PATCH 14/15] simplify constraints --- source/client-side-encryption/tests/README.md | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 04627388e6..1a2029194a 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3416,8 +3416,7 @@ Repeat this test with the `azure` and `gcp` masterKeys. ### 25. Test $lookup -Unless otherwise noted, these tests require: libmongocrypt 1.13.0, mongocryptd/crypt_shared 8.1+, and server 8.1+. Skip -on standalone. +All tests require libmongocrypt 1.13.0, server 7.0+, and must be skipped on standalone. Tests define more constraints. The syntax `` is used to refer to the content of the corresponding file in `../etc/data/lookup`. @@ -3460,6 +3459,8 @@ Insert documents with `encryptedClient`: #### Case 1: `db.csfle` joins `db.no_schema` +Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from impacting the test). @@ -3483,6 +3484,8 @@ Expect one document to be returned matching: `{"csfle" : "csfle", "matched" : [ #### Case 2: `db.qe` joins `db.no_schema` +Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from impacting the test). @@ -3507,6 +3510,8 @@ Expect one document to be returned matching: `{"qe" : "qe", "matched" : [ {"no_s #### Case 3: `db.no_schema` joins `db.csfle` +Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from impacting the test). @@ -3530,6 +3535,8 @@ Expect one document to be returned matching: `{"no_schema" : "no_schema", "match #### Case 4: `db.no_schema` joins `db.qe` +Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from impacting the test). @@ -3553,6 +3560,8 @@ Expect one document to be returned matching: `{"no_schema" : "no_schema", "match #### Case 5: `db.csfle` joins `db.csfle2` +Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from impacting the test). @@ -3576,6 +3585,8 @@ Expect one document to be returned matching: `{"csfle" : "csfle", "matched" : [ #### Case 6: `db.qe` joins `db.qe2` +Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from impacting the test). @@ -3599,6 +3610,8 @@ Expect one document to be returned matching: `{"qe" : "qe", "matched" : [ {"qe2" #### Case 7: `db.no_schema` joins `db.no_schema2` +Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from impacting the test). @@ -3623,6 +3636,8 @@ Expect one document to be returned matching: #### Case 8: `db.csfle` joins `db.qe` +Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+. + Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from impacting the test). @@ -3644,9 +3659,9 @@ Run an aggregate operation on `db.csfle` with the following pipeline: Expect an exception to be thrown with a message containing the substring `not supported`. -#### Case 9: test error with \<8.1 +#### Case 9: test error with pre-8.1 -This case requires mongocryptd/crypt_shared \<8.1. +This case requires mongocryptd/crypt_shared pre-8.1. Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from impacting the test). From 9ec176c0185bcc48df56bb81501cbbcb54bd4772 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Wed, 26 Feb 2025 18:53:24 +0000 Subject: [PATCH 15/15] fix link check? --- source/client-side-encryption/tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 1a2029194a..fda3fa9de3 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3659,9 +3659,9 @@ Run an aggregate operation on `db.csfle` with the following pipeline: Expect an exception to be thrown with a message containing the substring `not supported`. -#### Case 9: test error with pre-8.1 +#### Case 9: test error with \<8.1 -This case requires mongocryptd/crypt_shared pre-8.1. +This case requires mongocryptd/crypt_shared \<8.1. Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from impacting the test).