DRIVERS-3082 add prose tests for $lookup (#1757)

Co-authored-by: Maxim Katcharov <[email protected]>

Author: kevinAlbs

source/client-side-encryption/etc/data/lookup/key-doc.json

@@ -0,0 +1,30 @@
+{
+    "_id": {
+        "$binary": {
+            "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+            "subType": "04"
+        }
+    },
+    "keyMaterial": {
+        "$binary": {
+            "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==",
+            "subType": "00"
+        }
+    },
+    "creationDate": {
+        "$date": {
+            "$numberLong": "1648914851981"
+        }
+    },
+    "updateDate": {
+        "$date": {
+            "$numberLong": "1648914851981"
+        }
+    },
+    "status": {
+        "$numberInt": "0"
+    },
+    "masterKey": {
+        "provider": "local"
+    }
+}

source/client-side-encryption/etc/data/lookup/schema-csfle.json

@@ -0,0 +1,19 @@
+{
+    "properties": {
+        "csfle": {
+            "encrypt": {
+                "keyId": [
+                    {
+                        "$binary": {
+                            "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+                            "subType": "04"
+                        }
+                    }
+                ],
+                "bsonType": "string",
+                "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+            }
+        }
+    },
+    "bsonType": "object"
+}

source/client-side-encryption/etc/data/lookup/schema-csfle2.json

@@ -0,0 +1,19 @@
+{
+    "properties": {
+        "csfle2": {
+            "encrypt": {
+                "keyId": [
+                    {
+                        "$binary": {
+                            "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+                            "subType": "04"
+                        }
+                    }
+                ],
+                "bsonType": "string",
+                "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+            }
+        }
+    },
+    "bsonType": "object"
+}

source/client-side-encryption/etc/data/lookup/schema-qe.json

@@ -0,0 +1,20 @@
+{
+    "escCollection": "enxcol_.qe.esc",
+    "ecocCollection": "enxcol_.qe.ecoc",
+    "fields": [
+        {
+            "keyId": {
+                "$binary": {
+                    "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+                    "subType": "04"
+                }
+            },
+            "path": "qe",
+            "bsonType": "string",
+            "queries": {
+                "queryType": "equality",
+                "contention": 0
+            }
+        }
+    ]
+}

source/client-side-encryption/etc/data/lookup/schema-qe2.json

@@ -0,0 +1,20 @@
+{
+    "escCollection": "enxcol_.qe2.esc",
+    "ecocCollection": "enxcol_.qe2.ecoc",
+    "fields": [
+        {
+            "keyId": {
+                "$binary": {
+                    "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+                    "subType": "04"
+                }
+            },
+            "path": "qe2",
+            "bsonType": "string",
+            "queries": {
+                "queryType": "equality",
+                "contention": 0
+            }
+        }
+    ]
+}

source/client-side-encryption/tests/README.md

@@ -3413,3 +3413,273 @@ Repeat this test with the `azure` and `gcp` masterKeys.
 2. Call `client_encryption.createDataKey()` with "aws" as the provider. Expect this to fail.
 
 Repeat this test with the `azure` and `gcp` masterKeys.
+
+### 25. Test $lookup
+
+All tests require libmongocrypt 1.13.0, server 7.0+, and must be skipped on standalone. Tests define more constraints.
+
+The syntax `<filename.json>` is used to refer to the content of the corresponding file in `../etc/data/lookup`.
+
+#### Setup
+
+Create an encrypted MongoClient named `encryptedClient` configured with:
+
+```python
+AutoEncryptionOpts(
+    keyVaultNamespace="db.keyvault",
+    kmsProviders={"local": { "key": "<base64 decoding of LOCAL_MASTERKEY>" }}
+)
+```
+
+Use `encryptedClient` to drop `db.keyvault`. Insert `<key-doc.json>` into `db.keyvault` with majority write concern.
+
+Use `encryptedClient` to drop and create the following collections:
+
+- `db.csfle` with options: `{ "validator": { "$jsonSchema": "<schema-csfle.json>"}}`.
+- `db.csfle2` with options: `{ "validator": { "$jsonSchema": "<schema-csfle2.json>"}}`.
+- `db.qe` with options: `{ "encryptedFields": "<schema-qe.json>"}`.
+- `db.qe2` with options: `{ "encryptedFields": "<schema-qe2.json>"}`.
+- `db.no_schema` with no options.
+- `db.no_schema2` with no options.
+
+Create an unencrypted MongoClient named `unencryptedClient`.
+
+Insert documents with `encryptedClient`:
+
+- `{"csfle": "csfle"}` into `db.csfle`
+    - Use `unencryptedClient` to retrieve it. Assert the `csfle` field is BSON binary.
+- `{"csfle2": "csfle2"}` into `db.csfle2`
+    - Use `unencryptedClient` to retrieve it. Assert the `csfle2` field is BSON binary.
+- `{"qe": "qe"}` into `db.qe`
+    - Use `unencryptedClient` to retrieve it. Assert the `qe` field is BSON binary.
+- `{"qe2": "qe2"}` into `db.qe2`
+    - Use `unencryptedClient` to retrieve it. Assert the `qe2` field is BSON binary.
+- `{"no_schema": "no_schema"}` into `db.no_schema`
+- `{"no_schema2": "no_schema2"}` into `db.no_schema2`
+
+#### Case 1: `db.csfle` joins `db.no_schema`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.csfle` with the following pipeline:
+
+```json
+[
+    {"$match" : {"csfle" : "csfle"}},
+    {
+        "$lookup" : {
+            "from" : "no_schema",
+            "as" : "matched",
+            "pipeline" : [ {"$match" : {"no_schema" : "no_schema"}}, {"$project" : {"_id" : 0}} ]
+        }
+    },
+    {"$project" : {"_id" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"csfle" : "csfle", "matched" : [ {"no_schema" : "no_schema"} ]}`.
+
+#### Case 2: `db.qe` joins `db.no_schema`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.qe` with the following pipeline:
+
+```json
+[
+    {"$match" : {"qe" : "qe"}},
+    {
+       "$lookup" : {
+          "from" : "no_schema",
+          "as" : "matched",
+          "pipeline" :
+             [ {"$match" : {"no_schema" : "no_schema"}}, {"$project" : {"_id" : 0, "__safeContent__" : 0}} ]
+       }
+    },
+    {"$project" : {"_id" : 0, "__safeContent__" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"qe" : "qe", "matched" : [ {"no_schema" : "no_schema"} ]}`.
+
+#### Case 3: `db.no_schema` joins `db.csfle`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.no_schema` with the following pipeline:
+
+```json
+[
+    {"$match" : {"no_schema" : "no_schema"}},
+    {
+        "$lookup" : {
+            "from" : "csfle",
+            "as" : "matched",
+            "pipeline" : [ {"$match" : {"csfle" : "csfle"}}, {"$project" : {"_id" : 0}} ]
+        }
+    },
+    {"$project" : {"_id" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"no_schema" : "no_schema", "matched" : [ {"csfle" : "csfle"} ]}`.
+
+#### Case 4: `db.no_schema` joins `db.qe`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.no_schema` with the following pipeline:
+
+```json
+[
+   {"$match" : {"no_schema" : "no_schema"}},
+   {
+      "$lookup" : {
+         "from" : "qe",
+         "as" : "matched",
+         "pipeline" : [ {"$match" : {"qe" : "qe"}}, {"$project" : {"_id" : 0, "__safeContent__" : 0}} ]
+      }
+   },
+   {"$project" : {"_id" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"no_schema" : "no_schema", "matched" : [ {"qe" : "qe"} ]}`.
+
+#### Case 5: `db.csfle` joins `db.csfle2`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.csfle` with the following pipeline:
+
+```json
+[
+   {"$match" : {"csfle" : "csfle"}},
+   {
+      "$lookup" : {
+         "from" : "csfle2",
+         "as" : "matched",
+         "pipeline" : [ {"$match" : {"csfle2" : "csfle2"}}, {"$project" : {"_id" : 0}} ]
+      }
+   },
+   {"$project" : {"_id" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"csfle" : "csfle", "matched" : [ {"csfle2" : "csfle2"} ]}`.
+
+#### Case 6: `db.qe` joins `db.qe2`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.qe` with the following pipeline:
+
+```json
+[
+   {"$match" : {"qe" : "qe"}},
+   {
+      "$lookup" : {
+         "from" : "qe2",
+         "as" : "matched",
+         "pipeline" : [ {"$match" : {"qe2" : "qe2"}}, {"$project" : {"_id" : 0, "__safeContent__" : 0}} ]
+      }
+   },
+   {"$project" : {"_id" : 0, "__safeContent__" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"qe" : "qe", "matched" : [ {"qe2" : "qe2"} ]}`.
+
+#### Case 7: `db.no_schema` joins `db.no_schema2`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.no_schema` with the following pipeline:
+
+```json
+[
+    {"$match" : {"no_schema" : "no_schema"}},
+    {
+        "$lookup" : {
+            "from" : "no_schema2",
+            "as" : "matched",
+            "pipeline" : [ {"$match" : {"no_schema2" : "no_schema2"}}, {"$project" : {"_id" : 0}} ]
+        }
+    },
+    {"$project" : {"_id" : 0}}
+]
+```
+
+Expect one document to be returned matching:
+`{"no_schema" : "no_schema", "matched" : [ {"no_schema2" : "no_schema2"} ]}`.
+
+#### Case 8: `db.csfle` joins `db.qe`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.csfle` with the following pipeline:
+
+```json
+[
+    {"$match" : {"csfle" : "qe"}},
+    {
+        "$lookup" : {
+            "from" : "qe",
+            "as" : "matched",
+            "pipeline" : [ {"$match" : {"qe" : "qe"}}, {"$project" : {"_id" : 0}} ]
+        }
+    },
+    {"$project" : {"_id" : 0}}
+]
+```
+
+Expect an exception to be thrown with a message containing the substring `not supported`.
+
+#### Case 9: test error with \<8.1
+
+This case requires mongocryptd/crypt_shared \<8.1.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.csfle` with the following pipeline:
+
+```json
+[
+    {"$match" : {"csfle" : "csfle"}},
+    {
+        "$lookup" : {
+            "from" : "no_schema",
+            "as" : "matched",
+            "pipeline" : [ {"$match" : {"no_schema" : "no_schema"}}, {"$project" : {"_id" : 0}} ]
+        }
+    },
+    {"$project" : {"_id" : 0}}
+]
+```
+
+Expect an exception to be thrown with a message containing the substring `Upgrade`.