DRIVERS-555 Add spec and prose tests for CSOT

Author: Divjot Arora

.github/workflows/json-regenerate-check.yml

@@ -29,4 +29,5 @@ jobs:
           python3 ./source/server-discovery-and-monitoring/tests/errors/generate-error-tests.py
           python3 ./source/client-side-encryption/etc/generate-corpus.py ./source/client-side-encryption/corpus
           python3 ./source/client-side-encryption/etc/generate-test.py ./source/client-side-encryption/etc/test-templates/*.template ./source/client-side-encryption/tests
+          python3 ./source/client-side-operations-timeout/etc/generate-basic-tests.py ./source/client-side-operations-timeout/etc/templates ./source/client-side-operations-timeout/tests
           cd source && make && git diff --exit-code

source/client-side-encryption/etc/test-templates/timeoutMS.yml.template

@@ -0,0 +1,65 @@
+runOn:
+  - minServerVersion: "4.4"
+database_name: &database_name "cse-timeouts-db"
+collection_name: &collection_name "cse-timeouts-coll"
+
+data: []
+json_schema: {{schema()}}
+key_vault_data: [{{key()}}]
+
+tests:
+  - description: "timeoutMS applied to listCollections to get collection schema"
+    failPoint:
+      configureFailPoint: failCommand
+      mode: { times: 1 }
+      data:
+        failCommands: ["listCollections"]
+        blockConnection: true
+        blockTimeMS: 60
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          aws: {} # Credentials filled in from environment.
+      timeoutMS: 50
+    operations:
+      - name: insertOne
+        arguments:
+          document: &doc0 { _id: 1, encrypted_string: "string0", random: "abc" }
+        result:
+          isTimeoutError: true
+    expectations:
+      # Auto encryption will request the collection info.
+      - command_started_event:
+          command:
+            listCollections: 1
+            filter:
+              name: *collection_name
+            maxTimeMS: { $$type: ["int", "long"] }
+          command_name: listCollections
+
+  # Test that timeoutMS applies to the sum of all operations done for client-side encryption. This is done by blocking
+  # listCollections and find for 20ms each and running an insertOne with timeoutMS=50. There should be two
+  # listCollections commands and one "find" command, so the sum should take more than timeoutMS.
+  #
+  # This test does not include command monitoring expectations because the exact command sequence is dependent on the
+  # amount of time taken by mongocryptd communication. In slow runs, mongocryptd communication can breach the timeout
+  # and result in the final "find" not being sent.
+  - description: "remaining timeoutMS applied to find to get keyvault data"
+    failPoint:
+      configureFailPoint: failCommand
+      mode: { times: 3 }
+      data:
+        failCommands: ["listCollections", "find"]
+        blockConnection: true
+        blockTimeMS: 20
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          aws: {} # Credentials filled in from environment.
+      timeoutMS: 50
+    operations:
+      - name: insertOne
+        arguments:
+          document: *doc0
+        result:
+          isTimeoutError: true

source/client-side-encryption/tests/README.rst

@@ -9,8 +9,14 @@ Client Side Encryption Tests
 Introduction
 ============
 
-This document describes the format of the driver spec tests included in the JSON
-and YAML files included in this directory.
+This document describes the format of the driver spec tests included in the
+JSON and YAML files included in this directory. The
+``timeoutMS.yml``/``timeoutMS.json`` files in this directory contain tests
+for the ``timeoutMS`` option and its application to the client-side
+encryption feature. Drivers MUST only run these tests after implementing the
+`Client Side Operations Timeout
+<../client-side-operations-timeout/client-side-operations-timeout.rst>`__
+specification.
 
 Additional prose tests, that are not represented in the spec tests, are described
 and MUST be implemented by all drivers.
@@ -30,7 +36,8 @@ The spec tests format is an extension of `transactions spec tests <https://githu
 
 - Addition of `$$type` to command_started_event and outcome.
 
-The semantics of `$$type` is that any actual value matching the BSON type indicated by the BSON type string is considered a match.
+The semantics of `$$type` is that any actual value matching one of the types indicated by either a BSON type string
+or an array of BSON type strings is considered a match.
 
 For example, the following matches a command_started_event for an insert of a document where `random` must be of type ``binData``::
 
@@ -42,6 +49,16 @@ For example, the following matches a command_started_event for an insert of a do
         ordered: true
       command_name: insert
 
+The following matches a command_started_event for an insert of a document where ``random`` must be of type
+``binData`` or ``string``::
+
+  - command_started_event:
+      command:
+        insert: *collection_name
+        documents:
+          - { random: { $$type: ["binData", "string"] } }
+        ordered: true
+      command_name: insert
 
 The values of `$$type` correspond to `these documented string representations of BSON types <https://docs.mongodb.com/manual/reference/bson-types/>`_.
 
@@ -69,6 +86,10 @@ Each YAML file has the following keys:
 
   - ``skipReason``: |txn|
 
+  - ``useMultipleMongoses``: |txn|
+
+  - ``failPoint``: |txn|
+
   - ``clientOptions``: Optional, parameters to pass to MongoClient().
 
     - ``autoEncryptOpts``: Optional
@@ -104,7 +125,10 @@ Each YAML file has the following keys:
 
     - ``arguments``: |txn|
 
-    - ``result``: |txn|
+    - ``result``: Same as the Transactions spec test format with one addition: if the operation is expected to return
+      an error, the ``result`` document may contain an ``isTimeoutError`` boolean field. If ``true``, the test runner
+      MUST assert that the error represents a timeout due to the use of the ``timeoutMS`` operation. If ``false``, the
+      test runner MUST assert that the error does not represent a timeout.
 
   - ``expectations``: |txn|
 

source/client-side-encryption/tests/timeoutMS.yml

@@ -0,0 +1,65 @@
+runOn:
+  - minServerVersion: "4.4"
+database_name: &database_name "cse-timeouts-db"
+collection_name: &collection_name "cse-timeouts-coll"
+
+data: []
+json_schema: {'properties': {'encrypted_w_altname': {'encrypt': {'keyId': '/altname', 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'random': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string_equivalent': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'}
+key_vault_data: [{'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']}]
+
+tests:
+  - description: "timeoutMS applied to listCollections to get collection schema"
+    failPoint:
+      configureFailPoint: failCommand
+      mode: { times: 1 }
+      data:
+        failCommands: ["listCollections"]
+        blockConnection: true
+        blockTimeMS: 60
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          aws: {} # Credentials filled in from environment.
+      timeoutMS: 50
+    operations:
+      - name: insertOne
+        arguments:
+          document: &doc0 { _id: 1, encrypted_string: "string0", random: "abc" }
+        result:
+          isTimeoutError: true
+    expectations:
+      # Auto encryption will request the collection info.
+      - command_started_event:
+          command:
+            listCollections: 1
+            filter:
+              name: *collection_name
+            maxTimeMS: { $$type: ["int", "long"] }
+          command_name: listCollections
+
+  # Test that timeoutMS applies to the sum of all operations done for client-side encryption. This is done by blocking
+  # listCollections and find for 20ms each and running an insertOne with timeoutMS=50. There should be two
+  # listCollections commands and one "find" command, so the sum should take more than timeoutMS.
+  #
+  # This test does not include command monitoring expectations because the exact command sequence is dependent on the
+  # amount of time taken by mongocryptd communication. In slow runs, mongocryptd communication can breach the timeout
+  # and result in the final "find" not being sent.
+  - description: "remaining timeoutMS applied to find to get keyvault data"
+    failPoint:
+      configureFailPoint: failCommand
+      mode: { times: 3 }
+      data:
+        failCommands: ["listCollections", "find"]
+        blockConnection: true
+        blockTimeMS: 20
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          aws: {} # Credentials filled in from environment.
+      timeoutMS: 50
+    operations:
+      - name: insertOne
+        arguments:
+          document: *doc0
+        result:
+          isTimeoutError: true

source/client-side-operations-timeout/etc/generate-basic-tests.py

@@ -0,0 +1,123 @@
+from collections import namedtuple
+from jinja2 import Template
+import os
+import sys
+
+Operation = namedtuple('Operation', ['operation_name', 'command_name', 'object', 'arguments'])
+
+CLIENT_OPERATIONS = [
+    Operation('listDatabases', 'listDatabases', 'client', ['filter: {}']),
+    Operation('listDatabaseNames', 'listDatabases', 'client', ['filter: {}']),
+    Operation('createChangeStream', 'aggregate', 'client', ['pipeline: []'])
+]
+
+DB_OPERATIONS = [
+    Operation('aggregate', 'aggregate', 'database', ['pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ]']),
+    Operation('listCollections', 'listCollections', 'database', ['filter: {}']),
+    Operation('listCollectionNames', 'listCollections', 'database', ['filter: {}']),
+    Operation('runCommand', 'ping', 'database', ['command: { ping: 1 }']),
+    Operation('createChangeStream', 'aggregate', 'database', ['pipeline: []'])
+]
+
+INSERT_MANY_ARGUMENTS = '''documents:
+            - { x: 1 }'''
+
+BULK_WRITE_ARGUMENTS = '''requests:
+            - insertOne:
+                document: { _id: 1 }'''
+
+COLLECTION_OPERATIONS = [
+    Operation('aggregate', 'aggregate', 'collection', ['pipeline: []']),
+    Operation('count', 'count', 'collection', ['filter: {}']),
+    Operation('countDocuments', 'aggregate', 'collection', ['filter: {}']),
+    Operation('estimatedDocumentCount', 'count', 'collection', []),
+    Operation('distinct', 'distinct', 'collection', ['fieldName: x', 'filter: {}']),
+    Operation('find', 'find', 'collection', ['filter: {}']),
+    Operation('findOne', 'find', 'collection', ['filter: {}']),
+    Operation('listIndexes', 'listIndexes', 'collection', []),
+    Operation('listIndexNames', 'listIndexes', 'collection', []),
+    Operation('createChangeStream', 'aggregate', 'collection', ['pipeline: []']),
+    Operation('insertOne', 'insert', 'collection', ['document: { x: 1 }']),
+    Operation('insertMany', 'insert', 'collection', [INSERT_MANY_ARGUMENTS]),
+    Operation('deleteOne', 'delete', 'collection', ['filter: {}']),
+    Operation('deleteMany', 'delete', 'collection', ['filter: {}']),
+    Operation('replaceOne', 'update', 'collection', ['filter: {}', 'replacement: { x: 1 }']),
+    Operation('updateOne', 'update', 'collection', ['filter: {}', 'update: { $set: { x: 1 } }']),
+    Operation('updateMany', 'update', 'collection', ['filter: {}', 'update: { $set: { x: 1 } }']),
+    Operation('findOneAndDelete', 'findAndModify', 'collection', ['filter: {}']),
+    Operation('findOneAndReplace', 'findAndModify', 'collection', ['filter: {}', 'replacement: { x: 1 }']),
+    Operation('findOneAndUpdate', 'findAndModify', 'collection', ['filter: {}', 'update: { $set: { x: 1 } }']),
+    Operation('bulkWrite', 'insert', 'collection', [BULK_WRITE_ARGUMENTS]),
+    Operation('createIndex', 'createIndexes', 'collection', ['keys: { x: 1 }', 'name: "x_1"']),
+    Operation('dropIndex', 'dropIndexes', 'collection', ['name: "x_1"']),
+    Operation('dropIndexes', 'dropIndexes', 'collection', []),
+]
+
+# Session and GridFS operations are generally tested in other files, so they're not included in the list of all
+# operations. Individual generation functions can choose to include them if needed.
+OPERATIONS = CLIENT_OPERATIONS + DB_OPERATIONS + COLLECTION_OPERATIONS
+
+RETRYABLE_WRITE_OPERATIONS = [op for op in OPERATIONS if op.operation_name in 
+    ['insertOne', 'updateOne', 'deleteOne', 'replaceOne', 'findOneAndDelete', 'findOneAndUpdate', 'findOneAndReplace', 'insertMany', 'bulkWrite']
+]
+
+RETRYABLE_READ_OPERATIONS = [op for op in OPERATIONS if op.operation_name in
+    ['find', 'findOne', 'aggregate', 'distinct', 'count', 'estimatedDocumentCount', 'countDocuments', 'createChangeStream', 'listDatabases',
+    'listDatabaseNames', 'listCollections', 'listCollectionNames', 'listIndexes']
+]
+
+templates_dir = sys.argv[1]
+tests_dir = sys.argv[2]
+
+def get_template(file):
+    path = f'{templates_dir}/{file}.yml.template'
+    return Template(open(path, 'r').read())
+
+def write_yaml(file, template, injections):
+    rendered = template.render(**injections)
+    path = f'{tests_dir}/{file}.yml'
+    open(path, 'w').write(rendered)
+
+def get_command_object(object):
+    if object == 'client' or object == 'database':
+        return 1
+    return '*collectionName'
+
+def max_time_supported(operation_name):
+    return operation_name in ['aggregate', 'count', 'estimatedDocumentCount', 'distinct', 'find', 'findOne',
+    'findOneAndDelete', 'findOneAndReplace', 'findOneAndUpdate', 'createIndex', 'dropIndex', 'dropIndexes']
+
+def generate(name, operations):
+    template = get_template(name)
+    injections = {
+        'operations': operations,
+        'get_command_object': get_command_object,
+        'max_time_supported': max_time_supported,
+    }
+    write_yaml(name, template, injections)
+
+def generate_global_timeout_tests():
+    generate('global-timeoutMS', OPERATIONS)
+
+def generate_override_db():
+    generate('override-database-timeoutMS', DB_OPERATIONS + COLLECTION_OPERATIONS)
+
+def generate_override_coll():
+    generate('override-collection-timeoutMS', COLLECTION_OPERATIONS)
+
+def generate_override_operation():
+    generate('override-operation-timeoutMS', OPERATIONS)
+
+def generate_retryable():
+    generate('retryability-timeoutMS', RETRYABLE_WRITE_OPERATIONS + RETRYABLE_READ_OPERATIONS)
+    generate('retryability-legacy-timeouts', RETRYABLE_WRITE_OPERATIONS + RETRYABLE_READ_OPERATIONS)
+
+def generate_deprecated():
+    generate('deprecated-options', OPERATIONS)
+
+generate_global_timeout_tests()
+generate_override_db()
+generate_override_coll()
+generate_override_operation()
+generate_retryable()
+generate_deprecated()