From 42a2be2e44134f12d2258d95290401c991b4c971 Mon Sep 17 00:00:00 2001 From: James Kovacs Date: Fri, 25 Mar 2022 17:15:04 -0600 Subject: [PATCH 1/3] DRIVERS-1995: Do not error when parsing change stream event documents --- source/change-streams/change-streams.rst | 17 +++- source/change-streams/tests/README.rst | 4 + .../tests/unified/change-streams.json | 93 +++++++++++++++++++ .../tests/unified/change-streams.yml | 51 ++++++++++ 4 files changed, 164 insertions(+), 1 deletion(-) diff --git a/source/change-streams/change-streams.rst b/source/change-streams/change-streams.rst index 7c75cc4262..0552ee1af4 100644 --- a/source/change-streams/change-streams.rst +++ b/source/change-streams/change-streams.rst @@ -9,7 +9,7 @@ Change Streams :Status: Accepted :Type: Standards :Minimum Server Version: 3.6 -:Last Modified: 2022-02-10 +:Last Modified: 2022-03-25 :Version: 1.12 .. contents:: @@ -132,12 +132,20 @@ If an aggregate command with a ``$changeStream`` stage completes successfully, t * * @note: Whether a change is reported as an event of the operation type * `update` or `replace` is a server implementation detail. + * + * @note: Drivers MUST not err when an unknown operationType is encountered. + * The server will add new operationType values in the future and drivers + * must not err when they encounter a new operationType. Unknown operationType + * values may be represented by "unknown". */ operationType: "insert" | "update" | "replace" | "delete" | "invalidate" | "drop" | "dropDatabase" | "rename"; /** * Contains two fields: “db” and “coll” containing the database and * collection name in which the change happened. + * + * @note: Drivers MUST not err when extra fields are encountered in the ns document + * as the server may add new fields in the future such as viewOn. */ ns: Document; @@ -462,6 +470,11 @@ Presently change streams support only a subset of available aggregation stages: A driver MUST NOT throw an exception if any unsupported stage is provided, but instead depend on the server to return an error. +A driver MUST NOT throw an exception if a user adds, removes, or modifies fields using ``$project``. The server will produce an error if ``_id`` +is projected out, but a user should otherwise be able to modify the shape of the change stream event as desired. This may require the result +to be deserialized to a ``BsonDocument`` or custom-defined type rather than a ``ChangeStreamDocument``. It is the responsiblity of the +user to ensure that the deserialized type is compatible with the specified ``$project`` stage. + The aggregate helper methods MUST have no new logic related to the ``$changeStream`` stage. Drivers MUST be capable of handling `TAILABLE_AWAIT `_ cursors from the aggregate command in the same way they handle such cursors from find. ``Collection.watch`` helper @@ -921,3 +934,5 @@ Changelog | 2022-02-10 | Specified that ``getMore`` command must explicitly send | | | inherited ``comment``. | +------------+------------------------------------------------------------+ +| 2022-03-25 | Do not error when parsing change stream event documents. | ++------------+------------------------------------------------------------+ diff --git a/source/change-streams/tests/README.rst b/source/change-streams/tests/README.rst index e900dc88b8..c3ca715730 100644 --- a/source/change-streams/tests/README.rst +++ b/source/change-streams/tests/README.rst @@ -239,3 +239,7 @@ The following tests have not yet been automated, but MUST still be tested. All t #. **Removed** #. ``$changeStream`` stage for ``ChangeStream`` started with ``startAfter`` against a server ``>=4.1.1`` that has not received any results yet MUST include a ``startAfter`` option and MUST NOT include a ``resumeAfter`` option when resuming a change stream. #. ``$changeStream`` stage for ``ChangeStream`` started with ``startAfter`` against a server ``>=4.1.1`` that has received at least one result MUST include a ``resumeAfter`` option and MUST NOT include a ``startAfter`` option when resuming a change stream. + +#. When a ``ChangeStream`` returns a ``ChangeStreamDocument`` containing an ``operationType`` of ``addedInFutureMongoDBVersion``, the driver MUST NOT err. +#. When a ``ChangeStream`` returns a ``ChangeStreamDocument`` containing a ``newField`` with value ``newFieldValue``, the driver MUST NOT err. +#. When a ``ChangeStream`` returns a ``ChangeStreamDocument`` containing an ``ns`` with value ``{viewOn: "db.coll"}``, the driver MUST NOT err. diff --git a/source/change-streams/tests/unified/change-streams.json b/source/change-streams/tests/unified/change-streams.json index bc9d462ee3..30889f0add 100644 --- a/source/change-streams/tests/unified/change-streams.json +++ b/source/change-streams/tests/unified/change-streams.json @@ -333,6 +333,99 @@ ] } ] + }, + { + "description": "Test server error on projecting out _id", + "runOnRequirements": [ + { + "minServerVersion": "3.6" + } + ], + "operations": [ + { + "name": "createChangeStream", + "object": "collection0", + "arguments": { + "pipeline": [ + { + "$project": { + "_id": 0 + } + } + ] + }, + "saveResultAsEntity": "changeStream0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 1, + "a": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectError": { + "errorCode": 280, + "errorCodeName": "ChangeStreamFatalError", + "errorLabelsContain": [ + "NonResumableChangeStreamError" + ] + } + } + ] + }, + { + "description": "Test projection in change stream returns expected fields", + "runOnRequirements": [ + { + "minServerVersion": "3.6" + } + ], + "operations": [ + { + "name": "createChangeStream", + "object": "collection0", + "arguments": { + "pipeline": [ + { + "$project": { + "optype": "$operationType", + "ns": 1, + "newField": "value" + } + } + ] + }, + "saveResultAsEntity": "changeStream0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 1, + "a": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "optype": "insert", + "ns": { + "db": "database0", + "coll": "collection0" + }, + "newField": "value" + } + } + ] } ] } diff --git a/source/change-streams/tests/unified/change-streams.yml b/source/change-streams/tests/unified/change-streams.yml index a8468c9bc7..3d800b21ce 100644 --- a/source/change-streams/tests/unified/change-streams.yml +++ b/source/change-streams/tests/unified/change-streams.yml @@ -177,3 +177,54 @@ tests: comment: "comment" commandName: getMore databaseName: *database0 + + - description: "Test server error on projecting out _id" + runOnRequirements: + - minServerVersion: "3.6" + operations: + - name: createChangeStream + object: *collection0 + arguments: + pipeline: [ { $project: { _id: 0 } } ] + saveResultAsEntity: &changeStream0 changeStream0 + - name: insertOne + object: *collection0 + arguments: + document: { + "_id": 1, + "a": 1 + } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectError: + errorCode: 280 + errorCodeName: "ChangeStreamFatalError" + errorLabelsContain: [ "NonResumableChangeStreamError" ] + + + - description: "Test projection in change stream returns expected fields" + runOnRequirements: + - minServerVersion: "3.6" + operations: + - name: createChangeStream + object: *collection0 + arguments: + pipeline: [ { $project: { optype: "$operationType", ns: 1, newField: "value" } } ] + saveResultAsEntity: &changeStream0 changeStream0 + - name: insertOne + object: *collection0 + arguments: + document: { + "_id": 1, + "a": 1 + } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: { + "optype": "insert", + "ns": { + "db": "database0", + "coll": "collection0" + }, + "newField": "value" + } From 8783276e9d893e9cfb58ac339c8c4b394c1fa448 Mon Sep 17 00:00:00 2001 From: James Kovacs Date: Tue, 29 Mar 2022 15:43:34 -0600 Subject: [PATCH 2/3] Incorporated review feedback. --- source/change-streams/tests/README.rst | 4 - .../tests/unified/change-streams.json | 190 ++++++++++++++++++ .../tests/unified/change-streams.yml | 109 ++++++++++ 3 files changed, 299 insertions(+), 4 deletions(-) diff --git a/source/change-streams/tests/README.rst b/source/change-streams/tests/README.rst index c3ca715730..e900dc88b8 100644 --- a/source/change-streams/tests/README.rst +++ b/source/change-streams/tests/README.rst @@ -239,7 +239,3 @@ The following tests have not yet been automated, but MUST still be tested. All t #. **Removed** #. ``$changeStream`` stage for ``ChangeStream`` started with ``startAfter`` against a server ``>=4.1.1`` that has not received any results yet MUST include a ``startAfter`` option and MUST NOT include a ``resumeAfter`` option when resuming a change stream. #. ``$changeStream`` stage for ``ChangeStream`` started with ``startAfter`` against a server ``>=4.1.1`` that has received at least one result MUST include a ``resumeAfter`` option and MUST NOT include a ``startAfter`` option when resuming a change stream. - -#. When a ``ChangeStream`` returns a ``ChangeStreamDocument`` containing an ``operationType`` of ``addedInFutureMongoDBVersion``, the driver MUST NOT err. -#. When a ``ChangeStream`` returns a ``ChangeStreamDocument`` containing a ``newField`` with value ``newFieldValue``, the driver MUST NOT err. -#. When a ``ChangeStream`` returns a ``ChangeStreamDocument`` containing an ``ns`` with value ``{viewOn: "db.coll"}``, the driver MUST NOT err. diff --git a/source/change-streams/tests/unified/change-streams.json b/source/change-streams/tests/unified/change-streams.json index 30889f0add..1d2cc3ec30 100644 --- a/source/change-streams/tests/unified/change-streams.json +++ b/source/change-streams/tests/unified/change-streams.json @@ -334,6 +334,196 @@ } ] }, + { + "description": "Test unknown operationType MUST NOT err", + "runOnRequirements": [ + { + "minServerVersion": "3.6" + } + ], + "operations": [ + { + "name": "createChangeStream", + "object": "collection0", + "arguments": { + "pipeline": [ + { + "$project": { + "operationType": "addedInFutureMongoDBVersion", + "ns": 1 + } + } + ] + }, + "saveResultAsEntity": "changeStream0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 1, + "a": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "addedInFutureMongoDBVersion", + "ns": { + "db": "database0", + "coll": "collection0" + } + } + } + ] + }, + { + "description": "Test newField added in response MUST NOT err", + "runOnRequirements": [ + { + "minServerVersion": "3.6" + } + ], + "operations": [ + { + "name": "createChangeStream", + "object": "collection0", + "arguments": { + "pipeline": [ + { + "$project": { + "operationType": 1, + "ns": 1, + "newField": "newFieldValue" + } + } + ] + }, + "saveResultAsEntity": "changeStream0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 1, + "a": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "database0", + "coll": "collection0" + }, + "newField": "newFieldValue" + } + } + ] + }, + { + "description": "Test new structure in ns document MUST NOT err", + "runOnRequirements": [ + { + "minServerVersion": "3.6" + } + ], + "operations": [ + { + "name": "createChangeStream", + "object": "collection0", + "arguments": { + "pipeline": [ + { + "$project": { + "operationType": "insert", + "ns.viewOn": "db.coll" + } + } + ] + }, + "saveResultAsEntity": "changeStream0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 1, + "a": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "viewOn": "db.coll" + } + } + } + ] + }, + { + "description": "Test modified structure in ns document MUST NOT err", + "runOnRequirements": [ + { + "minServerVersion": "3.6" + } + ], + "operations": [ + { + "name": "createChangeStream", + "object": "collection0", + "arguments": { + "pipeline": [ + { + "$project": { + "operationType": "insert", + "ns": { + "db": "$ns.db", + "coll": "$ns.coll", + "viewOn": "db.coll" + } + } + } + ] + }, + "saveResultAsEntity": "changeStream0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 1, + "a": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "database0", + "coll": "collection0", + "viewOn": "db.coll" + } + } + } + ] + }, { "description": "Test server error on projecting out _id", "runOnRequirements": [ diff --git a/source/change-streams/tests/unified/change-streams.yml b/source/change-streams/tests/unified/change-streams.yml index 3d800b21ce..26d3a6f291 100644 --- a/source/change-streams/tests/unified/change-streams.yml +++ b/source/change-streams/tests/unified/change-streams.yml @@ -178,6 +178,115 @@ tests: commandName: getMore databaseName: *database0 + - description: "Test unknown operationType MUST NOT err" + runOnRequirements: + - minServerVersion: "3.6" + operations: + - name: createChangeStream + object: *collection0 + arguments: + # using $project to simulate future changes to ChangeStreamDocument structure + pipeline: [ { $project: { operationType: "addedInFutureMongoDBVersion", ns: 1 } } ] + saveResultAsEntity: &changeStream0 changeStream0 + - name: insertOne + object: *collection0 + arguments: + document: { + "_id": 1, + "a": 1 + } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: { + "operationType": "addedInFutureMongoDBVersion", + "ns": { + "db": "database0", + "coll": "collection0" + } + } + + - description: "Test newField added in response MUST NOT err" + runOnRequirements: + - minServerVersion: "3.6" + operations: + - name: createChangeStream + object: *collection0 + arguments: + # using $project to simulate future changes to ChangeStreamDocument structure + pipeline: [ { $project: { operationType: 1, ns: 1, newField: "newFieldValue" } } ] + saveResultAsEntity: &changeStream0 changeStream0 + - name: insertOne + object: *collection0 + arguments: + document: { + "_id": 1, + "a": 1 + } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: { + "operationType": "insert", + "ns": { + "db": "database0", + "coll": "collection0" + }, + "newField": "newFieldValue" + } + + - description: "Test new structure in ns document MUST NOT err" + runOnRequirements: + - minServerVersion: "3.6" + operations: + - name: createChangeStream + object: *collection0 + arguments: + # using $project to simulate future changes to ChangeStreamDocument structure + pipeline: [ { $project: { operationType: "insert", "ns.viewOn": "db.coll" } } ] + saveResultAsEntity: &changeStream0 changeStream0 + - name: insertOne + object: *collection0 + arguments: + document: { + "_id": 1, + "a": 1 + } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: { + "operationType": "insert", + "ns": { + "viewOn": "db.coll" + } + } + + - description: "Test modified structure in ns document MUST NOT err" + runOnRequirements: + - minServerVersion: "3.6" + operations: + - name: createChangeStream + object: *collection0 + arguments: + # using $project to simulate future changes to ChangeStreamDocument structure + pipeline: [ { $project: { operationType: "insert", ns: { db: "$ns.db", coll: "$ns.coll", viewOn: "db.coll" } } } ] + saveResultAsEntity: &changeStream0 changeStream0 + - name: insertOne + object: *collection0 + arguments: + document: { + "_id": 1, + "a": 1 + } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: { + "operationType": "insert", + "ns": { + "db": "database0", + "coll": "collection0", + "viewOn": "db.coll" + } + } + - description: "Test server error on projecting out _id" runOnRequirements: - minServerVersion: "3.6" From 5985672a294bf7fb8c55af22267ba1e2fab961ae Mon Sep 17 00:00:00 2001 From: James Kovacs Date: Wed, 30 Mar 2022 14:00:00 -0600 Subject: [PATCH 3/3] Incorporated feedback on YAML and unified test formatting. --- source/change-streams/change-streams.rst | 2 +- .../tests/unified/change-streams.json | 39 +----- .../tests/unified/change-streams.yml | 112 ++++++------------ 3 files changed, 39 insertions(+), 114 deletions(-) diff --git a/source/change-streams/change-streams.rst b/source/change-streams/change-streams.rst index 0552ee1af4..27cb704df6 100644 --- a/source/change-streams/change-streams.rst +++ b/source/change-streams/change-streams.rst @@ -10,7 +10,7 @@ Change Streams :Type: Standards :Minimum Server Version: 3.6 :Last Modified: 2022-03-25 -:Version: 1.12 +:Version: 1.12.1 .. contents:: diff --git a/source/change-streams/tests/unified/change-streams.json b/source/change-streams/tests/unified/change-streams.json index 1d2cc3ec30..4fc745f97e 100644 --- a/source/change-streams/tests/unified/change-streams.json +++ b/source/change-streams/tests/unified/change-streams.json @@ -2,6 +2,9 @@ "description": "change-streams", "schemaVersion": "1.0", "runOnRequirements": [ + { + "minServerVersion": "3.6" + }, { "topologies": [ "replicaset", @@ -167,7 +170,6 @@ "description": "Test with document comment - pre 4.4", "runOnRequirements": [ { - "minServerVersion": "3.6.0", "maxServerVersion": "4.2.99" } ], @@ -211,11 +213,6 @@ }, { "description": "Test with string comment", - "runOnRequirements": [ - { - "minServerVersion": "3.6.0" - } - ], "operations": [ { "name": "createChangeStream", @@ -336,11 +333,6 @@ }, { "description": "Test unknown operationType MUST NOT err", - "runOnRequirements": [ - { - "minServerVersion": "3.6" - } - ], "operations": [ { "name": "createChangeStream", @@ -382,11 +374,6 @@ }, { "description": "Test newField added in response MUST NOT err", - "runOnRequirements": [ - { - "minServerVersion": "3.6" - } - ], "operations": [ { "name": "createChangeStream", @@ -430,11 +417,6 @@ }, { "description": "Test new structure in ns document MUST NOT err", - "runOnRequirements": [ - { - "minServerVersion": "3.6" - } - ], "operations": [ { "name": "createChangeStream", @@ -475,11 +457,6 @@ }, { "description": "Test modified structure in ns document MUST NOT err", - "runOnRequirements": [ - { - "minServerVersion": "3.6" - } - ], "operations": [ { "name": "createChangeStream", @@ -526,11 +503,6 @@ }, { "description": "Test server error on projecting out _id", - "runOnRequirements": [ - { - "minServerVersion": "3.6" - } - ], "operations": [ { "name": "createChangeStream", @@ -571,11 +543,6 @@ }, { "description": "Test projection in change stream returns expected fields", - "runOnRequirements": [ - { - "minServerVersion": "3.6" - } - ], "operations": [ { "name": "createChangeStream", diff --git a/source/change-streams/tests/unified/change-streams.yml b/source/change-streams/tests/unified/change-streams.yml index 26d3a6f291..bca5c0d913 100644 --- a/source/change-streams/tests/unified/change-streams.yml +++ b/source/change-streams/tests/unified/change-streams.yml @@ -1,6 +1,7 @@ description: "change-streams" schemaVersion: "1.0" runOnRequirements: + - minServerVersion: "3.6" - topologies: [ replicaset, sharded-replicaset ] createEntities: - client: @@ -97,8 +98,7 @@ tests: - description: "Test with document comment - pre 4.4" runOnRequirements: - - minServerVersion: "3.6.0" - maxServerVersion: "4.2.99" + - maxServerVersion: "4.2.99" operations: - name: createChangeStream object: *collection0 @@ -118,8 +118,6 @@ tests: comment: *comment0 - description: "Test with string comment" - runOnRequirements: - - minServerVersion: "3.6.0" operations: - name: createChangeStream object: *collection0 @@ -179,8 +177,6 @@ tests: databaseName: *database0 - description: "Test unknown operationType MUST NOT err" - runOnRequirements: - - minServerVersion: "3.6" operations: - name: createChangeStream object: *collection0 @@ -191,23 +187,16 @@ tests: - name: insertOne object: *collection0 arguments: - document: { - "_id": 1, - "a": 1 - } + document: { "_id": 1, "a": 1 } - name: iterateUntilDocumentOrError object: *changeStream0 - expectResult: { - "operationType": "addedInFutureMongoDBVersion", - "ns": { - "db": "database0", - "coll": "collection0" - } - } + expectResult: + operationType: "addedInFutureMongoDBVersion" + ns: + db: *database0 + coll: *collection0 - description: "Test newField added in response MUST NOT err" - runOnRequirements: - - minServerVersion: "3.6" operations: - name: createChangeStream object: *collection0 @@ -218,24 +207,17 @@ tests: - name: insertOne object: *collection0 arguments: - document: { - "_id": 1, - "a": 1 - } + document: { "_id": 1, "a": 1 } - name: iterateUntilDocumentOrError object: *changeStream0 - expectResult: { - "operationType": "insert", - "ns": { - "db": "database0", - "coll": "collection0" - }, - "newField": "newFieldValue" - } + expectResult: + operationType: "insert" + ns: + db: *database0 + coll: *collection0 + newField: "newFieldValue" - description: "Test new structure in ns document MUST NOT err" - runOnRequirements: - - minServerVersion: "3.6" operations: - name: createChangeStream object: *collection0 @@ -246,22 +228,15 @@ tests: - name: insertOne object: *collection0 arguments: - document: { - "_id": 1, - "a": 1 - } + document: { "_id": 1, "a": 1 } - name: iterateUntilDocumentOrError object: *changeStream0 - expectResult: { - "operationType": "insert", - "ns": { - "viewOn": "db.coll" - } - } + expectResult: + operationType: "insert" + ns: + viewOn: "db.coll" - description: "Test modified structure in ns document MUST NOT err" - runOnRequirements: - - minServerVersion: "3.6" operations: - name: createChangeStream object: *collection0 @@ -272,24 +247,17 @@ tests: - name: insertOne object: *collection0 arguments: - document: { - "_id": 1, - "a": 1 - } + document: { "_id": 1, "a": 1 } - name: iterateUntilDocumentOrError object: *changeStream0 - expectResult: { - "operationType": "insert", - "ns": { - "db": "database0", - "coll": "collection0", - "viewOn": "db.coll" - } - } + expectResult: + operationType: "insert" + ns: + db: *database0 + coll: *collection0 + viewOn: "db.coll" - description: "Test server error on projecting out _id" - runOnRequirements: - - minServerVersion: "3.6" operations: - name: createChangeStream object: *collection0 @@ -299,10 +267,7 @@ tests: - name: insertOne object: *collection0 arguments: - document: { - "_id": 1, - "a": 1 - } + document: { "_id": 1, "a": 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectError: @@ -312,8 +277,6 @@ tests: - description: "Test projection in change stream returns expected fields" - runOnRequirements: - - minServerVersion: "3.6" operations: - name: createChangeStream object: *collection0 @@ -323,17 +286,12 @@ tests: - name: insertOne object: *collection0 arguments: - document: { - "_id": 1, - "a": 1 - } + document: { "_id": 1, "a": 1 } - name: iterateUntilDocumentOrError object: *changeStream0 - expectResult: { - "optype": "insert", - "ns": { - "db": "database0", - "coll": "collection0" - }, - "newField": "value" - } + expectResult: + optype: "insert" + ns: + db: *database0 + coll: *collection0 + newField: "value"