fix: 4.x-1.x interop (incl. ObjectID _bsontype)

Enables documents created by 4.x to be parsed/serialized/deserialized by 1.x BSON (e.g. the current Node driver) and vice-versa. Fiesx Object ID interop with the current Node driver and with existing duck-typing code that depends on ObjectId._bsontype === 'ObjectID'

Fixes NODE-1873

Author: justingrant

Diff for: lib/db_ref.js

@@ -68,4 +68,11 @@ class DBRef {
 }
 
 Object.defineProperty(DBRef.prototype, '_bsontype', { value: 'DBRef' });
+// the 1.x parser used a "namespace" property, while 4.x uses "collection". To ensure backwards
+// compatibility, let's expose "namespace"
+Object.defineProperty(DBRef.prototype, 'namespace', {
+  get() { return this.collection; },
+  set(val) { this.collection = val; },
+  configurable: false
+});
 module.exports = DBRef;

Diff for: lib/extended_json.js

@@ -70,7 +70,7 @@ function deserializeValue(self, key, value, options) {
     const date = new Date();
 
     if (typeof d === 'string') date.setTime(Date.parse(d));
-    else if (d instanceof Long) date.setTime(d.toNumber());
+    else if (Long.isLong(d)) date.setTime(d.toNumber());
     else if (typeof d === 'number' && options.relaxed) date.setTime(d);
     return date;
   }
@@ -265,38 +265,68 @@ function serializeValue(value, options) {
   return value;
 }
 
+const BSON_TYPE_MAPPINGS = {
+  Binary: o => new Binary(o.value(), o.subtype),
+  Code: o => new Code(o.code, o.scope),
+  DBRef: o => new DBRef(o.collection || o.namespace, o.oid, o.db, o.fields), // "namespace" for 1.x library backwards compat
+  Decimal128: o => new Decimal128(o.bytes),
+  Double: o => new Double(o.value),
+  Int32: o => new Int32(o.value),
+  Long: o => Long.fromBits(     // underscore variants for 1.x backwards compatibility
+    o.low != null ? o.low : o.low_,
+    o.low != null ? o.high : o.high_,
+    o.low != null ? o.unsigned : o.unsigned_
+  ),
+  MaxKey: o => new MaxKey(),
+  MinKey: o => new MinKey(),
+  ObjectID: o => new ObjectId(o),
+  ObjectId: o => new ObjectId(o), // support 4.0.0/4.0.1 before _bsontype was reverted back to ObjectID
+  BSONRegExp: o => new BSONRegExp(o.pattern, o.options),
+  Symbol: o => new Symbol(o.value),
+  Timestamp: o => Timestamp.fromBits(o.low, o.high)
+};
+
 function serializeDocument(doc, options) {
   if (doc == null || typeof doc !== 'object') throw new Error('not an object instance');
 
-  // the "document" is really just a BSON type
-  if (doc._bsontype) {
-    if (doc._bsontype === 'ObjectID') {
-      // Deprecated ObjectID class with capital "D" is still used (e.g. by 'mongodb' package). It has
-      // no "toExtendedJSON" method, so convert to new ObjectId (lowercase "d") class before serializing
-      doc = ObjectId.createFromHexString(doc.toString());
+  const bsontype = doc._bsontype;
+  if (typeof bsontype === 'undefined') {
+
+    // It's a regular object. Recursively serialize its property values.
+    const _doc = {};
+    for (let name in doc) {
+      _doc[name] = serializeValue(doc[name], options);
     }
-    if (typeof doc.toExtendedJSON === 'function') {
-      // TODO: the two cases below mutate the original document! Bad.  I don't know
-      // enough about these two BSON types to know how to safely clone these objects, but
-      // someone who knows MongoDB better should fix this to clone instead of mutating input objects.
-      if (doc._bsontype === 'Code' && doc.scope) {
-        doc.scope = serializeDocument(doc.scope, options);
-      } else if (doc._bsontype === 'DBRef' && doc.oid) {
-        doc.oid = serializeDocument(doc.oid, options);
+    return _doc;
+
+  } else if (typeof bsontype === 'string') {
+
+    // the "document" is really just a BSON type object
+    let _doc = doc;
+    if (typeof _doc.toExtendedJSON !== 'function') {
+      // There's no EJSON serialization function on the object. It's probably an
+      // object created by a previous version of this library (or another library)
+      // that's duck-typing objects to look like they were generated by this library).
+      // Copy the object into this library's version of that type.
+      const mapper = BSON_TYPE_MAPPINGS[bsontype];
+      if (!mapper) {
+        throw new TypeError('Unrecognized or invalid _bsontype: ' + bsontype);
       }
+      _doc = mapper(_doc);
+    }
 
-      return doc.toExtendedJSON(options);
+    // Two BSON types may have nested objects that may need to be serialized too 
+    if (bsontype === 'Code' && _doc.scope) {
+      _doc = new Code(_doc.code, serializeValue(_doc.scope, options));
+    } else if (bsontype === 'DBRef' && _doc.oid) {
+      _doc = new DBRef(_doc.collection, serializeValue(_doc.oid, options), _doc.db, _doc.fields);
     }
-    // TODO: should we throw an exception if there's a BSON type that has no toExtendedJSON method?
-  }
 
-  // Recursively serialize this document's property values.
-  const _doc = {};
-  for (let name in doc) {
-    _doc[name] = serializeValue(doc[name], options);
-  }
+    return _doc.toExtendedJSON(options);
 
-  return _doc;
+  } else {
+    throw new Error('_bsontype must be a string, but was: ' + typeof bsontype);
+  }
 }
 
 module.exports = {

Diff for: lib/objectid.js

@@ -52,7 +52,7 @@ class ObjectId {
   /**
    * Create an ObjectId type
    *
-   * @param {(string|number)} id Can be a 24 byte hex string, 12 byte binary string or a Number.
+   * @param {(string|Buffer|number)} id Can be a 24 byte hex string, 12 byte binary Buffer, or a Number.
    * @property {number} generationTime The generation time of this ObjectId instance
    * @return {ObjectId} instance of ObjectId.
    */
@@ -87,7 +87,7 @@ class ObjectId {
       this.id = id;
     } else if (id != null && id.toHexString) {
       // Duck-typing to support ObjectId from different npm packages
-      return id;
+      return ObjectId.createFromHexString(id.toHexString());
     } else {
       throw new TypeError(
         'Argument passed in must be a single String of 12 bytes or a string of 24 hex characters'
@@ -210,7 +210,7 @@ class ObjectId {
    * Compares the equality of this ObjectId with `otherID`.
    *
    * @method
-   * @param {object} otherID ObjectId instance to compare against.
+   * @param {object} otherId ObjectId instance to compare against.
    * @return {boolean} the result of comparing two ObjectId's
    */
   equals(otherId) {
@@ -246,7 +246,7 @@ class ObjectId {
    * Returns the generation date (accurate up to the second) that this ID was generated.
    *
    * @method
-   * @return {date} the generation date
+   * @return {Date} the generation date
    */
   getTimestamp() {
     const timestamp = new Date();
@@ -411,5 +411,8 @@ ObjectId.prototype[util.inspect.custom || 'inspect'] = ObjectId.prototype.toStri
  */
 ObjectId.index = ~~(Math.random() * 0xffffff);
 
-Object.defineProperty(ObjectId.prototype, '_bsontype', { value: 'ObjectId' });
+// In 4.0.0 and 4.0.1, this property name was changed to ObjectId to match the class name.
+// This caused interoperability problems with previous versions of the library, so in
+// later builds we changed it back to ObjectID (capital D) to match legacy implementations.
+Object.defineProperty(ObjectId.prototype, '_bsontype', { value: 'ObjectID' });
 module.exports = ObjectId;

Diff for: lib/parser/calculate_size.js

@@ -1,17 +1,6 @@
 'use strict';
 
 const Buffer = require('buffer').Buffer;
-const Long = require('../long');
-const Double = require('../double');
-const Timestamp = require('../timestamp');
-const ObjectId = require('../objectid');
-const BSONSymbol = require('../symbol');
-const BSONRegExp = require('../regexp');
-const Code = require('../code');
-const Decimal128 = require('../decimal128');
-const MinKey = require('../min_key');
-const MaxKey = require('../max_key');
-const DBRef = require('../db_ref');
 const Binary = require('../binary');
 const normalizedFunctionString = require('./utils').normalizedFunctionString;
 const constants = require('../constants');
@@ -86,15 +75,9 @@ function calculateElement(name, value, serializeFunctions, isArray, ignoreUndefi
     case 'boolean':
       return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + (1 + 1);
     case 'object':
-      if (
-        value == null ||
-        value instanceof MinKey ||
-        value instanceof MaxKey ||
-        value['_bsontype'] === 'MinKey' ||
-        value['_bsontype'] === 'MaxKey'
-      ) {
+      if (value == null || value['_bsontype'] === 'MinKey' || value['_bsontype'] === 'MaxKey') {
         return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + 1;
-      } else if (value instanceof ObjectId || value['_bsontype'] === 'ObjectId') {
+      } else if (value['_bsontype'] === 'ObjectId' || value['_bsontype'] === 'ObjectID') {
         return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + (12 + 1);
       } else if (value instanceof Date || isDate(value)) {
         return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + (8 + 1);
@@ -103,17 +86,14 @@ function calculateElement(name, value, serializeFunctions, isArray, ignoreUndefi
           (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + (1 + 4 + 1) + value.length
         );
       } else if (
-        value instanceof Long ||
-        value instanceof Double ||
-        value instanceof Timestamp ||
         value['_bsontype'] === 'Long' ||
         value['_bsontype'] === 'Double' ||
         value['_bsontype'] === 'Timestamp'
       ) {
         return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + (8 + 1);
-      } else if (value instanceof Decimal128 || value['_bsontype'] === 'Decimal128') {
+      } else if (value['_bsontype'] === 'Decimal128') {
         return (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + (16 + 1);
-      } else if (value instanceof Code || value['_bsontype'] === 'Code') {
+      } else if (value['_bsontype'] === 'Code') {
         // Calculate size depending on the availability of a scope
         if (value.scope != null && Object.keys(value.scope).length > 0) {
           return (
@@ -134,7 +114,7 @@ function calculateElement(name, value, serializeFunctions, isArray, ignoreUndefi
             1
           );
         }
-      } else if (value instanceof Binary || value['_bsontype'] === 'Binary') {
+      } else if (value['_bsontype'] === 'Binary') {
         // Check what kind of subtype we have
         if (value.sub_type === Binary.SUBTYPE_BYTE_ARRAY) {
           return (
@@ -146,15 +126,15 @@ function calculateElement(name, value, serializeFunctions, isArray, ignoreUndefi
             (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) + (value.position + 1 + 4 + 1)
           );
         }
-      } else if (value instanceof BSONSymbol || value['_bsontype'] === 'Symbol') {
+      } else if (value['_bsontype'] === 'Symbol') {
         return (
           (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) +
           Buffer.byteLength(value.value, 'utf8') +
           4 +
           1 +
           1
         );
-      } else if (value instanceof DBRef || value['_bsontype'] === 'DBRef') {
+      } else if (value['_bsontype'] === 'DBRef') {
         // Set up correct object for serialization
         const ordered_values = Object.assign(
           {
@@ -188,7 +168,7 @@ function calculateElement(name, value, serializeFunctions, isArray, ignoreUndefi
           (value.multiline ? 1 : 0) +
           1
         );
-      } else if (value instanceof BSONRegExp || value['_bsontype'] === 'BSONRegExp') {
+      } else if (value['_bsontype'] === 'BSONRegExp') {
         return (
           (name != null ? Buffer.byteLength(name, 'utf8') + 1 : 0) +
           1 +

Diff for: lib/parser/serializer.js

@@ -4,7 +4,6 @@ const Buffer = require('buffer').Buffer;
 const writeIEEE754 = require('../float_parser').writeIEEE754;
 const Long = require('../long');
 const Map = require('../map');
-const MinKey = require('../min_key');
 const Binary = require('../binary');
 const constants = require('../constants');
 const normalizedFunctionString = require('./utils').normalizedFunctionString;
@@ -254,7 +253,7 @@ function serializeMinMax(buffer, key, value, index, isArray) {
   // Write the type of either min or max key
   if (value === null) {
     buffer[index++] = constants.BSON_DATA_NULL;
-  } else if (value instanceof MinKey) {
+  } else if (value._bsontype === 'MinKey') {
     buffer[index++] = constants.BSON_DATA_MIN_KEY;
   } else {
     buffer[index++] = constants.BSON_DATA_MAX_KEY;
@@ -640,7 +639,7 @@ function serializeDBRef(buffer, key, value, index, depth, serializeFunctions, is
   let startIndex = index;
   let endIndex;
   let output = {
-    $ref: value.collection,
+    $ref: value.collection || value.namespace, // "namespace" was what library 1.x called "collection"
     $id: value.oid
   };
 
@@ -765,6 +764,8 @@ function serializeInto(
         index = serializeInt32(buffer, key, value, index, true);
       } else if (value['_bsontype'] === 'MinKey' || value['_bsontype'] === 'MaxKey') {
         index = serializeMinMax(buffer, key, value, index, true);
+      } else if (typeof value['_bsontype'] !== 'undefined') {
+        throw new TypeError('Unrecognized or invalid _bsontype: ' + value['_bsontype']);
       }
     }
   } else if (object instanceof Map) {
@@ -862,6 +863,8 @@ function serializeInto(
         index = serializeInt32(buffer, key, value, index);
       } else if (value['_bsontype'] === 'MinKey' || value['_bsontype'] === 'MaxKey') {
         index = serializeMinMax(buffer, key, value, index);
+      } else if (typeof value['_bsontype'] !== 'undefined') {
+        throw new TypeError('Unrecognized or invalid _bsontype: ' + value['_bsontype']);
       }
     }
   } else {
@@ -964,6 +967,8 @@ function serializeInto(
         index = serializeInt32(buffer, key, value, index);
       } else if (value['_bsontype'] === 'MinKey' || value['_bsontype'] === 'MaxKey') {
         index = serializeMinMax(buffer, key, value, index);
+      } else if (typeof value['_bsontype'] !== 'undefined') {
+        throw new TypeError('Unrecognized or invalid _bsontype: ' + value['_bsontype']);
       }
     }
   }

Diff for: lib/timestamp.js

@@ -10,7 +10,7 @@ const Long = require('./long');
  */
 class Timestamp extends Long {
   constructor(low, high) {
-    if (low instanceof Long) {
+    if (Long.isLong(low)) {
       super(low.low, low.high);
     } else {
       super(low, high);

Diff for: test/node/bson_test.js

@@ -2286,8 +2286,34 @@ describe('BSON', function() {
   });
 
   it('should serialize ObjectIds from old bson versions', function() {
-    // create a wrapper simulating the old ObjectID class
-    class ObjectID {
+    // In versions 4.0.0 and 4.0.1, we used _bsontype="ObjectId" which broke
+    // backwards compatibility with mongodb-core and other code. It was reverted
+    // back to "ObjectID" (capital D) in later library versions.
+    // The test below ensures that all three versions of Object ID work OK:
+    // 1. The current version's class
+    // 2. A simulation of the class from library 4.0.0
+    // 3. The class currently in use by mongodb (not tested in browser where mongodb is unavailable)
+
+    // test the old ObjectID class (in mongodb-core 3.1) because MongoDB drivers still return it
+    function getOldBSON() {
+      try {
+        // do a dynamic resolve to avoid exception when running browser tests
+        const file = require.resolve('mongodb-core');
+        const oldModule = require(file).BSON;
+        const funcs = new oldModule.BSON();
+        oldModule.serialize = funcs.serialize;
+        oldModule.deserialize = funcs.deserialize;
+        return oldModule;
+      } catch (e) {
+        return BSON; // if mongo is unavailable, e.g. browser tests, just re-use new BSON
+      }
+    }
+
+    const OldBSON = getOldBSON();
+    const OldObjectID = OldBSON === BSON ? BSON.ObjectId : OldBSON.ObjectID;
+
+    // create a wrapper simulating the old ObjectId class from v4.0.0
+    class ObjectIdv400 {
       constructor() {
         this.oid = new ObjectId();
       }
@@ -2298,10 +2324,10 @@ describe('BSON', function() {
         return this.oid.toString();
       }
     }
-    Object.defineProperty(ObjectID.prototype, '_bsontype', { value: 'ObjectID' });
+    Object.defineProperty(ObjectIdv400.prototype, '_bsontype', { value: 'ObjectId' });
 
     // Array
-    const array = [new ObjectID(), new ObjectId()];
+    const array = [new ObjectIdv400(), new OldObjectID(), new ObjectId()];
     const deserializedArrayAsMap = BSON.deserialize(BSON.serialize(array));
     const deserializedArray = Object.keys(deserializedArrayAsMap).map(
       x => deserializedArrayAsMap[x]
@@ -2310,7 +2336,8 @@ describe('BSON', function() {
 
     // Map
     const map = new Map();
-    map.set('oldBsonType', new ObjectID());
+    map.set('oldBsonType', new ObjectIdv400());
+    map.set('reallyOldBsonType', new OldObjectID());
     map.set('newBsonType', new ObjectId());
     const deserializedMapAsObject = BSON.deserialize(BSON.serialize(map), { relaxed: false });
     const deserializedMap = new Map(
@@ -2324,10 +2351,21 @@ describe('BSON', function() {
     });
 
     // Object
-    const record = { oldBsonType: new ObjectID(), newBsonType: new ObjectId() };
+    const record = { oldBsonType: new ObjectIdv400(), reallyOldBsonType: new OldObjectID, newBsonType: new ObjectId() };
     const deserializedObject = BSON.deserialize(BSON.serialize(record));
-    expect(deserializedObject).to.have.keys(['oldBsonType', 'newBsonType']);
+    expect(deserializedObject).to.have.keys(['oldBsonType', 'reallyOldBsonType', 'newBsonType']);
     expect(record.oldBsonType.toString()).to.equal(deserializedObject.oldBsonType.toString());
     expect(record.newBsonType.toString()).to.equal(deserializedObject.newBsonType.toString());
   });
+
+  it('should throw if invalid BSON types are input to BSON serializer', function() {
+    const oid = new ObjectId('111111111111111111111111');
+    const badBsonType = Object.assign({}, oid, { _bsontype: 'bogus' });
+    const badDoc = { bad: badBsonType };
+    const badArray = [oid, badDoc];
+    const badMap = new Map([['a', badBsonType], ['b', badDoc], ['c', badArray]]);
+    expect(() => BSON.serialize(badDoc)).to.throw();
+    expect(() => BSON.serialize(badArray)).to.throw();
+    expect(() => BSON.serialize(badMap)).to.throw();
+  });
 });