From d02dfc474d8a8dd2d0544a57cf69c70887dbb31c Mon Sep 17 00:00:00 2001 From: bailey Date: Thu, 23 Jan 2025 08:40:21 -0700 Subject: [PATCH 1/5] changes --- .eslintrc.js | 3 +- docs/field-level-encryption.md | 38 +++ lib/encryptionUtils.js | 72 +++++ lib/schema.js | 167 ++++++++-- test/encrypted_schema.test.js | 538 +++++++++++++++++++++++++++++++++ 5 files changed, 784 insertions(+), 34 deletions(-) create mode 100644 lib/encryptionUtils.js create mode 100644 test/encrypted_schema.test.js diff --git a/.eslintrc.js b/.eslintrc.js index 002d1e7b8b9..91b38166932 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -15,7 +15,8 @@ module.exports = { '!.*', 'node_modules', '.git', - 'data' + 'data', + '.config' ], overrides: [ { diff --git a/docs/field-level-encryption.md b/docs/field-level-encryption.md index 3531fca0218..13daef15dfe 100644 --- a/docs/field-level-encryption.md +++ b/docs/field-level-encryption.md @@ -112,3 +112,41 @@ With the above connection, if you create a model named 'Test' that uses the 'tes const Model = mongoose.model('Test', mongoose.Schema({ name: String })); await Model.create({ name: 'super secret' }); ``` + +## Automatic FLE in Mongoose + +Mongoose supports the declaration of encrypted schemas - schemas that, when connected to a model, utilize MongoDB's Client Side +Field Level Encryption or Queryable Encryption under the hood. Mongoose automatically generates either an `encryptedFieldsMap` or a +`schemaMap` when instantiating a MongoClient and encrypts fields on write and decrypts fields on reads. + +### Encryption types + +MongoDB has two different automatic encryption implementations: client side field level encryption (CSFLE) and queryable encryption (QE). +See [choosing an in-use encryption approach](https://www.mongodb.com/docs/v7.3/core/queryable-encryption/about-qe-csfle/#choosing-an-in-use-encryption-approach). + +### Declaring Encrypted Schemas + +The following schema declares two properties, `name` and `ssn`. `ssn` is encrypted using queryable encryption, and +is configured for equality queries: + +```javascript +const encryptedUserSchema = new Schema({ + name: String, + ssn: { + type: String, + // 1 + encrypt: { + keyId: '', + queries: 'equality' + } + } + // 2 +}, { encryptionType: 'queryableEncryption' }); +``` + +To declare a field as encrypted, you must: + +1. Annotate the field with encryption metadata in the schema definition +2. Choose an encryption type for the schema and configure the schema for the encryption type + +Not all schematypes are supported for CSFLE and QE. For an overview of valid schema types, refer to MongoDB's documentation. diff --git a/lib/encryptionUtils.js b/lib/encryptionUtils.js new file mode 100644 index 00000000000..916ef17a5cd --- /dev/null +++ b/lib/encryptionUtils.js @@ -0,0 +1,72 @@ +'use strict'; + +const schemaTypes = require('./schema/index.js'); +const SchemaBigInt = require('./schema/bigint'); +const SchemaBoolean = require('./schema/boolean'); +const SchemaBuffer = require('./schema/buffer'); +const SchemaDate = require('./schema/date'); +const SchemaDecimal128 = require('./schema/decimal128'); +const SchemaDouble = require('./schema/double'); +const SchemaInt32 = require('./schema/int32'); +const SchemaObjectId = require('./schema/objectId'); +const SchemaString = require('./schema/string'); + +/** + * Given a schema and a path to a field in the schema, this returns the + * BSON type of the field, if it can be determined. This method specifically + * **only** handles BSON types that are used for CSFLE and QE - any other + * BSON types will return `null`. (example: MinKey and MaxKey). + * + * @param {import('.').Schema} schema + * @param {string} path + * @returns + */ +function inferBSONType(schema, path) { + const type = schema.path(path); + + if (type instanceof SchemaString) { + return 'string'; + } + + if (type instanceof SchemaInt32) { + return 'int'; + } + + if (type instanceof SchemaBigInt) { + return 'long'; + } + + if (type instanceof SchemaBoolean) { + return 'bool'; + } + + if (type instanceof SchemaDate) { + return 'date'; + } + + if (type instanceof SchemaBuffer) { + return 'binData'; + } + + if (type instanceof SchemaObjectId) { + return 'objectId'; + } + + if (type instanceof SchemaDecimal128) { + return 'decimal'; + } + + if (type instanceof SchemaDouble) { + return 'double'; + } + + if (type instanceof schemaTypes.Array) { + return 'array'; + } + + return null; +} + +module.exports = { + inferBSONType +}; diff --git a/lib/schema.js b/lib/schema.js index 0204c6cc9c4..62053ed2511 100644 --- a/lib/schema.js +++ b/lib/schema.js @@ -25,6 +25,7 @@ const setPopulatedVirtualValue = require('./helpers/populate/setPopulatedVirtual const setupTimestamps = require('./helpers/timestamps/setupTimestamps'); const utils = require('./utils'); const validateRef = require('./helpers/populate/validateRef'); +const { inferBSONType } = require('./encryptionUtils'); const hasNumericSubpathRegex = /\.\d+(\.|$)/; @@ -86,6 +87,7 @@ const numberRE = /^\d+$/; * - [pluginTags](https://mongoosejs.com/docs/guide.html#pluginTags): array of strings - defaults to `undefined`. If set and plugin called with `tags` option, will only apply that plugin to schemas with a matching tag. * - [virtuals](https://mongoosejs.com/docs/tutorials/virtuals.html#virtuals-via-schema-options): object - virtuals to define, alias for [`.virtual`](https://mongoosejs.com/docs/api/schema.html#Schema.prototype.virtual()) * - [collectionOptions]: object with options passed to [`createCollection()`](https://www.mongodb.com/docs/manual/reference/method/db.createCollection/) when calling `Model.createCollection()` or `autoCreate` set to true. + * - [encryptionType]: the encryption type for the schema. Valid options are `csfle` or `queryableEncryption`. See https://mongoosejs.com/docs/field-level-encryption. * * #### Options for Nested Schemas: * @@ -128,6 +130,7 @@ function Schema(obj, options) { // For internal debugging. Do not use this to try to save a schema in MDB. this.$id = ++id; this.mapPaths = []; + this.encryptedFields = {}; this.s = { hooks: new Kareem() @@ -166,7 +169,7 @@ function Schema(obj, options) { // ensure the documents get an auto _id unless disabled const auto_id = !this.paths['_id'] && - (this.options._id) && !_idSubDoc; + (this.options._id) && !_idSubDoc; if (auto_id) { addAutoId(this); @@ -463,6 +466,8 @@ Schema.prototype._clone = function _clone(Constructor) { s.aliases = Object.assign({}, this.aliases); + s.encryptedFields = clone(this.encryptedFields); + return s; }; @@ -495,7 +500,17 @@ Schema.prototype.pick = function(paths, options) { } for (const path of paths) { - if (this.nested[path]) { + if (path in this.encryptedFields) { + const encrypt = this.encryptedFields[path]; + const schemaType = this.path(path); + newSchema.add({ + [path]: { + encrypt, + [this.options.typeKey]: schemaType + } + }); + } + else if (this.nested[path]) { newSchema.add({ [path]: get(this.tree, path) }); } else { const schematype = this.path(path); @@ -506,6 +521,10 @@ Schema.prototype.pick = function(paths, options) { } } + if (!this._hasEncryptedFields()) { + newSchema.options.encryptionType = null; + } + return newSchema; }; @@ -534,9 +553,9 @@ Schema.prototype.omit = function(paths, options) { if (!Array.isArray(paths)) { throw new MongooseError( 'Schema#omit() only accepts an array argument, ' + - 'got "' + - typeof paths + - '"' + 'got "' + + typeof paths + + '"' ); } @@ -667,6 +686,20 @@ Schema.prototype._defaultToObjectOptions = function(json) { return defaultOptions; }; +/** + * Sets the encryption type of the schema, if a value is provided, otherwise + * returns the encryption type. + * + * @param {'csfle' | 'queryableEncryption' | undefined} encryptionType plain object with paths to add, or another schema + */ +Schema.prototype.encryptionType = function encryptionType(encryptionType) { + if (typeof encryptionType === 'string' || encryptionType === null) { + this.options.encryptionType = encryptionType; + } else { + return this.options.encryptionType; + } +}; + /** * Adds key path / schema type pairs to this schema. * @@ -735,7 +768,7 @@ Schema.prototype.add = function add(obj, prefix) { if ( key !== '_id' && ((typeof val !== 'object' && typeof val !== 'function' && !isMongooseTypeString) || - val == null) + val == null) ) { throw new TypeError(`Invalid schema configuration: \`${val}\` is not ` + `a valid type at path \`${key}\`. See ` + @@ -818,15 +851,71 @@ Schema.prototype.add = function add(obj, prefix) { } } } + + if (val.instanceOfSchema && val.encryptionType() != null) { + // schema.add({ field: }) + if (this.encryptionType() != val.encryptionType()) { + throw new Error('encryptionType of a nested schema must match the encryption type of the parent schema.'); + } + + for (const [encryptedField, encryptedFieldConfig] of Object.entries(val.encryptedFields)) { + const path = fullPath + '.' + encryptedField; + this._addEncryptedField(path, encryptedFieldConfig); + } + } + else if (typeof val === 'object' && 'encrypt' in val) { + // schema.add({ field: { type: , encrypt: { ... }}}) + const { encrypt } = val; + + if (this.encryptionType() == null) { + throw new Error('encryptionType must be provided'); + } + + this._addEncryptedField(fullPath, encrypt); + } else { + // if the field was already encrypted and we re-configure it to be unencrypted, remove + // the encrypted field configuration + this._removeEncryptedField(fullPath); + } } const aliasObj = Object.fromEntries( Object.entries(obj).map(([key]) => ([prefix + key, null])) ); aliasFields(this, aliasObj); + return this; }; +/** + * @param {string} path + * @param {object} fieldConfig + * + * @api private + */ +Schema.prototype._addEncryptedField = function _addEncryptedField(path, fieldConfig) { + const type = inferBSONType(this, path); + if (type == null) { + throw new Error('unable to determine bson type for field `' + path + '`'); + } + + this.encryptedFields[path] = clone(fieldConfig); +}; + +/** + * @api private + */ +Schema.prototype._removeEncryptedField = function _removeEncryptedField(path) { + delete this.encryptedFields[path]; +}; + +/** + * @api private + */ +Schema.prototype._hasEncryptedFields = function _hasEncryptedFields() { + return Object.keys(this.encryptedFields).length > 0; +}; + /** * Add an alias for `path`. This means getting or setting the `alias` * is equivalent to getting or setting the `path`. @@ -1008,23 +1097,23 @@ Schema.prototype.reserved = Schema.reserved; const reserved = Schema.reserved; // Core object reserved['prototype'] = -// EventEmitter -reserved.emit = -reserved.listeners = -reserved.removeListener = - -// document properties and functions -reserved.collection = -reserved.errors = -reserved.get = -reserved.init = -reserved.isModified = -reserved.isNew = -reserved.populated = -reserved.remove = -reserved.save = -reserved.toObject = -reserved.validate = 1; + // EventEmitter + reserved.emit = + reserved.listeners = + reserved.removeListener = + + // document properties and functions + reserved.collection = + reserved.errors = + reserved.get = + reserved.init = + reserved.isModified = + reserved.isNew = + reserved.populated = + reserved.remove = + reserved.save = + reserved.toObject = + reserved.validate = 1; reserved.collection = 1; /** @@ -1104,10 +1193,10 @@ Schema.prototype.path = function(path, obj) { } if (typeof branch[sub] !== 'object') { const msg = 'Cannot set nested path `' + path + '`. ' - + 'Parent path `' - + fullPath - + '` already set to type ' + branch[sub].name - + '.'; + + 'Parent path `' + + fullPath + + '` already set to type ' + branch[sub].name + + '.'; throw new Error(msg); } branch = branch[sub]; @@ -1375,6 +1464,16 @@ Schema.prototype.interpretAsType = function(path, obj, options) { let type = obj[options.typeKey] && (obj[options.typeKey] instanceof Function || options.typeKey !== 'type' || !obj.type.type) ? obj[options.typeKey] : {}; + + if (type instanceof SchemaType) { + if (type.path === path) { + return type; + } + const clone = type.clone(); + clone.path = path; + return clone; + } + let name; if (utils.isPOJO(type) || type === 'mixed') { @@ -1404,8 +1503,8 @@ Schema.prototype.interpretAsType = function(path, obj, options) { return new MongooseTypes.DocumentArray(path, cast, obj); } if (cast && - cast[options.typeKey] && - cast[options.typeKey].instanceOfSchema) { + cast[options.typeKey] && + cast[options.typeKey].instanceOfSchema) { if (!(cast[options.typeKey] instanceof Schema)) { if (this.options._isMerging) { cast[options.typeKey] = new Schema(cast[options.typeKey]); @@ -1739,7 +1838,7 @@ Schema.prototype.hasMixedParent = function(path) { for (let i = 0; i < subpaths.length; ++i) { path = i > 0 ? path + '.' + subpaths[i] : subpaths[i]; if (this.paths.hasOwnProperty(path) && - this.paths[path] instanceof MongooseTypes.Mixed) { + this.paths[path] instanceof MongooseTypes.Mixed) { return this.paths[path]; } } @@ -2520,6 +2619,8 @@ Schema.prototype.remove = function(path) { delete this.paths[name]; _deletePath(this, name); + + this._removeEncryptedField(name); }, this); } return this; @@ -2615,9 +2716,9 @@ Schema.prototype.removeVirtual = function(path) { Schema.prototype.loadClass = function(model, virtualsOnly) { // Stop copying when hit certain base classes if (model === Object.prototype || - model === Function.prototype || - model.prototype.hasOwnProperty('$isMongooseModelPrototype') || - model.prototype.hasOwnProperty('$isMongooseDocumentPrototype')) { + model === Function.prototype || + model.prototype.hasOwnProperty('$isMongooseModelPrototype') || + model.prototype.hasOwnProperty('$isMongooseDocumentPrototype')) { return this; } diff --git a/test/encrypted_schema.test.js b/test/encrypted_schema.test.js new file mode 100644 index 00000000000..8f35dc97b73 --- /dev/null +++ b/test/encrypted_schema.test.js @@ -0,0 +1,538 @@ + +'use strict'; + +const assert = require('assert'); +const start = require('./common'); +const { ObjectId, Decimal128 } = require('../lib/types'); +const { Double, Int32, UUID } = require('bson'); + +const mongoose = start.mongoose; +const Schema = mongoose.Schema; + +/** + * + * @param {import('../lib').Schema} object + * @param {Array | string} path + * @returns + */ +function schemaHasEncryptedProperty(schema, path) { + path = [path].flat(); + path = path.join('.'); + + return path in schema.encryptedFields; +} + +const KEY_ID = new UUID(); +const algorithm = 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'; + +describe('encrypted schema declaration', function() { + describe('Tests that fields of valid schema types can be declared as encrypted schemas', function() { + const basicSchemaTypes = [ + { type: String, name: 'string' }, + { type: Schema.Types.Boolean, name: 'boolean' }, + { type: Schema.Types.Buffer, name: 'buffer' }, + { type: Date, name: 'date' }, + { type: ObjectId, name: 'objectid' }, + { type: BigInt, name: 'bigint' }, + { type: Decimal128, name: 'Decimal128' }, + { type: Int32, name: 'int32' }, + { type: Double, name: 'double' } + ]; + + for (const { type, name } of basicSchemaTypes) { + describe(`When a schema is instantiated with an encrypted field of type ${name}`, function() { + let schema; + beforeEach(function() { + schema = new Schema({ + field: { + type, encrypt: { keyId: KEY_ID, algorithm } + } + }, { + encryptionType: 'csfle' + }); + }); + + it(`Then the schema has an encrypted property of type ${name}`, function() { + assert.ok(schemaHasEncryptedProperty(schema, 'field')); + }); + }); + } + + describe('when a schema is instantiated with a nested encrypted schema', function() { + let schema; + beforeEach(function() { + const encryptedSchema = new Schema({ + encrypted: { + type: String, encrypt: { keyId: KEY_ID, algorithm } + } + }, { encryptionType: 'csfle' }); + schema = new Schema({ + field: encryptedSchema + }, { encryptionType: 'csfle' }); + }); + + + it('then the schema has a nested property that is encrypted', function() { + assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])); + }); + }); + + describe('when a schema is instantiated with a nested schema object', function() { + let schema; + beforeEach(function() { + schema = new Schema({ + field: { + encrypted: { + type: String, encrypt: { keyId: KEY_ID, algorithm } + } + } + }, { encryptionType: 'csfle' }); + }); + + it('then the schema has a nested property that is encrypted', function() { + assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])); + }); + }); + + describe('when a schema is instantiated as an Array', function() { + let schema; + beforeEach(function() { + schema = new Schema({ + encrypted: { + type: [Number], + encrypt: { keyId: KEY_ID, algorithm } + } + }, { encryptionType: 'csfle' }); + }); + + it('then the schema has a nested property that is encrypted', function() { + assert.ok(schemaHasEncryptedProperty(schema, 'encrypted')); + }); + }); + + }); + + describe('invalid schema types for encrypted schemas', function() { + describe('When a schema is instantiated with an encrypted field of type Number', function() { + it('Then an error is thrown', function() { + assert.throws(() => { + new Schema({ + field: { + type: Number, encrypt: { keyId: KEY_ID, algorithm } + } + }, { encryptionType: 'csfle' }); + }, /unable to determine bson type for field `field`/); + }); + }); + + describe('When a schema is instantiated with an encrypted field of type Mixed', function() { + it('Then an error is thrown', function() { + assert.throws(() => { + new Schema({ + field: { + type: Schema.Types.Mixed, encrypt: { keyId: KEY_ID, algorithm } + } + }, { encryptionType: 'csfle' }); + }, /unable to determine bson type for field `field`/); + }); + }); + + describe('When a schema is instantiated with a custom schema type plugin', function() { + class Int8 extends mongoose.SchemaType { + constructor(key, options) { + super(key, options, 'Int8'); + } + } + + beforeEach(function() { + // Don't forget to add `Int8` to the type registry + mongoose.Schema.Types.Int8 = Int8; + }); + afterEach(function() { + delete mongoose.Schema.Types.Int8; + }); + + it('Then an error is thrown', function() { + assert.throws(() => { + new Schema({ + field: { + type: Int8, encrypt: { keyId: KEY_ID, algorithm } + } + }, { encryptionType: 'csfle' }); + }, /unable to determine bson type for field `field`/); + }); + }); + + }); + + describe('options.encryptionType', function() { + describe('when an encrypted schema is instantiated and an encryptionType is not provided', function() { + it('an error is thrown', function() { + assert.throws( + () => { + new Schema({ + field: { + type: String, + encrypt: { keyId: KEY_ID, algorithm } + } + }); + }, /encryptionType must be provided/ + ); + + + }); + }); + + describe('when a nested encrypted schema is provided to schema constructor and the encryption types are different', function() { + it('then an error is thrown', function() { + const innerSchema = new Schema({ + field1: { + type: String, encrypt: { + keyId: KEY_ID, + queries: { type: 'equality' } + } + } + }, { encryptionType: 'csfle' }); + assert.throws(() => { + new Schema({ + field1: innerSchema + }, { encryptionType: 'queryableEncryption' }); + }, /encryptionType of a nested schema must match the encryption type of the parent schema/); + }); + }); + }); + + describe('tests for schema mutation methods', function() { + describe('Schema.prototype.add()', function() { + describe('Given a schema with no encrypted fields', function() { + describe('When an encrypted field is added', function() { + it('Then the encrypted field is added to the encrypted fields for the schema', function() { + const schema = new Schema({ + field1: Number + }); + schema.encryptionType('csfle'); + schema.add( + { name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } } + ); + assert.ok(schemaHasEncryptedProperty(schema, ['name'])); + }); + }); + }); + + describe('Given a schema with an encrypted field', function() { + describe('when an encrypted field is added', function() { + describe('and the encryption type matches the existing encryption type', function() { + it('Then the encrypted field is added to the encrypted fields for the schema', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.add( + { name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } } + ); + assert.ok(schemaHasEncryptedProperty(schema, ['name'])); + }); + }); + }); + }); + + describe('Given a schema with an encrypted field', function() { + describe('when an encrypted field is added with different encryption settings for the same field', function() { + it('The encryption settings for the field are overridden', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.add( + { name: { type: String, encrypt: { keyId: new UUID(), algorithm } } } + ); + assert.notEqual(schema.encryptedFields['name'].keyId, KEY_ID); + }); + + }); + + describe('When an unencrypted field is added for the same field', function() { + it('The field on the schema is overridden', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.add( + { field1: String } + ); + assert.equal(schemaHasEncryptedProperty(schema, ['field1']), false); + }); + + }); + }); + + describe('Given a schema', function() { + describe('When multiple encrypted fields are added to the schema in one call to add()', function() { + it('Then all the encrypted fields are added to the schema', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.add( + { + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + } + ); + + assert.ok(schemaHasEncryptedProperty(schema, ['name'])); + assert.ok(schemaHasEncryptedProperty(schema, ['age'])); + }); + }); + }); + }); + + describe('Schema.prototype.remove()', function() { + describe('Given a schema with one encrypted field', function() { + describe('When the encrypted field is removed', function() { + it('Then the encrypted fields on the schema does not contain the removed field', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.remove('field1'); + + assert.equal(schemaHasEncryptedProperty(schema, ['field1']), false); + }); + }); + }); + + describe('Given a schema with multiple encrypted fields', function() { + describe('When one encrypted field is removed', function() { + it('The encrypted fields on the schema does not contain the removed field', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.remove(['field1']); + + assert.equal(schemaHasEncryptedProperty(schema, ['field1']), false); + assert.equal(schemaHasEncryptedProperty(schema, ['name']), true); + assert.equal(schemaHasEncryptedProperty(schema, ['age']), true); + }); + }); + + describe('When all encrypted fields are removed', function() { + it('The encrypted fields on the schema does not contain the removed field', function() { + const schema = new Schema({ + field1: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + schema.remove(['field1', 'name', 'age']); + + assert.equal(schemaHasEncryptedProperty(schema, ['field1']), false); + assert.equal(schemaHasEncryptedProperty(schema, ['name']), false); + assert.equal(schemaHasEncryptedProperty(schema, ['age']), false); + }); + }); + }); + + describe('when a nested encrypted property is removed', function() { + it('the encrypted field is removed from the schema', function() { + const schema = new Schema({ + field1: { name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } } + }, { encryptionType: 'csfle' }); + + assert.equal(schemaHasEncryptedProperty(schema, ['field1.name']), true); + + schema.remove(['field1.name']); + + assert.equal(schemaHasEncryptedProperty(schema, ['field1.name']), false); + }); + }); + }); + }); + + describe('tests for schema copying methods', function() { + describe('Schema.prototype.clone()', function() { + describe('Given a schema with encrypted fields', function() { + describe('When the schema is cloned', function() { + it('The resultant schema contains all the same encrypted fields as the original schema', function() { + const schema1 = new Schema({ name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } }, { encryptionType: 'csfle' }); + const schema2 = schema1.clone(); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), true); + }); + it('The encryption type of the cloned schema is the same as the original', function() { + const schema1 = new Schema({ name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } }, { encryptionType: 'csfle' }); + const schema2 = schema1.clone(); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + describe('When the cloned schema is modified', function() { + it('The original is not modified', function() { + const schema1 = new Schema({ name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } }, { encryptionType: 'csfle' }); + const schema2 = schema1.clone(); + schema2.remove('name'); + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), false); + assert.equal(schemaHasEncryptedProperty(schema1, ['name']), true); + }); + }); + }); + }); + }); + + describe('Schema.prototype.pick()', function() { + describe('When pick() is used with only unencrypted fields', function() { + it('Then the resultant schema has none of the original schema’s encrypted fields', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name1', 'age1']); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), false); + assert.equal(schemaHasEncryptedProperty(schema2, ['age']), false); + }); + it('Then the encryption type is set to the cloned schemas encryptionType', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name1', 'age1']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + + describe('When pick() is used with some unencrypted fields', function() { + it('Then the resultant schema has the encrypted fields of the original schema that were specified to pick().', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name', 'age1']); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), true); + assert.equal(schemaHasEncryptedProperty(schema2, ['age']), false); + }); + it('Then the encryption type is the same as the original schema’s encryption type', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name', 'age1']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + + describe('When pick() is used with nested paths', function() { + it('Then the resultant schema has the encrypted fields of the original schema that were specified to pick().', function() { + const originalSchema = new Schema({ + name: { + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name.name', 'age1']); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name', 'name']), true); + assert.equal(schemaHasEncryptedProperty(schema2, ['age']), false); + }); + it('Then the encryption type is the same as the original schema’s encryption type', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.pick(['name', 'age1']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + }); + + describe('Schema.prototype.omit()', function() { + describe('When omit() is used with only unencrypted fields', function() { + it('Then the resultant schema has all the original schema’s encrypted fields', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.omit(['name1', 'age1']); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), true); + assert.equal(schemaHasEncryptedProperty(schema2, ['age']), true); + }); + it('Then the encryption type is the same as the original schema’s encryption type', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.omit(['name1', 'age1']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + + describe('When omit() is used with some unencrypted fields', function() { + it('Then the resultant schema has the encrypted fields of the original schema that were specified to omit()', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.omit(['name', 'age1']); + + assert.equal(schemaHasEncryptedProperty(schema2, ['name']), false); + assert.equal(schemaHasEncryptedProperty(schema2, ['age']), true); + }); + it('Then the encryption type is the same as the original schema’s encryption type', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.omit(['name', 'age1']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + + describe('When omit() is used with some all the encrypted fields', function() { + it('Then the encryption type is the same as the original schema’s encryption type', function() { + const originalSchema = new Schema({ + name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, + age: { type: Int32, encrypt: { keyId: KEY_ID, algorithm } }, + name1: String, + age1: Int32 + }, { encryptionType: 'csfle' }); + + const schema2 = originalSchema.omit(['name', 'age']); + + assert.equal(schema2.encryptionType(), 'csfle'); + }); + }); + }); + }); +}); From 4402f80ed8447b5cecce0dafd2abd891df7477db Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Thu, 23 Jan 2025 08:43:48 -0700 Subject: [PATCH 2/5] Update lib/encryptionUtils.js Co-authored-by: Aditi Khare <106987683+aditi-khare-mongoDB@users.noreply.github.com> --- lib/encryptionUtils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/encryptionUtils.js b/lib/encryptionUtils.js index 916ef17a5cd..f0c46dee716 100644 --- a/lib/encryptionUtils.js +++ b/lib/encryptionUtils.js @@ -19,7 +19,7 @@ const SchemaString = require('./schema/string'); * * @param {import('.').Schema} schema * @param {string} path - * @returns + * @returns {string} */ function inferBSONType(schema, path) { const type = schema.path(path); From a6a51bd56a75136127989d7586785e470ae30083 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Thu, 23 Jan 2025 08:44:21 -0700 Subject: [PATCH 3/5] Update test/encrypted_schema.test.js Co-authored-by: Aditi Khare <106987683+aditi-khare-mongoDB@users.noreply.github.com> --- test/encrypted_schema.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/encrypted_schema.test.js b/test/encrypted_schema.test.js index 8f35dc97b73..529ffe42ac0 100644 --- a/test/encrypted_schema.test.js +++ b/test/encrypted_schema.test.js @@ -519,7 +519,7 @@ describe('encrypted schema declaration', function() { }); }); - describe('When omit() is used with some all the encrypted fields', function() { + describe('When omit() is used with all the encrypted fields', function() { it('Then the encryption type is the same as the original schema’s encryption type', function() { const originalSchema = new Schema({ name: { type: String, encrypt: { keyId: KEY_ID, algorithm } }, From be4563e73069ac35f318fea2a61e1062d95ba44c Mon Sep 17 00:00:00 2001 From: bailey Date: Thu, 23 Jan 2025 15:47:35 -0700 Subject: [PATCH 4/5] Add support for encrypted models --- docs/field-level-encryption.md | 10 + lib/collection.js | 2 +- lib/connection.js | 20 +- lib/drivers/node-mongodb-native/connection.js | 52 +- lib/encryption_utils.js | 72 ++ lib/schema.js | 57 ++ lib/utils.js | 32 +- package.json | 2 +- scripts/configure-cluster-with-encryption.sh | 2 +- scripts/run-encryption-tests.sh | 39 + test/encrypted_schema.test.js | 677 ++++++++++++++++-- test/encryption/encryption.test.js | 629 ++++++++++++++-- test/model.test.js | 46 +- types/query.d.ts | 44 +- 14 files changed, 1473 insertions(+), 211 deletions(-) create mode 100644 lib/encryption_utils.js create mode 100755 scripts/run-encryption-tests.sh diff --git a/docs/field-level-encryption.md b/docs/field-level-encryption.md index 13daef15dfe..28ba0db9841 100644 --- a/docs/field-level-encryption.md +++ b/docs/field-level-encryption.md @@ -150,3 +150,13 @@ To declare a field as encrypted, you must: 2. Choose an encryption type for the schema and configure the schema for the encryption type Not all schematypes are supported for CSFLE and QE. For an overview of valid schema types, refer to MongoDB's documentation. + +### Registering Models + +Encrypted schemas must be registered on a connection, not the Mongoose global: + +```javascript + +const connection = mongoose.createConnection(); +const UserModel = connection.model('User', encryptedUserSchema); +``` diff --git a/lib/collection.js b/lib/collection.js index e6c365c9a13..9f60ba4c01b 100644 --- a/lib/collection.js +++ b/lib/collection.js @@ -81,7 +81,7 @@ Collection.prototype.onOpen = function() { * @api private */ -Collection.prototype.onClose = function() {}; +Collection.prototype.onClose = function() { }; /** * Queues a method for later execution when its diff --git a/lib/connection.js b/lib/connection.js index b747460083c..8ab0a76893c 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -607,7 +607,7 @@ Connection.prototype.bulkWrite = async function bulkWrite(ops, options) { Connection.prototype.createCollections = async function createCollections(options = {}) { const result = {}; - const errorsMap = { }; + const errorsMap = {}; const { continueOnError } = options; delete options.continueOnError; @@ -734,7 +734,7 @@ Connection.prototype.transaction = function transaction(fn, options) { throw err; }). finally(() => { - session.endSession().catch(() => {}); + session.endSession().catch(() => { }); }); }); }; @@ -1025,7 +1025,7 @@ Connection.prototype.openUri = async function openUri(uri, options) { for (const model of Object.values(this.models)) { // Errors handled internally, so safe to ignore error - model.init().catch(function $modelInitNoop() {}); + model.init().catch(function $modelInitNoop() { }); } // `createConnection()` calls this `openUri()` function without @@ -1061,7 +1061,7 @@ Connection.prototype.openUri = async function openUri(uri, options) { // to avoid uncaught exceptions when using `on('error')`. See gh-14377. Connection.prototype.on = function on(event, callback) { if (event === 'error' && this.$initialConnection) { - this.$initialConnection.catch(() => {}); + this.$initialConnection.catch(() => { }); } return EventEmitter.prototype.on.call(this, event, callback); }; @@ -1083,7 +1083,7 @@ Connection.prototype.on = function on(event, callback) { // to avoid uncaught exceptions when using `on('error')`. See gh-14377. Connection.prototype.once = function on(event, callback) { if (event === 'error' && this.$initialConnection) { - this.$initialConnection.catch(() => {}); + this.$initialConnection.catch(() => { }); } return EventEmitter.prototype.once.call(this, event, callback); }; @@ -1412,7 +1412,7 @@ Connection.prototype.model = function model(name, schema, collection, options) { } // Errors handled internally, so safe to ignore error - model.init().catch(function $modelInitNoop() {}); + model.init().catch(function $modelInitNoop() { }); return model; } @@ -1439,7 +1439,7 @@ Connection.prototype.model = function model(name, schema, collection, options) { } if (this === model.prototype.db - && (!collection || collection === model.collection.name)) { + && (!collection || collection === model.collection.name)) { // model already uses this connection. // only the first model with this name is cached to allow @@ -1626,8 +1626,8 @@ Connection.prototype.authMechanismDoesNotRequirePassword = function authMechanis */ Connection.prototype.optionsProvideAuthenticationData = function optionsProvideAuthenticationData(options) { return (options) && - (options.user) && - ((options.pass) || this.authMechanismDoesNotRequirePassword()); + (options.user) && + ((options.pass) || this.authMechanismDoesNotRequirePassword()); }; /** @@ -1689,7 +1689,7 @@ Connection.prototype.createClient = function createClient() { */ Connection.prototype.syncIndexes = async function syncIndexes(options = {}) { const result = {}; - const errorsMap = { }; + const errorsMap = {}; const { continueOnError } = options; delete options.continueOnError; diff --git a/lib/drivers/node-mongodb-native/connection.js b/lib/drivers/node-mongodb-native/connection.js index 0659ac4e647..0e6515a321f 100644 --- a/lib/drivers/node-mongodb-native/connection.js +++ b/lib/drivers/node-mongodb-native/connection.js @@ -315,6 +315,16 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio }; } + const { schemaMap, encryptedFieldsMap } = this._buildEncryptionSchemas(); + + if (Object.keys(schemaMap).length > 0) { + options.autoEncryption.schemaMap = schemaMap; + } + + if (Object.keys(encryptedFieldsMap).length > 0) { + options.autoEncryption.encryptedFieldsMap = encryptedFieldsMap; + } + this.readyState = STATES.connecting; this._connectionString = uri; @@ -338,6 +348,40 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio return this; }; +/** + * Given a connection, which may or may not have encrypted models, build + * a schemaMap and/or an encryptedFieldsMap for the connection, combining all models + * into a single schemaMap and encryptedFields map. + * + * @returns a copy of the options object with a schemaMap and/or an encryptedFieldsMap added to the options' autoEncryption + * options. + */ +NativeConnection.prototype._buildEncryptionSchemas = function() { + const schemaMap = Object.values(this.models).filter(model => model.schema.encryptionType() === 'csfle').reduce( + (schemaMap, model) => { + const { schema, collection: { collectionName } } = model; + const namespace = `${this.$dbName}.${collectionName}`; + schemaMap[namespace] = schema._buildSchemaMap(); + return schemaMap; + }, + {} + ); + + const encryptedFieldsMap = Object.values(this.models).filter(model => model.schema.encryptionType() === 'qe').reduce( + (encryptedFieldsMap, model) => { + const { schema, collection: { collectionName } } = model; + const namespace = `${this.$dbName}.${collectionName}`; + encryptedFieldsMap[namespace] = schema._buildEncryptedFields(); + return encryptedFieldsMap; + }, + {} + ); + + return { + schemaMap, encryptedFieldsMap + }; +}; + /*! * ignore */ @@ -358,7 +402,7 @@ NativeConnection.prototype.setClient = function setClient(client) { for (const model of Object.values(this.models)) { // Errors handled internally, so safe to ignore error - model.init().catch(function $modelInitNoop() {}); + model.init().catch(function $modelInitNoop() { }); } return this; @@ -401,9 +445,9 @@ function _setClient(conn, client, options, dbName) { }; const type = client && - client.topology && - client.topology.description && - client.topology.description.type || ''; + client.topology && + client.topology.description && + client.topology.description.type || ''; if (type === 'Single') { client.on('serverDescriptionChanged', ev => { diff --git a/lib/encryption_utils.js b/lib/encryption_utils.js new file mode 100644 index 00000000000..1f17fa5032b --- /dev/null +++ b/lib/encryption_utils.js @@ -0,0 +1,72 @@ +'use strict'; + +const { Array } = require('./schema/index.js'); +const SchemaBigInt = require('./schema/bigint'); +const SchemaBoolean = require('./schema/boolean'); +const SchemaBuffer = require('./schema/buffer'); +const SchemaDate = require('./schema/date'); +const SchemaDecimal128 = require('./schema/decimal128'); +const SchemaDouble = require('./schema/double'); +const SchemaInt32 = require('./schema/int32'); +const SchemaObjectId = require('./schema/objectId'); +const SchemaString = require('./schema/string'); + +/** + * Given a schema and a path to a field in the schema, this returns the + * BSON type of the field, if it can be determined. This method specifically + * **only** handles BSON types that are used for CSFLE and QE - any other + * BSON types will return `null`. (example: MinKey and MaxKey). + * + * @param {import('.').Schema} schema + * @param {string} path + * @returns + */ +function inferBSONType(schema, path) { + const type = schema.path(path); + + if (type instanceof SchemaString) { + return 'string'; + } + + if (type instanceof SchemaInt32) { + return 'int'; + } + + if (type instanceof SchemaBigInt) { + return 'long'; + } + + if (type instanceof SchemaBoolean) { + return 'bool'; + } + + if (type instanceof SchemaDate) { + return 'date'; + } + + if (type instanceof SchemaBuffer) { + return 'binData'; + } + + if (type instanceof SchemaObjectId) { + return 'objectId'; + } + + if (type instanceof SchemaDecimal128) { + return 'decimal'; + } + + if (type instanceof SchemaDouble) { + return 'double'; + } + + if (type instanceof Array) { + return 'array'; + } + + return null; +} + +module.exports = { + inferBSONType +}; diff --git a/lib/schema.js b/lib/schema.js index 62053ed2511..b0d992dd684 100644 --- a/lib/schema.js +++ b/lib/schema.js @@ -916,6 +916,63 @@ Schema.prototype._hasEncryptedFields = function _hasEncryptedFields() { return Object.keys(this.encryptedFields).length > 0; }; +Schema.prototype._buildEncryptedFields = function() { + const fields = Object.entries(this.encryptedFields).map( + ([path, config]) => { + const bsonType = inferBSONType(this, path); + // { path, bsonType, keyId, queries? } + return { path, bsonType, ...config }; + }); + + return { fields }; +}; + +Schema.prototype._buildSchemaMap = function() { + /** + * `schemaMap`s are JSON schemas, which use the following structure to represent objects: + * { field: { bsonType: 'object', properties: { ... } } } + * + * for example, a schema that looks like this `{ a: { b: int32 } }` would be encoded as + * `{ a: { bsonType: 'object', properties: { b: < encryption configuration > } } }` + * + * This function takes an array of path segments, an output object (that gets mutated) and + * a value to associated with the full path, and constructs a valid CSFLE JSON schema path for + * the object. This works for deeply nested properties as well. + * + * @param {string[]} path array of path components + * @param {object} object the object in which to build a JSON schema of `path`'s properties + * @param {object} value the value to associate with the path in object + */ + function buildNestedPath(path, object, value) { + let i = 0, component = path[i]; + for (; i < path.length - 1; ++i, component = path[i]) { + object[component] = object[component] == null ? { + bsonType: 'object', + properties: {} + } : object[component]; + object = object[component].properties; + } + object[component] = value; + } + + const schemaMapPropertyReducer = (accum, [path, propertyConfig]) => { + const bsonType = inferBSONType(this, path); + const pathComponents = path.split('.'); + const configuration = { encrypt: { ...propertyConfig, bsonType } }; + buildNestedPath(pathComponents, accum, configuration); + return accum; + }; + + const properties = Object.entries(this.encryptedFields).reduce( + schemaMapPropertyReducer, + {}); + + return { + bsonType: 'object', + properties + }; +}; + /** * Add an alias for `path`. This means getting or setting the `alias` * is equivalent to getting or setting the `path`. diff --git a/lib/utils.js b/lib/utils.js index 6fc5c335ef0..17ff140865a 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -89,19 +89,19 @@ exports.deepEqual = function deepEqual(a, b) { } if ((isBsonType(a, 'ObjectId') && isBsonType(b, 'ObjectId')) || - (isBsonType(a, 'Decimal128') && isBsonType(b, 'Decimal128'))) { + (isBsonType(a, 'Decimal128') && isBsonType(b, 'Decimal128'))) { return a.toString() === b.toString(); } if (a instanceof RegExp && b instanceof RegExp) { return a.source === b.source && - a.ignoreCase === b.ignoreCase && - a.multiline === b.multiline && - a.global === b.global && - a.dotAll === b.dotAll && - a.unicode === b.unicode && - a.sticky === b.sticky && - a.hasIndices === b.hasIndices; + a.ignoreCase === b.ignoreCase && + a.multiline === b.multiline && + a.global === b.global && + a.dotAll === b.dotAll && + a.unicode === b.unicode && + a.sticky === b.sticky && + a.hasIndices === b.hasIndices; } if (a == null || b == null) { @@ -287,8 +287,8 @@ exports.merge = function merge(to, from, options, path) { // base schema has a given path as a single nested but discriminator schema // has the path as a document array, or vice versa (gh-9534) if (options.isDiscriminatorSchemaMerge && - (from[key].$isSingleNested && to[key].$isMongooseDocumentArray) || - (from[key].$isMongooseDocumentArray && to[key].$isSingleNested)) { + (from[key].$isSingleNested && to[key].$isMongooseDocumentArray) || + (from[key].$isMongooseDocumentArray && to[key].$isSingleNested)) { continue; } else if (from[key].instanceOfSchema) { if (to[key].instanceOfSchema) { @@ -995,7 +995,7 @@ exports.getOption = function(name) { * ignore */ -exports.noop = function() {}; +exports.noop = function() { }; exports.errorToPOJO = function errorToPOJO(error) { const isError = error instanceof Error; @@ -1025,3 +1025,13 @@ exports.injectTimestampsOption = function injectTimestampsOption(writeOperation, } writeOperation.timestamps = timestampsOption; }; + +exports.print = function(...args) { + const { inspect } = require('util'); + console.error( + inspect( + ...args, + { depth: Infinity } + ) + ); +}; diff --git a/package.json b/package.json index ecade1a0589..86cd719b83b 100644 --- a/package.json +++ b/package.json @@ -146,4 +146,4 @@ "target": "ES2017" } } -} +} \ No newline at end of file diff --git a/scripts/configure-cluster-with-encryption.sh b/scripts/configure-cluster-with-encryption.sh index 8f366bc4bbc..bc21b6b5f62 100644 --- a/scripts/configure-cluster-with-encryption.sh +++ b/scripts/configure-cluster-with-encryption.sh @@ -51,4 +51,4 @@ if [ ! -d "data" ]; then echo 'Cluster Configuration Finished!' cd .. -fi \ No newline at end of file +fi diff --git a/scripts/run-encryption-tests.sh b/scripts/run-encryption-tests.sh new file mode 100755 index 00000000000..0209292168d --- /dev/null +++ b/scripts/run-encryption-tests.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# sets up mongodb cluster and encryption configuration, adds relevant variables to the environment, and runs encryption tests + +export CWD=$(pwd); + +# set up mongodb cluster and encryption configuration if the data/ folder does not exist +# note: for tooling, cluster set-up and configuration look into the 'scripts/configure-cluster-with-encryption.sh' script + +if [ -d "data" ]; then + cd data +else + source $CWD/scripts/configure-cluster-with-encryption.sh +fi + +# extracts MONGOOSE_TEST_URI and CRYPT_SHARED_LIB_PATH from .yml file into environment variables for this test run +read -r -d '' SOURCE_SCRIPT << EOM +const fs = require('fs'); +const file = fs.readFileSync('mo-expansion.yml', { encoding: 'utf-8' }) + .trim().split('\\n'); +const regex = /^(?.*): "(?.*)"$/; +const variables = file.map( + (line) => regex.exec(line.trim()).groups +).map( + ({key, value}) => \`export \${key}='\${value}'\` +).join('\n'); + +process.stdout.write(variables); +process.stdout.write('\n'); +EOM + +node --eval "$SOURCE_SCRIPT" | tee expansions.sh +source expansions.sh + +export MONGOOSE_TEST_URI=$MONGODB_URI + +# run encryption tests +cd .. +npx mocha --exit ./test/encryption/*.test.js diff --git a/test/encrypted_schema.test.js b/test/encrypted_schema.test.js index 529ffe42ac0..d5712aabe1a 100644 --- a/test/encrypted_schema.test.js +++ b/test/encrypted_schema.test.js @@ -22,94 +22,174 @@ function schemaHasEncryptedProperty(schema, path) { return path in schema.encryptedFields; } -const KEY_ID = new UUID(); +const KEY_ID = '9fbdace3-4e48-412d-88df-3807e8009522'; const algorithm = 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'; describe('encrypted schema declaration', function() { - describe('Tests that fields of valid schema types can be declared as encrypted schemas', function() { - const basicSchemaTypes = [ - { type: String, name: 'string' }, - { type: Schema.Types.Boolean, name: 'boolean' }, - { type: Schema.Types.Buffer, name: 'buffer' }, - { type: Date, name: 'date' }, - { type: ObjectId, name: 'objectid' }, - { type: BigInt, name: 'bigint' }, - { type: Decimal128, name: 'Decimal128' }, - { type: Int32, name: 'int32' }, - { type: Double, name: 'double' } - ]; - - for (const { type, name } of basicSchemaTypes) { - describe(`When a schema is instantiated with an encrypted field of type ${name}`, function() { + describe('schemaMap generation tests', function() { + for (const { type, name, encryptionType, schemaMap, encryptedFields } of primitiveSchemaMapTests()) { + describe(`When a schema is instantiated with an encrypted field of type ${name} for ${encryptionType}`, function() { let schema; + const encrypt = { + keyId: KEY_ID + }; + encryptionType === 'csfle' && (encrypt.algorithm = algorithm); + beforeEach(function() { schema = new Schema({ field: { - type, encrypt: { keyId: KEY_ID, algorithm } + type, encrypt } }, { - encryptionType: 'csfle' + encryptionType }); }); it(`Then the schema has an encrypted property of type ${name}`, function() { assert.ok(schemaHasEncryptedProperty(schema, 'field')); }); - }); - } - - describe('when a schema is instantiated with a nested encrypted schema', function() { - let schema; - beforeEach(function() { - const encryptedSchema = new Schema({ - encrypted: { - type: String, encrypt: { keyId: KEY_ID, algorithm } - } - }, { encryptionType: 'csfle' }); - schema = new Schema({ - field: encryptedSchema - }, { encryptionType: 'csfle' }); - }); + encryptionType === 'csfle' && it('then the generated schemaMap is correct', function() { + assert.deepEqual(schema._buildSchemaMap(), schemaMap); + }); - it('then the schema has a nested property that is encrypted', function() { - assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])); + encryptionType === 'qe' && it('then the generated encryptedFieldsMap is correct', function() { + assert.deepEqual(schema._buildEncryptedFields(), encryptedFields); + }); }); - }); + } + }); - describe('when a schema is instantiated with a nested schema object', function() { - let schema; - beforeEach(function() { - schema = new Schema({ - field: { + describe('Tests that fields of valid schema types can be declared as encrypted schemas', function() { + const tests = { + 'nested schema for csfle': + { + schemaFactory: () => { + const encryptedSchema = new Schema({ encrypted: { type: String, encrypt: { keyId: KEY_ID, algorithm } } + }, { encryptionType: 'csfle' }); + return new Schema({ + field: encryptedSchema + }, { encryptionType: 'csfle' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])), + schemaMap: { + bsonType: 'object', + properties: { + field: { + bsonType: 'object', + properties: { + encrypted: { encrypt: { bsonType: 'string', algorithm, keyId: KEY_ID } } + } + } } - }, { encryptionType: 'csfle' }); - }); - - it('then the schema has a nested property that is encrypted', function() { - assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])); - }); - }); - - describe('when a schema is instantiated as an Array', function() { - let schema; - beforeEach(function() { - schema = new Schema({ - encrypted: { - type: [Number], - encrypt: { keyId: KEY_ID, algorithm } + } + }, + 'nested schema for qe': { + schemaFactory: () => { + const encryptedSchema = new Schema({ + encrypted: { + type: String, encrypt: { keyId: KEY_ID } + } + }, { encryptionType: 'qe' }); + return new Schema({ + field: encryptedSchema + }, { encryptionType: 'qe' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])), + encryptedFields: { + fields: [ + { path: 'field.encrypted', keyId: KEY_ID, bsonType: 'string' } + ] + } + }, + 'nested object for csfle': + { + schemaFactory: () => { + return new Schema({ + field: { + encrypted: { + type: String, encrypt: { keyId: KEY_ID, algorithm } + } + } + }, { encryptionType: 'csfle' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])), + schemaMap: { + bsonType: 'object', + properties: { + field: { + bsonType: 'object', + properties: { + encrypted: { encrypt: { bsonType: 'string', algorithm, keyId: KEY_ID } } + } + } } - }, { encryptionType: 'csfle' }); - }); + } + }, + 'nested object for qe': { + schemaFactory: () => { + return new Schema({ + field: { + encrypted: { + type: String, encrypt: { keyId: KEY_ID } + } + } + }, { encryptionType: 'qe' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['field', 'encrypted'])), + encryptedFields: { + fields: [ + { path: 'field.encrypted', keyId: KEY_ID, bsonType: 'string' } + ] + } + }, + 'schema with encrypted array for csfle': { + schemaFactory: () => { + return new Schema({ + encrypted: { + type: [Number], + encrypt: { keyId: KEY_ID, algorithm } + } + }, { encryptionType: 'csfle' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['encrypted'])), + schemaMap: { + bsonType: 'object', + properties: { + encrypted: { + encrypt: { + bsonType: 'array', + keyId: KEY_ID, + algorithm + } + } + } + } + }, + 'schema with encrypted array for qe': { + schemaFactory: () => { + return new Schema({ + encrypted: { + type: [Number], + encrypt: { keyId: KEY_ID } + } + }, { encryptionType: 'qe' }); + }, predicate: (schema) => assert.ok(schemaHasEncryptedProperty(schema, ['encrypted'])), + encryptedFields: { + fields: [ + { path: 'encrypted', keyId: KEY_ID, bsonType: 'array' } + ] + } + } + }; - it('then the schema has a nested property that is encrypted', function() { - assert.ok(schemaHasEncryptedProperty(schema, 'encrypted')); - }); - }); + for (const [description, { schemaFactory, predicate, schemaMap, encryptedFields }] of Object.entries(tests)) { + it(description, function() { + const schema = schemaFactory(); + predicate(schema); + schemaMap && assert.deepEqual(schema._buildSchemaMap(), schemaMap); + encryptedFields && assert.deepEqual(schema._buildEncryptedFields(), encryptedFields); + }); + } }); describe('invalid schema types for encrypted schemas', function() { @@ -536,3 +616,476 @@ describe('encrypted schema declaration', function() { }); }); }); + +function primitiveSchemaMapTests() { + return [ + { + name: 'string', + type: String, + encryptionType: 'csfle', + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'string' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'string', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'string', + type: String, + encryptionType: 'qe', + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'string' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'string', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'boolean', + type: Schema.Types.Boolean, + encryptionType: 'csfle', + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'bool' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'bool', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'boolean', + encryptionType: 'qe', + type: Schema.Types.Boolean, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'bool' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'bool', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'buffer', + encryptionType: 'csfle', + type: Schema.Types.Buffer, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'binData' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'binData', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'buffer', + encryptionType: 'qe', + type: Schema.Types.Buffer, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'binData' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'binData', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'date', + encryptionType: 'csfle', + type: Date, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'date' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'date', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'date', + encryptionType: 'qe', + type: Date, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'date' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'date', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'objectid', + encryptionType: 'csfle', + type: ObjectId, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'objectId' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'objectId', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'objectid', + encryptionType: 'qe', + type: ObjectId, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'objectId' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'objectId', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'bigint', + encryptionType: 'csfle', + type: BigInt, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'long' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'long', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'bigint', + encryptionType: 'qe', + type: BigInt, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'long' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'long', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'Decimal128', + encryptionType: 'csfle', + type: Decimal128, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'decimal' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'decimal', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'Decimal128', + encryptionType: 'qe', + type: Decimal128, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'decimal' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'decimal', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'int32', + encryptionType: 'csfle', + type: Int32, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'int' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'int', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'int32', + encryptionType: 'qe', + type: Int32, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'int' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'int', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + }, + { + name: 'double', + encryptionType: 'csfle', + type: Double, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic', + bsonType: 'double' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'double', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + ] + } + }, + { + name: 'double', + encryptionType: 'qe', + type: Double, + schemaMap: { + bsonType: 'object', + properties: { + field: { + encrypt: { + keyId: '9fbdace3-4e48-412d-88df-3807e8009522', + bsonType: 'double' + } + } + } + }, + encryptedFields: { + fields: [ + { + path: 'field', + bsonType: 'double', + keyId: '9fbdace3-4e48-412d-88df-3807e8009522' + } + ] + } + } + ]; +} diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index a3b562e80aa..24abe7f89fc 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -1,31 +1,14 @@ 'use strict'; const assert = require('assert'); -const mongodb = require('mongodb'); -const fs = require('fs'); +const mdb = require('mongodb'); const isBsonType = require('../../lib/helpers/isBsonType'); +const { Schema, createConnection } = require('../../lib'); +const { ObjectId, Double, Int32, Decimal128 } = require('bson'); const LOCAL_KEY = Buffer.from('Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'base64'); describe('ci', () => { - - const cachedUri = process.env.MONGOOSE_TEST_URI; - const cachedLib = process.env.CRYPT_SHARED_LIB_PATH; - - before(function() { - const cwd = process.cwd(); - const file = fs.readFileSync(cwd + '/data/mo-expansion.yml', { encoding: 'utf-8' }).trim().split('\n'); - const regex = /^(?.*): "(?.*)"$/; - const variables = file.map((line) => regex.exec(line.trim()).groups).reduce((acc, { key, value }) => ({ ...acc, [key]: value }), {}); - process.env.CRYPT_SHARED_LIB_PATH = variables.CRYPT_SHARED_LIB_PATH; - process.env.MONGOOSE_TEST_URI = variables.MONGODB_URI; - }); - - after(function() { - process.env.CRYPT_SHARED_LIB_PATH = cachedLib; - process.env.MONGOOSE_TEST_URI = cachedUri; - }); - describe('environmental variables', () => { it('MONGOOSE_TEST_URI is set', async function() { const uri = process.env.MONGOOSE_TEST_URI; @@ -38,78 +21,572 @@ describe('ci', () => { }); }); - describe('basic integration', () => { - let keyVaultClient; - let dataKey; - let encryptedClient; - let unencryptedClient; - - beforeEach(async function() { - keyVaultClient = new mongodb.MongoClient(process.env.MONGOOSE_TEST_URI); - await keyVaultClient.connect(); - await keyVaultClient.db('keyvault').collection('datakeys'); - const clientEncryption = new mongodb.ClientEncryption(keyVaultClient, { - keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: LOCAL_KEY } } + let keyId, keyId2; + let utilClient; + + beforeEach(async function() { + const keyVaultClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + await keyVaultClient.connect(); + await keyVaultClient.db('keyvault').collection('datakeys'); + const clientEncryption = new mdb.ClientEncryption(keyVaultClient, { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } } + }); + keyId = await clientEncryption.createDataKey('local'); + keyId2 = await clientEncryption.createDataKey('local'); + await keyVaultClient.close(); + + utilClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + }); + + afterEach(async function() { + await utilClient.db('db').dropDatabase({ + w: 'majority' + }); + await utilClient.close(); + }); + + describe('Tests that fields of valid schema types can be declared as encrypted schemas', function() { + const algorithm = 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'; + let connection; + let schema; + let model; + + const basicSchemaTypes = [ + { type: String, name: 'string', input: 3, expected: 3 }, + { type: Schema.Types.Boolean, name: 'boolean', input: true, expected: true }, + { type: Schema.Types.Buffer, name: 'buffer', input: Buffer.from([1, 2, 3]) }, + { type: Date, name: 'date', input: new Date(12, 12, 2012), expected: new Date(12, 12, 2012) }, + { type: ObjectId, name: 'objectid', input: new ObjectId() }, + { type: BigInt, name: 'bigint', input: 3n }, + { type: Decimal128, name: 'Decimal128', input: new Decimal128('1.5') }, + { type: Int32, name: 'int32', input: new Int32(5), expected: 5 }, + { type: Double, name: 'double', input: new Double(1.5) } + ]; + + for (const { type, name, input, expected } of basicSchemaTypes) { + + this.afterEach(async function() { + await connection?.close(); }); - dataKey = await clientEncryption.createDataKey('local'); - encryptedClient = new mongodb.MongoClient( - process.env.MONGOOSE_TEST_URI, - { - autoEncryption: { - keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: LOCAL_KEY } }, - schemaMap: { - 'db.coll': { - bsonType: 'object', - encryptMetadata: { - keyId: [dataKey] - }, - properties: { - a: { - encrypt: { - bsonType: 'int', - algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random', - keyId: [dataKey] - } + // eslint-disable-next-line no-inner-declarations + async function test() { + const [{ _id }] = await model.insertMany([{ field: input }]); + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.field, 'Binary')); + assert.ok(encryptedDoc.field.sub_type === 6); + + const doc = await model.findOne({ _id }); + if (Buffer.isBuffer(input)) { + // mongoose's Buffer does not support deep equality - instead use the Buffer.equals method. + assert.ok(doc.field.equals(input)); + } else { + assert.deepEqual(doc.field, expected ?? input); + } + } + + describe('CSFLE', function() { + beforeEach(async function() { + schema = new Schema({ + field: { + type, encrypt: { keyId: [keyId], algorithm } + } + }, { + encryptionType: 'csfle' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + }); + + it(`${name} encrypts and decrypts`, test); + }); + + describe('QE', function() { + beforeEach(async function() { + schema = new Schema({ + field: { + type, encrypt: { keyId: keyId } + } + }, { + encryptionType: 'qe' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + }); + + it(`${name} encrypts and decrypts`, test); + }); + } + + describe('nested object schemas', function() { + const tests = { + 'nested object schemas for CSFLE': { + modelFactory: () => { + const schema = new Schema({ + a: { + b: { + c: { + type: String, + encrypt: { keyId: [keyId], algorithm } } } } - }, + }, { + encryptionType: 'csfle' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + + } + }, + 'nested object schemas for QE': { + modelFactory: () => { + const schema = new Schema({ + a: { + b: { + c: { + type: String, + encrypt: { keyId: keyId } + } + } + } + }, { + encryptionType: 'qe' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + + } + }, + 'nested schemas for csfle': { + modelFactory: () => { + const nestedSchema = new Schema({ + b: { + c: { + type: String, + encrypt: { keyId: [keyId], algorithm } + } + } + }, { + encryptionType: 'csfle' + }); + + const schema = new Schema({ + a: nestedSchema + }, { + encryptionType: 'csfle' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + + } + }, + 'nested schemas for QE': { + modelFactory: () => { + const nestedSchema = new Schema({ + b: { + c: { + type: String, + encrypt: { keyId: keyId } + } + } + }, { + encryptionType: 'qe' + }); + const schema = new Schema({ + a: nestedSchema + }, { + encryptionType: 'qe' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + + } + } + }; + + for (const [description, { modelFactory }] of Object.entries(tests)) { + describe(description, function() { + it('encrypts and decrypts', async function() { + const { model } = modelFactory(); + + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + + const [{ _id }] = await model.insertMany([{ a: { b: { c: 'hello' } } }]); + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.a.b.c, 'Binary')); + assert.ok(encryptedDoc.a.b.c.sub_type === 6); + + const doc = await model.findOne({ _id }); + assert.deepEqual(doc.a.b.c, 'hello'); + }); + }); + } + }); + + describe('array encrypted fields', function() { + const tests = { + 'array fields for CSFLE': { + modelFactory: () => { + const schema = new Schema({ + a: { + type: [Int32], + encrypt: { + keyId: [keyId], + algorithm + } + } + }, { + encryptionType: 'csfle' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + } + }, + 'array field for QE': { + modelFactory: () => { + const schema = new Schema({ + a: { + type: [Int32], + encrypt: { + keyId + } + } + }, { + encryptionType: 'qe' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + } + } + }; + + for (const [description, { modelFactory }] of Object.entries(tests)) { + describe(description, function() { + it('encrypts and decrypts', async function() { + const { model } = modelFactory(); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + + const [{ _id }] = await model.insertMany([{ a: [new Int32(3)] }]); + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.a, 'Binary')); + assert.ok(encryptedDoc.a.sub_type === 6); + + const doc = await model.findOne({ _id }); + assert.deepEqual(doc.a, [3]); + }); + }); + } + }); + + describe('multiple encrypted fields in a model', function() { + const tests = { + 'multiple fields in a schema for CSFLE': { + modelFactory: () => { + const encrypt = { + keyId: [keyId], + algorithm + }; + + const schema = new Schema({ + a: { + type: String, + encrypt + }, + b: { + type: BigInt + }, + c: { + d: { + type: String, + encrypt + } + } + }, { + encryptionType: 'csfle' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + } + }, + 'multiple fields in a schema for QE': { + modelFactory: () => { + const schema = new Schema({ + a: { + type: String, + encrypt: { + keyId + } + }, + b: { + type: BigInt + }, + c: { + d: { + type: String, + encrypt: { + keyId: keyId2 + } + } + } + }, { + encryptionType: 'qe' + }); + + connection = createConnection(); + model = connection.model('Schema', schema); + return { model }; + } + } + }; + + for (const [description, { modelFactory }] of Object.entries(tests)) { + describe(description, function() { + it('encrypts and decrypts', async function() { + const { model } = modelFactory(); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + + const [{ _id }] = await model.insertMany([{ a: 'hello', b: 1n, c: { d: 'world' } }]); + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.a, 'Binary')); + assert.ok(encryptedDoc.a.sub_type === 6); + assert.ok(typeof encryptedDoc.b === 'number'); + assert.ok(isBsonType(encryptedDoc.c.d, 'Binary')); + assert.ok(encryptedDoc.c.d.sub_type === 6); + + const doc = await model.findOne({ _id }, {}); + assert.deepEqual(doc.a, 'hello'); + assert.deepEqual(doc.b, 1n); + assert.deepEqual(doc.c, { d: 'world' }); + }); + }); + } + }); + + describe('multiple schemas', function() { + const tests = { + 'multiple schemas for CSFLE': { + modelFactory: () => { + connection = createConnection(); + const encrypt = { + keyId: [keyId], + algorithm + }; + const model1 = connection.model('Model1', new Schema({ + a: { + type: String, + encrypt + } + }, { + encryptionType: 'csfle' + })); + const model2 = connection.model('Model2', new Schema({ + b: { + type: String, + encrypt + } + }, { + encryptionType: 'csfle' + })); + + return { model1, model2 }; + } + }, + 'multiple schemas for QE': { + modelFactory: () => { + connection = createConnection(); + const model1 = connection.model('Model1', new Schema({ + a: { + type: String, + encrypt: { + keyId + } + } + }, { + encryptionType: 'qe' + })); + const model2 = connection.model('Model2', new Schema({ + b: { + type: String, + encrypt: { + keyId + } + } + }, { + encryptionType: 'qe' + })); + + return { model1, model2 }; + } + } + }; + + for (const [description, { modelFactory }] of Object.entries(tests)) { + describe(description, function() { + it('encrypts and decrypts', async function() { + const { model1, model2 } = modelFactory(); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + + { + const [{ _id }] = await model1.insertMany([{ a: 'hello' }]); + const encryptedDoc = await utilClient.db('db').collection('model1').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.a, 'Binary')); + assert.ok(encryptedDoc.a.sub_type === 6); + + const doc = await model1.findOne({ _id }); + assert.deepEqual(doc.a, 'hello'); + } + + { + const [{ _id }] = await model2.insertMany([{ b: 'world' }]); + const encryptedDoc = await utilClient.db('db').collection('model2').findOne({ _id }); + + assert.ok(isBsonType(encryptedDoc.b, 'Binary')); + assert.ok(encryptedDoc.b.sub_type === 6); + + const doc = await model2.findOne({ _id }); + assert.deepEqual(doc.b, 'world'); + } + }); + }); + } + }); + + describe('CSFLE and QE schemas on the same connection', function() { + it('encrypts and decrypts', async function() { + connection = createConnection(); + const model1 = connection.model('Model1', new Schema({ + a: { + type: String, + encrypt: { + keyId + } + } + }, { + encryptionType: 'qe' + })); + const model2 = connection.model('Model2', new Schema({ + b: { + type: String, + encrypt: { + keyId: [keyId], + algorithm + } + } + }, { + encryptionType: 'csfle' + })); + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, extraOptions: { cryptdSharedLibRequired: true, cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH } } - } - ); + }); - unencryptedClient = new mongodb.MongoClient(process.env.MONGOOSE_TEST_URI); - }); + { + const [{ _id }] = await model1.insertMany([{ a: 'hello' }]); + const encryptedDoc = await utilClient.db('db').collection('model1').findOne({ _id }); - afterEach(async function() { - await keyVaultClient.close(); - await encryptedClient.close(); - await unencryptedClient.close(); - }); + assert.ok(isBsonType(encryptedDoc.a, 'Binary')); + assert.ok(encryptedDoc.a.sub_type === 6); - it('ci set-up should support basic mongodb auto-encryption integration', async() => { - await encryptedClient.connect(); - const { insertedId } = await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + const doc = await model1.findOne({ _id }); + assert.deepEqual(doc.a, 'hello'); + } - // client not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded - const encryptedResult = await unencryptedClient.db('db').collection('coll').findOne({ _id: insertedId }); + { + const [{ _id }] = await model2.insertMany([{ b: 'world' }]); + const encryptedDoc = await utilClient.db('db').collection('model2').findOne({ _id }); - assert.ok(encryptedResult); - assert.ok(encryptedResult.a); - assert.ok(isBsonType(encryptedResult.a, 'Binary')); - assert.ok(encryptedResult.a.sub_type === 6); + assert.ok(isBsonType(encryptedDoc.b, 'Binary')); + assert.ok(encryptedDoc.b.sub_type === 6); - // when the encryptedClient runs a find, the original unencrypted value is returned - const unencryptedResult = await encryptedClient.db('db').collection('coll').findOne({ _id: insertedId }); - assert.ok(unencryptedResult); - assert.ok(unencryptedResult.a === 1); + const doc = await model2.findOne({ _id }); + assert.deepEqual(doc.b, 'world'); + } + }); }); }); }); diff --git a/test/model.test.js b/test/model.test.js index da870125e0d..07cea0d5ab3 100644 --- a/test/model.test.js +++ b/test/model.test.js @@ -368,7 +368,7 @@ describe('Model', function() { assert.equal(post.get('comments')[0].comments[0].isNew, true); post.invalidate('title'); // force error - await post.save().catch(() => {}); + await post.save().catch(() => { }); assert.equal(post.isNew, true); assert.equal(post.get('comments')[0].isNew, true); assert.equal(post.get('comments')[0].comments[0].isNew, true); @@ -2479,7 +2479,7 @@ describe('Model', function() { const DefaultErr = db.model('Test', DefaultErrSchema); - new DefaultErr().save().catch(() => {}); + new DefaultErr().save().catch(() => { }); await new Promise(resolve => { DefaultErr.once('error', function(err) { @@ -3043,7 +3043,7 @@ describe('Model', function() { const Location = db.model('Test', LocationSchema); - await Location.collection.drop().catch(() => {}); + await Location.collection.drop().catch(() => { }); await Location.init(); await Location.create({ @@ -3512,7 +3512,7 @@ describe('Model', function() { listener = null; // Change stream may still emit "MongoAPIError: ChangeStream is closed" because change stream // may still poll after close. - changeStream.on('error', () => {}); + changeStream.on('error', () => { }); changeStream.close(); changeStream = null; }); @@ -3664,7 +3664,7 @@ describe('Model', function() { // Change stream may still emit "MongoAPIError: ChangeStream is closed" because change stream // may still poll after close. - changeStream.on('error', () => {}); + changeStream.on('error', () => { }); await changeStream.close(); await db.close(); }); @@ -3682,7 +3682,7 @@ describe('Model', function() { // Change stream may still emit "MongoAPIError: ChangeStream is closed" because change stream // may still poll after close. - changeStream.on('error', () => {}); + changeStream.on('error', () => { }); const close = changeStream.close(); await db.asPromise(); @@ -3708,7 +3708,7 @@ describe('Model', function() { // Change stream may still emit "MongoAPIError: ChangeStream is closed" because change stream // may still poll after close. - changeStream.on('error', () => {}); + changeStream.on('error', () => { }); changeStream.close(); const closedData = await closed; @@ -5540,7 +5540,7 @@ describe('Model', function() { const Model = db.model('User', userSchema); - await Model.collection.drop().catch(() => {}); + await Model.collection.drop().catch(() => { }); await Model.createCollection(); const collectionName = Model.collection.name; @@ -5574,7 +5574,7 @@ describe('Model', function() { const Test = db.model('Test', schema, 'Test'); await Test.init(); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection(); const collections = await Test.db.db.listCollections().toArray(); @@ -5583,7 +5583,7 @@ describe('Model', function() { assert.equal(coll.type, 'timeseries'); assert.equal(coll.options.timeseries.timeField, 'timestamp'); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); }); it('createCollection() enforces expireAfterSeconds (gh-11229)', async function() { @@ -5604,7 +5604,7 @@ describe('Model', function() { const Test = db.model('TestGH11229Var1', schema); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection({ expireAfterSeconds: 5 }); const collOptions = await Test.collection.options(); @@ -5632,7 +5632,7 @@ describe('Model', function() { const Test = db.model('TestGH11229Var2', schema, 'TestGH11229Var2'); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection({ expires: '5 seconds' }); const collOptions = await Test.collection.options(); @@ -5660,7 +5660,7 @@ describe('Model', function() { const Test = db.model('TestGH11229Var3', schema); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection(); const collOptions = await Test.collection.options(); @@ -5688,7 +5688,7 @@ describe('Model', function() { const Test = db.model('TestGH11229Var4', schema); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection(); const collOptions = await Test.collection.options(); @@ -5716,7 +5716,7 @@ describe('Model', function() { const Test = db.model('Test', schema, 'Test'); await Test.init(); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection(); const collections = await Test.db.db.listCollections().toArray(); @@ -5725,7 +5725,7 @@ describe('Model', function() { assert.deepEqual(coll.options.clusteredIndex.key, { _id: 1 }); assert.equal(coll.options.clusteredIndex.name, 'clustered test'); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); }); it('mongodb actually removes expired documents (gh-11229)', async function() { @@ -5747,7 +5747,7 @@ describe('Model', function() { const Test = db.model('TestMongoDBExpireRemoval', schema); - await Test.collection.drop().catch(() => {}); + await Test.collection.drop().catch(() => { }); await Test.createCollection({ expireAfterSeconds: 5 }); await Test.insertMany([ @@ -5845,7 +5845,7 @@ describe('Model', function() { const Model = db.model('User', userSchema); - await Model.collection.drop().catch(() => {}); + await Model.collection.drop().catch(() => { }); await Model.createCollection(); await Model.createCollection(); @@ -6525,7 +6525,7 @@ describe('Model', function() { await User.bulkWrite([ { updateOne: { - filter: { }, + filter: {}, update: { friends: ['Sam'] }, upsert: true, setDefaultsOnInsert: true @@ -7002,7 +7002,7 @@ describe('Model', function() { }); it('insertMany should throw an error if there were operations that failed validation, ' + - 'but all operations that passed validation succeeded (gh-14572) (gh-13256)', async function() { + 'but all operations that passed validation succeeded (gh-14572) (gh-13256)', async function() { const userSchema = new Schema({ age: { type: Number } }); @@ -8020,7 +8020,7 @@ describe('Model', function() { decoratorSchema.loadClass(Decorator); // Define discriminated class before model is compiled - class Deco1 extends Decorator { whoAmI() { return 'I am Test1'; }} + class Deco1 extends Decorator { whoAmI() { return 'I am Test1'; } } const deco1Schema = new Schema({}); deco1Schema.loadClass(Deco1); decoratorSchema.discriminator('Test1', deco1Schema); @@ -8032,7 +8032,7 @@ describe('Model', function() { const shopModel = db.model('Test', shopSchema); // Define another discriminated class after the model is compiled - class Deco2 extends Decorator { whoAmI() { return 'I am Test2'; }} + class Deco2 extends Decorator { whoAmI() { return 'I am Test2'; } } const deco2Schema = new Schema({}); deco2Schema.loadClass(Deco2); decoratorSchema.discriminator('Test2', deco2Schema); @@ -8158,7 +8158,7 @@ describe('Model', function() { }); it('insertMany should throw an error if there were operations that failed validation, ' + - 'but all operations that passed validation succeeded (gh-13256)', async function() { + 'but all operations that passed validation succeeded (gh-13256)', async function() { const userSchema = new Schema({ age: { type: Number } }); diff --git a/types/query.d.ts b/types/query.d.ts index fbebf1b6467..77b3e47ad73 100644 --- a/types/query.d.ts +++ b/types/query.d.ts @@ -228,14 +228,14 @@ declare module 'mongoose' { type MergePopulatePaths> = QueryOp extends QueryOpThatReturnsDocument ? ResultType extends null - ? ResultType - : ResultType extends (infer U)[] - ? U extends Document - ? HydratedDocument, TDocOverrides, TQueryHelpers>[] - : (MergeType)[] - : ResultType extends Document - ? HydratedDocument, TDocOverrides, TQueryHelpers> - : MergeType + ? ResultType + : ResultType extends (infer U)[] + ? U extends Document + ? HydratedDocument, TDocOverrides, TQueryHelpers>[] + : (MergeType)[] + : ResultType extends Document + ? HydratedDocument, TDocOverrides, TQueryHelpers> + : MergeType : MergeType; class Query> implements SessionOperation { @@ -373,8 +373,8 @@ declare module 'mongoose' { ): QueryWithHelpers< Array< DocKey extends keyof WithLevel1NestedPaths - ? WithoutUndefined[DocKey]>> - : ResultType + ? WithoutUndefined[DocKey]>> + : ResultType >, DocType, THelpers, @@ -567,26 +567,26 @@ declare module 'mongoose' { val?: boolean | any ): QueryWithHelpers< ResultType extends null - ? GetLeanResultType | null - : GetLeanResultType, + ? GetLeanResultType | null + : GetLeanResultType, DocType, THelpers, RawDocType, QueryOp, TDocOverrides - >; + >; lean( val?: boolean | any ): QueryWithHelpers< ResultType extends null - ? LeanResultType | null - : LeanResultType, + ? LeanResultType | null + : LeanResultType, DocType, THelpers, RawDocType, QueryOp, TDocOverrides - >; + >; /** Specifies the maximum number of documents the query will return. */ limit(val: number): this; @@ -761,12 +761,12 @@ declare module 'mongoose' { {}, ResultType, ResultType extends any[] - ? ResultType extends HydratedDocument[] - ? HydratedDocument[] - : RawDocTypeOverride[] - : (ResultType extends HydratedDocument - ? HydratedDocument - : RawDocTypeOverride) | (null extends ResultType ? null : never) + ? ResultType extends HydratedDocument[] + ? HydratedDocument[] + : RawDocTypeOverride[] + : (ResultType extends HydratedDocument + ? HydratedDocument + : RawDocTypeOverride) | (null extends ResultType ? null : never) >, DocType, THelpers, From 63741394058686f4cd3394aaf35d67316b98f331 Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 25 Feb 2025 11:19:40 -0700 Subject: [PATCH 5/5] add support for discriminators --- lib/drivers/node-mongodb-native/connection.js | 32 ++- scripts/configure-cluster-with-encryption.sh | 2 +- test/encryption/encryption.test.js | 201 ++++++++++++++++-- 3 files changed, 214 insertions(+), 21 deletions(-) diff --git a/lib/drivers/node-mongodb-native/connection.js b/lib/drivers/node-mongodb-native/connection.js index 0e6515a321f..28bdf50b946 100644 --- a/lib/drivers/node-mongodb-native/connection.js +++ b/lib/drivers/node-mongodb-native/connection.js @@ -12,6 +12,7 @@ const pkg = require('../../../package.json'); const processConnectionOptions = require('../../helpers/processConnectionOptions'); const setTimeout = require('../../helpers/timers').setTimeout; const utils = require('../../utils'); +const { Schema } = require('../../mongoose'); /** * A [node-mongodb-native](https://github.com/mongodb/node-mongodb-native) connection implementation. @@ -357,20 +358,35 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio * options. */ NativeConnection.prototype._buildEncryptionSchemas = function() { - const schemaMap = Object.values(this.models).filter(model => model.schema.encryptionType() === 'csfle').reduce( - (schemaMap, model) => { - const { schema, collection: { collectionName } } = model; - const namespace = `${this.$dbName}.${collectionName}`; + const qeMappings = {}; + const csfleMappings = {}; + + // If discriminators are configured for the collection, there might be multiple models + // pointing to the same namespace. For this scenario, we merge all the schemas for each namespace + // into a single schema. + // Notably, this doesn't allow for discriminators to declare multiple values on the same fields. + for (const model of Object.values(this.models)) { + const { schema, collection: { collectionName } } = model; + const namespace = `${this.$dbName}.${collectionName}`; + if (schema.encryptionType() === 'csfle') { + csfleMappings[namespace] ??= new Schema({}, { encryptionType: 'csfle' }); + csfleMappings[namespace].add(schema); + } else if (schema.encryptionType() === 'queryableEncryption') { + qeMappings[namespace] ??= new Schema({}, { encryptionType: 'queryableEncryption' }); + qeMappings[namespace].add(schema); + } + } + + const schemaMap = Object.entries(csfleMappings).reduce( + (schemaMap, [namespace, schema]) => { schemaMap[namespace] = schema._buildSchemaMap(); return schemaMap; }, {} ); - const encryptedFieldsMap = Object.values(this.models).filter(model => model.schema.encryptionType() === 'qe').reduce( - (encryptedFieldsMap, model) => { - const { schema, collection: { collectionName } } = model; - const namespace = `${this.$dbName}.${collectionName}`; + const encryptedFieldsMap = Object.entries(qeMappings).reduce( + (encryptedFieldsMap, [namespace, schema]) => { encryptedFieldsMap[namespace] = schema._buildEncryptedFields(); return encryptedFieldsMap; }, diff --git a/scripts/configure-cluster-with-encryption.sh b/scripts/configure-cluster-with-encryption.sh index bc21b6b5f62..2a28555f7bb 100644 --- a/scripts/configure-cluster-with-encryption.sh +++ b/scripts/configure-cluster-with-encryption.sh @@ -7,7 +7,7 @@ export CWD=$(pwd); # install extra dependency -npm install mongodb-client-encryption +npm install --no-save mongodb-client-encryption # set up mongodb cluster and encryption configuration if the data/ folder does not exist if [ ! -d "data" ]; then diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index 24abe7f89fc..4efe48fae6b 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -8,6 +8,16 @@ const { ObjectId, Double, Int32, Decimal128 } = require('bson'); const LOCAL_KEY = Buffer.from('Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'base64'); +/** + * @param {object} object + * @param {string} property + */ +function isEncryptedValue(object, property) { + const value = object[property]; + assert.ok(isBsonType(value, 'Binary'), `auto encryption for property ${property} failed: not a BSON binary.`); + assert.ok(value.sub_type === 6, `auto encryption for property ${property} failed: not subtype 6.`); +} + describe('ci', () => { describe('environmental variables', () => { it('MONGOOSE_TEST_URI is set', async function() { @@ -21,7 +31,7 @@ describe('ci', () => { }); }); - let keyId, keyId2; + let keyId, keyId2, keyId3; let utilClient; beforeEach(async function() { @@ -34,6 +44,7 @@ describe('ci', () => { }); keyId = await clientEncryption.createDataKey('local'); keyId2 = await clientEncryption.createDataKey('local'); + keyId3 = await clientEncryption.createDataKey('local'); await keyVaultClient.close(); utilClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); @@ -65,7 +76,6 @@ describe('ci', () => { ]; for (const { type, name, input, expected } of basicSchemaTypes) { - this.afterEach(async function() { await connection?.close(); }); @@ -114,14 +124,14 @@ describe('ci', () => { it(`${name} encrypts and decrypts`, test); }); - describe('QE', function() { + describe('queryableEncryption', function() { beforeEach(async function() { schema = new Schema({ field: { type, encrypt: { keyId: keyId } } }, { - encryptionType: 'qe' + encryptionType: 'queryableEncryption' }); connection = createConnection(); @@ -177,7 +187,7 @@ describe('ci', () => { } } }, { - encryptionType: 'qe' + encryptionType: 'queryableEncryption' }); connection = createConnection(); @@ -221,12 +231,12 @@ describe('ci', () => { } } }, { - encryptionType: 'qe' + encryptionType: 'queryableEncryption' }); const schema = new Schema({ a: nestedSchema }, { - encryptionType: 'qe' + encryptionType: 'queryableEncryption' }); connection = createConnection(); @@ -297,7 +307,7 @@ describe('ci', () => { } } }, { - encryptionType: 'qe' + encryptionType: 'queryableEncryption' }); connection = createConnection(); @@ -388,7 +398,7 @@ describe('ci', () => { } } }, { - encryptionType: 'qe' + encryptionType: 'queryableEncryption' }); connection = createConnection(); @@ -471,7 +481,7 @@ describe('ci', () => { } } }, { - encryptionType: 'qe' + encryptionType: 'queryableEncryption' })); const model2 = connection.model('Model2', new Schema({ b: { @@ -481,7 +491,7 @@ describe('ci', () => { } } }, { - encryptionType: 'qe' + encryptionType: 'queryableEncryption' })); return { model1, model2 }; @@ -541,7 +551,7 @@ describe('ci', () => { } } }, { - encryptionType: 'qe' + encryptionType: 'queryableEncryption' })); const model2 = connection.model('Model2', new Schema({ b: { @@ -588,5 +598,172 @@ describe('ci', () => { } }); }); + + describe('Models with discriminators', function() { + let discrim1, discrim2, model; + + describe('csfle', function() { + beforeEach(async function() { + connection = createConnection(); + + const schema = new Schema({ + name: { + type: String, encrypt: { keyId: [keyId], algorithm } + } + }, { + encryptionType: 'csfle' + }); + model = connection.model('Schema', schema); + discrim1 = model.discriminator('Test', new Schema({ + age: { + type: Int32, encrypt: { keyId: [keyId], algorithm } + } + }, { + encryptionType: 'csfle' + })); + + discrim2 = model.discriminator('Test2', new Schema({ + dob: { + type: Int32, encrypt: { keyId: [keyId], algorithm } + } + }, { + encryptionType: 'csfle' + })); + + + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + }); + it('encrypts', async function() { + { + const doc = new discrim1({ name: 'bailey', age: 32 }); + await doc.save(); + + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id: doc._id }); + + isEncryptedValue(encryptedDoc, 'age'); + } + + { + const doc = new discrim2({ name: 'bailey', dob: 32 }); + await doc.save(); + + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id: doc._id }); + + isEncryptedValue(encryptedDoc, 'dob'); + } + }); + + it('decrypts', async function() { + { + const doc = new discrim1({ name: 'bailey', age: 32 }); + await doc.save(); + + const decryptedDoc = await discrim1.findOne({ _id: doc._id }); + + assert.equal(decryptedDoc.age, 32); + } + + { + const doc = new discrim2({ name: 'bailey', dob: 32 }); + await doc.save(); + + const decryptedDoc = await discrim2.findOne({ _id: doc._id }); + + assert.equal(decryptedDoc.dob, 32); + } + }); + }); + + + describe('queryableEncryption', function() { + beforeEach(async function() { + connection = createConnection(); + + const schema = new Schema({ + name: { + type: String, encrypt: { keyId } + } + }, { + encryptionType: 'queryableEncryption' + }); + model = connection.model('Schema', schema); + discrim1 = model.discriminator('Test', new Schema({ + age: { + type: Int32, encrypt: { keyId: keyId2 } + } + }, { + encryptionType: 'queryableEncryption' + })); + + discrim2 = model.discriminator('Test2', new Schema({ + dob: { + type: Int32, encrypt: { keyId: keyId3 } + } + }, { + encryptionType: 'queryableEncryption' + })); + + await connection.openUri(process.env.MONGOOSE_TEST_URI, { + dbName: 'db', autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + }); + }); + it('encrypts', async function() { + { + const doc = new discrim1({ name: 'bailey', age: 32 }); + await doc.save(); + + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id: doc._id }); + + isEncryptedValue(encryptedDoc, 'age'); + } + + { + const doc = new discrim2({ name: 'bailey', dob: 32 }); + await doc.save(); + + const encryptedDoc = await utilClient.db('db').collection('schemas').findOne({ _id: doc._id }); + + isEncryptedValue(encryptedDoc, 'dob'); + } + }); + + it('decrypts', async function() { + { + const doc = new discrim1({ name: 'bailey', age: 32 }); + await doc.save(); + + const decryptedDoc = await discrim1.findOne({ _id: doc._id }); + + assert.equal(decryptedDoc.age, 32); + } + + { + const doc = new discrim2({ name: 'bailey', dob: 32 }); + await doc.save(); + + const decryptedDoc = await discrim2.findOne({ _id: doc._id }); + + assert.equal(decryptedDoc.dob, 32); + } + }); + }); + + }); }); });