diff --git a/Changelog.md b/Changelog.md index 75db13d..49f406a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,14 @@ +## [Unreleased] +### Added + - The ```rekey``` command, to re-encrypt the project settings and encrypted submission data of a database with a new DB_SECRET. + +### Fixed + - The clone command did not re-encrypt encrypted fields within the rows of a datagrid or an editgrid. + - The clone command named custom submission collections after the unsanitized project and collection names, where + the server never reads them, and cloned their revisions from and to the default revisions collection. It now + uses the server's names (```myproject_records``` and ```myproject_records_revisions```), and warns when the + destination still has submissions in a collection an earlier clone misnamed. + ## 2.3.2 ### Changed - Fixing the cloner to not skip if the destination item does not exist diff --git a/Readme.md b/Readme.md index b241a27..4a90869 100644 --- a/Readme.md +++ b/Readme.md @@ -290,6 +290,37 @@ yarn test:cloner They cover every clone flag, the dry run output, resuming an interrupted clone, and cloning from an API source (served by a stub API), against a real Mongo database. +### Rekey + + formio rekey --new-db-secret=[NEW_SECRET] --old-db-secret=[OLD_SECRET] [--old-db-secret=[ANOTHER_OLD_SECRET]] + + Re-encrypts the project settings and the encrypted submission data of a database with a new ```DB_SECRET```, for + example after a clone was run with the wrong ```--dst-db-secret```. + + The project settings are encrypted with the ```DB_SECRET```. Encrypted submission data is encrypted with the + ```secret``` of its project settings or, when the project has none, with the ```DB_SECRET```. Every value is + first tried with the secret it should be encrypted with, and left alone when it already is. Otherwise each of + the old secrets is tried, and the value is re-encrypted and read back before it is written. Give every secret + the data may be encrypted with: after a clone that is both the wrong destination secret and the source secret. + + | Option | What it does | + | --- | --- | + | ```--new-db-secret ``` | The ```DB_SECRET``` the database should be encrypted with (required). | + | ```--old-db-secret ``` | A secret the data may currently be encrypted with. Give it more than once to try several. | + | ```-p, --project ``` | Only re-key these projects (comma separated). All projects by default. | + | ```--dry-run``` | Change nothing. Report how much would be re-keyed, and what cannot be decrypted. | + | ```--no-submission-revisions``` | Leave the submission revisions alone. | + | ```--batch-size ``` | The number of documents written at a time (default 1000). | + | ```--ca ```, ```--cert ``` | The TLS certificate authority and certificate key file for the connection. | + + Back up the database, and stop the servers that use it, before re-keying: a server running with the wrong + secret keeps writing data encrypted with it. Start with a ```--dry-run```. + + Values none of the secrets can decrypt are listed and left as they are, and the command exits with an error. + A project whose settings cannot be decrypted is skipped entirely, because the secret of its data is in those + settings. A document that changes while it is being re-keyed is never overwritten. Values that are already + encrypted with the new secret are skipped, so the command can be stopped and run again as often as needed. + ### Deploy ``` diff --git a/commands/commands.js b/commands/commands.js index d7e87a6..c1e5d42 100644 --- a/commands/commands.js +++ b/commands/commands.js @@ -6,6 +6,7 @@ module.exports = function(program, next) { deploy: require('./deploy.js')(program, next), copy: require('./copy.js')(program, next), migrate: require('./migrate.js')(program, next), + rekey: require('./rekey.js')(program, next), submissions: require('./submissions.js')(program, next) }; }; diff --git a/commands/rekey.js b/commands/rekey.js new file mode 100644 index 0000000..209de03 --- /dev/null +++ b/commands/rekey.js @@ -0,0 +1,26 @@ +/* eslint-disable max-len */ +'use strict'; + +/** + * Collect an option that can be given more than once. + * @param {string} value - This value. + * @param {string[]} previous - The values given before it. + */ +function collect(value, previous) { + return previous.concat([value]); +} + +module.exports = function(program, next) { + program + .command('rekey ') + .description('Re-encrypt the project settings and encrypted submission data of a database with a new DB_SECRET.') + .requiredOption('--new-db-secret ', 'The DB_SECRET the database should be encrypted with.') + .option('--old-db-secret ', 'A secret the data may currently be encrypted with. Give it more than once to try several.', collect, []) + .option('-p, --project ', 'Only re-key these projects (comma separated). All projects by default.') + .option('--dry-run', 'Change nothing. Report how much would be re-keyed, and what cannot be decrypted.', false) + .option('--no-submission-revisions', 'Leave the submission revisions alone.') + .option('--batch-size ', 'The number of documents written at a time (default - 1000).') + .option('--ca ', 'The TLS certificate authority for the mongo url.') + .option('--cert ', 'The TLS certificate key file for the mongo connection.') + .action((mongoUrl, options) => require('../src/rekey')(mongoUrl, options)); +}; diff --git a/src/Cloner.js b/src/Cloner.js index 83365f2..72a2927 100644 --- a/src/Cloner.js +++ b/src/Cloner.js @@ -3,9 +3,14 @@ const {MongoClient, ObjectId, BSON} = require('mongodb'); const fs = require('fs'); const _ = require('lodash'); -const crypto = require('crypto'); -const keygenerator = require('keygenerator'); -const {eachComponentAsync} = require('./eachComponentAsync'); +const encryption = require('./encryption'); +const { + redact, + tlsOptions, + submissionCollectionName, + legacySubmissionCollectionName, + idList +} = require('./db'); const {createHmac} = require('node:crypto'); const fetch = require('./fetch'); const Progress = require('./Progress'); @@ -36,22 +41,6 @@ const MAX_REPEATED_ERRORS = 5; const POSITION_COUNT_TIMEOUT = 2000; const BAR_COUNT_TIMEOUT = 60000; -/** - * Hide the credentials of a connection string, so that they never end up in a log. - * @param {string} uri - The connection string. - */ -function redact(uri) { - return String(uri).replace(/\/\/[^@/]+@/, '//***@'); -} - -/** - * Split a comma separated list of ids, ignoring the spaces around them. - * @param {string} value - The comma separated list. - */ -function idList(value) { - return String(value || '').split(',').map((id) => id.trim()).filter((id) => id); -} - class Cloner { /** * @param {*} srcPath - The source connection string. @@ -90,7 +79,6 @@ class Cloner { this.mongoDest = mongoDest; this.beforeAll = null; this.afterAll = null; - this.defaultSaltLength = 40; this.options = options; this.src = null; this.dest = null; @@ -371,111 +359,13 @@ class Cloner { } } - /** - * Derive the key and iv for a secret exactly the way the removed crypto.createCipher() did, which - * is OpenSSL's EVP_BytesToKey with MD5, a single iteration and no salt. This keeps us compatible - * with data that was encrypted by the servers, while still running on Node versions where - * crypto.createCipher() no longer exists. - * - * @param {string} secret - The secret to derive the key and iv from. - * @returns {{key: Buffer, iv: Buffer}} - The derived key and iv. - */ - legacyKey(secret) { - // createCipher() read the password as UTF-8; latin1 would derive a different key for any secret - // holding a non ASCII character, and nothing encrypted with it could be read back. - const password = Buffer.from(secret, 'utf8'); - const keyLength = 32; - const ivLength = 16; - let derived = Buffer.alloc(0); - let block = Buffer.alloc(0); - while (derived.length < keyLength + ivLength) { - block = crypto.createHash('md5').update(Buffer.concat([block, password])).digest(); - derived = Buffer.concat([derived, block]); - } - return { - key: derived.subarray(0, keyLength), - iv: derived.subarray(keyLength, keyLength + ivLength) - }; - } - - /** - * Encrypt data or settings using a secret. - * @param {*} secret - * @param {*} rawData - * @param {*} nosalt - * @returns - */ - encrypt(secret, rawData, nosalt) { - if (!secret || rawData === null || rawData === undefined) { - return null; - } - const salt = nosalt ? '' : keygenerator._({ - length: this.defaultSaltLength - }); - const {key, iv} = this.legacyKey(secret); - const cipher = crypto.createCipheriv('aes-256-cbc', key, iv); - const decryptedJSON = JSON.stringify(rawData) + salt; - return Buffer.concat([ - cipher.update(decryptedJSON), - cipher.final() - ]); - } - - /** - * Decrypt data or settings using a secret. - * @param {*} secret - * @param {*} cipherbuffer - * @param {*} nosalt - * @returns - */ - decrypt(secret, cipherbuffer, nosalt) { - if (!secret || !cipherbuffer) { - return null; - } - let data; - try { - const buffer = Buffer.isBuffer(cipherbuffer) ? cipherbuffer : cipherbuffer.buffer; - const {key, iv} = this.legacyKey(secret); - const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv); - const decryptedJSON = Buffer.concat([ - decipher.update(buffer), // Buffer contains encrypted utf8 - decipher.final() - ]); - data = JSON.parse(nosalt ? decryptedJSON : decryptedJSON.slice(0, -this.defaultSaltLength)); - } - catch (e) { - console.log(e); - data = null; - } - return data; - } - /** * Create the database configuration for the given database source type. * @param {*} type - The type of database source. * @returns MongoDB configuration. */ dbConfig(type) { - // The driver always uses the new url parser and the unified topology: from version 7 on it - // rejects the options that used to turn them on. - const config = {}; - const caFile = this.options[`${type}Ca`]; - const certFile = this.options[`${type}Cert`]; - if (caFile || certFile) { - config.tls = true; - if (caFile) { - config.tlsCAFile = `${process.cwd()}/${caFile}`; - - // A custom certificate authority is usually an internal one whose certificates do not name - // the host being connected to. A client certificate on its own is no reason to stop - // verifying who we are talking to. - config.tlsAllowInvalidHostnames = true; - } - if (certFile) { - config.tlsCertificateKeyFile = `${process.cwd()}/${certFile}`; - } - } - return config; + return tlsOptions(this.options[`${type}Ca`], this.options[`${type}Cert`]); } /** @@ -1500,25 +1390,57 @@ class Cloner { await this.flushWrites(); const subsCollection = _.get(form, 'settings.collection'); - if (!subsCollection) { - if (!this.options.apiSource) { - this.src.submissions = this.src.ogSubmissions; - } - this.dest.submissions = this.dest.ogSubmissions; - return; + if (!this.options.apiSource) { + this.useCollection(this.src, subsCollection && srcProj && submissionCollectionName(srcProj.name, subsCollection)); } + const destCollection = subsCollection && destProj && submissionCollectionName(destProj.name, subsCollection); + this.useCollection(this.dest, destCollection); + if (destCollection) { + console.log(`Using collection - ${destCollection}`); + await this.warnAboutMisnamedCollection(form, destProj, subsCollection, destCollection); + } + } - console.log(`Using collection - ${subsCollection}`); + /** + * Point the submissions, and submission revisions, of a database at the collections of a form. + * @param {object} database - The source or the destination. + * @param {string|null} name - The custom collection of the form, or null for the default ones. + */ + useCollection(database, name) { + database.submissions = name ? database.db.collection(name) : database.ogSubmissions; + database.submissionrevisions = name + ? database.db.collection(`${name}_revisions`) + : (database.ogSubmissionRevisions || database.submissionrevisions); + } - if (!this.options.apiSource) { - this.src.submissions = srcProj && subsCollection - ? this.src.db.collection(`${srcProj.name}_${subsCollection}`) - : this.src.ogSubmissions; + /** + * Earlier versions of the clone wrote the submissions of a custom collection to a collection named + * after the unsanitized project and collection names, where the server never reads them. Say so + * when the destination still has them, since they are not moved. + * + * @param {object} form - The form. + * @param {object} destProj - The destination project. + * @param {string} subsCollection - The collection setting of the form. + * @param {string} destCollection - The collection the submissions are cloned to. + */ + async warnAboutMisnamedCollection(form, destProj, subsCollection, destCollection) { + const misnamed = legacySubmissionCollectionName(destProj.name, subsCollection); + if (this.dryRun || misnamed === destCollection) { + return; + } + let found = 0; + try { + found = await this.dest.db.collection(misnamed).countDocuments({form: form._id}, {limit: 1}); + } + catch (err) { + // Only a warning: it is never a reason to stop the clone. + } + if (found) { + this.error( + `The destination has submissions of ${form.title || form._id} in ${misnamed}, where the server never ` + + `reads them. They are cloned to ${destCollection}; ${misnamed} can be dropped once that is done.` + ); } - - this.dest.submissions = destProj && subsCollection - ? this.dest.db.collection(`${destProj.name}_${subsCollection}`) - : this.dest.ogSubmissions; } /** @@ -1541,6 +1463,7 @@ class Cloner { forms: db.collection('forms'), ogSubmissions: db.collection('submissions'), submissions: db.collection('submissions'), + ogSubmissionRevisions: db.collection('submissionrevisions'), submissionrevisions: db.collection('submissionrevisions'), roles: db.collection('roles'), actions: db.collection('actions'), @@ -1729,36 +1652,38 @@ class Cloner { * Migrate the data encrypted within a submission to a new secret. * @param {*} src - The source record * @param {*} update - The updated record - * @param {string} decryptKey - The source key to decrypt data (either project.settings.secret or DB_SECRET). + * @param {object[]|null} encryptedComponents - The components of the form, when it has encrypted ones. */ - migrateDataEncryption(src, update, compsWithEncryptedData) { + migrateDataEncryption(src, update, encryptedComponents) { const srcSecret = this.srcSecret || this.options.srcDbSecret; const destSecret = this.dstSecret || this.options.dstDbSecret; if ( - !compsWithEncryptedData.length || + !encryptedComponents || !srcSecret || !destSecret || srcSecret === destSecret ) { return; } - compsWithEncryptedData.forEach((compPath) => { - if (_.get(src, `data.${compPath}`, false)) { - const decrypted = this.decrypt(srcSecret, _.get(src, `data.${compPath}`)); - - // A value that decrypts to 0, false or an empty string is still a value: only a decrypt - // that actually failed comes back as null. - const reEncrypted = (decrypted === null || decrypted === undefined) - ? null - : this.encrypt(destSecret, decrypted); - if (!reEncrypted) { - // Leave the value as it is rather than destroying it: it can still be read with the - // secret it was encrypted with. - this.error(`Could not re-encrypt ${compPath} of submission ${src._id}, it was left as it is.`); - return; - } - _.set(update, `data.${compPath}`, reEncrypted); + + // The values are collected first: the update can share its data with the source. + const values = []; + encryption.eachEncryptedValue(encryptedComponents, src.data, (value, path) => values.push({value, path})); + values.forEach(({value, path}) => { + const decrypted = encryption.decrypt(srcSecret, value); + + // A value that decrypts to 0, false or an empty string is still a value: only a decrypt + // that actually failed comes back as null. + const reEncrypted = (decrypted === null || decrypted === undefined) + ? null + : encryption.encrypt(destSecret, decrypted); + if (!reEncrypted) { + // Leave the value as it is rather than destroying it: it can still be read with the + // secret it was encrypted with. + this.error(`Could not re-encrypt ${path.join('.')} of submission ${src._id}, it was left as it is.`); + return; } + _.set(update, ['data', ...path], reEncrypted); }); } @@ -1766,9 +1691,9 @@ class Cloner { * Clone submissions from one form to another. * @param {*} srcForm - The source form from the database. * @param {*} destForm - The destination from from the database. - * @param {string[]} compsWithEncryptedData - Array with paths of components that are encrypted. + * @param {object[]|null} encryptedComponents - The components of the form, when it has encrypted ones. */ - async cloneSubmissions(srcForm, destForm, compsWithEncryptedData) { + async cloneSubmissions(srcForm, destForm, encryptedComponents) { // A dry run reports on every submission of the form, not only the ones a clone would write, so // it checks the destination for each of them and does not start from the last one cloned. const compareAll = this.options.updateExistingSubmissions || this.dryRun; @@ -1802,9 +1727,7 @@ class Cloner { await this.upsertAll('submissions', query, null, async(src, update) => { update.form = destForm._id; update.project = destForm.project; - if (compsWithEncryptedData.length) { - this.migrateDataEncryption(src, update, compsWithEncryptedData); - } + this.migrateDataEncryption(src, update, encryptedComponents); if (update.roles) { // Only the source roles are mapped: a submission without roles must not have the roles of // the destination document wiped, which are no longer fetched. @@ -1813,7 +1736,7 @@ class Cloner { this.migrateAccess(src.access, update.access); }, async(srcSubmission, destSubmission) => { this.currentSubmission = srcSubmission; - await this.cloneSubmissionRevisions(srcSubmission, destSubmission, compsWithEncryptedData); + await this.cloneSubmissionRevisions(srcSubmission, destSubmission, encryptedComponents); }, { find: compareAll ? null : false, scope: srcForm._id.toString(), @@ -1847,7 +1770,7 @@ class Cloner { * @param {*} srcSubmission - The source submission. * @param {*} destSubmission - The destination submission. */ - async cloneSubmissionRevisions(srcSubmission, destSubmission, compsWithEncryptedData) { + async cloneSubmissionRevisions(srcSubmission, destSubmission, encryptedComponents) { if (!this.options.includeSubmissionRevisions) { return; } @@ -1855,7 +1778,7 @@ class Cloner { _rid: srcSubmission._id }), null, async(src, update) => { update._rid = destSubmission._id; - this.migrateDataEncryption(src, update, compsWithEncryptedData); + this.migrateDataEncryption(src, update, encryptedComponents); }, null, { scope: srcSubmission._id.toString(), @@ -1937,7 +1860,7 @@ class Cloner { * @param {*} destProject - The destination project. */ async cloneForms(srcProject, destProject) { - let compsWithEncryptedData = []; + let encryptedComponents = null; const query = this.formQuery(srcProject); const total = this.dryRun ? null : await this.countSource('src.forms', query, POSITION_COUNT_TIMEOUT); let position = 0; @@ -1956,12 +1879,7 @@ class Cloner { // This mapping runs for every source form, including the ones that are not written to the // destination (--submissions-only, or a form that has not changed), because the submissions // of the form are cloned using these values. - compsWithEncryptedData = []; - await eachComponentAsync(update.components, async(component, path) => { - if (component.encrypted) { - compsWithEncryptedData.push(path); - } - }); + encryptedComponents = encryption.hasEncryptedComponents(update.components) ? update.components : null; await this.setCollection(update, srcProject, destProject); if (destProject) { @@ -1984,7 +1902,7 @@ class Cloner { } this.currentForm = srcForm; await this.flushWrites(); - await this.cloneSubmissions(srcForm, destForm, compsWithEncryptedData); + await this.cloneSubmissions(srcForm, destForm, encryptedComponents); await this.cloneActions(srcForm, destForm); await this.cloneFormRevisions(srcForm, destForm); await this.flushWrites(); @@ -2145,7 +2063,7 @@ class Cloner { if (!decryptedSrcSettings) { return !!dest; } - const reEncrypted = this.encrypt(this.options.dstDbSecret, decryptedSrcSettings, true); + const reEncrypted = encryption.encrypt(this.options.dstDbSecret, decryptedSrcSettings, true); if (reEncrypted) { update['settings_encrypted'] = reEncrypted; } @@ -2182,7 +2100,7 @@ class Cloner { // A destination project without settings has no secret of its own yet. const decryptedDstSettings = (destProject && destProject['settings_encrypted']) - ? this.decrypt(this.options.dstDbSecret, destProject['settings_encrypted'], true) + ? encryption.decrypt(this.options.dstDbSecret, destProject['settings_encrypted'], true) : {}; if (!decryptedDstSettings) { this.reportUndecryptable('destination', destProject, '--dst-db-secret'); @@ -2191,7 +2109,7 @@ class Cloner { if (decryptedDstSettings.secret) { this.dstSecret = decryptedDstSettings.secret; } - const decryptedSrcSettings = this.decrypt(this.options.srcDbSecret, srcProject['settings_encrypted'], true); + const decryptedSrcSettings = encryption.decrypt(this.options.srcDbSecret, srcProject['settings_encrypted'], true); if (!decryptedSrcSettings) { this.reportUndecryptable('source', srcProject, '--src-db-secret'); return null; diff --git a/src/Rekeyer.js b/src/Rekeyer.js new file mode 100644 index 0000000..b9d515a --- /dev/null +++ b/src/Rekeyer.js @@ -0,0 +1,523 @@ +'use strict'; +const {MongoClient, ObjectId} = require('mongodb'); +const _ = require('lodash'); +const Progress = require('./Progress'); +const { + redact, + tlsOptions, + submissionCollectionName, + legacySubmissionCollectionName, + idList +} = require('./db'); +const {encrypt, decrypt, eachEncryptedValue, hasEncryptedComponents} = require('./encryption'); + +const SETTINGS = 'settings_encrypted'; + +// How long, at most, counting the documents behind a progress bar may take. +const COUNT_TIMEOUT = 10000; + +// How many of the values that could not be decrypted are listed one by one, per collection. +const MAX_LISTED = 20; + +/** + * Re-encrypt the project settings and the encrypted submission data of a database with a new + * DB_SECRET. + * + * The project settings are encrypted with the DB_SECRET. The encrypted submission data is encrypted + * with the secret of its project (the "secret" of the project settings) or, when the project has + * none, with the DB_SECRET. Every value is first tried with the secret it should be encrypted with: + * the ones that already are are left alone, so this can be run again, as often as needed, until + * nothing is left to re-key. The others are tried with each of the old secrets, re-encrypted, and + * only written once the new value has been read back. A value none of the secrets can read is + * reported and left as it is. + */ +class Rekeyer { + /** + * @param {string} mongoUrl - The connection string of the database. + * @param {object} options - The options. + * @param {string} options.newDbSecret - The DB_SECRET everything should be encrypted with. + * @param {string|string[]} options.oldDbSecret - The secrets things may currently be encrypted with. + * @param {string} options.project - A comma separated list of the projects to re-key (all by default). + * @param {boolean} options.dryRun - Change nothing, only report what would be re-keyed. + * @param {number} options.batchSize - The number of documents written at a time (defaults to 1000). + * @param {boolean} options.submissionRevisions - Re-key the submission revisions too (defaults to true). + * @param {string} options.ca - The TLS certificate authority file. + * @param {string} options.cert - The TLS certificate key file. + */ + constructor(mongoUrl, options = {}) { + this.mongoUrl = mongoUrl; + this.options = options; + this.newSecret = options.newDbSecret; + this.oldSecrets = _.uniq([].concat(options.oldDbSecret || []).filter((secret) => secret)); + this.projectIds = idList(options.project); + this.dryRun = !!options.dryRun; + this.batchSize = Math.max(parseInt(options.batchSize, 10) || 1000, 1); + this.includeRevisions = options.submissionRevisions !== false; + this.interactive = (typeof options.interactive === 'boolean') + ? options.interactive + : !!process.stdout.isTTY; + this.activeProgress = null; + this.client = null; + this.db = null; + this.totals = { + settings: {rekeyed: 0, current: 0, undecryptable: 0}, + values: {rekeyed: 0, current: 0, undecryptable: 0}, + skippedProjects: 0, + conflicts: 0, + writeFailures: 0 + }; + } + + /** + * Make sure the options make sense before anything is touched. + */ + validate() { + if (!this.mongoUrl) { + throw new Error('The connection string of the database is required.'); + } + if (!this.newSecret) { + throw new Error('--new-db-secret is required.'); + } + if (!this.oldSecrets.length) { + throw new Error('At least one --old-db-secret is required.'); + } + if (this.oldSecrets.includes(this.newSecret)) { + throw new Error('The new secret is also given as an old secret.'); + } + const invalid = this.projectIds.filter((id) => !ObjectId.isValid(id)); + if (invalid.length) { + throw new Error(`Not a valid project id: ${invalid.join(', ')}`); + } + } + + /** + * Write a line, without it ending up in the middle of a progress line. + * @param {...*} args - What to write. + */ + log(...args) { + if (this.activeProgress) { + this.activeProgress.interrupt(); + } + console.log(...args); + } + + /** + * Report an error, without it ending up in the middle of a progress line. + * @param {...*} args - What to report. + */ + error(...args) { + if (this.activeProgress) { + this.activeProgress.interrupt(); + } + console.error(...args); + } + + async connect() { + try { + this.log(`Connecting to ${redact(this.mongoUrl)}`); + this.client = new MongoClient(this.mongoUrl, tlsOptions(this.options.ca, this.options.cert)); + await this.client.connect(); + this.db = this.client.db(); + } + catch (err) { + throw new Error(`Could not connect to database ${redact(this.mongoUrl)}: ${err.message}`, {cause: err}); + } + } + + async disconnect() { + if (this.client) { + await this.client.close(); + this.client = null; + } + } + + /** + * Re-key everything the options ask for. + */ + async rekey() { + this.validate(); + await this.connect(); + try { + if (this.dryRun) { + this.log('Dry run: nothing will be changed.'); + } + await this.rekeyProjects(); + } + finally { + await this.disconnect(); + } + this.report(); + const failures = this.failures(); + if (failures.length) { + throw new Error(`The re-key did not complete: ${failures.join(', ')}.`); + } + } + + /** + * What kept the re-key from completing, if anything. + * @returns {string[]} - The reasons. + */ + failures() { + const {settings, values, skippedProjects, conflicts, writeFailures} = this.totals; + const failures = []; + if (settings.undecryptable) { + failures.push(`${settings.undecryptable} project(s) had settings none of the secrets could decrypt`); + } + if (skippedProjects) { + failures.push(`${skippedProjects} project(s) were skipped`); + } + if (values.undecryptable) { + failures.push(`${values.undecryptable} encrypted value(s) could not be decrypted with any of the secrets`); + } + if (conflicts) { + failures.push(`${conflicts} document(s) changed while they were being re-keyed and were left as they are`); + } + if (writeFailures) { + failures.push(`${writeFailures} document(s) could not be written`); + } + return failures; + } + + /** + * Write the totals. + */ + report() { + const {settings, values} = this.totals; + const rekeyed = this.dryRun ? 'would be re-keyed' : 're-keyed'; + this.log(''); + this.log(this.dryRun ? 'Dry run complete. Nothing was changed.' : 'Done.'); + this.log(' Project settings'); + this.log(` ${rekeyed.padEnd(28)}: ${settings.rekeyed}`); + this.log(` ${'already on the new secret'.padEnd(28)}: ${settings.current}`); + this.log(` ${'could not be decrypted'.padEnd(28)}: ${settings.undecryptable}`); + this.log(' Encrypted submission values'); + this.log(` ${rekeyed.padEnd(28)}: ${values.rekeyed}`); + this.log(` ${'already on the new secret'.padEnd(28)}: ${values.current}`); + this.log(` ${'could not be decrypted'.padEnd(28)}: ${values.undecryptable}`); + } + + /** + * Try to re-key a single encrypted value. + * + * @param {Buffer|Binary} cipher - The encrypted value. + * @param {string} target - The secret the value should be encrypted with. + * @param {string[]} candidates - The secrets the value may currently be encrypted with. + * @param {boolean} nosalt - The value is encrypted without a salt, as the project settings are. + * @returns {{status: string, value: *, cipher: Buffer}} - The status is "current" when the value is + * already encrypted with the target secret, "rekeyed" when it was re-encrypted into + * cipher, and "undecryptable" when none of the secrets could read it. + */ + rekeyValue(cipher, target, candidates, nosalt = false) { + const current = decrypt(target, cipher, nosalt); + if (current !== null) { + return {status: 'current', value: current}; + } + for (const secret of candidates) { + if (secret === target) { + continue; + } + const value = decrypt(secret, cipher, nosalt); + if (value === null) { + continue; + } + const reEncrypted = encrypt(target, value, nosalt); + + // Never write a value that cannot be read back. + if (!reEncrypted || !_.isEqual(decrypt(target, reEncrypted, nosalt), value)) { + return {status: 'undecryptable'}; + } + return {status: 'rekeyed', value, cipher: reEncrypted}; + } + return {status: 'undecryptable'}; + } + + /** + * Re-key every project the options ask for. + */ + async rekeyProjects() { + const query = this.projectIds.length + ? {_id: {$in: this.projectIds.map((id) => new ObjectId(id))}} + : {}; + if (this.projectIds.length) { + const found = await this.db.collection('projects').countDocuments(query); + if (found !== this.projectIds.length) { + throw new Error(`Only ${found} of the ${this.projectIds.length} projects given were found.`); + } + } + // Only the ids are listed up front: a cursor left idle while the submissions of a project are + // re-keyed, which can take hours, is closed by the server before the next project is read. + const ids = await this.idsOf(this.db.collection('projects'), query); + for (const _id of ids) { + const project = await this.db.collection('projects').findOne({_id}); + if (project) { + await this.rekeyProject(project); + } + } + } + + /** + * Re-key the settings and the encrypted submission data of a project. + * @param {object} project - The project. + */ + async rekeyProject(project) { + this.log(`\n- Project ${project.title || project.name} (${project._id})${project.deleted ? ' [deleted]' : ''}`); + const settings = await this.rekeySettings(project); + if (!settings) { + this.totals.skippedProjects++; + this.error(' Its submission data was skipped: the secret it is encrypted with is in those settings.'); + return; + } + + // The data is encrypted with the secret of the project, or with the DB_SECRET when it has none. + // Any of the other secrets may have been used by mistake, so all of them are tried. + const target = settings.secret || this.newSecret; + const candidates = _.uniq([...this.oldSecrets, this.newSecret, settings.secret]) + .filter((secret) => secret && secret !== target); + + const formIds = await this.idsOf(this.db.collection('forms'), {project: project._id}); + for (const _id of formIds) { + const form = await this.db.collection('forms').findOne({_id}); + if (form && hasEncryptedComponents(form.components)) { + await this.rekeyForm(project, form, target, candidates); + } + } + } + + /** + * The ids of the documents a query matches, in the order they were created. + * @param {Collection} collection - The collection. + * @param {object} query - The query. + * @returns {ObjectId[]} - The ids. + */ + async idsOf(collection, query) { + const docs = await collection.find(query, {projection: {_id: 1}}).sort({created: 1, _id: 1}).toArray(); + return docs.map((doc) => doc._id); + } + + /** + * Re-key the settings of a project. + * @param {object} project - The project. + * @returns {object|null} - The decrypted settings, or null when they could not be re-keyed. + */ + async rekeySettings(project) { + if (!project[SETTINGS]) { + return {}; + } + const result = this.rekeyValue(project[SETTINGS], this.newSecret, this.oldSecrets, true); + if (result.status === 'undecryptable') { + this.totals.settings.undecryptable++; + this.error(' Its settings could not be decrypted with any of the secrets, they were left as they are.'); + return null; + } + this.totals.settings[result.status]++; + if (result.status === 'current') { + this.log(' Settings: already on the new secret.'); + return result.value; + } + if (this.dryRun) { + this.log(' Settings: would be re-keyed.'); + return result.value; + } + const written = await this.db.collection('projects').updateOne( + {_id: project._id, [SETTINGS]: project[SETTINGS]}, + {$set: {[SETTINGS]: result.cipher}} + ); + if (!written.matchedCount) { + this.totals.settings.rekeyed--; + this.totals.conflicts++; + this.error(' Its settings changed while they were being re-keyed, they were left as they are.'); + return null; + } + this.log(' Settings: re-keyed.'); + return result.value; + } + + /** + * The name of the collection the submissions of a form are kept in, the way the server names it. + * @param {object} project - The project. + * @param {object} form - The form. + */ + submissionCollection(project, form) { + const collection = _.get(form, 'settings.collection'); + return (collection && submissionCollectionName(project.name, collection)) || 'submissions'; + } + + /** + * Re-key the encrypted data of the submissions, and submission revisions, of a form. + * @param {object} project - The project. + * @param {object} form - The form. + * @param {string} target - The secret the data should be encrypted with. + * @param {string[]} candidates - The secrets the data may currently be encrypted with. + */ + async rekeyForm(project, form, target, candidates) { + this.log(` - Form ${form.title || form.name} (${form._id})${form.deleted ? ' [deleted]' : ''}`); + const collections = [this.submissionCollection(project, form)]; + + // Earlier versions of the clone command named a custom collection after the unsanitized project + // name, where the server never looks for them. Those submissions are re-keyed too, but they + // still have to be moved to where the server reads them from. + const custom = _.get(form, 'settings.collection'); + const unsanitized = custom ? legacySubmissionCollectionName(project.name, custom) : null; + const misplaced = unsanitized && unsanitized !== collections[0] && + await this.count(this.db.collection(unsanitized), {form: form._id}); + if (misplaced) { + this.error( + ` Submissions of this form are in ${unsanitized}, but the server reads them from ${collections[0]}. ` + + 'They are re-keyed where they are.' + ); + collections.push(unsanitized); + } + + const query = {form: form._id}; + for (const collection of collections) { + await this.rekeyCollection(collection, query, form.components, target, candidates, ' - Submissions:'); + } + if (!this.includeRevisions) { + return; + } + const revisions = collections.map((collection) => { + return (collection === 'submissions') ? 'submissionrevisions' : `${collection}_revisions`; + }); + + // Earlier versions of the clone command also wrote the revisions of a custom collection to the + // default revisions collection. The server never reads them there, but they are re-keyed all the + // same, so that nothing is left on the old secret. + if (custom && !revisions.includes('submissionrevisions') && + await this.db.collection('submissionrevisions').findOne(query, {projection: {_id: 1}})) { + this.error( + ` Revisions of this form are in submissionrevisions, but the server reads them from ${revisions[0]}. ` + + 'They are re-keyed where they are.' + ); + revisions.push('submissionrevisions'); + } + for (const collection of revisions) { + await this.rekeyCollection(collection, query, form.components, target, candidates, ' - Revisions:'); + } + } + + /** + * Count the documents behind a progress bar, giving up when it takes too long. + * @param {Collection} collection - The collection. + * @param {object} query - The query. + */ + async count(collection, query) { + try { + return await collection.countDocuments(query, {maxTimeMS: COUNT_TIMEOUT}); + } + catch (err) { + return null; + } + } + + /** + * Re-key the encrypted data of the documents of a collection. + * @param {string} name - The name of the collection. + * @param {object} query - The documents to re-key. + * @param {object[]} components - The components of the form the documents belong to. + * @param {string} target - The secret the data should be encrypted with. + * @param {string[]} candidates - The secrets the data may currently be encrypted with. + * @param {string} label - The label of the progress. + */ + async rekeyCollection(name, query, components, target, candidates, label) { + const collection = this.db.collection(name); + const total = await this.count(collection, query); + if (total === 0) { + return; + } + const progress = new Progress(label, {total, interactive: this.interactive}); + this.activeProgress = progress; + const counts = {rekeyed: 0, current: 0, undecryptable: 0}; + let operations = []; + + // What could not be decrypted is listed once the progress line is done, never in the middle of it. + const unreadable = []; + + const cursor = collection + .find(query, {projection: {data: 1}}) + .sort({_id: 1}) + .batchSize(Math.min(this.batchSize, 1000)); + for await (const doc of cursor) { + const filter = {_id: doc._id}; + const update = {}; + eachEncryptedValue(components, doc.data, (value, path) => { + const result = this.rekeyValue(value, target, candidates); + counts[result.status]++; + const key = ['data', ...path].join('.'); + if (result.status === 'rekeyed') { + // The old value is part of the filter, so that a value changed in the meantime is never + // overwritten with one re-keyed from what it used to be. + filter[key] = value; + update[key] = result.cipher; + } + else if (result.status === 'undecryptable') { + unreadable.push(`${key} of ${name} ${doc._id}`); + } + }); + if (!_.isEmpty(update)) { + operations.push({updateOne: {filter, update: {$set: update}}}); + } + if (operations.length >= this.batchSize) { + await this.write(collection, operations); + operations = []; + } + progress.tick(); + } + await this.write(collection, operations); + progress.done(); + this.activeProgress = null; + Object.keys(counts).forEach((status) => { + this.totals.values[status] += counts[status]; + }); + + // Finishes the progress line. + this.log( + ` - ${counts.rekeyed} ${this.dryRun ? 'would be re-keyed' : 're-keyed'}, ` + + `${counts.current} already on the new secret, ${counts.undecryptable} could not be decrypted` + ); + unreadable.slice(0, MAX_LISTED).forEach((value) => { + this.error(` ${value} could not be decrypted with any of the secrets.`); + }); + if (unreadable.length > MAX_LISTED) { + this.error(` ...and ${unreadable.length - MAX_LISTED} more.`); + } + } + + /** + * Write a batch of re-keyed documents. + * @param {Collection} collection - The collection. + * @param {object[]} operations - The bulk write operations. + */ + async write(collection, operations) { + if (!operations.length || this.dryRun) { + return; + } + let matched; + let failed = 0; + try { + const result = await collection.bulkWrite(operations, {ordered: false}); + matched = result.matchedCount; + } + catch (err) { + const writeErrors = [].concat(_.get(err, 'writeErrors', [])); + if (!err.result || !writeErrors.length) { + throw err; + } + failed = writeErrors.length; + matched = err.result.matchedCount; + this.error(`Failed to write ${failed} document(s) of ${collection.collectionName}: ${writeErrors[0].errmsg}`); + } + this.totals.writeFailures += failed; + + // A document whose encrypted values changed since it was read no longer matches its filter. + const conflicts = operations.length - matched - failed; + if (conflicts > 0) { + this.totals.conflicts += conflicts; + this.error( + `${conflicts} document(s) of ${collection.collectionName} changed while being re-keyed, ` + + 'run again to re-key them.' + ); + } + } +} + +module.exports = Rekeyer; diff --git a/src/db.js b/src/db.js new file mode 100644 index 0000000..8dbe46d --- /dev/null +++ b/src/db.js @@ -0,0 +1,77 @@ +'use strict'; +const path = require('path'); + +/** + * Hide the credentials of a connection string, so that they never end up in a log. + * @param {string} uri - The connection string. + */ +function redact(uri) { + return String(uri).replace(/\/\/[^@/]+@/, '//***@'); +} + +/** + * The mongo client options for a connection with a custom certificate authority or a client + * certificate. The driver always uses the new url parser and the unified topology: from version 7 on + * it rejects the options that used to turn them on. + * + * @param {string} caFile - The certificate authority file, absolute or relative to the working directory. + * @param {string} certFile - The certificate key file, absolute or relative to the working directory. + * @returns {object} - The client options. + */ +function tlsOptions(caFile, certFile) { + const config = {}; + if (caFile || certFile) { + config.tls = true; + if (caFile) { + config.tlsCAFile = path.resolve(caFile); + + // A custom certificate authority is usually an internal one whose certificates do not name + // the host being connected to. A client certificate on its own is no reason to stop + // verifying who we are talking to. + config.tlsAllowInvalidHostnames = true; + } + if (certFile) { + config.tlsCertificateKeyFile = path.resolve(certFile); + } + } + return config; +} + +/** + * The name of the collection the server keeps the submissions of a form with its own collection in. + * The server strips everything but letters and digits from both the project name and the collection. + * + * @param {string} projectName - The name of the project. + * @param {string} collection - The collection setting of the form. + * @returns {string|null} - The collection name, or null when the project name has nothing left. + */ +function submissionCollectionName(projectName, collection) { + const project = String(projectName || '').replace(/[^A-Za-z0-9]+/g, ''); + if (!project) { + return null; + } + return `${project}_${String(collection).replace(/[^A-Za-z0-9]+/g, '')}`; +} + +/** + * The name earlier versions of the clone command gave the collection of a form with its own + * collection: the unsanitized project and collection names, where the server never reads them. + * + * @param {string} projectName - The name of the project. + * @param {string} collection - The collection setting of the form. + * @returns {string} - The collection name. + */ +function legacySubmissionCollectionName(projectName, collection) { + return `${projectName}_${collection}`; +} + +/** + * Split a comma separated list of ids, ignoring the spaces around them and any repeated id. + * @param {string} value - The comma separated list. + * @returns {string[]} - The ids. + */ +function idList(value) { + return [...new Set(String(value || '').split(',').map((id) => id.trim()).filter((id) => id))]; +} + +module.exports = {redact, tlsOptions, submissionCollectionName, legacySubmissionCollectionName, idList}; diff --git a/src/eachComponentAsync.js b/src/eachComponentAsync.js deleted file mode 100644 index 8a4f816..0000000 --- a/src/eachComponentAsync.js +++ /dev/null @@ -1,92 +0,0 @@ -'use strict'; - -/** - * Iterate through each component within a form. - * - * @param {Object} components - * The components to iterate. - * @param {Function} fn - * The iteration function to invoke for each component. - * @param {Boolean} includeAll - * Whether or not to include layout components. - * @param {String} path - * The current data path of the element. Example: data.user.firstName - * @param {Object} parent - * The parent object. - */ - -const eachComponentAsync = async(components, fn, includeAll, path = '', parent) => { - if (!components || !components.length || !fn) { - return; - } - - for (const component of components) { - if (!component) { - return; - } - const hasColumns = component.columns && Array.isArray(component.columns); - const hasRows = component.rows && Array.isArray(component.rows); - const hasComps = component.components && Array.isArray(component.components); - let noRecurse = false; - const newPath = component.key ? (path ? (`${path}.${component.key}`) : component.key) : ''; - - // Keep track of parent references. - if (parent) { - // Ensure we don't create infinite JSON structures. - component.parent = {...parent}; - delete component.parent.components; - delete component.parent.componentMap; - delete component.parent.columns; - delete component.parent.rows; - } - - // there's no need to add other layout components here because we expect that those would either have columns, rows or components - const layoutTypes = ['htmlelement', 'content']; - const isLayoutComponent = hasColumns || hasRows || hasComps || layoutTypes.indexOf(component.type) > -1; - if (includeAll || component.tree || !isLayoutComponent) { - noRecurse = await fn(component, newPath, components); - } - - const subPath = () => { - if ( - component.key && - !['panel', 'table', 'well', 'columns', 'fieldset', 'tabs', 'form'].includes(component.type) && - ( - ['datagrid', 'container', 'editgrid', 'address', 'dynamicWizard'].includes(component.type) || - component.tree - ) - ) { - return newPath; - } - else if ( - component.key && - component.type === 'form' - ) { - return `${newPath}.data`; - } - return path; - }; - - if (!noRecurse) { - if (hasColumns) { - for (const column of component.columns) { - await eachComponentAsync(column.components, fn, includeAll, subPath(), parent ? component : null); - } - } - else if (hasRows) { - for (const row of component.rows) { - if (Array.isArray(row)) { - for (const column of row) { - await eachComponentAsync(column.components, fn, includeAll, subPath(), parent ? component : null); - } - } - } - } - else if (hasComps) { - await eachComponentAsync(component.components, fn, includeAll, subPath(), parent ? component : null); - } - } - } -}; - -module.exports = {eachComponentAsync}; diff --git a/src/encryption.js b/src/encryption.js new file mode 100644 index 0000000..444e3c7 --- /dev/null +++ b/src/encryption.js @@ -0,0 +1,232 @@ +'use strict'; +const crypto = require('crypto'); +const _ = require('lodash'); +const keygenerator = require('keygenerator'); + +// The number of random characters the server appends to an encrypted value (but not to encrypted +// settings) before encrypting it. +const SALT_LENGTH = 40; + +// The components whose data is nested under their own key. Layout components never nest data, even +// when they have a key. +const DATA_TYPES = ['datagrid', 'container', 'editgrid', 'address', 'dynamicWizard']; +const LAYOUT_TYPES = ['panel', 'table', 'well', 'columns', 'fieldset', 'tabs', 'form']; + +/** + * Derive the key and iv for a secret exactly the way the removed crypto.createCipher() did, which + * is OpenSSL's EVP_BytesToKey with MD5, a single iteration and no salt. This keeps us compatible + * with data that was encrypted by the servers, while still running on Node versions where + * crypto.createCipher() no longer exists. + * + * @param {string} secret - The secret to derive the key and iv from. + * @returns {{key: Buffer, iv: Buffer}} - The derived key and iv. + */ +function legacyKey(secret) { + // createCipher() read the password as UTF-8; latin1 would derive a different key for any secret + // holding a non ASCII character, and nothing encrypted with it could be read back. + const password = Buffer.from(secret, 'utf8'); + const keyLength = 32; + const ivLength = 16; + let derived = Buffer.alloc(0); + let block = Buffer.alloc(0); + while (derived.length < keyLength + ivLength) { + block = crypto.createHash('md5').update(Buffer.concat([block, password])).digest(); + derived = Buffer.concat([derived, block]); + } + return { + key: derived.subarray(0, keyLength), + iv: derived.subarray(keyLength, keyLength + ivLength) + }; +} + +/** + * Whether a value is a Buffer that went through JSON, which is how an API hands back the encrypted + * values it reads raw from the database. + * @param {*} value - The value. + */ +function isSerializedBuffer(value) { + return !!(value && value.type === 'Buffer' && Array.isArray(value.data)); +} + +/** + * Encrypt data or settings using a secret, the way the server does. + * @param {string} secret - The secret. + * @param {*} rawData - The data to encrypt. + * @param {boolean} nosalt - Leave out the salt, as the project settings do. + * @param {number} saltLength - The length of the salt. + * @returns {Buffer|null} - The encrypted data, or null when there is nothing to encrypt. + */ +function encrypt(secret, rawData, nosalt, saltLength = SALT_LENGTH) { + if (!secret || rawData === null || rawData === undefined) { + return null; + } + const salt = nosalt ? '' : keygenerator._({length: saltLength}); + const {key, iv} = legacyKey(secret); + const cipher = crypto.createCipheriv('aes-256-cbc', key, iv); + return Buffer.concat([ + cipher.update(JSON.stringify(rawData) + salt), + cipher.final() + ]); +} + +/** + * Decrypt data or settings using a secret. + * @param {string} secret - The secret. + * @param {Buffer|Binary|object} cipherbuffer - The encrypted data, as a Buffer, a mongo Binary or a + * Buffer serialized as JSON. + * @param {boolean} nosalt - The data was encrypted without a salt, as the project settings are. + * @param {number} saltLength - The length of the salt. + * @returns {*} - The decrypted data, or null when it cannot be decrypted with this secret. + */ +function decrypt(secret, cipherbuffer, nosalt, saltLength = SALT_LENGTH) { + if (!secret || !cipherbuffer) { + return null; + } + try { + const buffer = Buffer.isBuffer(cipherbuffer) + ? cipherbuffer + : (isSerializedBuffer(cipherbuffer) ? Buffer.from(cipherbuffer.data) : cipherbuffer.buffer); + const {key, iv} = legacyKey(secret); + const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv); + const decryptedJSON = Buffer.concat([ + decipher.update(buffer), + decipher.final() + ]); + return JSON.parse(nosalt ? decryptedJSON : decryptedJSON.slice(0, -saltLength)); + } + catch (err) { + // The wrong secret: either the padding does not check out, or what comes out is not JSON. + return null; + } +} + +/** + * Whether a stored value is encrypted data, rather than a plain value the server never encrypted + * (it leaves empty values, and the values saved before a component was marked encrypted, alone). + * @param {*} value - The stored value. + */ +function isEncryptedValue(value) { + return Buffer.isBuffer(value) || !!(value && value._bsontype === 'Binary') || isSerializedBuffer(value); +} + +/** + * Whether the server encrypts the value of a component. + * @param {object} component - The component. + */ +function isEncryptedComponent(component) { + return !!( + component && + component.key && + component.encrypted && + (!Object.prototype.hasOwnProperty.call(component, 'persistent') || component.persistent) + ); +} + +/** + * The lists of child components of a component. + * @param {object} component - The component. + */ +function childComponents(component) { + const lists = []; + if (Array.isArray(component.columns)) { + component.columns.forEach((column) => lists.push(column && column.components)); + } + else if (Array.isArray(component.rows)) { + component.rows.forEach((row) => { + if (Array.isArray(row)) { + row.forEach((column) => lists.push(column && column.components)); + } + else { + lists.push(row && row.components); + } + }); + } + else if (Array.isArray(component.components)) { + lists.push(component.components); + } + return lists.filter(Array.isArray); +} + +/** + * Whether the data of the children of a component is nested under the key of the component. + * @param {object} component - The component. + */ +function nestsData(component) { + return !!( + component.key && + !LAYOUT_TYPES.includes(component.type) && + (DATA_TYPES.includes(component.type) || component.tree) + ); +} + +/** + * Walk the encrypted values of a submission. The form is walked against the data itself, so the + * values of the rows of a datagrid or an editgrid are found as well. The children of an encrypted + * component are never visited: the server encrypts the whole value of the component at once. + * + * @param {object[]} components - The components of the form. + * @param {object} data - The data of the submission. + * @param {Function} fn - Called with each encrypted value and the path to it within the data, as + * an array of keys and row indexes. + * @param {Array} path - The path to the data, used while recursing. + */ +function eachEncryptedValue(components, data, fn, path = []) { + if (!Array.isArray(components) || !_.isPlainObject(data)) { + return; + } + for (const component of components) { + if (!component) { + continue; + } + if (isEncryptedComponent(component)) { + const value = data[component.key]; + if (isEncryptedValue(value)) { + fn(value, [...path, component.key]); + } + continue; + } + const children = childComponents(component); + if (!children.length) { + continue; + } + if (nestsData(component)) { + const value = data[component.key]; + if (Array.isArray(value)) { + value.forEach((row, index) => children.forEach((list) => { + eachEncryptedValue(list, row, fn, [...path, component.key, index]); + })); + } + else { + children.forEach((list) => eachEncryptedValue(list, value, fn, [...path, component.key])); + } + } + else { + children.forEach((list) => eachEncryptedValue(list, data, fn, path)); + } + } +} + +/** + * Whether a form has any encrypted component. + * @param {object[]} components - The components of the form. + */ +function hasEncryptedComponents(components) { + if (!Array.isArray(components)) { + return false; + } + return components.some((component) => component && ( + isEncryptedComponent(component) || + childComponents(component).some(hasEncryptedComponents) + )); +} + +module.exports = { + SALT_LENGTH, + legacyKey, + encrypt, + decrypt, + isEncryptedValue, + isEncryptedComponent, + eachEncryptedValue, + hasEncryptedComponents +}; diff --git a/src/rekey.js b/src/rekey.js new file mode 100644 index 0000000..ba89edc --- /dev/null +++ b/src/rekey.js @@ -0,0 +1,12 @@ +'use strict'; +const Rekeyer = require('./Rekeyer'); +module.exports = async function(mongoUrl, options) { + const rekeyer = new Rekeyer(mongoUrl, options); + try { + await rekeyer.rekey(); + } + catch (err) { + console.error(err.message.red); + process.exitCode = 1; + } +}; diff --git a/test/cli/index.js b/test/cli/index.js index 98e5fe0..fd256ca 100644 --- a/test/cli/index.js +++ b/test/cli/index.js @@ -50,8 +50,8 @@ function run(...args) { program.configureOutput({writeOut: () => {}, writeErr: () => {}}); require(path.join(root, 'commands/commands'))(program, (err) => (err ? reject(err) : resolve())); program.parseAsync(['node', 'formio', ...args]).then(() => { - // The clone command is not run through a series, so it is done once the action is. - if (args[0] === 'clone') { + // The clone and rekey commands are not run through a series, so they are done once the action is. + if (['clone', 'rekey'].includes(args[0])) { resolve(); } }, reject); @@ -71,6 +71,9 @@ describe('Command line', function() { stub('src/clone.js', async(source, destination, options) => { calls.push({name: 'clone', source, destination, options}); }); + stub('src/rekey.js', async(mongoUrl, options) => { + calls.push({name: 'rekey', mongoUrl, options}); + }); }); beforeEach(function() { @@ -127,6 +130,45 @@ describe('Command line', function() { }); }); + describe('rekey', function() { + it('Should hand every option to the re-key', async function() { + await run('rekey', 'mongodb://db', + '--new-db-secret', 'new', + '--old-db-secret', 'old1', + '--old-db-secret', 'old2', + '--project', 'aaa,bbb', + '--dry-run', + '--no-submission-revisions', + '--batch-size', '50', + '--ca', 'ca.pem', + '--cert', 'cert.pem'); + + assert.equal(calls.length, 1); + const {mongoUrl, options} = calls[0]; + assert.equal(mongoUrl, 'mongodb://db'); + assert.equal(options.newDbSecret, 'new'); + assert.deepEqual(options.oldDbSecret, ['old1', 'old2']); + assert.equal(options.project, 'aaa,bbb'); + assert.equal(options.dryRun, true); + assert.equal(options.submissionRevisions, false); + assert.equal(options.batchSize, '50'); + assert.equal(options.ca, 'ca.pem'); + assert.equal(options.cert, 'cert.pem'); + }); + + it('Should use the defaults when no options are given', async function() { + await run('rekey', 'mongodb://db', '--new-db-secret', 'new'); + const {options} = calls[0]; + assert.deepEqual(options.oldDbSecret, []); + assert.equal(options.dryRun, false); + assert.equal(options.submissionRevisions, true); + }); + + it('Should require the new secret', async function() { + await assert.rejects(() => run('rekey', 'mongodb://db'), /required option '--new-db-secret '/); + }); + }); + describe('commands run as a series', function() { it('Should hand the options and arguments of deploy to every step', async function() { await run('deploy', 'https://src.example.com', 'https://dst.example.com', @@ -208,12 +250,16 @@ describe('Command line', function() { assert.match(formio('copy', '--help'), /--full/); assert.match(formio('migrate', '--help'), /--delete-before/); assert.match(formio('submissions', '--help'), /--admin-key/); + assert.match(formio('rekey', '--help'), /--old-db-secret/); }); }); describe('modules', function() { it('Should load every command implementation', function() { - for (const file of ['src/migrate.js', 'src/submissions.js', 'src/copy.js', 'src/deploy.js', 'src/clone.js']) { + const files = [ + 'src/migrate.js', 'src/submissions.js', 'src/copy.js', 'src/deploy.js', 'src/clone.js', 'src/rekey.js' + ]; + for (const file of files) { const resolved = require.resolve(path.join(root, file)); const cached = require.cache[resolved]; delete require.cache[resolved]; diff --git a/test/cloner/index.js b/test/cloner/index.js index 5d041f8..1483d94 100644 --- a/test/cloner/index.js +++ b/test/cloner/index.js @@ -18,4 +18,5 @@ describe('Cloner', function() { require('./api'); require('./operations'); require('./progress'); + require('./rekey'); }); diff --git a/test/cloner/operations.js b/test/cloner/operations.js index 4bdcdb4..1f5ea1f 100644 --- a/test/cloner/operations.js +++ b/test/cloner/operations.js @@ -4,7 +4,6 @@ const assert = require('assert'); const fs = require('fs'); const path = require('path'); const {ObjectId, Collection} = require('mongodb'); -const Cloner = require('../../src/Cloner'); const {seedProject, at, project, role, form, formRevision, submissionRevision} = require('./fixtures'); const { useTestDatabases, @@ -17,12 +16,10 @@ const { } = require('./helper'); /** - * Build a bare cloner instance that can encrypt/decrypt without connecting to anything. + * The helpers that encrypt and decrypt the way the server does. */ function crypter() { - const instance = Object.create(Cloner.prototype); - instance.defaultSaltLength = 40; - return instance; + return require('../../src/encryption'); } /** diff --git a/test/cloner/projects.js b/test/cloner/projects.js index e24ab41..19d4f91 100644 --- a/test/cloner/projects.js +++ b/test/cloner/projects.js @@ -3,17 +3,14 @@ const assert = require('assert'); const _ = require('lodash'); const {ObjectId} = require('mongodb'); -const Cloner = require('../../src/Cloner'); -const {seedProject, at, project, role, form, submission} = require('./fixtures'); +const {seedProject, at, project, role, form, submission, submissionRevision} = require('./fixtures'); const {useTestDatabases, runClone, quiet} = require('./helper'); /** - * Build a bare cloner instance that can encrypt/decrypt without connecting to anything. + * The helpers that encrypt and decrypt the way the server does. */ function crypter() { - const instance = Object.create(Cloner.prototype); - instance.defaultSaltLength = 40; - return instance; + return require('../../src/encryption'); } describe('Project targeting', function() { @@ -287,6 +284,74 @@ describe('Project targeting', function() { }); }); + describe('Custom submission collections of projects with symbols in their name', function() { + it('Should clone submissions and revisions to the collections the server reads', async function() { + const seed = await seedProject(dbs.src, { + forms: 1, + submissions: 0, + project: {name: 'src-project'}, + formOverrides: [{settings: {collection: 'custom-records'}}] + }); + const destProject = await dbs.dst.insert('projects', project({title: 'Destination', name: 'dst-project'})); + const sub = await dbs.src.insert('srcproject_customrecords', submission({ + form: seed.forms[0]._id, + project: seed.project._id, + data: {firstName: 'Custom'}, + created: at(20), + modified: at(20) + })); + await dbs.src.insert('srcproject_customrecords_revisions', submissionRevision({ + _rid: sub._id, + form: seed.forms[0]._id, + project: seed.project._id, + data: {firstName: 'Custom v1'}, + created: at(10), + modified: at(10) + })); + + await quiet(() => runClone({ + srcProject: seed.project._id.toString(), + dstProject: destProject._id.toString(), + includeSubmissionRevisions: true + })); + + const custom = await dbs.dst.all('dstproject_customrecords'); + assert.equal(custom.length, 1, 'Submissions must land where the server reads them.'); + assert.equal(custom[0].data.firstName, 'Custom'); + assert.equal(await dbs.dst.count('dst-project_custom-records'), 0); + const revisions = await dbs.dst.all('dstproject_customrecords_revisions'); + assert.equal(revisions.length, 1, 'Revisions of a custom collection must be cloned to its revisions.'); + assert.equal(revisions[0]._rid.toString(), sub._id.toString()); + assert.equal(await dbs.dst.count('submissionrevisions'), 0); + }); + + it('Should warn about submissions an earlier clone left where the server never reads them', async function() { + const seed = await seedProject(dbs.src, { + forms: 1, + submissions: 0, + project: {name: 'src-project'}, + formOverrides: [{settings: {collection: 'custom'}}] + }); + const destProject = await dbs.dst.insert('projects', project({title: 'Destination', name: 'dst-project'})); + await dbs.dst.insert('dst-project_custom', submission({form: seed.forms[0]._id, project: destProject._id})); + + const warnings = []; + const consoleError = console.error; + console.error = (...args) => warnings.push(args.map(String).join(' ')); + try { + await quiet(() => runClone({ + srcProject: seed.project._id.toString(), + dstProject: destProject._id.toString() + })); + } + finally { + console.error = consoleError; + } + assert.ok(warnings.some((line) => /in dst-project_custom, where the server never reads them/.test(line)), + 'The operator must be told about the misplaced submissions.'); + }); + }); + describe('Encrypted submission data', function() { const srcDbSecret = 'source-db-secret'; const dstDbSecret = 'destination-db-secret'; @@ -328,6 +393,49 @@ describe('Project targeting', function() { 'Encrypted data must be readable with the destination project secret.'); }); + it('Should re-encrypt encrypted components within the rows of a datagrid', async function() { + const crypt = crypter(); + const srcProject = project({ + title: 'Encrypted', + name: 'encrypted-grid', + ['settings_encrypted']: crypt.encrypt(srcDbSecret, {}, true) + }); + await dbs.src.insert('projects', srcProject); + const srcForm = await dbs.src.insert('forms', form({ + project: srcProject._id, + components: [{ + type: 'panel', + key: 'panel', + components: [{ + type: 'datagrid', + key: 'people', + input: true, + components: [ + {type: 'textfield', key: 'name', input: true}, + {type: 'textfield', key: 'ssn', input: true, encrypted: true} + ] + }] + }] + })); + await dbs.src.insert('submissions', submission({ + form: srcForm._id, + project: srcProject._id, + data: {people: [ + {name: 'One', ssn: crypt.encrypt(srcDbSecret, '111-11-1111')}, + {name: 'Two', ssn: crypt.encrypt(srcDbSecret, '222-22-2222')} + ]}, + created: at(20), + modified: at(20) + })); + + await quiet(() => runClone({srcDbSecret, dstDbSecret})); + + const cloned = (await dbs.dst.all('submissions'))[0]; + assert.deepEqual(cloned.data.people.map((row) => crypt.decrypt(dstDbSecret, row.ssn)), + ['111-11-1111', '222-22-2222'], 'Every row must be readable with the destination secret.'); + assert.deepEqual(cloned.data.people.map((row) => row.name), ['One', 'Two']); + }); + it('Should leave unencrypted components untouched', async function() { const crypt = crypter(); const srcProject = project({ diff --git a/test/cloner/rekey.js b/test/cloner/rekey.js new file mode 100644 index 0000000..e4c3b51 --- /dev/null +++ b/test/cloner/rekey.js @@ -0,0 +1,408 @@ +/* globals describe, it, before, after, beforeEach */ +'use strict'; +const assert = require('assert'); +const {ObjectId} = require('mongodb'); +const Rekeyer = require('../../src/Rekeyer'); +const {encrypt, decrypt, eachEncryptedValue, hasEncryptedComponents} = require('../../src/encryption'); +const {tlsOptions} = require('../../src/db'); +const {DST_URI, TestDb, project, form, submission, submissionRevision} = require('./fixtures'); +const {quiet} = require('./helper'); + +const NEW_SECRET = 'the-right-secret'; +const WRONG_SECRET = 'the-wrong-destination-secret'; +const SOURCE_SECRET = 'the-source-secret'; +const PROJECT_SECRET = 'the-project-secret'; + +// A form with an encrypted field at the top, one within the rows of a datagrid (inside a panel), one +// within a container, and an encrypted datagrid whose rows are encrypted as a whole. +const COMPONENTS = [ + {type: 'textfield', key: 'name', input: true}, + {type: 'textfield', key: 'ssn', input: true, encrypted: true}, + {type: 'panel', key: 'panel', components: [{ + type: 'datagrid', + key: 'people', + input: true, + components: [ + {type: 'textfield', key: 'name', input: true}, + {type: 'textfield', key: 'ssn', input: true, encrypted: true} + ] + }]}, + {type: 'container', key: 'details', input: true, components: [ + {type: 'columns', key: 'columns', columns: [{components: [ + {type: 'textfield', key: 'pin', input: true, encrypted: true} + ]}]} + ]}, + {type: 'datagrid', key: 'secrets', input: true, encrypted: true, components: [ + {type: 'textfield', key: 'value', input: true, encrypted: true} + ]} +]; + +/** + * Submission data with every encrypted value of COMPONENTS encrypted with a secret. + * @param {string} secret - The secret. + */ +function encryptedData(secret) { + return { + name: 'Plain', + ssn: encrypt(secret, '123-45-6789'), + people: [ + {name: 'One', ssn: encrypt(secret, '111-11-1111')}, + {name: 'Two', ssn: encrypt(secret, '222-22-2222')} + ], + details: {pin: encrypt(secret, 1234)}, + secrets: encrypt(secret, [{value: 'a'}, {value: 'b'}]) + }; +} + +/** + * Decrypt every encrypted value of submission data built by encryptedData(). + * @param {string} secret - The secret to decrypt with. + * @param {object} data - The data. + */ +function decryptedData(secret, data) { + return { + name: data.name, + ssn: decrypt(secret, data.ssn), + people: data.people.map((row) => ({name: row.name, ssn: decrypt(secret, row.ssn)})), + details: {pin: decrypt(secret, data.details.pin)}, + secrets: decrypt(secret, data.secrets) + }; +} + +const PLAIN = { + name: 'Plain', + ssn: '123-45-6789', + people: [{name: 'One', ssn: '111-11-1111'}, {name: 'Two', ssn: '222-22-2222'}], + details: {pin: 1234}, + secrets: [{value: 'a'}, {value: 'b'}] +}; + +/** + * Run a re-key, returning what it reported as errors, what it printed, and the error it ended with. + * @param {object} options - The re-key options. + */ +async function runRekey(options) { + const warnings = []; + const consoleError = console.error; + console.error = (...args) => warnings.push(args.map(String).join(' ')); + const rekeyer = new Rekeyer(DST_URI, Object.assign({ + newDbSecret: NEW_SECRET, + oldDbSecret: [WRONG_SECRET, SOURCE_SECRET], + interactive: false + }, options)); + let error = null; + try { + await quiet(() => rekeyer.rekey()); + } + catch (err) { + error = err; + } + finally { + console.error = consoleError; + } + return {rekeyer, warnings, error}; +} + +describe('Rekey', function() { + describe('eachEncryptedValue', function() { + it('Should find the encrypted values within rows, containers and layout components', function() { + const found = []; + eachEncryptedValue(COMPONENTS, encryptedData('x'), (value, path) => found.push(path.join('.'))); + assert.deepEqual(found, ['ssn', 'people.0.ssn', 'people.1.ssn', 'details.pin', 'secrets']); + }); + + it('Should skip plain values, and components that are not saved', function() { + const found = []; + eachEncryptedValue([ + {type: 'textfield', key: 'plain', encrypted: true}, + {type: 'textfield', key: 'empty', encrypted: true}, + {type: 'textfield', key: 'transient', encrypted: true, persistent: false} + ], {plain: 'not encrypted', empty: '', transient: encrypt('x', 'value')}, (value, path) => found.push(path)); + assert.deepEqual(found, []); + }); + + it('Should find and read values that went through JSON, the way an API returns them', function() { + const serialized = JSON.parse(JSON.stringify({ssn: encrypt('x', '123-45-6789')})); + const found = []; + eachEncryptedValue([{type: 'textfield', key: 'ssn', encrypted: true}], serialized, (value) => found.push(value)); + assert.equal(found.length, 1); + assert.equal(decrypt('x', found[0]), '123-45-6789'); + }); + + it('Should tell whether a form has encrypted components', function() { + assert.equal(hasEncryptedComponents(COMPONENTS), true); + assert.equal(hasEncryptedComponents([{type: 'panel', components: [{type: 'textfield', key: 'a'}]}]), false); + assert.equal(hasEncryptedComponents(undefined), false); + }); + }); + + describe('rekeyValue', function() { + const rekeyer = new Rekeyer(DST_URI, {newDbSecret: NEW_SECRET, oldDbSecret: [WRONG_SECRET]}); + + it('Should leave a value that is already encrypted with the target secret', function() { + const result = rekeyer.rekeyValue(encrypt(NEW_SECRET, 'value'), NEW_SECRET, [WRONG_SECRET]); + assert.equal(result.status, 'current'); + }); + + it('Should re-encrypt a value encrypted with one of the candidates', function() { + const result = rekeyer.rekeyValue(encrypt(WRONG_SECRET, 0), NEW_SECRET, [SOURCE_SECRET, WRONG_SECRET]); + assert.equal(result.status, 'rekeyed'); + assert.strictEqual(decrypt(NEW_SECRET, result.cipher), 0, 'A falsy value is still a value.'); + }); + + it('Should re-encrypt settings without a salt', function() { + const result = rekeyer.rekeyValue(encrypt(WRONG_SECRET, {secret: 'a'}, true), NEW_SECRET, [WRONG_SECRET], true); + assert.equal(result.status, 'rekeyed'); + assert.deepEqual(decrypt(NEW_SECRET, result.cipher, true), {secret: 'a'}); + }); + + it('Should report a value none of the secrets can read', function() { + const result = rekeyer.rekeyValue(encrypt('unknown', 'value'), NEW_SECRET, [WRONG_SECRET]); + assert.equal(result.status, 'undecryptable'); + }); + }); + + describe('options', function() { + it('Should refuse to run without the secrets', async function() { + assert.throws(() => new Rekeyer(DST_URI, {oldDbSecret: ['a']}).validate(), /--new-db-secret is required/); + assert.throws(() => new Rekeyer(DST_URI, {newDbSecret: 'a'}).validate(), /At least one --old-db-secret/); + assert.throws(() => new Rekeyer(DST_URI, {newDbSecret: 'a', oldDbSecret: ['a']}).validate(), + /also given as an old secret/); + assert.throws(() => new Rekeyer(DST_URI, {newDbSecret: 'a', oldDbSecret: ['b'], project: 'nope'}).validate(), + /Not a valid project id: nope/); + }); + + it('Should accept a single old secret as a string', function() { + assert.deepEqual(new Rekeyer(DST_URI, {newDbSecret: 'a', oldDbSecret: 'b'}).oldSecrets, ['b']); + }); + + it('Should ignore a project id that is given twice', function() { + const id = new ObjectId().toString(); + assert.deepEqual(new Rekeyer(DST_URI, {project: `${id}, ${id}`}).projectIds, [id]); + }); + + it('Should accept absolute and relative certificate paths', function() { + assert.equal(tlsOptions('/etc/ssl/ca.pem').tlsCAFile, '/etc/ssl/ca.pem'); + assert.equal(tlsOptions(null, '/etc/ssl/cert.pem').tlsCertificateKeyFile, '/etc/ssl/cert.pem'); + assert.equal(tlsOptions('ca.pem').tlsCAFile, require('path').join(process.cwd(), 'ca.pem')); + }); + }); + + describe('a database', function() { + let db = null; + let seed = null; + + before('Connect to the test database', async function() { + db = await new TestDb(DST_URI).connect(); + }); + + after('Disconnect from the test database', async function() { + await db.close(); + }); + + /** + * Seed the database the way a clone with the wrong destination secret leaves it. + */ + beforeEach('Seed a database cloned with the wrong secret', async function() { + await db.reset(); + + // A project without a secret of its own: its data is encrypted with the DB_SECRET, which the + // clone got wrong. The values within datagrid rows were never migrated, and still have the + // source secret. + const plain = await db.insert('projects', project({ + title: 'No project secret', + name: 'no-secret', + ['settings_encrypted']: encrypt(WRONG_SECRET, {cors: '*'}, true) + })); + const plainForm = await db.insert('forms', form({project: plain._id, components: COMPONENTS})); + const wrongData = encryptedData(WRONG_SECRET); + wrongData.people = encryptedData(SOURCE_SECRET).people; + const plainSubmission = await db.insert('submissions', submission({ + form: plainForm._id, + project: plain._id, + data: wrongData + })); + await db.insert('submissionrevisions', submissionRevision({ + _rid: plainSubmission._id, + form: plainForm._id, + project: plain._id, + data: encryptedData(WRONG_SECRET) + })); + const unreadable = await db.insert('submissions', submission({ + form: plainForm._id, + project: plain._id, + data: {ssn: encrypt('a-secret-nobody-knows', 'lost')} + })); + + // A form of the same project with its own collection, which the server names after the + // project name without its dashes. + const customForm = await db.insert('forms', form({ + project: plain._id, + components: COMPONENTS, + settings: {collection: 'people-records'} + })); + await db.insert('nosecret_peoplerecords', submission({ + form: customForm._id, + project: plain._id, + data: encryptedData(WRONG_SECRET) + })); + + // A project with a secret of its own: its data is encrypted with that, so only its settings + // need to be re-keyed. + const own = await db.insert('projects', project({ + title: 'Own secret', + name: 'own-secret', + ['settings_encrypted']: encrypt(WRONG_SECRET, {secret: PROJECT_SECRET}, true) + })); + const ownForm = await db.insert('forms', form({project: own._id, components: COMPONENTS})); + const ownSubmission = await db.insert('submissions', submission({ + form: ownForm._id, + project: own._id, + data: encryptedData(PROJECT_SECRET) + })); + + // The project the destination server created itself, already on the right secret. + const formio = await db.insert('projects', project({ + title: 'Form.io', + name: 'formio', + ['settings_encrypted']: encrypt(NEW_SECRET, {secret: 'formio-secret'}, true) + })); + + seed = {plain, plainForm, plainSubmission, unreadable, customForm, own, ownForm, ownSubmission, formio}; + }); + + /** + * The decrypted settings of a project. + * @param {object} proj - The project. + */ + async function settings(proj) { + return decrypt(NEW_SECRET, (await db.findOne('projects', {_id: proj._id}))['settings_encrypted'], true); + } + + it('Should change nothing on a dry run', async function() { + const before = await db.all('submissions'); + const {rekeyer, error} = await runRekey({dryRun: true}); + + assert.deepEqual(await db.all('submissions'), before, 'The submissions must be left as they are.'); + assert.equal(await settings(seed.plain), null, 'The settings must be left as they are.'); + assert.deepEqual(rekeyer.totals.settings, {rekeyed: 2, current: 1, undecryptable: 0}); + // The plain project has 5 values in its submission, 5 in its revision and 5 in its custom + // collection. The own secret project has 5 that are already right. One value is lost. + assert.deepEqual(rekeyer.totals.values, {rekeyed: 15, current: 5, undecryptable: 1}); + assert.match(String(error && error.message), /1 encrypted value\(s\) could not be decrypted/); + }); + + it('Should re-key the settings and the encrypted data', async function() { + const {rekeyer, warnings, error} = await runRekey({}); + + assert.deepEqual(await settings(seed.plain), {cors: '*'}); + assert.deepEqual(await settings(seed.own), {secret: PROJECT_SECRET}); + assert.deepEqual(await settings(seed.formio), {secret: 'formio-secret'}); + + const sub = await db.findOne('submissions', {_id: seed.plainSubmission._id}); + assert.deepEqual(decryptedData(NEW_SECRET, sub.data), PLAIN, + 'Every value, including the ones within rows, must be readable with the new secret.'); + const revision = await db.findOne('submissionrevisions', {_rid: seed.plainSubmission._id}); + assert.deepEqual(decryptedData(NEW_SECRET, revision.data), PLAIN, 'Revisions must be re-keyed as well.'); + const custom = (await db.all('nosecret_peoplerecords'))[0]; + assert.deepEqual(decryptedData(NEW_SECRET, custom.data), PLAIN, 'Custom collections must be re-keyed as well.'); + + const own = await db.findOne('submissions', {_id: seed.ownSubmission._id}); + assert.deepEqual(decryptedData(PROJECT_SECRET, own.data), PLAIN); + assert.ok(Buffer.from(own.data.ssn.buffer).equals(seed.ownSubmission.data.ssn), + 'Data encrypted with the project secret must not be touched.'); + + const unreadable = await db.findOne('submissions', {_id: seed.unreadable._id}); + assert.ok(Buffer.from(unreadable.data.ssn.buffer).equals(seed.unreadable.data.ssn), + 'A value no secret can read must be left as it is.'); + assert.ok(warnings.some((line) => line.includes(`data.ssn of submissions ${seed.unreadable._id}`)), + 'The value that could not be decrypted must be reported.'); + assert.match(String(error && error.message), /1 encrypted value\(s\) could not be decrypted/, + 'The re-key must not report success.'); + assert.equal(rekeyer.totals.conflicts, 0); + }); + + it('Should find nothing left to do when it is run again', async function() { + await runRekey({}); + const {rekeyer} = await runRekey({}); + assert.deepEqual(rekeyer.totals.settings, {rekeyed: 0, current: 3, undecryptable: 0}); + assert.deepEqual(rekeyer.totals.values, {rekeyed: 0, current: 20, undecryptable: 1}); + }); + + it('Should re-key data encrypted with the wrong secret into the project secret', async function() { + await db.collection('submissions').updateOne( + {_id: seed.ownSubmission._id}, + {$set: {'data.ssn': encrypt(WRONG_SECRET, 'moved')}} + ); + await runRekey({project: seed.own._id.toString()}); + const own = await db.findOne('submissions', {_id: seed.ownSubmission._id}); + assert.equal(decrypt(PROJECT_SECRET, own.data.ssn), 'moved'); + }); + + it('Should only re-key the projects it is given', async function() { + const {rekeyer, error} = await runRekey({project: seed.own._id.toString()}); + assert.equal(error, null); + assert.deepEqual(rekeyer.totals.settings, {rekeyed: 1, current: 0, undecryptable: 0}); + assert.deepEqual(await settings(seed.own), {secret: PROJECT_SECRET}); + assert.equal(await settings(seed.plain), null, 'Other projects must be left alone.'); + }); + + it('Should refuse projects that do not exist', async function() { + const {error} = await runRekey({project: new ObjectId().toString()}); + assert.match(String(error && error.message), /Only 0 of the 1 projects given were found/); + }); + + it('Should skip the data of a project whose settings none of the secrets can read', async function() { + const {rekeyer, error} = await runRekey({oldDbSecret: ['something-else'], project: seed.plain._id.toString()}); + assert.deepEqual(rekeyer.totals.settings, {rekeyed: 0, current: 0, undecryptable: 1}); + assert.deepEqual(rekeyer.totals.values, {rekeyed: 0, current: 0, undecryptable: 0}, + 'Without the settings, the secret of the data is not known.'); + assert.match(String(error && error.message), /had settings none of the secrets could decrypt/); + }); + + it('Should re-key the revisions of a custom collection left in the default collection', async function() { + // Earlier versions of the clone wrote them there, instead of to the revisions of the collection. + const stray = await db.insert('submissionrevisions', submissionRevision({ + _rid: new ObjectId(), + form: seed.customForm._id, + project: seed.plain._id, + data: encryptedData(WRONG_SECRET) + })); + const {warnings} = await runRekey({}); + + const revision = await db.findOne('submissionrevisions', {_id: stray._id}); + assert.deepEqual(decryptedData(NEW_SECRET, revision.data), PLAIN); + assert.ok(warnings.some((line) => /Revisions of this form are in submissionrevisions/.test(line)), + 'The operator should be told the revisions are where the server does not read them.'); + }); + + it('Should leave the submission revisions alone when asked to', async function() { + await runRekey({submissionRevisions: false}); + const revision = await db.findOne('submissionrevisions', {_rid: seed.plainSubmission._id}); + assert.deepEqual(decryptedData(WRONG_SECRET, revision.data), PLAIN); + }); + + it('Should never overwrite a value that changed after it was read', async function() { + const rekeyer = new Rekeyer(DST_URI, {newDbSecret: NEW_SECRET, oldDbSecret: [WRONG_SECRET]}); + const collection = db.collection('submissions'); + const changed = encrypt(NEW_SECRET, 'changed by someone else'); + await collection.updateOne({_id: seed.plainSubmission._id}, {$set: {'data.ssn': changed}}); + + const consoleError = console.error; + console.error = () => {}; + try { + await rekeyer.write(collection, [{updateOne: { + filter: {_id: seed.plainSubmission._id, 'data.ssn': seed.plainSubmission.data.ssn}, + update: {$set: {'data.ssn': encrypt(NEW_SECRET, '123-45-6789')}} + }}]); + } + finally { + console.error = consoleError; + } + + assert.equal(rekeyer.totals.conflicts, 1); + const sub = await db.findOne('submissions', {_id: seed.plainSubmission._id}); + assert.equal(decrypt(NEW_SECRET, sub.data.ssn), 'changed by someone else'); + }); + }); +}); diff --git a/test/cloner/units.js b/test/cloner/units.js index b93efda..d655420 100644 --- a/test/cloner/units.js +++ b/test/cloner/units.js @@ -3,6 +3,7 @@ const assert = require('assert'); const {ObjectId} = require('mongodb'); const Cloner = require('../../src/Cloner'); +const {encrypt, decrypt} = require('../../src/encryption'); const {quiet} = require('./helper'); /** @@ -12,7 +13,6 @@ const {quiet} = require('./helper'); function bare(options = {}) { const instance = Object.create(Cloner.prototype); instance.options = options; - instance.defaultSaltLength = 40; instance.exclude = []; instance.dryRun = !!options.dryRun; instance.dryRunBuffer = []; @@ -31,7 +31,7 @@ describe('Cloner units', function() { it('Should produce the legacy OpenSSL compatible ciphertext', function() { // Generated with: printf '{"a":1}' | openssl enc -aes-256-cbc -nosalt -md md5 -pass pass:mysecret -base64 // which is the exact format the removed crypto.createCipher() produced. - const encrypted = bare().encrypt('mysecret', {a: 1}, true); + const encrypted = encrypt('mysecret', {a: 1}, true); assert.equal(encrypted.toString('base64'), 'aqN/mkNcqrTMql3ObHRt6Q=='); }); @@ -39,49 +39,55 @@ describe('Cloner units', function() { // printf '{"a":1}' | openssl enc -aes-256-cbc -nosalt -md md5 -pass pass:sécret -base64 // Reading the secret as latin1 instead derives a different key, and nothing encrypted with a // non ASCII secret could be decrypted again. - const encrypted = bare().encrypt('sécret', {a: 1}, true); + const encrypted = encrypt('sécret', {a: 1}, true); assert.equal(encrypted.toString('base64'), 's332i21yTmVluklx2eqRaA=='); - assert.deepEqual(bare().decrypt('sécret', encrypted, true), {a: 1}); + assert.deepEqual(decrypt('sécret', encrypted, true), {a: 1}); }); it('Should decrypt legacy ciphertext', function() { const buffer = Buffer.from('aqN/mkNcqrTMql3ObHRt6Q==', 'base64'); - assert.deepEqual(bare().decrypt('mysecret', buffer, true), {a: 1}); + assert.deepEqual(decrypt('mysecret', buffer, true), {a: 1}); }); it('Should round trip salted data', function() { - const instance = bare(); - const encrypted = instance.encrypt('a-secret', {ssn: '123-45-6789'}); - assert.deepEqual(instance.decrypt('a-secret', encrypted), {ssn: '123-45-6789'}); + const encrypted = encrypt('a-secret', {ssn: '123-45-6789'}); + assert.deepEqual(decrypt('a-secret', encrypted), {ssn: '123-45-6789'}); }); it('Should round trip a binary buffer wrapper', function() { - const instance = bare(); - const encrypted = instance.encrypt('a-secret', 'plain value'); - assert.equal(instance.decrypt('a-secret', {buffer: encrypted}), 'plain value'); + const encrypted = encrypt('a-secret', 'plain value'); + assert.equal(decrypt('a-secret', {buffer: encrypted}), 'plain value'); }); it('Should encrypt values that are falsy but real', function() { - const instance = bare(); for (const value of [0, false, '']) { - const encrypted = instance.encrypt('a-secret', value); + const encrypted = encrypt('a-secret', value); assert.ok(encrypted, `${JSON.stringify(value)} is a value and must be encrypted.`); - assert.deepEqual(instance.decrypt('a-secret', encrypted), value); + assert.deepEqual(decrypt('a-secret', encrypted), value); } }); it('Should return null when a secret or payload is missing', function() { - const instance = bare(); - assert.equal(instance.encrypt('', {a: 1}), null); - assert.equal(instance.encrypt('secret', null), null); - assert.equal(instance.decrypt('', Buffer.from('x')), null); - assert.equal(instance.decrypt('secret', null), null); + assert.equal(encrypt('', {a: 1}), null); + assert.equal(encrypt('secret', null), null); + assert.equal(decrypt('', Buffer.from('x')), null); + assert.equal(decrypt('secret', null), null); }); it('Should return null when decrypting with the wrong secret', function() { - const instance = bare(); - const encrypted = instance.encrypt('right-secret', {a: 1}, true); - assert.equal(instance.decrypt('wrong-secret', encrypted, true), null); + const encrypted = encrypt('right-secret', {a: 1}, true); + assert.equal(decrypt('wrong-secret', encrypted, true), null); + }); + }); + + describe('migrateDataEncryption', function() { + it('Should re-encrypt values an API returned as JSON', function() { + const instance = bare({srcDbSecret: 'source', dstDbSecret: 'destination'}); + const src = {_id: 'a', data: JSON.parse(JSON.stringify({ssn: encrypt('source', '123-45-6789')}))}; + const update = {data: Object.assign({}, src.data)}; + instance.migrateDataEncryption(src, update, [{type: 'textfield', key: 'ssn', encrypted: true}]); + assert.ok(Buffer.isBuffer(update.data.ssn), 'It should be written back as binary data.'); + assert.equal(decrypt('destination', update.data.ssn), '123-45-6789'); }); }); @@ -609,6 +615,38 @@ describe('Cloner units', function() { assert.equal(instance.src.submissions.name, 'a_custom'); assert.equal(instance.dest.submissions.name, 'b_custom'); }); + + it('Should name custom collections, and their revisions, the way the server does', async function() { + const instance = bare({}); + instance.writeBuffer = {}; + instance.cloneState = {}; + instance.pendingState = {}; + const defaults = () => ({ + db: database(), + ogSubmissions: {name: 'submissions'}, + ogSubmissionRevisions: {name: 'submissionrevisions'} + }); + instance.src = defaults(); + instance.dest = defaults(); + + // The server strips everything but letters and digits from both names. + await quiet(() => instance.setCollection( + {settings: {collection: 'custom-records'}}, + {name: 'my-source'}, + {name: 'my_destination.v2'} + )); + assert.equal(instance.src.submissions.name, 'mysource_customrecords'); + assert.equal(instance.src.submissionrevisions.name, 'mysource_customrecords_revisions'); + assert.equal(instance.dest.submissions.name, 'mydestinationv2_customrecords'); + assert.equal(instance.dest.submissionrevisions.name, 'mydestinationv2_customrecords_revisions'); + + // The next form without a collection of its own goes back to the default ones. + await quiet(() => instance.setCollection({settings: {}}, {name: 'my-source'}, {name: 'my_destination.v2'})); + assert.equal(instance.src.submissions.name, 'submissions'); + assert.equal(instance.src.submissionrevisions.name, 'submissionrevisions'); + assert.equal(instance.dest.submissions.name, 'submissions'); + assert.equal(instance.dest.submissionrevisions.name, 'submissionrevisions'); + }); }); describe('roles and access', function() {