From bf6fc2966c1482993e6530fe2615e0180eb0bba2 Mon Sep 17 00:00:00 2001 From: Travis Tidwell Date: Thu, 22 Oct 2026 10:59:13 -0500 Subject: [PATCH 1/3] Fix clone encryption of nested fields and custom collection names - Re-encrypt encrypted fields within the rows of a datagrid or editgrid. The encrypted paths were looked up as flat paths (data.grid.field) against array data, so those values silently kept the source secret. The form is now walked against the submission data itself. - Name custom submission collections the way the server does, with everything but letters and digits stripped from the project and collection names. Projects with a dash in their name had their submissions cloned where the server never reads them. - Clone the revisions of a custom collection from and to its _revisions collection instead of submissionrevisions. - Warn when the destination still holds submissions in a collection an earlier clone misnamed. - Move the crypto helpers to src/encryption.js and the connection helpers to src/db.js. decrypt() no longer logs every failed attempt. - Remove src/eachComponentAsync.js, which is no longer used. Co-Authored-By: Claude Opus 5.5 --- src/Cloner.js | 218 +++++++++++++++---------------------- src/db.js | 55 ++++++++++ src/eachComponentAsync.js | 92 ---------------- src/encryption.js | 220 ++++++++++++++++++++++++++++++++++++++ test/cloner/projects.js | 113 +++++++++++++++++++- test/cloner/units.js | 32 ++++++ 6 files changed, 504 insertions(+), 226 deletions(-) create mode 100644 src/db.js delete mode 100644 src/eachComponentAsync.js create mode 100644 src/encryption.js diff --git a/src/Cloner.js b/src/Cloner.js index 84c2748..a6272ae 100644 --- a/src/Cloner.js +++ b/src/Cloner.js @@ -3,9 +3,8 @@ const {MongoClient, ObjectId, BSON} = require('mongodb'); const fs = require('fs'); const _ = require('lodash'); -const crypto = require('crypto'); -const keygenerator = require('keygenerator'); -const {eachComponentAsync} = require('./eachComponentAsync'); +const encryption = require('./encryption'); +const {redact, tlsOptions, submissionCollectionName} = require('./db'); const {createHmac} = require('node:crypto'); const fetch = require('./fetch'); const Progress = require('./Progress'); @@ -36,14 +35,6 @@ const MAX_REPEATED_ERRORS = 5; const POSITION_COUNT_TIMEOUT = 2000; const BAR_COUNT_TIMEOUT = 60000; -/** - * Hide the credentials of a connection string, so that they never end up in a log. - * @param {string} uri - The connection string. - */ -function redact(uri) { - return String(uri).replace(/\/\/[^@/]+@/, '//***@'); -} - /** * Split a comma separated list of ids, ignoring the spaces around them. * @param {string} value - The comma separated list. @@ -372,30 +363,12 @@ class Cloner { } /** - * Derive the key and iv for a secret exactly the way the removed crypto.createCipher() did, which - * is OpenSSL's EVP_BytesToKey with MD5, a single iteration and no salt. This keeps us compatible - * with data that was encrypted by the servers, while still running on Node versions where - * crypto.createCipher() no longer exists. - * + * Derive the key and iv for a secret the way the server does. * @param {string} secret - The secret to derive the key and iv from. * @returns {{key: Buffer, iv: Buffer}} - The derived key and iv. */ legacyKey(secret) { - // createCipher() read the password as UTF-8; latin1 would derive a different key for any secret - // holding a non ASCII character, and nothing encrypted with it could be read back. - const password = Buffer.from(secret, 'utf8'); - const keyLength = 32; - const ivLength = 16; - let derived = Buffer.alloc(0); - let block = Buffer.alloc(0); - while (derived.length < keyLength + ivLength) { - block = crypto.createHash('md5').update(Buffer.concat([block, password])).digest(); - derived = Buffer.concat([derived, block]); - } - return { - key: derived.subarray(0, keyLength), - iv: derived.subarray(keyLength, keyLength + ivLength) - }; + return encryption.legacyKey(secret); } /** @@ -406,19 +379,7 @@ class Cloner { * @returns */ encrypt(secret, rawData, nosalt) { - if (!secret || rawData === null || rawData === undefined) { - return null; - } - const salt = nosalt ? '' : keygenerator._({ - length: this.defaultSaltLength - }); - const {key, iv} = this.legacyKey(secret); - const cipher = crypto.createCipheriv('aes-256-cbc', key, iv); - const decryptedJSON = JSON.stringify(rawData) + salt; - return Buffer.concat([ - cipher.update(decryptedJSON), - cipher.final() - ]); + return encryption.encrypt(secret, rawData, nosalt, this.defaultSaltLength); } /** @@ -429,25 +390,7 @@ class Cloner { * @returns */ decrypt(secret, cipherbuffer, nosalt) { - if (!secret || !cipherbuffer) { - return null; - } - let data; - try { - const buffer = Buffer.isBuffer(cipherbuffer) ? cipherbuffer : cipherbuffer.buffer; - const {key, iv} = this.legacyKey(secret); - const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv); - const decryptedJSON = Buffer.concat([ - decipher.update(buffer), // Buffer contains encrypted utf8 - decipher.final() - ]); - data = JSON.parse(nosalt ? decryptedJSON : decryptedJSON.slice(0, -this.defaultSaltLength)); - } - catch (e) { - console.log(e); - data = null; - } - return data; + return encryption.decrypt(secret, cipherbuffer, nosalt, this.defaultSaltLength); } /** @@ -456,26 +399,7 @@ class Cloner { * @returns MongoDB configuration. */ dbConfig(type) { - // The driver always uses the new url parser and the unified topology: from version 7 on it - // rejects the options that used to turn them on. - const config = {}; - const caFile = this.options[`${type}Ca`]; - const certFile = this.options[`${type}Cert`]; - if (caFile || certFile) { - config.tls = true; - if (caFile) { - config.tlsCAFile = `${process.cwd()}/${caFile}`; - - // A custom certificate authority is usually an internal one whose certificates do not name - // the host being connected to. A client certificate on its own is no reason to stop - // verifying who we are talking to. - config.tlsAllowInvalidHostnames = true; - } - if (certFile) { - config.tlsCertificateKeyFile = `${process.cwd()}/${certFile}`; - } - } - return config; + return tlsOptions(this.options[`${type}Ca`], this.options[`${type}Cert`]); } /** @@ -1496,25 +1420,57 @@ class Cloner { await this.flushWrites(); const subsCollection = _.get(form, 'settings.collection'); - if (!subsCollection) { - if (!this.options.apiSource) { - this.src.submissions = this.src.ogSubmissions; - } - this.dest.submissions = this.dest.ogSubmissions; - return; + if (!this.options.apiSource) { + this.useCollection(this.src, subsCollection && srcProj && submissionCollectionName(srcProj.name, subsCollection)); + } + const destCollection = subsCollection && destProj && submissionCollectionName(destProj.name, subsCollection); + this.useCollection(this.dest, destCollection); + if (destCollection) { + console.log(`Using collection - ${destCollection}`); + await this.warnAboutMisnamedCollection(form, destProj, subsCollection, destCollection); } + } - console.log(`Using collection - ${subsCollection}`); + /** + * Point the submissions, and submission revisions, of a database at the collections of a form. + * @param {object} database - The source or the destination. + * @param {string|null} name - The custom collection of the form, or null for the default ones. + */ + useCollection(database, name) { + database.submissions = name ? database.db.collection(name) : database.ogSubmissions; + database.submissionrevisions = name + ? database.db.collection(`${name}_revisions`) + : (database.ogSubmissionRevisions || database.submissionrevisions); + } - if (!this.options.apiSource) { - this.src.submissions = srcProj && subsCollection - ? this.src.db.collection(`${srcProj.name}_${subsCollection}`) - : this.src.ogSubmissions; + /** + * Earlier versions of the clone wrote the submissions of a custom collection to a collection named + * after the unsanitized project and collection names, where the server never reads them. Say so + * when the destination still has them, since they are not moved. + * + * @param {object} form - The form. + * @param {object} destProj - The destination project. + * @param {string} subsCollection - The collection setting of the form. + * @param {string} destCollection - The collection the submissions are cloned to. + */ + async warnAboutMisnamedCollection(form, destProj, subsCollection, destCollection) { + const misnamed = `${destProj.name}_${subsCollection}`; + if (this.dryRun || misnamed === destCollection) { + return; + } + let found = 0; + try { + found = await this.dest.db.collection(misnamed).countDocuments({form: form._id}, {limit: 1}); + } + catch (err) { + // Only a warning: it is never a reason to stop the clone. + } + if (found) { + this.error( + `The destination has submissions of ${form.title || form._id} in ${misnamed}, where the server never ` + + `reads them. They are cloned to ${destCollection}; ${misnamed} can be dropped once that is done.` + ); } - - this.dest.submissions = destProj && subsCollection - ? this.dest.db.collection(`${destProj.name}_${subsCollection}`) - : this.dest.ogSubmissions; } /** @@ -1537,6 +1493,7 @@ class Cloner { forms: db.collection('forms'), ogSubmissions: db.collection('submissions'), submissions: db.collection('submissions'), + ogSubmissionRevisions: db.collection('submissionrevisions'), submissionrevisions: db.collection('submissionrevisions'), roles: db.collection('roles'), actions: db.collection('actions'), @@ -1725,36 +1682,38 @@ class Cloner { * Migrate the data encrypted within a submission to a new secret. * @param {*} src - The source record * @param {*} update - The updated record - * @param {string} decryptKey - The source key to decrypt data (either project.settings.secret or DB_SECRET). + * @param {object[]|null} encryptedComponents - The components of the form, when it has encrypted ones. */ - migrateDataEncryption(src, update, compsWithEncryptedData) { + migrateDataEncryption(src, update, encryptedComponents) { const srcSecret = this.srcSecret || this.options.srcDbSecret; const destSecret = this.dstSecret || this.options.dstDbSecret; if ( - !compsWithEncryptedData.length || + !encryptedComponents || !srcSecret || !destSecret || srcSecret === destSecret ) { return; } - compsWithEncryptedData.forEach((compPath) => { - if (_.get(src, `data.${compPath}`, false)) { - const decrypted = this.decrypt(srcSecret, _.get(src, `data.${compPath}`)); - - // A value that decrypts to 0, false or an empty string is still a value: only a decrypt - // that actually failed comes back as null. - const reEncrypted = (decrypted === null || decrypted === undefined) - ? null - : this.encrypt(destSecret, decrypted); - if (!reEncrypted) { - // Leave the value as it is rather than destroying it: it can still be read with the - // secret it was encrypted with. - this.error(`Could not re-encrypt ${compPath} of submission ${src._id}, it was left as it is.`); - return; - } - _.set(update, `data.${compPath}`, reEncrypted); + + // The values are collected first: the update can share its data with the source. + const values = []; + encryption.eachEncryptedValue(encryptedComponents, src.data, (value, path) => values.push({value, path})); + values.forEach(({value, path}) => { + const decrypted = this.decrypt(srcSecret, value); + + // A value that decrypts to 0, false or an empty string is still a value: only a decrypt + // that actually failed comes back as null. + const reEncrypted = (decrypted === null || decrypted === undefined) + ? null + : this.encrypt(destSecret, decrypted); + if (!reEncrypted) { + // Leave the value as it is rather than destroying it: it can still be read with the + // secret it was encrypted with. + this.error(`Could not re-encrypt ${path.join('.')} of submission ${src._id}, it was left as it is.`); + return; } + _.set(update, ['data', ...path], reEncrypted); }); } @@ -1762,9 +1721,9 @@ class Cloner { * Clone submissions from one form to another. * @param {*} srcForm - The source form from the database. * @param {*} destForm - The destination from from the database. - * @param {string[]} compsWithEncryptedData - Array with paths of components that are encrypted. + * @param {object[]|null} encryptedComponents - The components of the form, when it has encrypted ones. */ - async cloneSubmissions(srcForm, destForm, compsWithEncryptedData) { + async cloneSubmissions(srcForm, destForm, encryptedComponents) { // A dry run reports on every submission of the form, not only the ones a clone would write, so // it checks the destination for each of them and does not start from the last one cloned. const compareAll = this.options.updateExistingSubmissions || this.dryRun; @@ -1789,9 +1748,7 @@ class Cloner { await this.upsertAll('submissions', query, null, async(src, update) => { update.form = destForm._id; update.project = destForm.project; - if (compsWithEncryptedData.length) { - this.migrateDataEncryption(src, update, compsWithEncryptedData); - } + this.migrateDataEncryption(src, update, encryptedComponents); if (update.roles) { // Only the source roles are mapped: a submission without roles must not have the roles of // the destination document wiped, which are no longer fetched. @@ -1800,7 +1757,7 @@ class Cloner { this.migrateAccess(src.access, update.access); }, async(srcSubmission, destSubmission) => { this.currentSubmission = srcSubmission; - await this.cloneSubmissionRevisions(srcSubmission, destSubmission, compsWithEncryptedData); + await this.cloneSubmissionRevisions(srcSubmission, destSubmission, encryptedComponents); }, { find: compareAll ? null : false, scope: srcForm._id.toString(), @@ -1823,7 +1780,7 @@ class Cloner { * @param {*} srcSubmission - The source submission. * @param {*} destSubmission - The destination submission. */ - async cloneSubmissionRevisions(srcSubmission, destSubmission, compsWithEncryptedData) { + async cloneSubmissionRevisions(srcSubmission, destSubmission, encryptedComponents) { if (!this.options.includeSubmissionRevisions) { return; } @@ -1831,7 +1788,7 @@ class Cloner { _rid: srcSubmission._id }), null, async(src, update) => { update._rid = destSubmission._id; - this.migrateDataEncryption(src, update, compsWithEncryptedData); + this.migrateDataEncryption(src, update, encryptedComponents); }, null, { scope: srcSubmission._id.toString(), @@ -1913,7 +1870,7 @@ class Cloner { * @param {*} destProject - The destination project. */ async cloneForms(srcProject, destProject) { - let compsWithEncryptedData = []; + let encryptedComponents = null; const query = this.formQuery(srcProject); const total = this.dryRun ? null : await this.countSource('src.forms', query, POSITION_COUNT_TIMEOUT); let position = 0; @@ -1932,12 +1889,7 @@ class Cloner { // This mapping runs for every source form, including the ones that are not written to the // destination (--submissions-only, or a form that has not changed), because the submissions // of the form are cloned using these values. - compsWithEncryptedData = []; - await eachComponentAsync(update.components, async(component, path) => { - if (component.encrypted) { - compsWithEncryptedData.push(path); - } - }); + encryptedComponents = encryption.hasEncryptedComponents(update.components) ? update.components : null; await this.setCollection(update, srcProject, destProject); if (destProject) { @@ -1955,7 +1907,7 @@ class Cloner { } this.currentForm = srcForm; await this.flushWrites(); - await this.cloneSubmissions(srcForm, destForm, compsWithEncryptedData); + await this.cloneSubmissions(srcForm, destForm, encryptedComponents); await this.cloneActions(srcForm, destForm); await this.cloneFormRevisions(srcForm, destForm); await this.flushWrites(); diff --git a/src/db.js b/src/db.js new file mode 100644 index 0000000..3cccb57 --- /dev/null +++ b/src/db.js @@ -0,0 +1,55 @@ +'use strict'; + +/** + * Hide the credentials of a connection string, so that they never end up in a log. + * @param {string} uri - The connection string. + */ +function redact(uri) { + return String(uri).replace(/\/\/[^@/]+@/, '//***@'); +} + +/** + * The mongo client options for a connection with a custom certificate authority or a client + * certificate. The driver always uses the new url parser and the unified topology: from version 7 on + * it rejects the options that used to turn them on. + * + * @param {string} caFile - The certificate authority file, relative to the working directory. + * @param {string} certFile - The certificate key file, relative to the working directory. + * @returns {object} - The client options. + */ +function tlsOptions(caFile, certFile) { + const config = {}; + if (caFile || certFile) { + config.tls = true; + if (caFile) { + config.tlsCAFile = `${process.cwd()}/${caFile}`; + + // A custom certificate authority is usually an internal one whose certificates do not name + // the host being connected to. A client certificate on its own is no reason to stop + // verifying who we are talking to. + config.tlsAllowInvalidHostnames = true; + } + if (certFile) { + config.tlsCertificateKeyFile = `${process.cwd()}/${certFile}`; + } + } + return config; +} + +/** + * The name of the collection the server keeps the submissions of a form with its own collection in. + * The server strips everything but letters and digits from both the project name and the collection. + * + * @param {string} projectName - The name of the project. + * @param {string} collection - The collection setting of the form. + * @returns {string|null} - The collection name, or null when the project name has nothing left. + */ +function submissionCollectionName(projectName, collection) { + const project = String(projectName || '').replace(/[^A-Za-z0-9]+/g, ''); + if (!project) { + return null; + } + return `${project}_${String(collection).replace(/[^A-Za-z0-9]+/g, '')}`; +} + +module.exports = {redact, tlsOptions, submissionCollectionName}; diff --git a/src/eachComponentAsync.js b/src/eachComponentAsync.js deleted file mode 100644 index 8a4f816..0000000 --- a/src/eachComponentAsync.js +++ /dev/null @@ -1,92 +0,0 @@ -'use strict'; - -/** - * Iterate through each component within a form. - * - * @param {Object} components - * The components to iterate. - * @param {Function} fn - * The iteration function to invoke for each component. - * @param {Boolean} includeAll - * Whether or not to include layout components. - * @param {String} path - * The current data path of the element. Example: data.user.firstName - * @param {Object} parent - * The parent object. - */ - -const eachComponentAsync = async(components, fn, includeAll, path = '', parent) => { - if (!components || !components.length || !fn) { - return; - } - - for (const component of components) { - if (!component) { - return; - } - const hasColumns = component.columns && Array.isArray(component.columns); - const hasRows = component.rows && Array.isArray(component.rows); - const hasComps = component.components && Array.isArray(component.components); - let noRecurse = false; - const newPath = component.key ? (path ? (`${path}.${component.key}`) : component.key) : ''; - - // Keep track of parent references. - if (parent) { - // Ensure we don't create infinite JSON structures. - component.parent = {...parent}; - delete component.parent.components; - delete component.parent.componentMap; - delete component.parent.columns; - delete component.parent.rows; - } - - // there's no need to add other layout components here because we expect that those would either have columns, rows or components - const layoutTypes = ['htmlelement', 'content']; - const isLayoutComponent = hasColumns || hasRows || hasComps || layoutTypes.indexOf(component.type) > -1; - if (includeAll || component.tree || !isLayoutComponent) { - noRecurse = await fn(component, newPath, components); - } - - const subPath = () => { - if ( - component.key && - !['panel', 'table', 'well', 'columns', 'fieldset', 'tabs', 'form'].includes(component.type) && - ( - ['datagrid', 'container', 'editgrid', 'address', 'dynamicWizard'].includes(component.type) || - component.tree - ) - ) { - return newPath; - } - else if ( - component.key && - component.type === 'form' - ) { - return `${newPath}.data`; - } - return path; - }; - - if (!noRecurse) { - if (hasColumns) { - for (const column of component.columns) { - await eachComponentAsync(column.components, fn, includeAll, subPath(), parent ? component : null); - } - } - else if (hasRows) { - for (const row of component.rows) { - if (Array.isArray(row)) { - for (const column of row) { - await eachComponentAsync(column.components, fn, includeAll, subPath(), parent ? component : null); - } - } - } - } - else if (hasComps) { - await eachComponentAsync(component.components, fn, includeAll, subPath(), parent ? component : null); - } - } - } -}; - -module.exports = {eachComponentAsync}; diff --git a/src/encryption.js b/src/encryption.js new file mode 100644 index 0000000..95dd737 --- /dev/null +++ b/src/encryption.js @@ -0,0 +1,220 @@ +'use strict'; +const crypto = require('crypto'); +const _ = require('lodash'); +const keygenerator = require('keygenerator'); + +// The number of random characters the server appends to an encrypted value (but not to encrypted +// settings) before encrypting it. +const SALT_LENGTH = 40; + +// The components whose data is nested under their own key. Layout components never nest data, even +// when they have a key. +const DATA_TYPES = ['datagrid', 'container', 'editgrid', 'address', 'dynamicWizard']; +const LAYOUT_TYPES = ['panel', 'table', 'well', 'columns', 'fieldset', 'tabs', 'form']; + +/** + * Derive the key and iv for a secret exactly the way the removed crypto.createCipher() did, which + * is OpenSSL's EVP_BytesToKey with MD5, a single iteration and no salt. This keeps us compatible + * with data that was encrypted by the servers, while still running on Node versions where + * crypto.createCipher() no longer exists. + * + * @param {string} secret - The secret to derive the key and iv from. + * @returns {{key: Buffer, iv: Buffer}} - The derived key and iv. + */ +function legacyKey(secret) { + // createCipher() read the password as UTF-8; latin1 would derive a different key for any secret + // holding a non ASCII character, and nothing encrypted with it could be read back. + const password = Buffer.from(secret, 'utf8'); + const keyLength = 32; + const ivLength = 16; + let derived = Buffer.alloc(0); + let block = Buffer.alloc(0); + while (derived.length < keyLength + ivLength) { + block = crypto.createHash('md5').update(Buffer.concat([block, password])).digest(); + derived = Buffer.concat([derived, block]); + } + return { + key: derived.subarray(0, keyLength), + iv: derived.subarray(keyLength, keyLength + ivLength) + }; +} + +/** + * Encrypt data or settings using a secret, the way the server does. + * @param {string} secret - The secret. + * @param {*} rawData - The data to encrypt. + * @param {boolean} nosalt - Leave out the salt, as the project settings do. + * @param {number} saltLength - The length of the salt. + * @returns {Buffer|null} - The encrypted data, or null when there is nothing to encrypt. + */ +function encrypt(secret, rawData, nosalt, saltLength = SALT_LENGTH) { + if (!secret || rawData === null || rawData === undefined) { + return null; + } + const salt = nosalt ? '' : keygenerator._({length: saltLength}); + const {key, iv} = legacyKey(secret); + const cipher = crypto.createCipheriv('aes-256-cbc', key, iv); + return Buffer.concat([ + cipher.update(JSON.stringify(rawData) + salt), + cipher.final() + ]); +} + +/** + * Decrypt data or settings using a secret. + * @param {string} secret - The secret. + * @param {Buffer|Binary} cipherbuffer - The encrypted data, as a Buffer or as a mongo Binary. + * @param {boolean} nosalt - The data was encrypted without a salt, as the project settings are. + * @param {number} saltLength - The length of the salt. + * @returns {*} - The decrypted data, or null when it cannot be decrypted with this secret. + */ +function decrypt(secret, cipherbuffer, nosalt, saltLength = SALT_LENGTH) { + if (!secret || !cipherbuffer) { + return null; + } + try { + const buffer = Buffer.isBuffer(cipherbuffer) ? cipherbuffer : cipherbuffer.buffer; + const {key, iv} = legacyKey(secret); + const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv); + const decryptedJSON = Buffer.concat([ + decipher.update(buffer), + decipher.final() + ]); + return JSON.parse(nosalt ? decryptedJSON : decryptedJSON.slice(0, -saltLength)); + } + catch (err) { + // The wrong secret: either the padding does not check out, or what comes out is not JSON. + return null; + } +} + +/** + * Whether a stored value is encrypted data, rather than a plain value the server never encrypted + * (it leaves empty values, and the values saved before a component was marked encrypted, alone). + * @param {*} value - The stored value. + */ +function isEncryptedValue(value) { + return Buffer.isBuffer(value) || !!(value && value._bsontype === 'Binary'); +} + +/** + * Whether the server encrypts the value of a component. + * @param {object} component - The component. + */ +function isEncryptedComponent(component) { + return !!( + component && + component.key && + component.encrypted && + (!Object.prototype.hasOwnProperty.call(component, 'persistent') || component.persistent) + ); +} + +/** + * The lists of child components of a component. + * @param {object} component - The component. + */ +function childComponents(component) { + const lists = []; + if (Array.isArray(component.columns)) { + component.columns.forEach((column) => lists.push(column && column.components)); + } + else if (Array.isArray(component.rows)) { + component.rows.forEach((row) => { + if (Array.isArray(row)) { + row.forEach((column) => lists.push(column && column.components)); + } + else { + lists.push(row && row.components); + } + }); + } + else if (Array.isArray(component.components)) { + lists.push(component.components); + } + return lists.filter(Array.isArray); +} + +/** + * Whether the data of the children of a component is nested under the key of the component. + * @param {object} component - The component. + */ +function nestsData(component) { + return !!( + component.key && + !LAYOUT_TYPES.includes(component.type) && + (DATA_TYPES.includes(component.type) || component.tree) + ); +} + +/** + * Walk the encrypted values of a submission. The form is walked against the data itself, so the + * values of the rows of a datagrid or an editgrid are found as well. The children of an encrypted + * component are never visited: the server encrypts the whole value of the component at once. + * + * @param {object[]} components - The components of the form. + * @param {object} data - The data of the submission. + * @param {Function} fn - Called with each encrypted value and the path to it within the data, as + * an array of keys and row indexes. + * @param {Array} path - The path to the data, used while recursing. + */ +function eachEncryptedValue(components, data, fn, path = []) { + if (!Array.isArray(components) || !_.isPlainObject(data)) { + return; + } + for (const component of components) { + if (!component) { + continue; + } + if (isEncryptedComponent(component)) { + const value = data[component.key]; + if (isEncryptedValue(value)) { + fn(value, [...path, component.key]); + } + continue; + } + const children = childComponents(component); + if (!children.length) { + continue; + } + if (nestsData(component)) { + const value = data[component.key]; + if (Array.isArray(value)) { + value.forEach((row, index) => children.forEach((list) => { + eachEncryptedValue(list, row, fn, [...path, component.key, index]); + })); + } + else { + children.forEach((list) => eachEncryptedValue(list, value, fn, [...path, component.key])); + } + } + else { + children.forEach((list) => eachEncryptedValue(list, data, fn, path)); + } + } +} + +/** + * Whether a form has any encrypted component. + * @param {object[]} components - The components of the form. + */ +function hasEncryptedComponents(components) { + if (!Array.isArray(components)) { + return false; + } + return components.some((component) => component && ( + isEncryptedComponent(component) || + childComponents(component).some(hasEncryptedComponents) + )); +} + +module.exports = { + SALT_LENGTH, + legacyKey, + encrypt, + decrypt, + isEncryptedValue, + isEncryptedComponent, + eachEncryptedValue, + hasEncryptedComponents +}; diff --git a/test/cloner/projects.js b/test/cloner/projects.js index f22fda4..1a3de55 100644 --- a/test/cloner/projects.js +++ b/test/cloner/projects.js @@ -4,7 +4,7 @@ const assert = require('assert'); const _ = require('lodash'); const {ObjectId} = require('mongodb'); const Cloner = require('../../src/Cloner'); -const {seedProject, at, project, role, form, submission} = require('./fixtures'); +const {seedProject, at, project, role, form, submission, submissionRevision} = require('./fixtures'); const {useTestDatabases, runClone, quiet} = require('./helper'); /** @@ -287,6 +287,74 @@ describe('Project targeting', function() { }); }); + describe('Custom submission collections of projects with symbols in their name', function() { + it('Should clone submissions and revisions to the collections the server reads', async function() { + const seed = await seedProject(dbs.src, { + forms: 1, + submissions: 0, + project: {name: 'src-project'}, + formOverrides: [{settings: {collection: 'custom-records'}}] + }); + const destProject = await dbs.dst.insert('projects', project({title: 'Destination', name: 'dst-project'})); + const sub = await dbs.src.insert('srcproject_customrecords', submission({ + form: seed.forms[0]._id, + project: seed.project._id, + data: {firstName: 'Custom'}, + created: at(20), + modified: at(20) + })); + await dbs.src.insert('srcproject_customrecords_revisions', submissionRevision({ + _rid: sub._id, + form: seed.forms[0]._id, + project: seed.project._id, + data: {firstName: 'Custom v1'}, + created: at(10), + modified: at(10) + })); + + await quiet(() => runClone({ + srcProject: seed.project._id.toString(), + dstProject: destProject._id.toString(), + includeSubmissionRevisions: true + })); + + const custom = await dbs.dst.all('dstproject_customrecords'); + assert.equal(custom.length, 1, 'Submissions must land where the server reads them.'); + assert.equal(custom[0].data.firstName, 'Custom'); + assert.equal(await dbs.dst.count('dst-project_custom-records'), 0); + const revisions = await dbs.dst.all('dstproject_customrecords_revisions'); + assert.equal(revisions.length, 1, 'Revisions of a custom collection must be cloned to its revisions.'); + assert.equal(revisions[0]._rid.toString(), sub._id.toString()); + assert.equal(await dbs.dst.count('submissionrevisions'), 0); + }); + + it('Should warn about submissions an earlier clone left where the server never reads them', async function() { + const seed = await seedProject(dbs.src, { + forms: 1, + submissions: 0, + project: {name: 'src-project'}, + formOverrides: [{settings: {collection: 'custom'}}] + }); + const destProject = await dbs.dst.insert('projects', project({title: 'Destination', name: 'dst-project'})); + await dbs.dst.insert('dst-project_custom', submission({form: seed.forms[0]._id, project: destProject._id})); + + const warnings = []; + const consoleError = console.error; + console.error = (...args) => warnings.push(args.map(String).join(' ')); + try { + await quiet(() => runClone({ + srcProject: seed.project._id.toString(), + dstProject: destProject._id.toString() + })); + } + finally { + console.error = consoleError; + } + assert.ok(warnings.some((line) => /in dst-project_custom, where the server never reads them/.test(line)), + 'The operator must be told about the misplaced submissions.'); + }); + }); + describe('Encrypted submission data', function() { const srcDbSecret = 'source-db-secret'; const dstDbSecret = 'destination-db-secret'; @@ -328,6 +396,49 @@ describe('Project targeting', function() { 'Encrypted data must be readable with the destination project secret.'); }); + it('Should re-encrypt encrypted components within the rows of a datagrid', async function() { + const crypt = crypter(); + const srcProject = project({ + title: 'Encrypted', + name: 'encrypted-grid', + ['settings_encrypted']: crypt.encrypt(srcDbSecret, {}, true) + }); + await dbs.src.insert('projects', srcProject); + const srcForm = await dbs.src.insert('forms', form({ + project: srcProject._id, + components: [{ + type: 'panel', + key: 'panel', + components: [{ + type: 'datagrid', + key: 'people', + input: true, + components: [ + {type: 'textfield', key: 'name', input: true}, + {type: 'textfield', key: 'ssn', input: true, encrypted: true} + ] + }] + }] + })); + await dbs.src.insert('submissions', submission({ + form: srcForm._id, + project: srcProject._id, + data: {people: [ + {name: 'One', ssn: crypt.encrypt(srcDbSecret, '111-11-1111')}, + {name: 'Two', ssn: crypt.encrypt(srcDbSecret, '222-22-2222')} + ]}, + created: at(20), + modified: at(20) + })); + + await quiet(() => runClone({srcDbSecret, dstDbSecret})); + + const cloned = (await dbs.dst.all('submissions'))[0]; + assert.deepEqual(cloned.data.people.map((row) => crypt.decrypt(dstDbSecret, row.ssn)), + ['111-11-1111', '222-22-2222'], 'Every row must be readable with the destination secret.'); + assert.deepEqual(cloned.data.people.map((row) => row.name), ['One', 'Two']); + }); + it('Should leave unencrypted components untouched', async function() { const crypt = crypter(); const srcProject = project({ diff --git a/test/cloner/units.js b/test/cloner/units.js index b93efda..910a7f1 100644 --- a/test/cloner/units.js +++ b/test/cloner/units.js @@ -609,6 +609,38 @@ describe('Cloner units', function() { assert.equal(instance.src.submissions.name, 'a_custom'); assert.equal(instance.dest.submissions.name, 'b_custom'); }); + + it('Should name custom collections, and their revisions, the way the server does', async function() { + const instance = bare({}); + instance.writeBuffer = {}; + instance.cloneState = {}; + instance.pendingState = {}; + const defaults = () => ({ + db: database(), + ogSubmissions: {name: 'submissions'}, + ogSubmissionRevisions: {name: 'submissionrevisions'} + }); + instance.src = defaults(); + instance.dest = defaults(); + + // The server strips everything but letters and digits from both names. + await quiet(() => instance.setCollection( + {settings: {collection: 'custom-records'}}, + {name: 'my-source'}, + {name: 'my_destination.v2'} + )); + assert.equal(instance.src.submissions.name, 'mysource_customrecords'); + assert.equal(instance.src.submissionrevisions.name, 'mysource_customrecords_revisions'); + assert.equal(instance.dest.submissions.name, 'mydestinationv2_customrecords'); + assert.equal(instance.dest.submissionrevisions.name, 'mydestinationv2_customrecords_revisions'); + + // The next form without a collection of its own goes back to the default ones. + await quiet(() => instance.setCollection({settings: {}}, {name: 'my-source'}, {name: 'my_destination.v2'})); + assert.equal(instance.src.submissions.name, 'submissions'); + assert.equal(instance.src.submissionrevisions.name, 'submissionrevisions'); + assert.equal(instance.dest.submissions.name, 'submissions'); + assert.equal(instance.dest.submissionrevisions.name, 'submissionrevisions'); + }); }); describe('roles and access', function() { From 2447000ca64814f0cf61f803c26db106f751b99d Mon Sep 17 00:00:00 2001 From: Travis Tidwell Date: Thu, 22 Oct 2026 10:59:38 -0500 Subject: [PATCH 2/3] Add the rekey command Re-encrypt the project settings and encrypted submission data of a database with a new DB_SECRET, for example after a clone was run with the wrong --dst-db-secret. formio rekey --new-db-secret NEW \ --old-db-secret OLD1 [--old-db-secret OLD2] [--dry-run] - Settings are re-encrypted with the new DB_SECRET. Field values are re-encrypted with the project's own secret when it has one, otherwise with the new DB_SECRET, the same way the server picks the key. - Each value is first tried with its target secret and left alone when it already opens, so the command can be re-run until nothing is left. Otherwise every old secret is tried, and the re-encrypted value is read back before it is written. - Covers fields within datagrid and editgrid rows and containers, custom submission collections, and submission revisions (unless --no-submission-revisions). - Writes only apply when the stored ciphertext is unchanged since it was read, so a value edited mid-run is never overwritten. - Values no secret can open, and projects whose settings cannot be decrypted, are reported and left as they are, and the command exits with an error. The server's own DB_SECRET_OLD rotation is not a substitute: it misses nested values, skips falsy values and revisions, and runs its field updates without awaiting them. Co-Authored-By: Claude Opus 5.5 --- Changelog.md | 11 + Readme.md | 31 +++ commands/commands.js | 1 + commands/rekey.js | 26 +++ src/Rekeyer.js | 491 +++++++++++++++++++++++++++++++++++++++++++ src/rekey.js | 12 ++ test/cli/index.js | 52 ++++- test/cloner/index.js | 1 + test/cloner/rekey.js | 372 ++++++++++++++++++++++++++++++++ 9 files changed, 994 insertions(+), 3 deletions(-) create mode 100644 commands/rekey.js create mode 100644 src/Rekeyer.js create mode 100644 src/rekey.js create mode 100644 test/cloner/rekey.js diff --git a/Changelog.md b/Changelog.md index 75db13d..49f406a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,14 @@ +## [Unreleased] +### Added + - The ```rekey``` command, to re-encrypt the project settings and encrypted submission data of a database with a new DB_SECRET. + +### Fixed + - The clone command did not re-encrypt encrypted fields within the rows of a datagrid or an editgrid. + - The clone command named custom submission collections after the unsanitized project and collection names, where + the server never reads them, and cloned their revisions from and to the default revisions collection. It now + uses the server's names (```myproject_records``` and ```myproject_records_revisions```), and warns when the + destination still has submissions in a collection an earlier clone misnamed. + ## 2.3.2 ### Changed - Fixing the cloner to not skip if the destination item does not exist diff --git a/Readme.md b/Readme.md index b241a27..4a90869 100644 --- a/Readme.md +++ b/Readme.md @@ -290,6 +290,37 @@ yarn test:cloner They cover every clone flag, the dry run output, resuming an interrupted clone, and cloning from an API source (served by a stub API), against a real Mongo database. +### Rekey + + formio rekey --new-db-secret=[NEW_SECRET] --old-db-secret=[OLD_SECRET] [--old-db-secret=[ANOTHER_OLD_SECRET]] + + Re-encrypts the project settings and the encrypted submission data of a database with a new ```DB_SECRET```, for + example after a clone was run with the wrong ```--dst-db-secret```. + + The project settings are encrypted with the ```DB_SECRET```. Encrypted submission data is encrypted with the + ```secret``` of its project settings or, when the project has none, with the ```DB_SECRET```. Every value is + first tried with the secret it should be encrypted with, and left alone when it already is. Otherwise each of + the old secrets is tried, and the value is re-encrypted and read back before it is written. Give every secret + the data may be encrypted with: after a clone that is both the wrong destination secret and the source secret. + + | Option | What it does | + | --- | --- | + | ```--new-db-secret ``` | The ```DB_SECRET``` the database should be encrypted with (required). | + | ```--old-db-secret ``` | A secret the data may currently be encrypted with. Give it more than once to try several. | + | ```-p, --project ``` | Only re-key these projects (comma separated). All projects by default. | + | ```--dry-run``` | Change nothing. Report how much would be re-keyed, and what cannot be decrypted. | + | ```--no-submission-revisions``` | Leave the submission revisions alone. | + | ```--batch-size ``` | The number of documents written at a time (default 1000). | + | ```--ca ```, ```--cert ``` | The TLS certificate authority and certificate key file for the connection. | + + Back up the database, and stop the servers that use it, before re-keying: a server running with the wrong + secret keeps writing data encrypted with it. Start with a ```--dry-run```. + + Values none of the secrets can decrypt are listed and left as they are, and the command exits with an error. + A project whose settings cannot be decrypted is skipped entirely, because the secret of its data is in those + settings. A document that changes while it is being re-keyed is never overwritten. Values that are already + encrypted with the new secret are skipped, so the command can be stopped and run again as often as needed. + ### Deploy ``` diff --git a/commands/commands.js b/commands/commands.js index d7e87a6..c1e5d42 100644 --- a/commands/commands.js +++ b/commands/commands.js @@ -6,6 +6,7 @@ module.exports = function(program, next) { deploy: require('./deploy.js')(program, next), copy: require('./copy.js')(program, next), migrate: require('./migrate.js')(program, next), + rekey: require('./rekey.js')(program, next), submissions: require('./submissions.js')(program, next) }; }; diff --git a/commands/rekey.js b/commands/rekey.js new file mode 100644 index 0000000..209de03 --- /dev/null +++ b/commands/rekey.js @@ -0,0 +1,26 @@ +/* eslint-disable max-len */ +'use strict'; + +/** + * Collect an option that can be given more than once. + * @param {string} value - This value. + * @param {string[]} previous - The values given before it. + */ +function collect(value, previous) { + return previous.concat([value]); +} + +module.exports = function(program, next) { + program + .command('rekey ') + .description('Re-encrypt the project settings and encrypted submission data of a database with a new DB_SECRET.') + .requiredOption('--new-db-secret ', 'The DB_SECRET the database should be encrypted with.') + .option('--old-db-secret ', 'A secret the data may currently be encrypted with. Give it more than once to try several.', collect, []) + .option('-p, --project ', 'Only re-key these projects (comma separated). All projects by default.') + .option('--dry-run', 'Change nothing. Report how much would be re-keyed, and what cannot be decrypted.', false) + .option('--no-submission-revisions', 'Leave the submission revisions alone.') + .option('--batch-size ', 'The number of documents written at a time (default - 1000).') + .option('--ca ', 'The TLS certificate authority for the mongo url.') + .option('--cert ', 'The TLS certificate key file for the mongo connection.') + .action((mongoUrl, options) => require('../src/rekey')(mongoUrl, options)); +}; diff --git a/src/Rekeyer.js b/src/Rekeyer.js new file mode 100644 index 0000000..550eb54 --- /dev/null +++ b/src/Rekeyer.js @@ -0,0 +1,491 @@ +'use strict'; +const {MongoClient, ObjectId} = require('mongodb'); +const _ = require('lodash'); +const Progress = require('./Progress'); +const {redact, tlsOptions, submissionCollectionName} = require('./db'); +const {encrypt, decrypt, eachEncryptedValue, hasEncryptedComponents} = require('./encryption'); + +const SETTINGS = 'settings_encrypted'; + +// How long, at most, counting the documents behind a progress bar may take. +const COUNT_TIMEOUT = 10000; + +// How many of the values that could not be decrypted are listed one by one, per collection. +const MAX_LISTED = 20; + +/** + * Split a comma separated list of ids, ignoring the spaces around them. + * @param {string} value - The comma separated list. + */ +function idList(value) { + return String(value || '').split(',').map((id) => id.trim()).filter((id) => id); +} + +/** + * Re-encrypt the project settings and the encrypted submission data of a database with a new + * DB_SECRET. + * + * The project settings are encrypted with the DB_SECRET. The encrypted submission data is encrypted + * with the secret of its project (the "secret" of the project settings) or, when the project has + * none, with the DB_SECRET. Every value is first tried with the secret it should be encrypted with: + * the ones that already are are left alone, so this can be run again, as often as needed, until + * nothing is left to re-key. The others are tried with each of the old secrets, re-encrypted, and + * only written once the new value has been read back. A value none of the secrets can read is + * reported and left as it is. + */ +class Rekeyer { + /** + * @param {string} mongoUrl - The connection string of the database. + * @param {object} options - The options. + * @param {string} options.newDbSecret - The DB_SECRET everything should be encrypted with. + * @param {string|string[]} options.oldDbSecret - The secrets things may currently be encrypted with. + * @param {string} options.project - A comma separated list of the projects to re-key (all by default). + * @param {boolean} options.dryRun - Change nothing, only report what would be re-keyed. + * @param {number} options.batchSize - The number of documents written at a time (defaults to 1000). + * @param {boolean} options.submissionRevisions - Re-key the submission revisions too (defaults to true). + * @param {string} options.ca - The TLS certificate authority file. + * @param {string} options.cert - The TLS certificate key file. + */ + constructor(mongoUrl, options = {}) { + this.mongoUrl = mongoUrl; + this.options = options; + this.newSecret = options.newDbSecret; + this.oldSecrets = _.uniq([].concat(options.oldDbSecret || []).filter((secret) => secret)); + this.projectIds = idList(options.project); + this.dryRun = !!options.dryRun; + this.batchSize = Math.max(parseInt(options.batchSize, 10) || 1000, 1); + this.includeRevisions = options.submissionRevisions !== false; + this.interactive = (typeof options.interactive === 'boolean') + ? options.interactive + : !!process.stdout.isTTY; + this.activeProgress = null; + this.client = null; + this.db = null; + this.totals = { + settings: {rekeyed: 0, current: 0, undecryptable: 0}, + values: {rekeyed: 0, current: 0, undecryptable: 0}, + skippedProjects: 0, + conflicts: 0, + writeFailures: 0 + }; + } + + /** + * Make sure the options make sense before anything is touched. + */ + validate() { + if (!this.mongoUrl) { + throw new Error('The connection string of the database is required.'); + } + if (!this.newSecret) { + throw new Error('--new-db-secret is required.'); + } + if (!this.oldSecrets.length) { + throw new Error('At least one --old-db-secret is required.'); + } + if (this.oldSecrets.includes(this.newSecret)) { + throw new Error('The new secret is also given as an old secret.'); + } + const invalid = this.projectIds.filter((id) => !ObjectId.isValid(id)); + if (invalid.length) { + throw new Error(`Not a valid project id: ${invalid.join(', ')}`); + } + } + + /** + * Write a line, without it ending up in the middle of a progress line. + * @param {...*} args - What to write. + */ + log(...args) { + if (this.activeProgress) { + this.activeProgress.interrupt(); + } + console.log(...args); + } + + /** + * Report an error, without it ending up in the middle of a progress line. + * @param {...*} args - What to report. + */ + error(...args) { + if (this.activeProgress) { + this.activeProgress.interrupt(); + } + console.error(...args); + } + + async connect() { + try { + this.log(`Connecting to ${redact(this.mongoUrl)}`); + this.client = new MongoClient(this.mongoUrl, tlsOptions(this.options.ca, this.options.cert)); + await this.client.connect(); + this.db = this.client.db(); + } + catch (err) { + throw new Error(`Could not connect to database ${redact(this.mongoUrl)}: ${err.message}`, {cause: err}); + } + } + + async disconnect() { + if (this.client) { + await this.client.close(); + this.client = null; + } + } + + /** + * Re-key everything the options ask for. + */ + async rekey() { + this.validate(); + await this.connect(); + try { + if (this.dryRun) { + this.log('Dry run: nothing will be changed.'); + } + await this.rekeyProjects(); + } + finally { + await this.disconnect(); + } + this.report(); + const failures = this.failures(); + if (failures.length) { + throw new Error(`The re-key did not complete: ${failures.join(', ')}.`); + } + } + + /** + * What kept the re-key from completing, if anything. + * @returns {string[]} - The reasons. + */ + failures() { + const {settings, values, skippedProjects, conflicts, writeFailures} = this.totals; + const failures = []; + if (settings.undecryptable) { + failures.push(`${settings.undecryptable} project(s) had settings none of the secrets could decrypt`); + } + if (skippedProjects) { + failures.push(`${skippedProjects} project(s) were skipped`); + } + if (values.undecryptable) { + failures.push(`${values.undecryptable} encrypted value(s) could not be decrypted with any of the secrets`); + } + if (conflicts) { + failures.push(`${conflicts} document(s) changed while they were being re-keyed and were left as they are`); + } + if (writeFailures) { + failures.push(`${writeFailures} document(s) could not be written`); + } + return failures; + } + + /** + * Write the totals. + */ + report() { + const {settings, values} = this.totals; + const rekeyed = this.dryRun ? 'would be re-keyed' : 're-keyed'; + this.log(''); + this.log(this.dryRun ? 'Dry run complete. Nothing was changed.' : 'Done.'); + this.log(' Project settings'); + this.log(` ${rekeyed.padEnd(28)}: ${settings.rekeyed}`); + this.log(` ${'already on the new secret'.padEnd(28)}: ${settings.current}`); + this.log(` ${'could not be decrypted'.padEnd(28)}: ${settings.undecryptable}`); + this.log(' Encrypted submission values'); + this.log(` ${rekeyed.padEnd(28)}: ${values.rekeyed}`); + this.log(` ${'already on the new secret'.padEnd(28)}: ${values.current}`); + this.log(` ${'could not be decrypted'.padEnd(28)}: ${values.undecryptable}`); + } + + /** + * Try to re-key a single encrypted value. + * + * @param {Buffer|Binary} cipher - The encrypted value. + * @param {string} target - The secret the value should be encrypted with. + * @param {string[]} candidates - The secrets the value may currently be encrypted with. + * @param {boolean} nosalt - The value is encrypted without a salt, as the project settings are. + * @returns {{status: string, value: *, cipher: Buffer}} - The status is "current" when the value is + * already encrypted with the target secret, "rekeyed" when it was re-encrypted into + * cipher, and "undecryptable" when none of the secrets could read it. + */ + rekeyValue(cipher, target, candidates, nosalt = false) { + const current = decrypt(target, cipher, nosalt); + if (current !== null) { + return {status: 'current', value: current}; + } + for (const secret of candidates) { + if (secret === target) { + continue; + } + const value = decrypt(secret, cipher, nosalt); + if (value === null) { + continue; + } + const reEncrypted = encrypt(target, value, nosalt); + + // Never write a value that cannot be read back. + if (!reEncrypted || !_.isEqual(decrypt(target, reEncrypted, nosalt), value)) { + return {status: 'undecryptable'}; + } + return {status: 'rekeyed', value, cipher: reEncrypted}; + } + return {status: 'undecryptable'}; + } + + /** + * Re-key every project the options ask for. + */ + async rekeyProjects() { + const query = this.projectIds.length + ? {_id: {$in: this.projectIds.map((id) => new ObjectId(id))}} + : {}; + if (this.projectIds.length) { + const found = await this.db.collection('projects').countDocuments(query); + if (found !== this.projectIds.length) { + throw new Error(`Only ${found} of the ${this.projectIds.length} projects given were found.`); + } + } + const cursor = this.db.collection('projects').find(query).sort({created: 1, _id: 1}); + for await (const project of cursor) { + await this.rekeyProject(project); + } + } + + /** + * Re-key the settings and the encrypted submission data of a project. + * @param {object} project - The project. + */ + async rekeyProject(project) { + this.log(`\n- Project ${project.title || project.name} (${project._id})${project.deleted ? ' [deleted]' : ''}`); + const settings = await this.rekeySettings(project); + if (!settings) { + this.totals.skippedProjects++; + this.error(' Its submission data was skipped: the secret it is encrypted with is in those settings.'); + return; + } + + // The data is encrypted with the secret of the project, or with the DB_SECRET when it has none. + // Any of the other secrets may have been used by mistake, so all of them are tried. + const target = settings.secret || this.newSecret; + const candidates = _.uniq([...this.oldSecrets, this.newSecret, settings.secret]) + .filter((secret) => secret && secret !== target); + + const forms = this.db.collection('forms').find({project: project._id}).sort({created: 1, _id: 1}); + for await (const form of forms) { + if (hasEncryptedComponents(form.components)) { + await this.rekeyForm(project, form, target, candidates); + } + } + } + + /** + * Re-key the settings of a project. + * @param {object} project - The project. + * @returns {object|null} - The decrypted settings, or null when they could not be re-keyed. + */ + async rekeySettings(project) { + if (!project[SETTINGS]) { + return {}; + } + const result = this.rekeyValue(project[SETTINGS], this.newSecret, this.oldSecrets, true); + if (result.status === 'undecryptable') { + this.totals.settings.undecryptable++; + this.error(' Its settings could not be decrypted with any of the secrets, they were left as they are.'); + return null; + } + this.totals.settings[result.status]++; + if (result.status === 'current') { + this.log(' Settings: already on the new secret.'); + return result.value; + } + if (this.dryRun) { + this.log(' Settings: would be re-keyed.'); + return result.value; + } + const written = await this.db.collection('projects').updateOne( + {_id: project._id, [SETTINGS]: project[SETTINGS]}, + {$set: {[SETTINGS]: result.cipher}} + ); + if (!written.matchedCount) { + this.totals.settings.rekeyed--; + this.totals.conflicts++; + this.error(' Its settings changed while they were being re-keyed, they were left as they are.'); + return null; + } + this.log(' Settings: re-keyed.'); + return result.value; + } + + /** + * The name of the collection the submissions of a form are kept in, the way the server names it. + * @param {object} project - The project. + * @param {object} form - The form. + */ + submissionCollection(project, form) { + const collection = _.get(form, 'settings.collection'); + return (collection && submissionCollectionName(project.name, collection)) || 'submissions'; + } + + /** + * Re-key the encrypted data of the submissions, and submission revisions, of a form. + * @param {object} project - The project. + * @param {object} form - The form. + * @param {string} target - The secret the data should be encrypted with. + * @param {string[]} candidates - The secrets the data may currently be encrypted with. + */ + async rekeyForm(project, form, target, candidates) { + this.log(` - Form ${form.title || form.name} (${form._id})${form.deleted ? ' [deleted]' : ''}`); + const collections = [this.submissionCollection(project, form)]; + + // Earlier versions of the clone command named a custom collection after the unsanitized project + // name, where the server never looks for them. Those submissions are re-keyed too, but they + // still have to be moved to where the server reads them from. + const custom = _.get(form, 'settings.collection'); + const unsanitized = custom ? `${project.name}_${custom}` : null; + const misplaced = unsanitized && unsanitized !== collections[0] && + await this.count(this.db.collection(unsanitized), {form: form._id}); + if (misplaced) { + this.error( + ` Submissions of this form are in ${unsanitized}, but the server reads them from ${collections[0]}. ` + + 'They are re-keyed where they are.' + ); + collections.push(unsanitized); + } + + const query = {form: form._id}; + for (const collection of collections) { + await this.rekeyCollection(collection, query, form.components, target, candidates, ' - Submissions:'); + if (this.includeRevisions) { + const revisions = (collection === 'submissions') ? 'submissionrevisions' : `${collection}_revisions`; + await this.rekeyCollection(revisions, query, form.components, target, candidates, ' - Revisions:'); + } + } + } + + /** + * Count the documents behind a progress bar, giving up when it takes too long. + * @param {Collection} collection - The collection. + * @param {object} query - The query. + */ + async count(collection, query) { + try { + return await collection.countDocuments(query, {maxTimeMS: COUNT_TIMEOUT}); + } + catch (err) { + return null; + } + } + + /** + * Re-key the encrypted data of the documents of a collection. + * @param {string} name - The name of the collection. + * @param {object} query - The documents to re-key. + * @param {object[]} components - The components of the form the documents belong to. + * @param {string} target - The secret the data should be encrypted with. + * @param {string[]} candidates - The secrets the data may currently be encrypted with. + * @param {string} label - The label of the progress. + */ + async rekeyCollection(name, query, components, target, candidates, label) { + const collection = this.db.collection(name); + const total = await this.count(collection, query); + if (total === 0) { + return; + } + const progress = new Progress(label, {total, interactive: this.interactive}); + this.activeProgress = progress; + const counts = {rekeyed: 0, current: 0, undecryptable: 0}; + let operations = []; + + // What could not be decrypted is listed once the progress line is done, never in the middle of it. + const unreadable = []; + + const cursor = collection + .find(query, {projection: {data: 1}}) + .sort({_id: 1}) + .batchSize(Math.min(this.batchSize, 1000)); + for await (const doc of cursor) { + const filter = {_id: doc._id}; + const update = {}; + eachEncryptedValue(components, doc.data, (value, path) => { + const result = this.rekeyValue(value, target, candidates); + counts[result.status]++; + const key = ['data', ...path].join('.'); + if (result.status === 'rekeyed') { + // The old value is part of the filter, so that a value changed in the meantime is never + // overwritten with one re-keyed from what it used to be. + filter[key] = value; + update[key] = result.cipher; + } + else if (result.status === 'undecryptable') { + unreadable.push(`${key} of ${name} ${doc._id}`); + } + }); + if (!_.isEmpty(update)) { + operations.push({updateOne: {filter, update: {$set: update}}}); + } + if (operations.length >= this.batchSize) { + await this.write(collection, operations); + operations = []; + } + progress.tick(); + } + await this.write(collection, operations); + progress.done(); + this.activeProgress = null; + Object.keys(counts).forEach((status) => { + this.totals.values[status] += counts[status]; + }); + + // Finishes the progress line. + this.log( + ` - ${counts.rekeyed} ${this.dryRun ? 'would be re-keyed' : 're-keyed'}, ` + + `${counts.current} already on the new secret, ${counts.undecryptable} could not be decrypted` + ); + unreadable.slice(0, MAX_LISTED).forEach((value) => { + this.error(` ${value} could not be decrypted with any of the secrets.`); + }); + if (unreadable.length > MAX_LISTED) { + this.error(` ...and ${unreadable.length - MAX_LISTED} more.`); + } + } + + /** + * Write a batch of re-keyed documents. + * @param {Collection} collection - The collection. + * @param {object[]} operations - The bulk write operations. + */ + async write(collection, operations) { + if (!operations.length || this.dryRun) { + return; + } + let matched; + let failed = 0; + try { + const result = await collection.bulkWrite(operations, {ordered: false}); + matched = result.matchedCount; + } + catch (err) { + const writeErrors = [].concat(_.get(err, 'writeErrors', [])); + if (!err.result || !writeErrors.length) { + throw err; + } + failed = writeErrors.length; + matched = err.result.matchedCount; + this.error(`Failed to write ${failed} document(s) of ${collection.collectionName}: ${writeErrors[0].errmsg}`); + } + this.totals.writeFailures += failed; + + // A document whose encrypted values changed since it was read no longer matches its filter. + const conflicts = operations.length - matched - failed; + if (conflicts > 0) { + this.totals.conflicts += conflicts; + this.error( + `${conflicts} document(s) of ${collection.collectionName} changed while being re-keyed, ` + + 'run again to re-key them.' + ); + } + } +} + +module.exports = Rekeyer; diff --git a/src/rekey.js b/src/rekey.js new file mode 100644 index 0000000..ba89edc --- /dev/null +++ b/src/rekey.js @@ -0,0 +1,12 @@ +'use strict'; +const Rekeyer = require('./Rekeyer'); +module.exports = async function(mongoUrl, options) { + const rekeyer = new Rekeyer(mongoUrl, options); + try { + await rekeyer.rekey(); + } + catch (err) { + console.error(err.message.red); + process.exitCode = 1; + } +}; diff --git a/test/cli/index.js b/test/cli/index.js index 98e5fe0..fd256ca 100644 --- a/test/cli/index.js +++ b/test/cli/index.js @@ -50,8 +50,8 @@ function run(...args) { program.configureOutput({writeOut: () => {}, writeErr: () => {}}); require(path.join(root, 'commands/commands'))(program, (err) => (err ? reject(err) : resolve())); program.parseAsync(['node', 'formio', ...args]).then(() => { - // The clone command is not run through a series, so it is done once the action is. - if (args[0] === 'clone') { + // The clone and rekey commands are not run through a series, so they are done once the action is. + if (['clone', 'rekey'].includes(args[0])) { resolve(); } }, reject); @@ -71,6 +71,9 @@ describe('Command line', function() { stub('src/clone.js', async(source, destination, options) => { calls.push({name: 'clone', source, destination, options}); }); + stub('src/rekey.js', async(mongoUrl, options) => { + calls.push({name: 'rekey', mongoUrl, options}); + }); }); beforeEach(function() { @@ -127,6 +130,45 @@ describe('Command line', function() { }); }); + describe('rekey', function() { + it('Should hand every option to the re-key', async function() { + await run('rekey', 'mongodb://db', + '--new-db-secret', 'new', + '--old-db-secret', 'old1', + '--old-db-secret', 'old2', + '--project', 'aaa,bbb', + '--dry-run', + '--no-submission-revisions', + '--batch-size', '50', + '--ca', 'ca.pem', + '--cert', 'cert.pem'); + + assert.equal(calls.length, 1); + const {mongoUrl, options} = calls[0]; + assert.equal(mongoUrl, 'mongodb://db'); + assert.equal(options.newDbSecret, 'new'); + assert.deepEqual(options.oldDbSecret, ['old1', 'old2']); + assert.equal(options.project, 'aaa,bbb'); + assert.equal(options.dryRun, true); + assert.equal(options.submissionRevisions, false); + assert.equal(options.batchSize, '50'); + assert.equal(options.ca, 'ca.pem'); + assert.equal(options.cert, 'cert.pem'); + }); + + it('Should use the defaults when no options are given', async function() { + await run('rekey', 'mongodb://db', '--new-db-secret', 'new'); + const {options} = calls[0]; + assert.deepEqual(options.oldDbSecret, []); + assert.equal(options.dryRun, false); + assert.equal(options.submissionRevisions, true); + }); + + it('Should require the new secret', async function() { + await assert.rejects(() => run('rekey', 'mongodb://db'), /required option '--new-db-secret '/); + }); + }); + describe('commands run as a series', function() { it('Should hand the options and arguments of deploy to every step', async function() { await run('deploy', 'https://src.example.com', 'https://dst.example.com', @@ -208,12 +250,16 @@ describe('Command line', function() { assert.match(formio('copy', '--help'), /--full/); assert.match(formio('migrate', '--help'), /--delete-before/); assert.match(formio('submissions', '--help'), /--admin-key/); + assert.match(formio('rekey', '--help'), /--old-db-secret/); }); }); describe('modules', function() { it('Should load every command implementation', function() { - for (const file of ['src/migrate.js', 'src/submissions.js', 'src/copy.js', 'src/deploy.js', 'src/clone.js']) { + const files = [ + 'src/migrate.js', 'src/submissions.js', 'src/copy.js', 'src/deploy.js', 'src/clone.js', 'src/rekey.js' + ]; + for (const file of files) { const resolved = require.resolve(path.join(root, file)); const cached = require.cache[resolved]; delete require.cache[resolved]; diff --git a/test/cloner/index.js b/test/cloner/index.js index 5d041f8..1483d94 100644 --- a/test/cloner/index.js +++ b/test/cloner/index.js @@ -18,4 +18,5 @@ describe('Cloner', function() { require('./api'); require('./operations'); require('./progress'); + require('./rekey'); }); diff --git a/test/cloner/rekey.js b/test/cloner/rekey.js new file mode 100644 index 0000000..6d2046c --- /dev/null +++ b/test/cloner/rekey.js @@ -0,0 +1,372 @@ +/* globals describe, it, before, after, beforeEach */ +'use strict'; +const assert = require('assert'); +const {ObjectId} = require('mongodb'); +const Rekeyer = require('../../src/Rekeyer'); +const {encrypt, decrypt, eachEncryptedValue, hasEncryptedComponents} = require('../../src/encryption'); +const {DST_URI, TestDb, project, form, submission, submissionRevision} = require('./fixtures'); +const {quiet} = require('./helper'); + +const NEW_SECRET = 'the-right-secret'; +const WRONG_SECRET = 'the-wrong-destination-secret'; +const SOURCE_SECRET = 'the-source-secret'; +const PROJECT_SECRET = 'the-project-secret'; + +// A form with an encrypted field at the top, one within the rows of a datagrid (inside a panel), one +// within a container, and an encrypted datagrid whose rows are encrypted as a whole. +const COMPONENTS = [ + {type: 'textfield', key: 'name', input: true}, + {type: 'textfield', key: 'ssn', input: true, encrypted: true}, + {type: 'panel', key: 'panel', components: [{ + type: 'datagrid', + key: 'people', + input: true, + components: [ + {type: 'textfield', key: 'name', input: true}, + {type: 'textfield', key: 'ssn', input: true, encrypted: true} + ] + }]}, + {type: 'container', key: 'details', input: true, components: [ + {type: 'columns', key: 'columns', columns: [{components: [ + {type: 'textfield', key: 'pin', input: true, encrypted: true} + ]}]} + ]}, + {type: 'datagrid', key: 'secrets', input: true, encrypted: true, components: [ + {type: 'textfield', key: 'value', input: true, encrypted: true} + ]} +]; + +/** + * Submission data with every encrypted value of COMPONENTS encrypted with a secret. + * @param {string} secret - The secret. + */ +function encryptedData(secret) { + return { + name: 'Plain', + ssn: encrypt(secret, '123-45-6789'), + people: [ + {name: 'One', ssn: encrypt(secret, '111-11-1111')}, + {name: 'Two', ssn: encrypt(secret, '222-22-2222')} + ], + details: {pin: encrypt(secret, 1234)}, + secrets: encrypt(secret, [{value: 'a'}, {value: 'b'}]) + }; +} + +/** + * Decrypt every encrypted value of submission data built by encryptedData(). + * @param {string} secret - The secret to decrypt with. + * @param {object} data - The data. + */ +function decryptedData(secret, data) { + return { + name: data.name, + ssn: decrypt(secret, data.ssn), + people: data.people.map((row) => ({name: row.name, ssn: decrypt(secret, row.ssn)})), + details: {pin: decrypt(secret, data.details.pin)}, + secrets: decrypt(secret, data.secrets) + }; +} + +const PLAIN = { + name: 'Plain', + ssn: '123-45-6789', + people: [{name: 'One', ssn: '111-11-1111'}, {name: 'Two', ssn: '222-22-2222'}], + details: {pin: 1234}, + secrets: [{value: 'a'}, {value: 'b'}] +}; + +/** + * Run a re-key, returning what it reported as errors, what it printed, and the error it ended with. + * @param {object} options - The re-key options. + */ +async function runRekey(options) { + const warnings = []; + const consoleError = console.error; + console.error = (...args) => warnings.push(args.map(String).join(' ')); + const rekeyer = new Rekeyer(DST_URI, Object.assign({ + newDbSecret: NEW_SECRET, + oldDbSecret: [WRONG_SECRET, SOURCE_SECRET], + interactive: false + }, options)); + let error = null; + try { + await quiet(() => rekeyer.rekey()); + } + catch (err) { + error = err; + } + finally { + console.error = consoleError; + } + return {rekeyer, warnings, error}; +} + +describe('Rekey', function() { + describe('eachEncryptedValue', function() { + it('Should find the encrypted values within rows, containers and layout components', function() { + const found = []; + eachEncryptedValue(COMPONENTS, encryptedData('x'), (value, path) => found.push(path.join('.'))); + assert.deepEqual(found, ['ssn', 'people.0.ssn', 'people.1.ssn', 'details.pin', 'secrets']); + }); + + it('Should skip plain values, and components that are not saved', function() { + const found = []; + eachEncryptedValue([ + {type: 'textfield', key: 'plain', encrypted: true}, + {type: 'textfield', key: 'empty', encrypted: true}, + {type: 'textfield', key: 'transient', encrypted: true, persistent: false} + ], {plain: 'not encrypted', empty: '', transient: encrypt('x', 'value')}, (value, path) => found.push(path)); + assert.deepEqual(found, []); + }); + + it('Should tell whether a form has encrypted components', function() { + assert.equal(hasEncryptedComponents(COMPONENTS), true); + assert.equal(hasEncryptedComponents([{type: 'panel', components: [{type: 'textfield', key: 'a'}]}]), false); + assert.equal(hasEncryptedComponents(undefined), false); + }); + }); + + describe('rekeyValue', function() { + const rekeyer = new Rekeyer(DST_URI, {newDbSecret: NEW_SECRET, oldDbSecret: [WRONG_SECRET]}); + + it('Should leave a value that is already encrypted with the target secret', function() { + const result = rekeyer.rekeyValue(encrypt(NEW_SECRET, 'value'), NEW_SECRET, [WRONG_SECRET]); + assert.equal(result.status, 'current'); + }); + + it('Should re-encrypt a value encrypted with one of the candidates', function() { + const result = rekeyer.rekeyValue(encrypt(WRONG_SECRET, 0), NEW_SECRET, [SOURCE_SECRET, WRONG_SECRET]); + assert.equal(result.status, 'rekeyed'); + assert.strictEqual(decrypt(NEW_SECRET, result.cipher), 0, 'A falsy value is still a value.'); + }); + + it('Should re-encrypt settings without a salt', function() { + const result = rekeyer.rekeyValue(encrypt(WRONG_SECRET, {secret: 'a'}, true), NEW_SECRET, [WRONG_SECRET], true); + assert.equal(result.status, 'rekeyed'); + assert.deepEqual(decrypt(NEW_SECRET, result.cipher, true), {secret: 'a'}); + }); + + it('Should report a value none of the secrets can read', function() { + const result = rekeyer.rekeyValue(encrypt('unknown', 'value'), NEW_SECRET, [WRONG_SECRET]); + assert.equal(result.status, 'undecryptable'); + }); + }); + + describe('options', function() { + it('Should refuse to run without the secrets', async function() { + assert.throws(() => new Rekeyer(DST_URI, {oldDbSecret: ['a']}).validate(), /--new-db-secret is required/); + assert.throws(() => new Rekeyer(DST_URI, {newDbSecret: 'a'}).validate(), /At least one --old-db-secret/); + assert.throws(() => new Rekeyer(DST_URI, {newDbSecret: 'a', oldDbSecret: ['a']}).validate(), + /also given as an old secret/); + assert.throws(() => new Rekeyer(DST_URI, {newDbSecret: 'a', oldDbSecret: ['b'], project: 'nope'}).validate(), + /Not a valid project id: nope/); + }); + + it('Should accept a single old secret as a string', function() { + assert.deepEqual(new Rekeyer(DST_URI, {newDbSecret: 'a', oldDbSecret: 'b'}).oldSecrets, ['b']); + }); + }); + + describe('a database', function() { + let db = null; + let seed = null; + + before('Connect to the test database', async function() { + db = await new TestDb(DST_URI).connect(); + }); + + after('Disconnect from the test database', async function() { + await db.close(); + }); + + /** + * Seed the database the way a clone with the wrong destination secret leaves it. + */ + beforeEach('Seed a database cloned with the wrong secret', async function() { + await db.reset(); + + // A project without a secret of its own: its data is encrypted with the DB_SECRET, which the + // clone got wrong. The values within datagrid rows were never migrated, and still have the + // source secret. + const plain = await db.insert('projects', project({ + title: 'No project secret', + name: 'no-secret', + ['settings_encrypted']: encrypt(WRONG_SECRET, {cors: '*'}, true) + })); + const plainForm = await db.insert('forms', form({project: plain._id, components: COMPONENTS})); + const wrongData = encryptedData(WRONG_SECRET); + wrongData.people = encryptedData(SOURCE_SECRET).people; + const plainSubmission = await db.insert('submissions', submission({ + form: plainForm._id, + project: plain._id, + data: wrongData + })); + await db.insert('submissionrevisions', submissionRevision({ + _rid: plainSubmission._id, + form: plainForm._id, + project: plain._id, + data: encryptedData(WRONG_SECRET) + })); + const unreadable = await db.insert('submissions', submission({ + form: plainForm._id, + project: plain._id, + data: {ssn: encrypt('a-secret-nobody-knows', 'lost')} + })); + + // A form of the same project with its own collection, which the server names after the + // project name without its dashes. + const customForm = await db.insert('forms', form({ + project: plain._id, + components: COMPONENTS, + settings: {collection: 'people-records'} + })); + await db.insert('nosecret_peoplerecords', submission({ + form: customForm._id, + project: plain._id, + data: encryptedData(WRONG_SECRET) + })); + + // A project with a secret of its own: its data is encrypted with that, so only its settings + // need to be re-keyed. + const own = await db.insert('projects', project({ + title: 'Own secret', + name: 'own-secret', + ['settings_encrypted']: encrypt(WRONG_SECRET, {secret: PROJECT_SECRET}, true) + })); + const ownForm = await db.insert('forms', form({project: own._id, components: COMPONENTS})); + const ownSubmission = await db.insert('submissions', submission({ + form: ownForm._id, + project: own._id, + data: encryptedData(PROJECT_SECRET) + })); + + // The project the destination server created itself, already on the right secret. + const formio = await db.insert('projects', project({ + title: 'Form.io', + name: 'formio', + ['settings_encrypted']: encrypt(NEW_SECRET, {secret: 'formio-secret'}, true) + })); + + seed = {plain, plainForm, plainSubmission, unreadable, customForm, own, ownForm, ownSubmission, formio}; + }); + + /** + * The decrypted settings of a project. + * @param {object} proj - The project. + */ + async function settings(proj) { + return decrypt(NEW_SECRET, (await db.findOne('projects', {_id: proj._id}))['settings_encrypted'], true); + } + + it('Should change nothing on a dry run', async function() { + const before = await db.all('submissions'); + const {rekeyer, error} = await runRekey({dryRun: true}); + + assert.deepEqual(await db.all('submissions'), before, 'The submissions must be left as they are.'); + assert.equal(await settings(seed.plain), null, 'The settings must be left as they are.'); + assert.deepEqual(rekeyer.totals.settings, {rekeyed: 2, current: 1, undecryptable: 0}); + // The plain project has 5 values in its submission, 5 in its revision and 5 in its custom + // collection. The own secret project has 5 that are already right. One value is lost. + assert.deepEqual(rekeyer.totals.values, {rekeyed: 15, current: 5, undecryptable: 1}); + assert.match(String(error && error.message), /1 encrypted value\(s\) could not be decrypted/); + }); + + it('Should re-key the settings and the encrypted data', async function() { + const {rekeyer, warnings, error} = await runRekey({}); + + assert.deepEqual(await settings(seed.plain), {cors: '*'}); + assert.deepEqual(await settings(seed.own), {secret: PROJECT_SECRET}); + assert.deepEqual(await settings(seed.formio), {secret: 'formio-secret'}); + + const sub = await db.findOne('submissions', {_id: seed.plainSubmission._id}); + assert.deepEqual(decryptedData(NEW_SECRET, sub.data), PLAIN, + 'Every value, including the ones within rows, must be readable with the new secret.'); + const revision = await db.findOne('submissionrevisions', {_rid: seed.plainSubmission._id}); + assert.deepEqual(decryptedData(NEW_SECRET, revision.data), PLAIN, 'Revisions must be re-keyed as well.'); + const custom = (await db.all('nosecret_peoplerecords'))[0]; + assert.deepEqual(decryptedData(NEW_SECRET, custom.data), PLAIN, 'Custom collections must be re-keyed as well.'); + + const own = await db.findOne('submissions', {_id: seed.ownSubmission._id}); + assert.deepEqual(decryptedData(PROJECT_SECRET, own.data), PLAIN); + assert.ok(Buffer.from(own.data.ssn.buffer).equals(seed.ownSubmission.data.ssn), + 'Data encrypted with the project secret must not be touched.'); + + const unreadable = await db.findOne('submissions', {_id: seed.unreadable._id}); + assert.ok(Buffer.from(unreadable.data.ssn.buffer).equals(seed.unreadable.data.ssn), + 'A value no secret can read must be left as it is.'); + assert.ok(warnings.some((line) => line.includes(`data.ssn of submissions ${seed.unreadable._id}`)), + 'The value that could not be decrypted must be reported.'); + assert.match(String(error && error.message), /1 encrypted value\(s\) could not be decrypted/, + 'The re-key must not report success.'); + assert.equal(rekeyer.totals.conflicts, 0); + }); + + it('Should find nothing left to do when it is run again', async function() { + await runRekey({}); + const {rekeyer} = await runRekey({}); + assert.deepEqual(rekeyer.totals.settings, {rekeyed: 0, current: 3, undecryptable: 0}); + assert.deepEqual(rekeyer.totals.values, {rekeyed: 0, current: 20, undecryptable: 1}); + }); + + it('Should re-key data encrypted with the wrong secret into the project secret', async function() { + await db.collection('submissions').updateOne( + {_id: seed.ownSubmission._id}, + {$set: {'data.ssn': encrypt(WRONG_SECRET, 'moved')}} + ); + await runRekey({project: seed.own._id.toString()}); + const own = await db.findOne('submissions', {_id: seed.ownSubmission._id}); + assert.equal(decrypt(PROJECT_SECRET, own.data.ssn), 'moved'); + }); + + it('Should only re-key the projects it is given', async function() { + const {rekeyer, error} = await runRekey({project: seed.own._id.toString()}); + assert.equal(error, null); + assert.deepEqual(rekeyer.totals.settings, {rekeyed: 1, current: 0, undecryptable: 0}); + assert.deepEqual(await settings(seed.own), {secret: PROJECT_SECRET}); + assert.equal(await settings(seed.plain), null, 'Other projects must be left alone.'); + }); + + it('Should refuse projects that do not exist', async function() { + const {error} = await runRekey({project: new ObjectId().toString()}); + assert.match(String(error && error.message), /Only 0 of the 1 projects given were found/); + }); + + it('Should skip the data of a project whose settings none of the secrets can read', async function() { + const {rekeyer, error} = await runRekey({oldDbSecret: ['something-else'], project: seed.plain._id.toString()}); + assert.deepEqual(rekeyer.totals.settings, {rekeyed: 0, current: 0, undecryptable: 1}); + assert.deepEqual(rekeyer.totals.values, {rekeyed: 0, current: 0, undecryptable: 0}, + 'Without the settings, the secret of the data is not known.'); + assert.match(String(error && error.message), /had settings none of the secrets could decrypt/); + }); + + it('Should leave the submission revisions alone when asked to', async function() { + await runRekey({submissionRevisions: false}); + const revision = await db.findOne('submissionrevisions', {_rid: seed.plainSubmission._id}); + assert.deepEqual(decryptedData(WRONG_SECRET, revision.data), PLAIN); + }); + + it('Should never overwrite a value that changed after it was read', async function() { + const rekeyer = new Rekeyer(DST_URI, {newDbSecret: NEW_SECRET, oldDbSecret: [WRONG_SECRET]}); + const collection = db.collection('submissions'); + const changed = encrypt(NEW_SECRET, 'changed by someone else'); + await collection.updateOne({_id: seed.plainSubmission._id}, {$set: {'data.ssn': changed}}); + + const consoleError = console.error; + console.error = () => {}; + try { + await rekeyer.write(collection, [{updateOne: { + filter: {_id: seed.plainSubmission._id, 'data.ssn': seed.plainSubmission.data.ssn}, + update: {$set: {'data.ssn': encrypt(NEW_SECRET, '123-45-6789')}} + }}]); + } + finally { + console.error = consoleError; + } + + assert.equal(rekeyer.totals.conflicts, 1); + const sub = await db.findOne('submissions', {_id: seed.plainSubmission._id}); + assert.equal(decrypt(NEW_SECRET, sub.data.ssn), 'changed by someone else'); + }); + }); +}); From 083638a8b4869d250910634dea022c7914ff9ec4 Mon Sep 17 00:00:00 2001 From: Travis Tidwell Date: Wed, 28 Oct 2026 09:33:17 -0500 Subject: [PATCH 3/3] Fix rekey gaps found in review and share the clone helpers - rekey kept the projects and forms cursors open while it re-keyed the submissions of each form. On a large database the server closes an idle cursor, and the rekey stopped part way. It now lists the ids first and reads each project and form on its own. - rekey never re-keyed the revisions of a custom collection that earlier clones wrote to the default submissionrevisions collection. It now re-keys them where they are and says so. - Encrypted values an API returns as JSON serialized Buffers were skipped with no message when cloning from an API source. They are now read and re-encrypted like any other encrypted value. - An absolute --ca or --cert path was prefixed with the working directory. - idList (which now ignores repeated ids) and the legacy collection name live in src/db.js, shared by Cloner and Rekeyer. - Removed the encrypt/decrypt/legacyKey wrappers and defaultSaltLength from Cloner, which call src/encryption.js directly. Co-Authored-By: Claude Opus 5.5 --- src/Cloner.js | 60 +++++++------------------------- src/Rekeyer.js | 72 ++++++++++++++++++++++++++++----------- src/db.js | 32 ++++++++++++++--- src/encryption.js | 18 ++++++++-- test/cloner/operations.js | 7 ++-- test/cloner/projects.js | 7 ++-- test/cloner/rekey.js | 36 ++++++++++++++++++++ test/cloner/units.js | 50 +++++++++++++++------------ 8 files changed, 175 insertions(+), 107 deletions(-) diff --git a/src/Cloner.js b/src/Cloner.js index bd99212..72a2927 100644 --- a/src/Cloner.js +++ b/src/Cloner.js @@ -4,7 +4,13 @@ const {MongoClient, ObjectId, BSON} = require('mongodb'); const fs = require('fs'); const _ = require('lodash'); const encryption = require('./encryption'); -const {redact, tlsOptions, submissionCollectionName} = require('./db'); +const { + redact, + tlsOptions, + submissionCollectionName, + legacySubmissionCollectionName, + idList +} = require('./db'); const {createHmac} = require('node:crypto'); const fetch = require('./fetch'); const Progress = require('./Progress'); @@ -35,14 +41,6 @@ const MAX_REPEATED_ERRORS = 5; const POSITION_COUNT_TIMEOUT = 2000; const BAR_COUNT_TIMEOUT = 60000; -/** - * Split a comma separated list of ids, ignoring the spaces around them. - * @param {string} value - The comma separated list. - */ -function idList(value) { - return String(value || '').split(',').map((id) => id.trim()).filter((id) => id); -} - class Cloner { /** * @param {*} srcPath - The source connection string. @@ -81,7 +79,6 @@ class Cloner { this.mongoDest = mongoDest; this.beforeAll = null; this.afterAll = null; - this.defaultSaltLength = 40; this.options = options; this.src = null; this.dest = null; @@ -362,37 +359,6 @@ class Cloner { } } - /** - * Derive the key and iv for a secret the way the server does. - * @param {string} secret - The secret to derive the key and iv from. - * @returns {{key: Buffer, iv: Buffer}} - The derived key and iv. - */ - legacyKey(secret) { - return encryption.legacyKey(secret); - } - - /** - * Encrypt data or settings using a secret. - * @param {*} secret - * @param {*} rawData - * @param {*} nosalt - * @returns - */ - encrypt(secret, rawData, nosalt) { - return encryption.encrypt(secret, rawData, nosalt, this.defaultSaltLength); - } - - /** - * Decrypt data or settings using a secret. - * @param {*} secret - * @param {*} cipherbuffer - * @param {*} nosalt - * @returns - */ - decrypt(secret, cipherbuffer, nosalt) { - return encryption.decrypt(secret, cipherbuffer, nosalt, this.defaultSaltLength); - } - /** * Create the database configuration for the given database source type. * @param {*} type - The type of database source. @@ -1458,7 +1424,7 @@ class Cloner { * @param {string} destCollection - The collection the submissions are cloned to. */ async warnAboutMisnamedCollection(form, destProj, subsCollection, destCollection) { - const misnamed = `${destProj.name}_${subsCollection}`; + const misnamed = legacySubmissionCollectionName(destProj.name, subsCollection); if (this.dryRun || misnamed === destCollection) { return; } @@ -1704,13 +1670,13 @@ class Cloner { const values = []; encryption.eachEncryptedValue(encryptedComponents, src.data, (value, path) => values.push({value, path})); values.forEach(({value, path}) => { - const decrypted = this.decrypt(srcSecret, value); + const decrypted = encryption.decrypt(srcSecret, value); // A value that decrypts to 0, false or an empty string is still a value: only a decrypt // that actually failed comes back as null. const reEncrypted = (decrypted === null || decrypted === undefined) ? null - : this.encrypt(destSecret, decrypted); + : encryption.encrypt(destSecret, decrypted); if (!reEncrypted) { // Leave the value as it is rather than destroying it: it can still be read with the // secret it was encrypted with. @@ -2097,7 +2063,7 @@ class Cloner { if (!decryptedSrcSettings) { return !!dest; } - const reEncrypted = this.encrypt(this.options.dstDbSecret, decryptedSrcSettings, true); + const reEncrypted = encryption.encrypt(this.options.dstDbSecret, decryptedSrcSettings, true); if (reEncrypted) { update['settings_encrypted'] = reEncrypted; } @@ -2134,7 +2100,7 @@ class Cloner { // A destination project without settings has no secret of its own yet. const decryptedDstSettings = (destProject && destProject['settings_encrypted']) - ? this.decrypt(this.options.dstDbSecret, destProject['settings_encrypted'], true) + ? encryption.decrypt(this.options.dstDbSecret, destProject['settings_encrypted'], true) : {}; if (!decryptedDstSettings) { this.reportUndecryptable('destination', destProject, '--dst-db-secret'); @@ -2143,7 +2109,7 @@ class Cloner { if (decryptedDstSettings.secret) { this.dstSecret = decryptedDstSettings.secret; } - const decryptedSrcSettings = this.decrypt(this.options.srcDbSecret, srcProject['settings_encrypted'], true); + const decryptedSrcSettings = encryption.decrypt(this.options.srcDbSecret, srcProject['settings_encrypted'], true); if (!decryptedSrcSettings) { this.reportUndecryptable('source', srcProject, '--src-db-secret'); return null; diff --git a/src/Rekeyer.js b/src/Rekeyer.js index 550eb54..b9d515a 100644 --- a/src/Rekeyer.js +++ b/src/Rekeyer.js @@ -2,7 +2,13 @@ const {MongoClient, ObjectId} = require('mongodb'); const _ = require('lodash'); const Progress = require('./Progress'); -const {redact, tlsOptions, submissionCollectionName} = require('./db'); +const { + redact, + tlsOptions, + submissionCollectionName, + legacySubmissionCollectionName, + idList +} = require('./db'); const {encrypt, decrypt, eachEncryptedValue, hasEncryptedComponents} = require('./encryption'); const SETTINGS = 'settings_encrypted'; @@ -13,14 +19,6 @@ const COUNT_TIMEOUT = 10000; // How many of the values that could not be decrypted are listed one by one, per collection. const MAX_LISTED = 20; -/** - * Split a comma separated list of ids, ignoring the spaces around them. - * @param {string} value - The comma separated list. - */ -function idList(value) { - return String(value || '').split(',').map((id) => id.trim()).filter((id) => id); -} - /** * Re-encrypt the project settings and the encrypted submission data of a database with a new * DB_SECRET. @@ -246,9 +244,14 @@ class Rekeyer { throw new Error(`Only ${found} of the ${this.projectIds.length} projects given were found.`); } } - const cursor = this.db.collection('projects').find(query).sort({created: 1, _id: 1}); - for await (const project of cursor) { - await this.rekeyProject(project); + // Only the ids are listed up front: a cursor left idle while the submissions of a project are + // re-keyed, which can take hours, is closed by the server before the next project is read. + const ids = await this.idsOf(this.db.collection('projects'), query); + for (const _id of ids) { + const project = await this.db.collection('projects').findOne({_id}); + if (project) { + await this.rekeyProject(project); + } } } @@ -271,14 +274,26 @@ class Rekeyer { const candidates = _.uniq([...this.oldSecrets, this.newSecret, settings.secret]) .filter((secret) => secret && secret !== target); - const forms = this.db.collection('forms').find({project: project._id}).sort({created: 1, _id: 1}); - for await (const form of forms) { - if (hasEncryptedComponents(form.components)) { + const formIds = await this.idsOf(this.db.collection('forms'), {project: project._id}); + for (const _id of formIds) { + const form = await this.db.collection('forms').findOne({_id}); + if (form && hasEncryptedComponents(form.components)) { await this.rekeyForm(project, form, target, candidates); } } } + /** + * The ids of the documents a query matches, in the order they were created. + * @param {Collection} collection - The collection. + * @param {object} query - The query. + * @returns {ObjectId[]} - The ids. + */ + async idsOf(collection, query) { + const docs = await collection.find(query, {projection: {_id: 1}}).sort({created: 1, _id: 1}).toArray(); + return docs.map((doc) => doc._id); + } + /** * Re-key the settings of a project. * @param {object} project - The project. @@ -342,7 +357,7 @@ class Rekeyer { // name, where the server never looks for them. Those submissions are re-keyed too, but they // still have to be moved to where the server reads them from. const custom = _.get(form, 'settings.collection'); - const unsanitized = custom ? `${project.name}_${custom}` : null; + const unsanitized = custom ? legacySubmissionCollectionName(project.name, custom) : null; const misplaced = unsanitized && unsanitized !== collections[0] && await this.count(this.db.collection(unsanitized), {form: form._id}); if (misplaced) { @@ -356,10 +371,27 @@ class Rekeyer { const query = {form: form._id}; for (const collection of collections) { await this.rekeyCollection(collection, query, form.components, target, candidates, ' - Submissions:'); - if (this.includeRevisions) { - const revisions = (collection === 'submissions') ? 'submissionrevisions' : `${collection}_revisions`; - await this.rekeyCollection(revisions, query, form.components, target, candidates, ' - Revisions:'); - } + } + if (!this.includeRevisions) { + return; + } + const revisions = collections.map((collection) => { + return (collection === 'submissions') ? 'submissionrevisions' : `${collection}_revisions`; + }); + + // Earlier versions of the clone command also wrote the revisions of a custom collection to the + // default revisions collection. The server never reads them there, but they are re-keyed all the + // same, so that nothing is left on the old secret. + if (custom && !revisions.includes('submissionrevisions') && + await this.db.collection('submissionrevisions').findOne(query, {projection: {_id: 1}})) { + this.error( + ` Revisions of this form are in submissionrevisions, but the server reads them from ${revisions[0]}. ` + + 'They are re-keyed where they are.' + ); + revisions.push('submissionrevisions'); + } + for (const collection of revisions) { + await this.rekeyCollection(collection, query, form.components, target, candidates, ' - Revisions:'); } } diff --git a/src/db.js b/src/db.js index 3cccb57..8dbe46d 100644 --- a/src/db.js +++ b/src/db.js @@ -1,4 +1,5 @@ 'use strict'; +const path = require('path'); /** * Hide the credentials of a connection string, so that they never end up in a log. @@ -13,8 +14,8 @@ function redact(uri) { * certificate. The driver always uses the new url parser and the unified topology: from version 7 on * it rejects the options that used to turn them on. * - * @param {string} caFile - The certificate authority file, relative to the working directory. - * @param {string} certFile - The certificate key file, relative to the working directory. + * @param {string} caFile - The certificate authority file, absolute or relative to the working directory. + * @param {string} certFile - The certificate key file, absolute or relative to the working directory. * @returns {object} - The client options. */ function tlsOptions(caFile, certFile) { @@ -22,7 +23,7 @@ function tlsOptions(caFile, certFile) { if (caFile || certFile) { config.tls = true; if (caFile) { - config.tlsCAFile = `${process.cwd()}/${caFile}`; + config.tlsCAFile = path.resolve(caFile); // A custom certificate authority is usually an internal one whose certificates do not name // the host being connected to. A client certificate on its own is no reason to stop @@ -30,7 +31,7 @@ function tlsOptions(caFile, certFile) { config.tlsAllowInvalidHostnames = true; } if (certFile) { - config.tlsCertificateKeyFile = `${process.cwd()}/${certFile}`; + config.tlsCertificateKeyFile = path.resolve(certFile); } } return config; @@ -52,4 +53,25 @@ function submissionCollectionName(projectName, collection) { return `${project}_${String(collection).replace(/[^A-Za-z0-9]+/g, '')}`; } -module.exports = {redact, tlsOptions, submissionCollectionName}; +/** + * The name earlier versions of the clone command gave the collection of a form with its own + * collection: the unsanitized project and collection names, where the server never reads them. + * + * @param {string} projectName - The name of the project. + * @param {string} collection - The collection setting of the form. + * @returns {string} - The collection name. + */ +function legacySubmissionCollectionName(projectName, collection) { + return `${projectName}_${collection}`; +} + +/** + * Split a comma separated list of ids, ignoring the spaces around them and any repeated id. + * @param {string} value - The comma separated list. + * @returns {string[]} - The ids. + */ +function idList(value) { + return [...new Set(String(value || '').split(',').map((id) => id.trim()).filter((id) => id))]; +} + +module.exports = {redact, tlsOptions, submissionCollectionName, legacySubmissionCollectionName, idList}; diff --git a/src/encryption.js b/src/encryption.js index 95dd737..444e3c7 100644 --- a/src/encryption.js +++ b/src/encryption.js @@ -39,6 +39,15 @@ function legacyKey(secret) { }; } +/** + * Whether a value is a Buffer that went through JSON, which is how an API hands back the encrypted + * values it reads raw from the database. + * @param {*} value - The value. + */ +function isSerializedBuffer(value) { + return !!(value && value.type === 'Buffer' && Array.isArray(value.data)); +} + /** * Encrypt data or settings using a secret, the way the server does. * @param {string} secret - The secret. @@ -63,7 +72,8 @@ function encrypt(secret, rawData, nosalt, saltLength = SALT_LENGTH) { /** * Decrypt data or settings using a secret. * @param {string} secret - The secret. - * @param {Buffer|Binary} cipherbuffer - The encrypted data, as a Buffer or as a mongo Binary. + * @param {Buffer|Binary|object} cipherbuffer - The encrypted data, as a Buffer, a mongo Binary or a + * Buffer serialized as JSON. * @param {boolean} nosalt - The data was encrypted without a salt, as the project settings are. * @param {number} saltLength - The length of the salt. * @returns {*} - The decrypted data, or null when it cannot be decrypted with this secret. @@ -73,7 +83,9 @@ function decrypt(secret, cipherbuffer, nosalt, saltLength = SALT_LENGTH) { return null; } try { - const buffer = Buffer.isBuffer(cipherbuffer) ? cipherbuffer : cipherbuffer.buffer; + const buffer = Buffer.isBuffer(cipherbuffer) + ? cipherbuffer + : (isSerializedBuffer(cipherbuffer) ? Buffer.from(cipherbuffer.data) : cipherbuffer.buffer); const {key, iv} = legacyKey(secret); const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv); const decryptedJSON = Buffer.concat([ @@ -94,7 +106,7 @@ function decrypt(secret, cipherbuffer, nosalt, saltLength = SALT_LENGTH) { * @param {*} value - The stored value. */ function isEncryptedValue(value) { - return Buffer.isBuffer(value) || !!(value && value._bsontype === 'Binary'); + return Buffer.isBuffer(value) || !!(value && value._bsontype === 'Binary') || isSerializedBuffer(value); } /** diff --git a/test/cloner/operations.js b/test/cloner/operations.js index 4bdcdb4..1f5ea1f 100644 --- a/test/cloner/operations.js +++ b/test/cloner/operations.js @@ -4,7 +4,6 @@ const assert = require('assert'); const fs = require('fs'); const path = require('path'); const {ObjectId, Collection} = require('mongodb'); -const Cloner = require('../../src/Cloner'); const {seedProject, at, project, role, form, formRevision, submissionRevision} = require('./fixtures'); const { useTestDatabases, @@ -17,12 +16,10 @@ const { } = require('./helper'); /** - * Build a bare cloner instance that can encrypt/decrypt without connecting to anything. + * The helpers that encrypt and decrypt the way the server does. */ function crypter() { - const instance = Object.create(Cloner.prototype); - instance.defaultSaltLength = 40; - return instance; + return require('../../src/encryption'); } /** diff --git a/test/cloner/projects.js b/test/cloner/projects.js index fe712fd..19d4f91 100644 --- a/test/cloner/projects.js +++ b/test/cloner/projects.js @@ -3,17 +3,14 @@ const assert = require('assert'); const _ = require('lodash'); const {ObjectId} = require('mongodb'); -const Cloner = require('../../src/Cloner'); const {seedProject, at, project, role, form, submission, submissionRevision} = require('./fixtures'); const {useTestDatabases, runClone, quiet} = require('./helper'); /** - * Build a bare cloner instance that can encrypt/decrypt without connecting to anything. + * The helpers that encrypt and decrypt the way the server does. */ function crypter() { - const instance = Object.create(Cloner.prototype); - instance.defaultSaltLength = 40; - return instance; + return require('../../src/encryption'); } describe('Project targeting', function() { diff --git a/test/cloner/rekey.js b/test/cloner/rekey.js index 6d2046c..e4c3b51 100644 --- a/test/cloner/rekey.js +++ b/test/cloner/rekey.js @@ -4,6 +4,7 @@ const assert = require('assert'); const {ObjectId} = require('mongodb'); const Rekeyer = require('../../src/Rekeyer'); const {encrypt, decrypt, eachEncryptedValue, hasEncryptedComponents} = require('../../src/encryption'); +const {tlsOptions} = require('../../src/db'); const {DST_URI, TestDb, project, form, submission, submissionRevision} = require('./fixtures'); const {quiet} = require('./helper'); @@ -120,6 +121,14 @@ describe('Rekey', function() { assert.deepEqual(found, []); }); + it('Should find and read values that went through JSON, the way an API returns them', function() { + const serialized = JSON.parse(JSON.stringify({ssn: encrypt('x', '123-45-6789')})); + const found = []; + eachEncryptedValue([{type: 'textfield', key: 'ssn', encrypted: true}], serialized, (value) => found.push(value)); + assert.equal(found.length, 1); + assert.equal(decrypt('x', found[0]), '123-45-6789'); + }); + it('Should tell whether a form has encrypted components', function() { assert.equal(hasEncryptedComponents(COMPONENTS), true); assert.equal(hasEncryptedComponents([{type: 'panel', components: [{type: 'textfield', key: 'a'}]}]), false); @@ -166,6 +175,17 @@ describe('Rekey', function() { it('Should accept a single old secret as a string', function() { assert.deepEqual(new Rekeyer(DST_URI, {newDbSecret: 'a', oldDbSecret: 'b'}).oldSecrets, ['b']); }); + + it('Should ignore a project id that is given twice', function() { + const id = new ObjectId().toString(); + assert.deepEqual(new Rekeyer(DST_URI, {project: `${id}, ${id}`}).projectIds, [id]); + }); + + it('Should accept absolute and relative certificate paths', function() { + assert.equal(tlsOptions('/etc/ssl/ca.pem').tlsCAFile, '/etc/ssl/ca.pem'); + assert.equal(tlsOptions(null, '/etc/ssl/cert.pem').tlsCertificateKeyFile, '/etc/ssl/cert.pem'); + assert.equal(tlsOptions('ca.pem').tlsCAFile, require('path').join(process.cwd(), 'ca.pem')); + }); }); describe('a database', function() { @@ -340,6 +360,22 @@ describe('Rekey', function() { assert.match(String(error && error.message), /had settings none of the secrets could decrypt/); }); + it('Should re-key the revisions of a custom collection left in the default collection', async function() { + // Earlier versions of the clone wrote them there, instead of to the revisions of the collection. + const stray = await db.insert('submissionrevisions', submissionRevision({ + _rid: new ObjectId(), + form: seed.customForm._id, + project: seed.plain._id, + data: encryptedData(WRONG_SECRET) + })); + const {warnings} = await runRekey({}); + + const revision = await db.findOne('submissionrevisions', {_id: stray._id}); + assert.deepEqual(decryptedData(NEW_SECRET, revision.data), PLAIN); + assert.ok(warnings.some((line) => /Revisions of this form are in submissionrevisions/.test(line)), + 'The operator should be told the revisions are where the server does not read them.'); + }); + it('Should leave the submission revisions alone when asked to', async function() { await runRekey({submissionRevisions: false}); const revision = await db.findOne('submissionrevisions', {_rid: seed.plainSubmission._id}); diff --git a/test/cloner/units.js b/test/cloner/units.js index 910a7f1..d655420 100644 --- a/test/cloner/units.js +++ b/test/cloner/units.js @@ -3,6 +3,7 @@ const assert = require('assert'); const {ObjectId} = require('mongodb'); const Cloner = require('../../src/Cloner'); +const {encrypt, decrypt} = require('../../src/encryption'); const {quiet} = require('./helper'); /** @@ -12,7 +13,6 @@ const {quiet} = require('./helper'); function bare(options = {}) { const instance = Object.create(Cloner.prototype); instance.options = options; - instance.defaultSaltLength = 40; instance.exclude = []; instance.dryRun = !!options.dryRun; instance.dryRunBuffer = []; @@ -31,7 +31,7 @@ describe('Cloner units', function() { it('Should produce the legacy OpenSSL compatible ciphertext', function() { // Generated with: printf '{"a":1}' | openssl enc -aes-256-cbc -nosalt -md md5 -pass pass:mysecret -base64 // which is the exact format the removed crypto.createCipher() produced. - const encrypted = bare().encrypt('mysecret', {a: 1}, true); + const encrypted = encrypt('mysecret', {a: 1}, true); assert.equal(encrypted.toString('base64'), 'aqN/mkNcqrTMql3ObHRt6Q=='); }); @@ -39,49 +39,55 @@ describe('Cloner units', function() { // printf '{"a":1}' | openssl enc -aes-256-cbc -nosalt -md md5 -pass pass:sécret -base64 // Reading the secret as latin1 instead derives a different key, and nothing encrypted with a // non ASCII secret could be decrypted again. - const encrypted = bare().encrypt('sécret', {a: 1}, true); + const encrypted = encrypt('sécret', {a: 1}, true); assert.equal(encrypted.toString('base64'), 's332i21yTmVluklx2eqRaA=='); - assert.deepEqual(bare().decrypt('sécret', encrypted, true), {a: 1}); + assert.deepEqual(decrypt('sécret', encrypted, true), {a: 1}); }); it('Should decrypt legacy ciphertext', function() { const buffer = Buffer.from('aqN/mkNcqrTMql3ObHRt6Q==', 'base64'); - assert.deepEqual(bare().decrypt('mysecret', buffer, true), {a: 1}); + assert.deepEqual(decrypt('mysecret', buffer, true), {a: 1}); }); it('Should round trip salted data', function() { - const instance = bare(); - const encrypted = instance.encrypt('a-secret', {ssn: '123-45-6789'}); - assert.deepEqual(instance.decrypt('a-secret', encrypted), {ssn: '123-45-6789'}); + const encrypted = encrypt('a-secret', {ssn: '123-45-6789'}); + assert.deepEqual(decrypt('a-secret', encrypted), {ssn: '123-45-6789'}); }); it('Should round trip a binary buffer wrapper', function() { - const instance = bare(); - const encrypted = instance.encrypt('a-secret', 'plain value'); - assert.equal(instance.decrypt('a-secret', {buffer: encrypted}), 'plain value'); + const encrypted = encrypt('a-secret', 'plain value'); + assert.equal(decrypt('a-secret', {buffer: encrypted}), 'plain value'); }); it('Should encrypt values that are falsy but real', function() { - const instance = bare(); for (const value of [0, false, '']) { - const encrypted = instance.encrypt('a-secret', value); + const encrypted = encrypt('a-secret', value); assert.ok(encrypted, `${JSON.stringify(value)} is a value and must be encrypted.`); - assert.deepEqual(instance.decrypt('a-secret', encrypted), value); + assert.deepEqual(decrypt('a-secret', encrypted), value); } }); it('Should return null when a secret or payload is missing', function() { - const instance = bare(); - assert.equal(instance.encrypt('', {a: 1}), null); - assert.equal(instance.encrypt('secret', null), null); - assert.equal(instance.decrypt('', Buffer.from('x')), null); - assert.equal(instance.decrypt('secret', null), null); + assert.equal(encrypt('', {a: 1}), null); + assert.equal(encrypt('secret', null), null); + assert.equal(decrypt('', Buffer.from('x')), null); + assert.equal(decrypt('secret', null), null); }); it('Should return null when decrypting with the wrong secret', function() { - const instance = bare(); - const encrypted = instance.encrypt('right-secret', {a: 1}, true); - assert.equal(instance.decrypt('wrong-secret', encrypted, true), null); + const encrypted = encrypt('right-secret', {a: 1}, true); + assert.equal(decrypt('wrong-secret', encrypted, true), null); + }); + }); + + describe('migrateDataEncryption', function() { + it('Should re-encrypt values an API returned as JSON', function() { + const instance = bare({srcDbSecret: 'source', dstDbSecret: 'destination'}); + const src = {_id: 'a', data: JSON.parse(JSON.stringify({ssn: encrypt('source', '123-45-6789')}))}; + const update = {data: Object.assign({}, src.data)}; + instance.migrateDataEncryption(src, update, [{type: 'textfield', key: 'ssn', encrypted: true}]); + assert.ok(Buffer.isBuffer(update.data.ssn), 'It should be written back as binary data.'); + assert.equal(decrypt('destination', update.data.ssn), '123-45-6789'); }); });