Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: Support encrypted backups and fix restoring to renamed databases (#29902)

Browse Source 
Fixes: [31004](https://github.com/appsmithorg/appsmith/issues/31004)


Co-authored-by: Shrikant Sharat Kandula <shrikant@appsmith.com>
This commit is contained in:
Goutham Pratapa 2024-03-27 18:09:02 +05:30 committed by GitHub
parent 92ff6a9c7d
commit f85d64d775
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 289 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
deploy/docker/fs/opt/appsmith/utils/bin/backup.js
View File

@ -6,12 +6,16 @@ const utils = require('./utils');
const Constants = require('./constants');
const logger = require('./logger');
const mailer = require('./mailer');
const tty = require('tty');
const readlineSync = require('readline-sync');
const command_args = process.argv.slice(3);
async function run() {
const timestamp = getTimeStampInISO();
let errorCode = 0;
let backupRootPath, archivePath, encryptionPassword;
let encryptArchive = false;
try {
const check_supervisord_status_cmd = '/usr/bin/supervisorctl >/dev/null 2>&1';
shell.exec(check_supervisord_status_cmd, function (code) {
@ -37,13 +41,36 @@ async function run() {
await createGitStorageArchive(backupContentsPath);
await createManifestFile(backupContentsPath);
await exportDockerEnvFile(backupContentsPath);
if (!command_args.includes('--non-interactive') && (tty.isatty(process.stdout.fd))){
encryptionPassword = getEncryptionPasswordFromUser();
if (encryptionPassword == -1){
throw new Error('Backup process aborted because a valid enctyption password could not be obtained from the user');
}
encryptArchive = true;
}
await exportDockerEnvFile(backupContentsPath, encryptArchive);
const archivePath = await createFinalArchive(backupRootPath, timestamp);
// shell.exec("openssl enc -aes-256-cbc -pbkdf2 -iter 100000 -in " + archivePath + " -out " + archivePath + ".enc");
if (encryptArchive){
const encryptedArchivePath = await encryptBackupArchive(archivePath,encryptionPassword);
logger.backup_info('Finished creating an encrypted a backup archive at ' + encryptedArchivePath);
if (archivePath != null) {
await fsPromises.rm(archivePath, { recursive: true, force: true });
}
}
else {
logger.backup_info('Finished creating a backup archive at ' + archivePath);
console.log('********************************************************* IMPORTANT!!! *************************************************************');
console.log('*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file **')
console.log('*** These values are not included in the backup export. **');
console.log('************************************************************************************************************************************');
}
await fsPromises.rm(backupRootPath, { recursive: true, force: true });
logger.backup_info('Finished taking a backup at' + archivePath);
logger.backup_info('Finished taking a backup at ' + archivePath);
} catch (err) {
errorCode = 1;
@ -58,11 +85,44 @@ async function run() {
}
}
} finally {
if (backupRootPath != null) {
await fsPromises.rm(backupRootPath, { recursive: true, force: true });
}
if (encryptArchive) {
if (archivePath != null) {
await fsPromises.rm(archivePath, { recursive: true, force: true });
}
}
await postBackupCleanup();
process.exit(errorCode);
}
}
async function encryptBackupArchive(archivePath, encryptionPassword){
const encryptedArchivePath = archivePath + '.enc';
await utils.execCommand(['openssl', 'enc', '-aes-256-cbc', '-pbkdf2', '-iter', 100000, '-in', archivePath, '-out', encryptedArchivePath, '-k', encryptionPassword ])
return encryptedArchivePath;
}
function getEncryptionPasswordFromUser(){
for (const _ of [1, 2, 3])
{
const encryptionPwd1 = readlineSync.question('Enter a password to encrypt the backup archive: ', { hideEchoBack: true });
const encryptionPwd2 = readlineSync.question('Enter the above password again: ', { hideEchoBack: true });
if (encryptionPwd1 === encryptionPwd2){
if (encryptionPwd1){
return encryptionPwd1;
}
console.error("Invalid input. Empty password is not allowed, please try again.")
}
else {
console.error("The passwords do not match, please try again.");
}
}
console.error("Aborting backup process, failed to obtain valid encryption password.");
return -1
}
async function exportDatabase(destFolder) {
console.log('Exporting database');
await executeMongoDumpCMD(destFolder, process.env.APPSMITH_MONGODB_URI)
@ -81,19 +141,20 @@ async function createGitStorageArchive(destFolder) {
async function createManifestFile(path) {
const version = await utils.getCurrentAppsmithVersion()
const manifest_data = { "appsmithVersion": version }
const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromMongoURI(process.env.APPSMITH_MONGODB_URI) }
await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data));
}
async function exportDockerEnvFile(destFolder) {
async function exportDockerEnvFile(destFolder, encryptArchive) {
console.log('Exporting docker environment file');
const content = await fsPromises.readFile('/appsmith-stacks/configuration/docker.env', { encoding: 'utf8' });
const cleaned_content = removeSensitiveEnvData(content)
let cleaned_content = removeSensitiveEnvData(content);
if (encryptArchive){
cleaned_content += '\nAPPSMITH_ENCRYPTION_SALT=' + process.env.APPSMITH_ENCRYPTION_SALT +
'\nAPPSMITH_ENCRYPTION_PASSWORD=' + process.env.APPSMITH_ENCRYPTION_PASSWORD
}
await fsPromises.writeFile(destFolder + '/docker.env', cleaned_content);
console.log('Exporting docker environment file done.');
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!! Important !!!!!!!!!!!!!!!!!!!!!!!!!!');
console.log('!!! Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file because those values are not included in the backup export.');
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
}
async function executeMongoDumpCMD(destFolder, appsmithMongoURI) {
@ -195,5 +256,7 @@ module.exports = {
executeCopyCMD,
removeSensitiveEnvData,
getBackupArchiveLimit,
removeOldBackups
removeOldBackups,
getEncryptionPasswordFromUser,
encryptBackupArchive,
};

82
deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js
View File

@ -4,6 +4,7 @@ const os = require('os');
const fsPromises = require('fs/promises');
const utils = require('./utils');
const shell = require('shelljs');
const readlineSync = require('readline-sync');
describe('Backup Tests', () => {
@ -78,16 +79,18 @@ test('Test ln command generation', async () => {
})
it('Checks for the current Appsmith Version.', async () => {
fsPromises.readFile = jest.fn().mockImplementation(async (a) =>
`Object.defineProperty(exports, "__esModule", { value: true });
exports.VERSION = void 0;
exports.VERSION = "v0.0.0-SNAPSHOT";`);
const res = await utils.getCurrentAppsmithVersion()
expect(res).toBe("v0.0.0-SNAPSHOT")
console.log(res)
fsPromises.readFile = jest.fn().mockImplementation(async () => `{"githubRef":"refs/tags/v1.2.3"}`);
await expect(utils.getCurrentAppsmithVersion()).resolves.toBe("v1.2.3")
})
test('If Encryption env values are being removed', () => {
expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_INSTANCE_NAME=Appsmith\n
`)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`)
});
test('If MONGODB env values are being removed', () => {
test('If MONGODB and Encryption env values are being removed', () => {
expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_MONGODB_URI=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n
`)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`)
});
@ -182,5 +185,72 @@ test('Cleanup Backups when limit is 2 and there is no file', async () => {
console.log(res)
expect(res).toEqual(expectedBackupFiles)
})
test('Test get encryption password from user prompt whene both passords are the same', async () => {
const password = 'password#4321'
readlineSync.question = jest.fn().mockImplementation((a) => {return password});
const password_res = backup.getEncryptionPasswordFromUser()
expect(password_res).toEqual(password)
})
test('Test get encryption password from user prompt when both passords are the different', async () => {
const password = 'password#4321'
readlineSync.question = jest.fn().mockImplementation((a) => {
if (a=='Enter the above password again: '){
return 'pass';
}
return password});
const password_res = backup.getEncryptionPasswordFromUser()
expect(password_res).toEqual(-1)
})
test('Get encrypted archive path', async () => {
const archivePath = '/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z';
const encryptionPassword = 'password#4321'
utils.execCommand = jest.fn().mockImplementation( async (a) => console.log(a));
const encArchivePath = await backup.encryptBackupArchive(archivePath, encryptionPassword)
expect(encArchivePath).toEqual('/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z' + '.enc')
})
test('Test backup encryption function', async () => {
utils.execCommand= jest.fn().mockImplementation(async (a) => console.log(a));
const archivePath = '/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z'
const encryptionPassword = 'password#123'
const res = await backup.encryptBackupArchive(archivePath,encryptionPassword)
console.log(res)
expect(res).toEqual('/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z.enc')
})
});
test('Get DB name from Mongo URI 1', async () => {
var mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/my_db_name?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"
var expectedDBName = 'my_db_name'
const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
expect(dbName).toEqual(expectedDBName)
})
test('Get DB name from Mongo URI 2', async () => {
var mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/test123?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"
var expectedDBName = 'test123'
const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
expect(dbName).toEqual(expectedDBName)
})
test('Get DB name from Mongo URI 3', async () => {
var mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/test123"
var expectedDBName = 'test123'
const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
expect(dbName).toEqual(expectedDBName)
})
test('Get DB name from Mongo URI 4', async () => {
var mongodb_uri = "mongodb://appsmith:pAssW0rd!@localhost:27017/appsmith"
var expectedDBName = 'appsmith'
const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
expect(dbName).toEqual(expectedDBName)
})

139
deploy/docker/fs/opt/appsmith/utils/bin/restore.js
View File

@ -7,6 +7,7 @@ const shell = require('shelljs');
const utils = require('./utils');
const Constants = require('./constants');
const command_args = process.argv.slice(3);
const {getCurrentAppsmithVersion} = require("./utils")
async function getBackupFileName() {
@ -19,7 +20,7 @@ async function getBackupFileName() {
console.log('----------------------------------------------------------------');
console.log('Index\t|\tAppsmith Backup Archive File');
console.log('----------------------------------------------------------------');
for (var i = 0; i < backupFiles.length; i++) {
for (let i = 0; i < backupFiles.length; i++) {
if (i === backupFiles.length - 1)
console.log(i + '\t|\t' + backupFiles[i] + ' <--Most recent backup');
else
@ -27,15 +28,27 @@ async function getBackupFileName() {
}
console.log('----------------------------------------------------------------');
var backupFileIndex = parseInt(readlineSync.question('Please enter the backup file index: '), 10);
const backupFileIndex = parseInt(readlineSync.question('Please enter the backup file index: '), 10);
if (!isNaN(backupFileIndex) && Number.isInteger(backupFileIndex) && (backupFileIndex >= 0) && (backupFileIndex < backupFiles.length)) {
return backupFiles[parseInt(backupFileIndex, 10)];
}
else {
console.log('Invalid input, please try the command again with a valid option');
return;
}
}
async function decryptArchive(encryptedFilePath, backupFilePath){
console.log('Enter the password to decrypt the backup archive:')
for (const _ of [1, 2, 3]) {
const decryptionPwd = readlineSync.question('', { hideEchoBack: true });
try{
await utils.execCommandSilent(['openssl', 'enc', '-d', '-aes-256-cbc', '-pbkdf2', '-iter', 100000, '-in', encryptedFilePath, '-out', backupFilePath, '-k', decryptionPwd])
return true
} catch (error) {
console.log('Invalid password. Please try again:');
}
}
return false
}
async function extractArchive(backupFilePath, restoreRootPath) {
@ -45,25 +58,46 @@ async function extractArchive(backupFilePath, restoreRootPath) {
}
async function restoreDatabase(restoreContentsPath) {
console.log('Restoring database ....');
await utils.execCommand(['mongorestore', `--uri=${process.env.APPSMITH_MONGODB_URI}`, '--drop', `--archive=${restoreContentsPath}/mongodb-data.gz`, '--gzip']);
console.log('Restoring database...');
const cmd = ['mongorestore', `--uri=${process.env.APPSMITH_MONGODB_URI}`, '--drop', `--archive=${restoreContentsPath}/mongodb-data.gz`, '--gzip']
try {
const fromDbName = await getBackupDatabaseName(restoreContentsPath);
const toDbName = utils.getDatabaseNameFromMongoURI(process.env.APPSMITH_MONGODB_URI);
console.log("Restoring database from " + fromDbName + " to " + toDbName)
cmd.push('--nsInclude=*', `--nsFrom=${fromDbName}.*`, `--nsTo=${toDbName}.*`)
} catch (error) {
console.warn('Error reading manifest file. Assuming same database name.', error);
}
await utils.execCommand(cmd);
console.log('Restoring database completed');
}
async function restoreDockerEnvFile(restoreContentsPath, backupName) {
async function restoreDockerEnvFile(restoreContentsPath, backupName, overwriteEncryptionKeys) {
console.log('Restoring docker environment file');
const dockerEnvFile = '/appsmith-stacks/configuration/docker.env';
var encryptionPwd = process.env.APPSMITH_ENCRYPTION_PASSWORD;
var encryptionSalt = process.env.APPSMITH_ENCRYPTION_SALT;
let encryptionPwd = process.env.APPSMITH_ENCRYPTION_PASSWORD;
let encryptionSalt = process.env.APPSMITH_ENCRYPTION_SALT;
await utils.execCommand(['mv', dockerEnvFile, dockerEnvFile + '.' + backupName]);
await utils.execCommand(['cp', restoreContentsPath + '/docker.env', dockerEnvFile]);
if (encryptionPwd && encryptionSalt) {
const input = readlineSync.question('If you are restoring to the same Appsmith deployment which generated the backup archive, you can use the existing encryption keys on the instance.\n\
Press Enter to continue with existing encryption keys\n\
Or Type "n"/"No" to provide encryption key & password corresponding to the original Appsmith instance that is being restored.\n');
const answer = input && input.toLocaleUpperCase();
if (answer === 'N' || answer === 'NO') {
if (overwriteEncryptionKeys){
if (encryptionPwd && encryptionSalt) {
const input = readlineSync.question('If you are restoring to the same Appsmith deployment which generated the backup archive, you can use the existing encryption keys on the instance.\n\
Press Enter to continue with existing encryption keys\n\
Or Type "n"/"No" to provide encryption key & password corresponding to the original Appsmith instance that is being restored.\n');
const answer = input && input.toLocaleUpperCase();
if (answer === 'N' || answer === 'NO') {
encryptionPwd = readlineSync.question('Enter the APPSMITH_ENCRYPTION_PASSWORD: ', {
hideEchoBack: true
});
encryptionSalt = readlineSync.question('Enter the APPSMITH_ENCRYPTION_SALT: ', {
hideEchoBack: true
});
}
else {
console.log('Restoring docker environment file with existing encryption password & salt');
}
}
else {
encryptionPwd = readlineSync.question('Enter the APPSMITH_ENCRYPTION_PASSWORD: ', {
hideEchoBack: true
});
@ -71,24 +105,13 @@ async function restoreDockerEnvFile(restoreContentsPath, backupName) {
hideEchoBack: true
});
}
else {
console.log('Restoring docker environment file with existing encryption password & salt');
}
}
else {
encryptionPwd = readlineSync.question('Enter the APPSMITH_ENCRYPTION_PASSWORD: ', {
hideEchoBack: true
});
encryptionSalt = readlineSync.question('Enter the APPSMITH_ENCRYPTION_SALT: ', {
hideEchoBack: true
});
}
await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_ENCRYPTION_PASSWORD=' + encryptionPwd +
'\nAPPSMITH_ENCRYPTION_SALT=' + encryptionSalt + '\nAPPSMITH_MONGODB_URI=' + process.env.APPSMITH_MONGODB_URI +
await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_ENCRYPTION_PASSWORD=' + encryptionPwd + '\nAPPSMITH_ENCRYPTION_SALT=' + encryptionSalt + '\nAPPSMITH_MONGODB_URI=' + process.env.APPSMITH_MONGODB_URI +
'\nAPPSMITH_MONGODB_USER=' + process.env.APPSMITH_MONGODB_USER + '\nAPPSMITH_MONGODB_PASSWORD=' + process.env.APPSMITH_MONGODB_PASSWORD ) ;
console.log('Restoring docker environment file completed');
} else {
await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_MONGODB_URI=' + process.env.APPSMITH_MONGODB_URI +
'\nAPPSMITH_MONGODB_USER=' + process.env.APPSMITH_MONGODB_USER + '\nAPPSMITH_MONGODB_PASSWORD=' + process.env.APPSMITH_MONGODB_PASSWORD ) ;
}
console.log('Restoring docker environment file completed');
}
async function restoreGitStorageArchive(restoreContentsPath, backupName) {
@ -124,22 +147,58 @@ async function checkRestoreVersionCompatability(restoreContentsPath) {
}
}
async function getBackupDatabaseName(restoreContentsPath) {
let db_name = "appsmith"
if (command_args.includes('--backup-db-name')) {
for (let i = 0; i < command_args.length; i++) {
if (command_args[i].startsWith('--backup-db-name')) {
db_name = command_args[i].split("=")[1];
}
}
}
else {
const manifest_data = await fsPromises.readFile(restoreContentsPath + '/manifest.json', { encoding: 'utf8' });
const manifest_json = JSON.parse(manifest_data);
if ("dbName" in manifest_json){
db_name = manifest_json["dbName"];
}
}
console.log('Backup Database Name: ' + db_name);
return db_name
}
async function run() {
let errorCode = 0;
let cleanupArchive = false;
let overwriteEncryptionKeys = true;
let backupFilePath;
try {
check_supervisord_status_cmd = '/usr/bin/supervisorctl >/dev/null 2>&1';
shell.exec(check_supervisord_status_cmd, function (code) {
shell.exec('/usr/bin/supervisorctl >/dev/null 2>&1', function (code) {
if (code > 0) {
shell.echo('application is not running, starting supervisord');
shell.exec('/usr/bin/supervisord');
}
});
const backupFileName = await getBackupFileName();
let backupFileName = await getBackupFileName();
if (backupFileName == null) {
process.exit(errorCode);
} else {
const backupFilePath = path.join(Constants.BACKUP_PATH, backupFileName);
backupFilePath = path.join(Constants.BACKUP_PATH, backupFileName);
if (isArchiveEncrypted(backupFileName)){
const encryptedBackupFilePath = path.join(Constants.BACKUP_PATH, backupFileName);
backupFileName = backupFileName.replace('.enc', '');
backupFilePath = path.join(Constants.BACKUP_PATH, backupFileName);
cleanupArchive = true;
overwriteEncryptionKeys = false;
const decryptSuccess = await decryptArchive(encryptedBackupFilePath, backupFilePath);
if (!decryptSuccess){
console.log('You have entered the incorrect password multiple times. Aborting the restore process.')
await fsPromises.rm(backupFilePath, { force: true });
process.exit(errorCode)
}
}
const backupName = backupFileName.replace(/\.tar\.gz$/, "");
const restoreRootPath = await fsPromises.mkdtemp(os.tmpdir());
const restoreContentsPath = path.join(restoreRootPath, backupName);
@ -151,7 +210,7 @@ async function run() {
console.log('Restoring Appsmith instance from the backup at ' + backupFilePath);
utils.stop(['backend', 'rts']);
await restoreDatabase(restoreContentsPath);
await restoreDockerEnvFile(restoreContentsPath, backupName);
await restoreDockerEnvFile(restoreContentsPath, backupName, overwriteEncryptionKeys);
await restoreGitStorageArchive(restoreContentsPath, backupName);
console.log('Appsmith instance successfully restored.');
await fsPromises.rm(restoreRootPath, { recursive: true, force: true });
@ -161,12 +220,18 @@ async function run() {
errorCode = 1;
} finally {
if (cleanupArchive){
await fsPromises.rm(backupFilePath, { force: true });
}
utils.start(['backend', 'rts']);
process.exit(errorCode);
}
}
function isArchiveEncrypted(backupFilePath){
return backupFilePath.endsWith('.enc');
}
module.exports = {
run,

39
deploy/docker/fs/opt/appsmith/utils/bin/utils.js
View File

@ -71,7 +71,7 @@ async function listLocalBackupFiles() {
.readdir(Constants.BACKUP_PATH)
.then((filenames) => {
for (let filename of filenames) {
if (filename.match(/^appsmith-backup-.*\.tar\.gz$/)) {
if (filename.match(/^appsmith-backup-.*\.tar\.gz(\.enc)?$/)) {
backupFiles.push(filename);
}
}
@ -129,6 +129,41 @@ function preprocessMongoDBURI(uri /* string */) {
return cs.toString();
}
function execCommandSilent(cmd, options) {
return new Promise((resolve, reject) => {
let isPromiseDone = false;
const p = childProcess.spawn(cmd[0], cmd.slice(1), {
...options,
});
p.on("exit", (code) => {
if (isPromiseDone) {
return;
}
isPromiseDone = true;
if (code === 0) {
resolve();
} else {
reject();
}
});
p.on("error", (err) => {
if (isPromiseDone) {
return;
}
isPromiseDone = true;
console.error("Error running command", err);
reject();
});
});
}
function getDatabaseNameFromMongoURI(uri) {
const uriParts = uri.split("/");
return uriParts[uriParts.length - 1].split("?")[0];
}
module.exports = {
showHelp,
@ -140,4 +175,6 @@ module.exports = {
getLastBackupErrorMailSentInMilliSec,
getCurrentAppsmithVersion,
preprocessMongoDBURI,
execCommandSilent,
getDatabaseNameFromMongoURI,
};

2
deploy/docker/fs/watchtower-hooks/pre-update.sh
View File

@ -1,2 +1,2 @@
#!/usr/bin/env bash
appsmithctl backup --error-mail || exit 1
appsmithctl backup --non-interactive --error-mail || exit 1