From 2af62a8dfe9d168eb11bea70f3c0aee054d489de Mon Sep 17 00:00:00 2001 From: Sumesh Pradhan Date: Tue, 18 Apr 2023 06:52:36 +0530 Subject: [PATCH] fix: appsmithctl restore overwrites the MONGODB URI (#22229) **Fixes:** - Overwriting MongoDB env data on appsmithctl restore. - appsmithctl backup does not include the MongoDB env vars. **Features:** - appsmithctl backup now does not stop backend and rts service during backup operation. _Note:- It will be the responsibility of the Admin to make sure that a manual backup is really a desired snapshot when done via the appsmithctl backup util command To ensure this, Admins may require to stop the backend and rts processes in the container before a manual backup( We will need to document this) [ In future we can add an appsmithctl cmd to put appsmith to maintenance mode with a maintenance page ]_ Fixes # [21603](https://github.com/appsmithorg/appsmith/issues/21603) --- deploy/docker/utils/bin/backup.js | 20 +++++++++----------- deploy/docker/utils/bin/backup.test.js | 17 ++++++++++++++--- deploy/docker/utils/bin/constants.js | 3 +++ deploy/docker/utils/bin/restore.js | 9 +++++---- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/deploy/docker/utils/bin/backup.js b/deploy/docker/utils/bin/backup.js index 22bf67d484..7cc8a7f43a 100644 --- a/deploy/docker/utils/bin/backup.js +++ b/deploy/docker/utils/bin/backup.js @@ -21,8 +21,6 @@ async function run() { } }); - utils.stop(['backend', 'rts']); - console.log('Available free space at /appsmith-stacks'); const availSpaceInBytes = getAvailableBackupSpaceInBytes(); console.log('\n'); @@ -46,7 +44,6 @@ async function run() { await fsPromises.rm(backupRootPath, { recursive: true, force: true }); logger.backup_info('Finished taking a backup at' + archivePath); - await postBackupCleanup(); } catch (err) { errorCode = 1; @@ -61,7 +58,7 @@ async function run() { } } } finally { - utils.start(['backend', 'rts']); + await postBackupCleanup(); process.exit(errorCode); } } @@ -91,7 +88,7 @@ async function createManifestFile(path) { async function exportDockerEnvFile(destFolder) { console.log('Exporting docker environment file'); const content = await fsPromises.readFile('/appsmith-stacks/configuration/docker.env', { encoding: 'utf8' }); - const cleaned_content = removeEncryptionEnvData(content) + const cleaned_content = removeSensitiveEnvData(content) await fsPromises.writeFile(destFolder + '/docker.env', cleaned_content); console.log('Exporting docker environment file done.'); console.log('!!!!!!!!!!!!!!!!!!!!!!!!!! Important !!!!!!!!!!!!!!!!!!!!!!!!!!'); @@ -145,11 +142,12 @@ function getBackupContentsPath(backupRootPath, timestamp) { return backupRootPath + '/appsmith-backup-' + timestamp; } -function removeEncryptionEnvData(content) { +function removeSensitiveEnvData(content) { + // Remove encryption and Mongodb data from docker.env const output_lines = [] content.split(/\r?\n/).forEach(line => { - if (!line.startsWith("APPSMITH_ENCRYPTION")) { - output_lines.push(line) + if (!line.startsWith("APPSMITH_ENCRYPTION") && !line.startsWith("APPSMITH_MONGODB")) { + output_lines.push(line); } }); return output_lines.join('\n') @@ -157,7 +155,7 @@ function removeEncryptionEnvData(content) { function getBackupArchiveLimit(backupArchivesLimit) { if (!backupArchivesLimit) - backupArchivesLimit = 4; + backupArchivesLimit = Constants.APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT; return backupArchivesLimit } @@ -179,7 +177,7 @@ function getAvailableBackupSpaceInBytes() { function checkAvailableBackupSpace(availSpaceInBytes) { if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) { - throw new Error('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.'); + throw new Error('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 2GB to backup successfully.'); } } @@ -195,7 +193,7 @@ module.exports = { executeMongoDumpCMD, getGitRoot, executeCopyCMD, - removeEncryptionEnvData, + removeSensitiveEnvData, getBackupArchiveLimit, removeOldBackups }; diff --git a/deploy/docker/utils/bin/backup.test.js b/deploy/docker/utils/bin/backup.test.js index 6eb8278f9c..de6ebbb553 100644 --- a/deploy/docker/utils/bin/backup.test.js +++ b/deploy/docker/utils/bin/backup.test.js @@ -24,7 +24,7 @@ it('Checkx the constant is 2 GB', () => { }); it('Should throw Error when the available size is below MIN_REQUIRED_DISK_SPACE_IN_BYTES', () => { let size = Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES - 1; - expect(() => {backup.checkAvailableBackupSpace(size)}).toThrow('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.'); + expect(() => {backup.checkAvailableBackupSpace(size)}).toThrow('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 2GB to backup successfully.'); }); it('Should not hould throw Error when the available size is >= MIN_REQUIRED_DISK_SPACE_IN_BYTES', () => { @@ -88,11 +88,22 @@ it('Checks for the current Appsmith Version.', async () => { console.log(res) }) -test('If encriytpion env values are being removed', () => { - expect(backup.removeEncryptionEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_INSTANCE_NAME=Appsmith\n +test('If Encryption env values are being removed', () => { + expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_INSTANCE_NAME=Appsmith\n `)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`) }); +test('If MONGODB env values are being removed', () => { + expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_MONGODB_URI=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n + `)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`) +}); + +test('If MONGODB and Encryption env values are being removed', () => { + expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_MONGODB_URI=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n + `)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`) +}); + + test('Backup Archive Limit when env APPSMITH_BACKUP_ARCHIVE_LIMIT is null', () => { expect(backup.getBackupArchiveLimit()).toBe(4) }); diff --git a/deploy/docker/utils/bin/constants.js b/deploy/docker/utils/bin/constants.js index 08f2d5c78a..bbce704de5 100644 --- a/deploy/docker/utils/bin/constants.js +++ b/deploy/docker/utils/bin/constants.js @@ -13,6 +13,8 @@ const MIN_REQUIRED_DISK_SPACE_IN_BYTES = 2147483648 // 2GB const DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC = 21600000 // 6 hrs +const APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT = 4 // 4 backup archives + module.exports = { BACKUP_PATH, RESTORE_PATH, @@ -21,4 +23,5 @@ module.exports = { APPSMITHCTL_LOG_PATH, MIN_REQUIRED_DISK_SPACE_IN_BYTES, DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC, + APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT } diff --git a/deploy/docker/utils/bin/restore.js b/deploy/docker/utils/bin/restore.js index 93e43cfbd4..9fb080f98e 100644 --- a/deploy/docker/utils/bin/restore.js +++ b/deploy/docker/utils/bin/restore.js @@ -58,9 +58,9 @@ async function restoreDockerEnvFile(restoreContentsPath, backupName) { await utils.execCommand(['cp', restoreContentsPath + '/docker.env', dockerEnvFile]); if (encryptionPwd && encryptionSalt) { - const input = readlineSync.question('Existing encryption env values of the previous instance were found.\n\ - Press Enter to continue with existing encryption values\n\ - Or Type "n"/"No" to provide encryption key & password for the new restore instance.\n'); + const input = readlineSync.question('If you are restoring to the same Appsmith deployment which generated the backup archive, you can use the existing encryption keys on the instance.\n\ + Press Enter to continue with existing encryption keys\n\ + Or Type "n"/"No" to provide encryption key & password corresponding to the original Appsmith instance that is being restored.\n'); const answer = input && input.toLocaleUpperCase(); if (answer === 'N' || answer === 'NO') { encryptionPwd = readlineSync.question('Enter the APPSMITH_ENCRYPTION_PASSWORD: ', { @@ -84,7 +84,8 @@ async function restoreDockerEnvFile(restoreContentsPath, backupName) { } await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_ENCRYPTION_PASSWORD=' + encryptionPwd + - '\nAPPSMITH_ENCRYPTION_SALT=' + encryptionSalt); + '\nAPPSMITH_ENCRYPTION_SALT=' + encryptionSalt + '\nAPPSMITH_MONGODB_URI=' + process.env.APPSMITH_MONGODB_URI + + '\nAPPSMITH_MONGODB_USER=' + process.env.APPSMITH_MONGODB_USER + '\nAPPSMITH_MONGODB_PASSWORD=' + process.env.APPSMITH_MONGODB_PASSWORD ) ; console.log('Restoring docker environment file completed'); }