diff --git a/deploy/docker/utils/bin/backup.js b/deploy/docker/utils/bin/backup.js new file mode 100644 index 0000000000..a56f629e2f --- /dev/null +++ b/deploy/docker/utils/bin/backup.js @@ -0,0 +1,111 @@ +const fsPromises = require('fs/promises'); +const path = require('path'); +const os = require('os'); + +const shell = require('shelljs'); + +const utils = require('./utils'); +const Constants = require('./constants'); + + +async function run() { + let errorCode = 0; + try { + const check_supervisord_status_cmd = '/usr/bin/supervisorctl >/dev/null 2>&1'; + shell.exec(check_supervisord_status_cmd, function (code) { + if (code > 0) { + shell.echo('application is not running, starting supervisord'); + shell.exec('/usr/bin/supervisord'); + } + }); + + utils.stop(['backend', 'rts']); + + const timestamp = new Date().toISOString().replace(/:/g, '-') + const backupRootPath = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'appsmithctl-backup-')); + const backupContentsPath = backupRootPath + '/appsmith-backup-' + timestamp; + + await fsPromises.mkdir(backupContentsPath); + + await exportDatabase(backupContentsPath); + + await createGitStorageArchive(backupContentsPath); + + await createManifestFile(backupContentsPath); + await exportDockerEnvFile(backupContentsPath); + + const archivePath = await createFinalArchive(backupRootPath, timestamp); + + await fsPromises.rm(backupRootPath, {recursive: true, force: true}); + + console.log('Finished taking a baceup at', archivePath); + // console.log('Please remember to also take the `docker.env` separately since it includes sensitive, but critical information.') + + } catch (err) { + console.log(err); + errorCode = 1; + + } finally { + utils.start(['backend', 'rts']); + process.exit(errorCode); + + } +} + +async function exportDatabase(destFolder) { + console.log('Exporting database'); + await utils.execCommand(['mongodump', `--uri=${process.env.APPSMITH_MONGODB_URI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']); + console.log('Exporting database done.'); +} + +async function createGitStorageArchive(destFolder) { + console.log('Creating git-storage archive'); + + let gitRoot = process.env.APPSMITH_GIT_ROOT; + if (gitRoot == null || gitRoot === '') { + gitRoot = '/appsmith-stacks/git-storage'; + } + + await utils.execCommand(['ln', '-s', gitRoot, destFolder + '/git-storage']) + + console.log('Created git-storage archive'); +} + +async function createManifestFile(path) { + const content = await fsPromises.readFile('/opt/appsmith/rts/version.js', {encoding: 'utf8'}); + const version = content.match(/\bexports\.VERSION\s*=\s*["']([^"]+)["']/)[1]; + const manifest_data = {"appsmithVersion": version} + await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data)); +} + +async function exportDockerEnvFile(destFolder) { + console.log('Exporting docker environment file'); + const content = await fsPromises.readFile('/appsmith-stacks/configuration/docker.env', {encoding: 'utf8'}); + const output_lines = [] + content.split(/\r?\n/).forEach(line => { + if (!line.startsWith("APPSMITH_ENCRYPTION")) { + output_lines.push(line) + } + }); + await fsPromises.writeFile(destFolder + '/docker.env', output_lines.join('\n')); + console.log('Exporting docker environment file done.'); + + console.log('!!!!!!!!!!!!!!!!!!!!!!!!!! Important !!!!!!!!!!!!!!!!!!!!!!!!!!'); + console.log('!!! Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file because those values are not included in the backup export.'); + console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'); +} + +async function createFinalArchive(destFolder, timestamp) { + console.log('Creating final archive'); + + const archive = `${Constants.BACKUP_PATH}/appsmith-backup-${timestamp}.tar.gz`; + await utils.execCommand(['tar', '-cah', '-C', destFolder, '-f', archive, '.']); + + console.log('Created final archive'); + + return archive; +} + +module.exports = { + run, +}; diff --git a/deploy/docker/utils/bin/export_db.js b/deploy/docker/utils/bin/export_db.js index f0836a004e..d4554a6881 100644 --- a/deploy/docker/utils/bin/export_db.js +++ b/deploy/docker/utils/bin/export_db.js @@ -23,7 +23,7 @@ function start_application() { } // Main application workflow -function main() { +function run() { let errorCode = 0; try { check_supervisord_status_cmd = '/usr/bin/supervisorctl >/dev/null 2>&1 '; @@ -57,7 +57,7 @@ function main() { } module.exports = { - runExportDatabase: main, + run, exportDatabase: export_database, stopApplication: stop_application, startApplication: start_application, diff --git a/deploy/docker/utils/bin/index.js b/deploy/docker/utils/bin/index.js index 2de9a73465..5655cdbef7 100755 --- a/deploy/docker/utils/bin/index.js +++ b/deploy/docker/utils/bin/index.js @@ -1,4 +1,4 @@ -#! /usr/bin/env node +#!/usr/bin/env node const process = require('process'); const utils = require('./utils'); @@ -12,12 +12,12 @@ const APPLICATION_CONFIG_PATH = '/appsmith-stacks/configuration/docker.env'; // Loading latest application configuration require('dotenv').config({ path: APPLICATION_CONFIG_PATH }); -const command = process.argv[2] -console.log("command", command) +const command = process.argv[2]; +console.log('command', command); if (['export-db', 'export_db', 'ex'].includes(command)) { console.log('Exporting database'); - export_db.runExportDatabase(); + export_db.run(); console.log('Export database done'); return; } @@ -43,4 +43,9 @@ if (['check-replica-set', 'check_replica_set', 'crs'].includes(command)) { return; } +if (['backup', 'restore'].includes(command)) { + require(`./${command}.js`).run(process.argv.slice(3)); + return; +} + utils.showHelp(); diff --git a/deploy/docker/utils/bin/restore.js b/deploy/docker/utils/bin/restore.js new file mode 100644 index 0000000000..035c71fcec --- /dev/null +++ b/deploy/docker/utils/bin/restore.js @@ -0,0 +1,173 @@ +const fsPromises = require('fs/promises'); +const path = require('path'); +const os = require('os'); +const readlineSync = require('readline-sync'); + +const shell = require('shelljs'); + +const utils = require('./utils'); +const Constants = require('./constants'); + +async function getBackupFileName(){ + const backupFiles = []; + await fsPromises.readdir(Constants.BACKUP_PATH).then(filenames => { + for (let filename of filenames) { + if (filename.match(/^appsmith-backup-.*\.tar\.gz$/)) { + backupFiles.push(filename); + }}}).catch(err => { + console.log(err); + }); + console.log("\n" + backupFiles.length + " Appsmith backup file(s) found: "); + if (backupFiles.length == 0){ + return + } + console.log('----------------------------------------------------------------'); + console.log('Index\t|\tAppsmith Backup Archive File'); + console.log('----------------------------------------------------------------'); + for (var i=0; i= 0) && (backupFileIndex < backupFiles.length)){ + return backupFiles[Number(backupFileIndex)]; + } + else { + console.log('Invalid input, please try the command again with a valid option'); + return + } + +} + +async function extractArchive(backupFilePath, restoreRootPath){ + console.log('Extracting the Appsmith backup archive at ' + backupFilePath); + await utils.execCommand(['tar', '-C', restoreRootPath, '-xf', backupFilePath]); + console.log('Extracting the backup archive completed'); +} + +async function restoreDatabase(restoreContentsPath) { + console.log('Restoring database ....'); + await utils.execCommand(['mongorestore', `--uri=${process.env.APPSMITH_MONGODB_URI}`, '--drop', `--archive=${restoreContentsPath}/mongodb-data.gz`, '--gzip']); + console.log('Restoring database completed'); +} + +async function restoreDockerEnvFile(restoreContentsPath, backupName){ + console.log('Restoring docker environment file'); + const dockerEnvFile = '/appsmith-stacks/configuration/docker.env'; + var encryptionPwd= process.env.APPSMITH_ENCRYPTION_PASSWORD; + var encryptionSalt = process.env.APPSMITH_ENCRYPTION_SALT; + await utils.execCommand(['mv', dockerEnvFile, dockerEnvFile + '.' + backupName]); + await utils.execCommand(['cp', restoreContentsPath + '/docker.env', dockerEnvFile]); + console.log("Your current Appsmith instance has the following encryption values:\n" + + "APPSMITH_ENCRYPTION_PASSWORD=" + encryptionPwd + "\n" + + "APPSMITH_ENCRYPTION_SALT" + encryptionSalt); + + if ((encryptionPwd != null) && (encryptionSalt != null)){ + const input = readlineSync.question('Would you like to proceed using the existing encryption values?\n\ + Press Enter to continue with existing encryption values\n\ + Or Type "n"/"No" to provide encryption key & password for the restore instance.\n'); + const answer = input && input.toLocaleUpperCase(); + if (answer === 'N' || answer === 'NO') { + encryptionPwd = readlineSync.question('Enter the APPSMITH_ENCRYPTION_PASSWORD: ', { + hideEchoBack: true + }); + encryptionSalt = readlineSync.question('Enter the APPSMITH_ENCRYPTION_SALT: ', { + hideEchoBack: true + }); + } + else { + console.log('Restoring docker environment file with existing encryption password & salt'); + } + } + else { + encryptionPwd = readlineSync.question('Enter the APPSMITH_ENCRYPTION_PASSWORD: ', { + hideEchoBack: true + }); + encryptionSalt = readlineSync.question('Enter the APPSMITH_ENCRYPTION_SALT: ', { + hideEchoBack: true + }); + } + + await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_ENCRYPTION_PASSWORD=' + encryptionPwd + + '\nAPPSMITH_ENCRYPTION_SALT=' + encryptionPwd); + + console.log('Restoring docker environment file completed'); +} +async function restoreGitStorageArchive(restoreContentsPath, backupName){ + console.log('Restoring git-storage archive'); + // TODO: Consider APPSMITH_GIT_ROOT env for later iterations + const gitRoot = '/appsmith-stacks/git-storage'; + await utils.execCommand(['mv', gitRoot, gitRoot + '.' + backupName]); + await utils.execCommand(['mv', restoreContentsPath + '/git-storage', '/appsmith-stacks']); + console.log('Restoring git-storage archive completed'); + +} +async function checkRestoreVersionCompatability(restoreContentsPath){ + const content = await fsPromises.readFile('/opt/appsmith/rts/version.js', {encoding: 'utf8'}); + const currentVersion = content.match(/\bexports\.VERSION\s*=\s*["']([^"]+)["']/)[1]; + const manifest_data = await fsPromises.readFile(restoreContentsPath + '/manifest.json', {encoding: 'utf8'}); + const manifest_json = JSON.parse(manifest_data) + const restoreVersion = manifest_json["appsmithVersion"] + console.log('Current Appsmith Version: ' + currentVersion); + console.log('Restore Appsmith Version: ' + restoreVersion); + + if (currentVersion === restoreVersion){ + console.log('The restore instance is compatible with the current appsmith version'); + } else { + console.log('**************************** WARNING ****************************'); + console.log('The Appsmith instance to be restored is not compatible with the current version.'); + console.log('Please update your appsmith image to \"index.docker.io/appsmith/appsmith-ce:' + restoreVersion + + '\" in the \"docker-compose.yml\" file\nand run the cmd: \"docker-compose restart\" ' + + 'after the restore process is completed, to ensure the restored instance runs successfully.'); + const confirm = readlineSync.question('Press Enter to continue \nOr Type "c" to cancel the restore process.\n'); + if (confirm.toLowerCase() === 'c') { + process.exit(0); + } + } +} + +async function run() { + let errorCode = 0; + try { + check_supervisord_status_cmd = '/usr/bin/supervisorctl >/dev/null 2>&1'; + shell.exec(check_supervisord_status_cmd, function (code) { + if (code > 0) { + shell.echo('application is not running, starting supervisord'); + shell.exec('/usr/bin/supervisord'); + } + }); + + const backupFileName = await getBackupFileName(); + if (backupFileName == null) { + process.exit(errorCode); + } else { + const backupFilePath = path.join(Constants.BACKUP_PATH, backupFileName); + const backupName = backupFileName.replace(/\.tar\.gz$/, ""); + const restoreRootPath = await fsPromises.mkdtemp(os.tmpdir()); + const restoreContentsPath = path.join(restoreRootPath, backupName); + + await extractArchive(backupFilePath, restoreRootPath); + await checkRestoreVersionCompatability(restoreContentsPath); + + console.log('****************************************************************'); + console.log('Restoring Appsmith instance from the backup at ' + backupFilePath); + utils.stop(['backend', 'rts']); + await restoreDatabase(restoreContentsPath); + await restoreDockerEnvFile(restoreContentsPath, backupName); + await restoreGitStorageArchive(restoreContentsPath, backupName); + } + } catch (err) { + console.log(err); + errorCode = 1; + + } finally { + utils.start(['backend', 'rts']); + process.exit(errorCode); + + } +} + + +module.exports = { + run, +}; diff --git a/deploy/docker/utils/bin/utils.js b/deploy/docker/utils/bin/utils.js index ce9e69ba81..86b6a54fe7 100644 --- a/deploy/docker/utils/bin/utils.js +++ b/deploy/docker/utils/bin/utils.js @@ -1,4 +1,5 @@ -module.exports = { showHelp: showHelp }; +const shell = require('shelljs'); +const childProcess = require('child_process'); function showHelp() { console.log('\nUsage: appsmith to interactive with appsmith utils tool'); @@ -9,3 +10,56 @@ function showHelp() { console.log('\tcrs, check_replica_set\t\tcheck replica set mongoDB.\r'); console.log('\t--help\t\t\t' + 'Show help.' + '\t\t\t' + '[boolean]\n'); } + +function stop(apps) { + const appsStr = apps.join(' ') + console.log('Stopping ' + appsStr); + shell.exec('/usr/bin/supervisorctl stop ' + appsStr); + console.log('Stopped ' + appsStr); +} + +function start(apps) { + const appsStr = apps.join(' ') + console.log('Starting ' + appsStr); + shell.exec('/usr/bin/supervisorctl start ' + appsStr); + console.log('Started ' + appsStr); +} + +function execCommand(cmd, options) { + return new Promise((resolve, reject) => { + let isPromiseDone = false; + + const p = childProcess.spawn(cmd[0], cmd.slice(1), { + stdio: 'inherit', + ...options, + }); + + p.on('exit', (code) => { + if (isPromiseDone) { + return; + } + isPromiseDone = true; + if (code === 0) { + resolve(); + } else { + reject(); + } + }) + + p.on('error', (err) => { + if (isPromiseDone) { + return; + } + isPromiseDone = true; + log.error('Error rynning command', err); + reject(); + }) + }) +} + +module.exports = { + showHelp, + start, + stop, + execCommand, +};