Auto-backup and cleanup with appsmithctl (#15203)

- Auto-cleanup of backup files using env variable APPSMITH_BACKUP_ARCHIVE_LIMIT (default value is 4) after every backup.
- Updated docker file to include watchtower hook scripts.
- Error mail interval to 6hrs.

Dockerfile

@@ -90,6 +90,12 @@ RUN find / \( -path /proc -prune \) -o \( \( -perm -2000 -o -perm -4000 \) -prin
 
 # Update path to load appsmith utils tool as default
 ENV PATH /opt/appsmith/utils/node_modules/.bin:$PATH
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-check=/watchtower-hooks/pre-check.sh
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-update=/watchtower-hooks/pre-update.sh
+COPY ./deploy/docker/watchtower-hooks /watchtower-hooks
+RUN chmod +x /watchtower-hooks/pre-check.sh
+RUN chmod +x /watchtower-hooks/pre-update.sh
+
 
 EXPOSE 80
 EXPOSE 443

deploy/docker/utils/bin/backup.js

@@ -50,15 +50,20 @@ async function run() {
 
     await fsPromises.rm(backupRootPath, { recursive: true, force: true });
 
-    console.log('Finished taking a baceup at', archivePath);
+    console.log('Finished taking a backup at', archivePath);
-    // console.log('Please remember to also take the `docker.env` separately since it includes sensitive, but critical information.')
+    await postBackupCleanup();
 
   } catch (err) {
     errorCode = 1;
     await logger.backup_error(err.stack);
 
     if (command_args.includes('--error-mail')) {
-      await mailer.sendBackupErrorToAdmins(err, timestamp);
+      const currentTS = new Date().getTime();
+      const lastMailTS = await utils.getLastBackupErrorMailSentInMilliSec();
+      if ((lastMailTS + Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC) < currentTS){
+        await mailer.sendBackupErrorToAdmins(err, timestamp);
+        await utils.updateLastBackupErrorMailSentInMilliSec(currentTS);
+      }
     }
   } finally {
     utils.start(['backend', 'rts']);
@@ -121,6 +126,20 @@ async function createFinalArchive(destFolder, timestamp) {
   return archive;
 }
 
+async function postBackupCleanup(){
+  console.log('Starting the cleanup task after taking a backup.');
+  let backupArchivesLimit = process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT;
+  if(!backupArchivesLimit)
+    backupArchivesLimit = 4;
+  const backupFiles = await utils.listLocalBackupFiles();
+  while (backupFiles.length > backupArchivesLimit){
+    const fileName = backupFiles.shift();
+    await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
+  }
+  console.log('Cleanup task completed.');
+
+}
+
 module.exports = {
   run,
 };

deploy/docker/utils/bin/constants.js

@@ -5,14 +5,20 @@ const RESTORE_PATH = "/appsmith-stacks/data/restore"
 
 const DUMP_FILE_NAME = "appsmith-data.archive"
 
-const BACKUP_ERROR_LOG_PATH = '/appsmith-stacks/logs/backup'
+const BACKUP_ERROR_LOG_PATH = "/appsmith-stacks/logs/backup"
 
-const MIN_REQUIRED_DISK_SPACE_IN_BYTES = 5368709120 // 5GB
+const LAST_ERROR_MAIL_TS = "/appsmith-stacks/data/backup/last-error-mail-ts"
+
+const MIN_REQUIRED_DISK_SPACE_IN_BYTES = 2147483648 // 2GB
+
+const DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC = 21600000 // 6 hrs
 
 module.exports = {
     BACKUP_PATH,
     RESTORE_PATH,
     DUMP_FILE_NAME,
+    LAST_ERROR_MAIL_TS,
     BACKUP_ERROR_LOG_PATH,
     MIN_REQUIRED_DISK_SPACE_IN_BYTES,
+    DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC,
 }

deploy/docker/utils/bin/utils.js

@@ -82,10 +82,27 @@ async function listLocalBackupFiles() {
   return backupFiles;
 }
 
+
+async function updateLastBackupErrorMailSentInMilliSec(ts) {
+  await fsPromises.mkdir(Constants.BACKUP_PATH, { recursive: true });
+  await fsPromises.writeFile(Constants.LAST_ERROR_MAIL_TS, ts.toString());
+}
+
+async function getLastBackupErrorMailSentInMilliSec() {
+  try {
+    const ts = await fsPromises.readFile(Constants.LAST_ERROR_MAIL_TS);
+    return parseInt(ts, 10);
+  } catch (error) {
+    return 0;
+  }
+}
+
 module.exports = {
   showHelp,
   start,
   stop,
   execCommand,
   listLocalBackupFiles,
+  updateLastBackupErrorMailSentInMilliSec,
+  getLastBackupErrorMailSentInMilliSec,
 };

deploy/docker/watchtower-hooks/pre-check.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+exit 0

deploy/docker/watchtower-hooks/pre-update.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+appsmithctl backup --error-mail || exit 1