Protei/PromucFlow_constructor
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore: Modular backup implementation (#37715)

Browse Source 
Backup implementation changed to be modular, and made up of separate
pieces. Each piece (link in a chain) is responsible for one
component/functionality of all the data that's being backed up. This PR
introduces the framework for this modularization. The next PR will
finish migration to that architecture.


## Automation

/test sanity

### 🔍 Cypress test results
<!-- This is an auto-generated comment: Cypress test results  -->
> [!TIP]
> 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉
> Workflow run:
<https://github.com/appsmithorg/appsmith/actions/runs/12063841586>
> Commit: 75c1d787c874ff1dd398b7e7228d062a2c66c141
> <a
href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=12063841586&attempt=1"
target="_blank">Cypress dashboard</a>.
> Tags: `@tag.Sanity`
> Spec:
> <hr>Thu, 28 Nov 2024 07:23:30 UTC
<!-- end of auto-generated comment: Cypress test results  -->


## Communication
Should the DevRel and Marketing teams inform users about this change?
- [ ] Yes
- [x] No


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

## Release Notes

- **New Features**
- Introduced a modular backup system with `BackupState`,
`DiskSpaceLink`, `EncryptionLink`, and `ManifestLink` classes to manage
backup operations more efficiently.
- Added command-line argument support for the backup command, enhancing
flexibility.
- Improved user interaction during backup restoration with prompts for
encryption passwords.

- **Bug Fixes**
- Enhanced error handling and clarity in the restoration process,
particularly for manifest file reading.

- **Documentation**
- Updated test structure to reflect new directory organization and
improved focus on backup cleanup logic.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Shrikant Sharat Kandula 2024-11-28 13:05:31 +05:30 committed by GitHub
parent 5e89edf8c4
commit 6a31cacba5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 232 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
app/client/packages/rts/src/ctl/backup/BackupState.ts Normal file
View File

@ -0,0 +1,25 @@
import { getTimeStampInISO } from "./index";
export class BackupState {
readonly args: string[];
readonly initAt: string = getTimeStampInISO();
readonly errors: string[] = [];
backupRootPath: string = "";
archivePath: string = "";
encryptionPassword: string = "";
constructor(args: string[]) {
this.args = args;
// We seal `this` so that no link in the chain can "add" new properties to the state. This is intentional. If any
// link wants to save data in the `BackupState`, which shouldn't even be needed in most cases, it should do so by
// explicitly declaring a property in this class. No surprises.
Object.seal(this);
}
isEncryptionEnabled() {
return !!this.encryptionPassword;
}
}

90
app/client/packages/rts/src/ctl/backup.test.ts → app/client/packages/rts/src/ctl/backup/backup.test.ts
View File

@ -1,15 +1,14 @@
jest.mock("./utils", () => ({ import fsPromises from "fs/promises";
...jest.requireActual("./utils"), import * as backup from ".";
import * as Constants from "../constants";
import * as utils from "../utils";
import readlineSync from "readline-sync";
jest.mock("../utils", () => ({
...jest.requireActual("../utils"),
execCommand: jest.fn().mockImplementation(async (a) => a.join(" ")), execCommand: jest.fn().mockImplementation(async (a) => a.join(" ")),
})); }));
import * as backup from "./backup";
import * as Constants from "./constants";
import os from "os";
import fsPromises from "fs/promises";
import * as utils from "./utils";
import readlineSync from "readline-sync";
describe("Backup Tests", () => { describe("Backup Tests", () => {
test("Timestamp string in ISO format", () => { test("Timestamp string in ISO format", () => {
console.log(backup.getTimeStampInISO()); console.log(backup.getTimeStampInISO());
@ -46,14 +45,6 @@ describe("Backup Tests", () => {
); );
}); });
it("Generates t", async () => {
os.tmpdir = jest.fn().mockReturnValue("temp/dir");
fsPromises.mkdtemp = jest.fn().mockImplementation((a) => a);
const res = await backup.generateBackupRootPath();
expect(res).toBe("temp/dir/appsmithctl-backup-");
});
test("Test backup contents path generation", () => { test("Test backup contents path generation", () => {
const root = "/rootDir"; const root = "/rootDir";
const timestamp = "0000-00-0T00-00-00.00Z"; const timestamp = "0000-00-0T00-00-00.00Z";
@ -136,67 +127,60 @@ describe("Backup Tests", () => {
}); });
test("Cleanup Backups when limit is 4 and there are 5 files", async () => { test("Cleanup Backups when limit is 4 and there are 5 files", async () => {
const backupArchivesLimit = 4; fsPromises.rm = jest.fn().mockImplementation();
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
const backupFiles = ["file1", "file2", "file3", "file4", "file5"]; const backupFiles = ["file1", "file2", "file3", "file4", "file5"];
const expectedBackupFiles = ["file2", "file3", "file4", "file5"];
const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit);
console.log(res); await backup.removeOldBackups(backupFiles, 4);
expect(res).toEqual(expectedBackupFiles); expect(fsPromises.rm).toHaveBeenCalledTimes(1);
expect(fsPromises.rm).toHaveBeenCalledWith(
Constants.BACKUP_PATH + "/file1",
);
}); });
test("Cleanup Backups when limit is 2 and there are 5 files", async () => { test("Cleanup Backups when limit is 2 and there are 5 files", async () => {
const backupArchivesLimit = 2; fsPromises.rm = jest.fn().mockImplementation();
const backupFiles = ["file1", "file4", "file3", "file2", "file5"];
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); await backup.removeOldBackups(backupFiles, 2);
const backupFiles = ["file1", "file2", "file3", "file4", "file5"];
const expectedBackupFiles = ["file4", "file5"];
const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit);
console.log(res); expect(fsPromises.rm).toHaveBeenCalledTimes(3);
expect(fsPromises.rm).toHaveBeenCalledWith(
expect(res).toEqual(expectedBackupFiles); Constants.BACKUP_PATH + "/file1",
);
expect(fsPromises.rm).toHaveBeenCalledWith(
Constants.BACKUP_PATH + "/file2",
);
expect(fsPromises.rm).toHaveBeenCalledWith(
Constants.BACKUP_PATH + "/file3",
);
}); });
test("Cleanup Backups when limit is 4 and there are 4 files", async () => { test("Cleanup Backups when limit is 4 and there are 4 files", async () => {
const backupArchivesLimit = 4; fsPromises.rm = jest.fn().mockImplementation();
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
const backupFiles = ["file1", "file2", "file3", "file4"]; const backupFiles = ["file1", "file2", "file3", "file4"];
const expectedBackupFiles = ["file1", "file2", "file3", "file4"];
const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit);
console.log(res); await backup.removeOldBackups(backupFiles, 4);
expect(res).toEqual(expectedBackupFiles); expect(fsPromises.rm).not.toHaveBeenCalled();
}); });
test("Cleanup Backups when limit is 4 and there are 2 files", async () => { test("Cleanup Backups when limit is 4 and there are 2 files", async () => {
const backupArchivesLimit = 4; fsPromises.rm = jest.fn().mockImplementation();
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
const backupFiles = ["file1", "file2"]; const backupFiles = ["file1", "file2"];
const expectedBackupFiles = ["file1", "file2"];
const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit);
console.log(res); await backup.removeOldBackups(backupFiles, 4);
expect(res).toEqual(expectedBackupFiles); expect(fsPromises.rm).not.toHaveBeenCalled();
}); });
test("Cleanup Backups when limit is 2 and there is 1 file", async () => { test("Cleanup Backups when limit is 2 and there is 1 file", async () => {
const backupArchivesLimit = 4; fsPromises.rm = jest.fn().mockImplementation();
fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a));
const backupFiles = ["file1"]; const backupFiles = ["file1"];
const expectedBackupFiles = ["file1"];
const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit);
console.log(res); await backup.removeOldBackups(backupFiles, 4);
expect(res).toEqual(expectedBackupFiles);
expect(fsPromises.rm).not.toHaveBeenCalled();
}); });
test("Cleanup Backups when limit is 2 and there is no file", async () => { test("Cleanup Backups when limit is 2 and there is no file", async () => {

159
app/client/packages/rts/src/ctl/backup.ts → app/client/packages/rts/src/ctl/backup/index.ts
View File

@ -1,64 +1,50 @@
import fsPromises from "fs/promises"; import fsPromises from "fs/promises";
import path from "path"; import path from "path";
import os from "os"; import os from "os";
import * as utils from "./utils"; import * as utils from "../utils";
import * as Constants from "./constants"; import * as Constants from "../constants";
import * as logger from "./logger"; import * as logger from "../logger";
import * as mailer from "./mailer"; import * as mailer from "../mailer";
import tty from "tty";
import readlineSync from "readline-sync"; import readlineSync from "readline-sync";
import { DiskSpaceLink } from "./links/DiskSpaceLink";
import type { Link } from "./links";
import { EncryptionLink, ManifestLink } from "./links";
import { BackupState } from "./BackupState";
const command_args = process.argv.slice(3); export async function run(args: string[]) {
class BackupState {
readonly initAt: string = getTimeStampInISO();
readonly errors: string[] = [];
backupRootPath: string = "";
archivePath: string = "";
encryptionPassword: string = "";
isEncryptionEnabled() {
return !!this.encryptionPassword;
}
}
export async function run() {
await utils.ensureSupervisorIsRunning(); await utils.ensureSupervisorIsRunning();
const state: BackupState = new BackupState(); const state: BackupState = new BackupState(args);
const chain: Link[] = [
new DiskSpaceLink(),
new ManifestLink(state),
new EncryptionLink(state),
];
try { try {
// PRE-BACKUP // PRE-BACKUP
const availSpaceInBytes: number = for (const link of chain) {
await getAvailableBackupSpaceInBytes("/appsmith-stacks"); await link.preBackup?.();
checkAvailableBackupSpace(availSpaceInBytes);
if (
!command_args.includes("--non-interactive") &&
tty.isatty((process.stdout as any).fd)
) {
state.encryptionPassword = getEncryptionPasswordFromUser();
} }
state.backupRootPath = await generateBackupRootPath(); // BACKUP
const backupContentsPath: string = getBackupContentsPath( state.backupRootPath = await fsPromises.mkdtemp(
state.backupRootPath, path.join(os.tmpdir(), "appsmithctl-backup-"),
state.initAt,
); );
// BACKUP await exportDatabase(state.backupRootPath);
await fsPromises.mkdir(backupContentsPath);
await exportDatabase(backupContentsPath); await createGitStorageArchive(state.backupRootPath);
await createGitStorageArchive(backupContentsPath); await exportDockerEnvFile(
state.backupRootPath,
state.isEncryptionEnabled(),
);
await createManifestFile(backupContentsPath); for (const link of chain) {
await link.doBackup?.();
await exportDockerEnvFile(backupContentsPath, state.isEncryptionEnabled()); }
state.archivePath = await createFinalArchive( state.archivePath = await createFinalArchive(
state.backupRootPath, state.backupRootPath,
@ -66,27 +52,13 @@ export async function run() {
); );
// POST-BACKUP // POST-BACKUP
if (state.isEncryptionEnabled()) { for (const link of chain) {
const encryptedArchivePath = await encryptBackupArchive( await link.postBackup?.();
state.archivePath,
state.encryptionPassword,
);
await logger.backup_info(
"Finished creating an encrypted a backup archive at " +
encryptedArchivePath,
);
if (state.archivePath != null) {
await fsPromises.rm(state.archivePath, {
recursive: true,
force: true,
});
} }
} else {
await logger.backup_info( console.log("Post-backup done. Final archive at", state.archivePath);
"Finished creating a backup archive at " + state.archivePath,
); if (!state.isEncryptionEnabled()) {
console.log( console.log(
"********************************************************* IMPORTANT!!! *************************************************************", "********************************************************* IMPORTANT!!! *************************************************************",
); );
@ -110,7 +82,7 @@ export async function run() {
process.exitCode = 1; process.exitCode = 1;
await logger.backup_error(err.stack); await logger.backup_error(err.stack);
if (command_args.includes("--error-mail")) { if (state.args.includes("--error-mail")) {
const currentTS = new Date().getTime(); const currentTS = new Date().getTime();
const lastMailTS = await utils.getLastBackupErrorMailSentInMilliSec(); const lastMailTS = await utils.getLastBackupErrorMailSentInMilliSec();
@ -123,21 +95,20 @@ export async function run() {
await utils.updateLastBackupErrorMailSentInMilliSec(currentTS); await utils.updateLastBackupErrorMailSentInMilliSec(currentTS);
} }
} }
} finally {
if (state.backupRootPath != null) {
await fsPromises.rm(state.backupRootPath, {
recursive: true,
force: true,
});
}
if (state.isEncryptionEnabled()) { // Delete the archive, if exists, since its existence may mislead the user.
if (state.archivePath != null) { if (state.archivePath != null) {
await fsPromises.rm(state.archivePath, { await fsPromises.rm(state.archivePath, {
recursive: true, recursive: true,
force: true, force: true,
}); });
} }
} finally {
if (state.backupRootPath != null) {
await fsPromises.rm(state.backupRootPath, {
recursive: true,
force: true,
});
} }
await postBackupCleanup(); await postBackupCleanup();
@ -222,19 +193,6 @@ async function createGitStorageArchive(destFolder: string) {
console.log("Created git-storage archive"); console.log("Created git-storage archive");
} }
async function createManifestFile(path: string) {
const version = await utils.getCurrentAppsmithVersion();
const manifest_data = {
appsmithVersion: version,
dbName: utils.getDatabaseNameFromMongoURI(utils.getDburl()),
};
await fsPromises.writeFile(
path + "/manifest.json",
JSON.stringify(manifest_data),
);
}
async function exportDockerEnvFile( async function exportDockerEnvFile(
destFolder: string, destFolder: string,
encryptArchive: boolean, encryptArchive: boolean,
@ -291,19 +249,15 @@ async function createFinalArchive(destFolder: string, timestamp: string) {
} }
async function postBackupCleanup() { async function postBackupCleanup() {
console.log("Starting the cleanup task after taking a backup."); console.log("Starting cleanup.");
const backupArchivesLimit = getBackupArchiveLimit( const backupArchivesLimit = getBackupArchiveLimit(
parseInt(process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT, 10), parseInt(process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT, 10),
); );
const backupFiles = await utils.listLocalBackupFiles(); const backupFiles = await utils.listLocalBackupFiles();
while (backupFiles.length > backupArchivesLimit) { await removeOldBackups(backupFiles, backupArchivesLimit);
const fileName = backupFiles.shift();
await fsPromises.rm(Constants.BACKUP_PATH + "/" + fileName); console.log("Cleanup completed.");
}
console.log("Cleanup task completed.");
} }
export async function executeCopyCMD(srcFolder: string, destFolder: string) { export async function executeCopyCMD(srcFolder: string, destFolder: string) {
@ -323,10 +277,6 @@ export function getGitRoot(gitRoot?: string | undefined) {
return gitRoot; return gitRoot;
} }
export async function generateBackupRootPath() {
return fsPromises.mkdtemp(path.join(os.tmpdir(), "appsmithctl-backup-"));
}
export function getBackupContentsPath( export function getBackupContentsPath(
backupRootPath: string, backupRootPath: string,
timestamp: string, timestamp: string,
@ -359,13 +309,14 @@ export async function removeOldBackups(
backupFiles: string[], backupFiles: string[],
backupArchivesLimit: number, backupArchivesLimit: number,
) { ) {
while (backupFiles.length > backupArchivesLimit) { return Promise.all(
const fileName = backupFiles.shift(); backupFiles
.sort()
await fsPromises.rm(Constants.BACKUP_PATH + "/" + fileName); .reverse()
} .slice(backupArchivesLimit)
.map((file) => Constants.BACKUP_PATH + "/" + file)
return backupFiles; .map(async (file) => fsPromises.rm(file)),
);
} }
export function getTimeStampInISO() { export function getTimeStampInISO() {

11
app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts Normal file
View File

@ -0,0 +1,11 @@
import { checkAvailableBackupSpace, getAvailableBackupSpaceInBytes } from "..";
import type { Link } from ".";
export class DiskSpaceLink implements Link {
async preBackup() {
const availSpaceInBytes: number =
await getAvailableBackupSpaceInBytes("/appsmith-stacks");
checkAvailableBackupSpace(availSpaceInBytes);
}
}

36
app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts Normal file
View File

@ -0,0 +1,36 @@
import type { Link } from "./index";
import tty from "tty";
import fsPromises from "fs/promises";
import { encryptBackupArchive, getEncryptionPasswordFromUser } from "../index";
import type { BackupState } from "../BackupState";
export class EncryptionLink implements Link {
constructor(private readonly state: BackupState) {}
async preBackup() {
if (
!this.state.args.includes("--non-interactive") &&
tty.isatty((process.stdout as any).fd)
) {
this.state.encryptionPassword = getEncryptionPasswordFromUser();
}
}
async postBackup() {
if (!this.state.isEncryptionEnabled()) {
return;
}
const unencryptedArchivePath = this.state.archivePath;
this.state.archivePath = await encryptBackupArchive(
unencryptedArchivePath,
this.state.encryptionPassword,
);
await fsPromises.rm(unencryptedArchivePath, {
recursive: true,
force: true,
});
}
}

22
app/client/packages/rts/src/ctl/backup/links/ManifestLink.ts Normal file
View File

@ -0,0 +1,22 @@
import type { Link } from "./index";
import type { BackupState } from "../BackupState";
import * as utils from "../../utils";
import fsPromises from "fs/promises";
import path from "path";
export class ManifestLink implements Link {
constructor(private readonly state: BackupState) {}
async doBackup() {
const version = await utils.getCurrentAppsmithVersion();
const manifestData = {
appsmithVersion: version,
dbName: utils.getDatabaseNameFromMongoURI(utils.getDburl()),
};
await fsPromises.writeFile(
path.join(this.state.backupRootPath, "/manifest.json"),
JSON.stringify(manifestData, null, 2),
);
}
}

13
app/client/packages/rts/src/ctl/backup/links/index.ts Normal file
View File

@ -0,0 +1,13 @@
export interface Link {
// Called before the backup folder is created.
preBackup?(): Promise<void>;
// Called after backup folder is created. Expected to copy/create any backup files in the backup folder.
doBackup?(): Promise<void>;
// Called after backup archive is created. The archive location is available now.
postBackup?(): Promise<void>;
}
export { EncryptionLink } from "./EncryptionLink";
export { ManifestLink } from "./ManifestLink";

2
app/client/packages/rts/src/ctl/index.ts
View File

@ -51,7 +51,7 @@ if (["export-db", "export_db", "ex"].includes(command)) {
) { ) {
check_replica_set.exec(); check_replica_set.exec();
} else if (["backup"].includes(command)) { } else if (["backup"].includes(command)) {
backup.run(); backup.run(process.argv.slice(3));
} else if (["restore"].includes(command)) { } else if (["restore"].includes(command)) {
restore.run(); restore.run();
} else if ( } else if (

31
app/client/packages/rts/src/ctl/restore.ts
View File

@ -61,14 +61,15 @@ async function decryptArchive(
encryptedFilePath: string, encryptedFilePath: string,
backupFilePath: string, backupFilePath: string,
) { ) {
console.log("Enter the password to decrypt the backup archive:");
for (const attempt of [1, 2, 3]) { for (const attempt of [1, 2, 3]) {
if (attempt > 1) { if (attempt > 1) {
console.log("Retry attempt", attempt); console.log("Retry attempt", attempt);
} }
const decryptionPwd = readlineSync.question("", { hideEchoBack: true }); const decryptionPwd = readlineSync.question(
"Enter the password to decrypt the backup archive: ",
{ hideEchoBack: true },
);
try { try {
await utils.execCommandSilent([ await utils.execCommandSilent([
@ -150,15 +151,15 @@ async function restoreDockerEnvFile(
let encryptionSalt = process.env.APPSMITH_ENCRYPTION_SALT; let encryptionSalt = process.env.APPSMITH_ENCRYPTION_SALT;
await utils.execCommand([ await utils.execCommand([
"mv", "cp",
dockerEnvFile, dockerEnvFile,
dockerEnvFile + "." + backupName, dockerEnvFile + "." + backupName,
]); ]);
await utils.execCommand([
"cp", let dockerEnvContent = await fsPromises.readFile(
restoreContentsPath + "/docker.env", restoreContentsPath + "/docker.env",
dockerEnvFile, "utf8",
]); );
if (overwriteEncryptionKeys) { if (overwriteEncryptionKeys) {
if (encryptionPwd && encryptionSalt) { if (encryptionPwd && encryptionSalt) {
@ -202,8 +203,7 @@ async function restoreDockerEnvFile(
); );
} }
await fsPromises.appendFile( dockerEnvContent +=
dockerEnvFile,
"\nAPPSMITH_ENCRYPTION_PASSWORD=" + "\nAPPSMITH_ENCRYPTION_PASSWORD=" +
encryptionPwd + encryptionPwd +
"\nAPPSMITH_ENCRYPTION_SALT=" + "\nAPPSMITH_ENCRYPTION_SALT=" +
@ -213,20 +213,19 @@ async function restoreDockerEnvFile(
"\nAPPSMITH_MONGODB_USER=" + "\nAPPSMITH_MONGODB_USER=" +
process.env.APPSMITH_MONGODB_USER + process.env.APPSMITH_MONGODB_USER +
"\nAPPSMITH_MONGODB_PASSWORD=" + "\nAPPSMITH_MONGODB_PASSWORD=" +
process.env.APPSMITH_MONGODB_PASSWORD, process.env.APPSMITH_MONGODB_PASSWORD;
);
} else { } else {
await fsPromises.appendFile( dockerEnvContent +=
dockerEnvFile,
"\nAPPSMITH_DB_URL=" + "\nAPPSMITH_DB_URL=" +
updatedbUrl + updatedbUrl +
"\nAPPSMITH_MONGODB_USER=" + "\nAPPSMITH_MONGODB_USER=" +
process.env.APPSMITH_MONGODB_USER + process.env.APPSMITH_MONGODB_USER +
"\nAPPSMITH_MONGODB_PASSWORD=" + "\nAPPSMITH_MONGODB_PASSWORD=" +
process.env.APPSMITH_MONGODB_PASSWORD, process.env.APPSMITH_MONGODB_PASSWORD;
);
} }
await fsPromises.writeFile(dockerEnvFile, dockerEnvContent, "utf8");
console.log("Restoring docker environment file completed"); console.log("Restoring docker environment file completed");
} }

5
deploy/docker/fs/opt/bin/ctl
View File

@ -3,4 +3,7 @@
# Do NOT change working directory with `cd`, so that the command being run has access to the working directory where # Do NOT change working directory with `cd`, so that the command being run has access to the working directory where
# the command was invoked by the user. # the command was invoked by the user.
exec node /opt/appsmith/rts/bundle/ctl/index.js "$@" exec node \
--enable-source-maps \
/opt/appsmith/rts/bundle/ctl/index.js \
"$@"